diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 00000000000..68db3a9248b --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,85 @@ +((java-mode + . + ((eval + . + (progn + (defun my/point-in-defun-declaration-p () + (let ((bod (save-excursion (c-beginning-of-defun) + (point)))) + (<= bod + (point) + (save-excursion (goto-char bod) + (re-search-forward "{") + (point))))) + + (defun my/is-string-concatenation-p () + "Returns true if the previous line is a string concatenation" + (save-excursion + (let ((start (point))) + (forward-line -1) + (if (re-search-forward " \\\+$" start t) t nil)))) + + (defun my/inside-java-lambda-p () + "Returns true if point is the first statement inside of a lambda" + (save-excursion + (c-beginning-of-statement-1) + (let ((start (point))) + (forward-line -1) + (if (search-forward " -> {" start t) t nil)))) + + (defun my/trailing-paren-p () + "Returns true if point is a training paren and semicolon" + (save-excursion + (end-of-line) + (let ((endpoint (point))) + (beginning-of-line) + (if (re-search-forward "[ ]*);$" endpoint t) t nil)))) + + (defun my/prev-line-call-with-no-args-p () + "Return true if the previous line is a function call with no arguments" + (save-excursion + (let ((start (point))) + (forward-line -1) + (if (re-search-forward ".($" start t) t nil)))) + + (defun my/arglist-cont-nonempty-indentation (arg) + (if (my/inside-java-lambda-p) + '+ + (if (my/is-string-concatenation-p) + 16 + (unless (my/point-in-defun-declaration-p) '++)))) + + (defun my/statement-block-intro (arg) + (if (and (c-at-statement-start-p) (my/inside-java-lambda-p)) 0 '+)) + + (defun my/block-close (arg) + (if (my/inside-java-lambda-p) '- 0)) + + (defun my/arglist-close (arg) (if (my/trailing-paren-p) 0 '--)) + + (defun my/arglist-intro (arg) + (if (my/prev-line-call-with-no-args-p) '++ 0)) + + (c-set-offset 'inline-open 0) + (c-set-offset 'topmost-intro-cont '+) + (c-set-offset 'statement-block-intro 'my/statement-block-intro) + (c-set-offset 'block-close 'my/block-close) + (c-set-offset 'knr-argdecl-intro '+) + (c-set-offset 'substatement-open '+) + (c-set-offset 'substatement-label '+) + (c-set-offset 'case-label '+) + (c-set-offset 'label '+) + (c-set-offset 'statement-case-open '+) + (c-set-offset 'statement-cont '++) + (c-set-offset 'arglist-intro 'my/arglist-intro) + (c-set-offset 'arglist-cont-nonempty '(my/arglist-cont-nonempty-indentation c-lineup-arglist)) + (c-set-offset 'arglist-close 'my/arglist-close) + (c-set-offset 'inexpr-class 0) + (c-set-offset 'access-label 0) + (c-set-offset 'inher-intro '++) + (c-set-offset 'inher-cont '++) + (c-set-offset 'brace-list-intro '+) + (c-set-offset 'func-decl-cont '++) + )) + (c-basic-offset . 4) + (c-comment-only-line-offset . (0 . 0))))) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000..9d4bfbf55d3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +# EditorConfig: http://editorconfig.org/ + +root = true + +[*.java] +charset = utf-8 +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/.gitignore b/.gitignore index d8798cd9694..31f2aa5fc66 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,8 @@ work/ logs/ .DS_Store build/ -target/ -*-execution-hints.log +generated-resources/ +**/.local* docs/html/ docs/build.log /tmp/ @@ -31,3 +31,7 @@ nb-configuration.xml nbactions.xml dependency-reduced-pom.xml + +# old patterns specific to maven +*-execution-hints.log +target/ diff --git a/.projectile b/.projectile new file mode 100644 index 00000000000..d2a5e762a88 --- /dev/null +++ b/.projectile @@ -0,0 +1,32 @@ +-/target +-/core/target +-/qa/target +-/rest-api-spec/target +-/test-framework/target +-/plugins/target +-/plugins/analysis-icu/target +-/plugins/analysis-kuromoji/target +-/plugins/analysis-phonetic/target +-/plugins/analysis-smartcn/target +-/plugins/analysis-stempel/target +-/plugins/cloud-aws/target +-/plugins/cloud-azure/target +-/plugins/cloud-gce/target +-/plugins/delete-by-query/target +-/plugins/discovery-azure/target +-/plugins/discovery-ec2/target +-/plugins/discovery-gce/target +-/plugins/discovery-multicast/target +-/plugins/jvm-example/target +-/plugins/lang-expression/target +-/plugins/lang-groovy/target +-/plugins/lang-javascript/target +-/plugins/lang-python/target +-/plugins/mapper-murmur3/target +-/plugins/mapper-size/target +-/plugins/repository-azure/target +-/plugins/repository-s3/target +-/plugins/site-example/target +-/plugins/store-smb/target +-/plugins/target +-*.class diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index f1163fdd583..00000000000 --- a/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,18 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=enabled -org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore -org.eclipse.jdt.core.compiler.annotation.nullable=org.elasticsearch.common.Nullable -org.eclipse.jdt.core.compiler.annotation.nullanalysis=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.compliance=1.7 -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning -org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=warning -org.eclipse.jdt.core.compiler.problem.nullReference=warning -org.eclipse.jdt.core.compiler.problem.nullSpecViolation=warning -org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning -org.eclipse.jdt.core.compiler.problem.potentialNullReference=warning -org.eclipse.jdt.core.compiler.source=1.7 -org.eclipse.jdt.core.formatter.lineSplit=140 -org.eclipse.jdt.core.formatter.tabulation.char=space -org.eclipse.jdt.core.formatter.tabulation.size=4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4491737e68e..507a27a5912 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,9 +76,7 @@ Contributing to the Elasticsearch codebase **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch) -Make sure you have [Maven](http://maven.apache.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE by running `mvn eclipse:eclipse` and then importing the project into their workspace: `File > Import > Existing project into workspace` and make sure to select `Search for nested projects...` option as Elasticsearch is a multi-module maven project. Additionally you will want to ensure that Eclipse is using 2048m of heap by modifying `eclipse.ini` accordingly to avoid GC overhead errors. Please make sure the [m2e-connector](http://marketplace.eclipse.org/content/m2e-connector-maven-dependency-plugin) is not installed in your Eclipse distribution as it will interfere with setup performed by `mvn eclipse:eclipse`. - -Elasticsearch also works perfectly with Eclipse's [m2e](http://www.eclipse.org/m2e/). Once you've installed m2e you can import Elasticsearch as an `Existing Maven Project`. +Make sure you have [Gradle](http://gradle.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE: `gradle eclipse` then `File: Import: Existing Projects into Workspace`. Select the option `Search for nested projects`. Additionally you will want to ensure that Eclipse is using 2048m of heap by modifying `eclipse.ini` accordingly to avoid GC overhead errors. Please follow these formatting guidelines: @@ -92,15 +90,15 @@ To create a distribution from the source, simply run: ```sh cd elasticsearch/ -mvn clean package -DskipTests +gradle assemble ``` -You will find the newly built packages under: `./target/releases/`. +You will find the newly built packages under: `./distribution/build/distributions/`. Before submitting your changes, run the test suite to make sure that nothing is broken, with: ```sh -mvn clean test -Dtests.slow=true +gradle check ``` Source: [Contributing to elasticsearch](https://www.elastic.co/contributing-to-elasticsearch/) diff --git a/GRADLE.CHEATSHEET b/GRADLE.CHEATSHEET new file mode 100644 index 00000000000..3362b8571e7 --- /dev/null +++ b/GRADLE.CHEATSHEET @@ -0,0 +1,7 @@ +As a quick helper, below are the equivalent commands from maven to gradle (TESTING.md has also been updated). You can also run "gradle tasks" to see all tasks that are available to run. +clean -> clean +test -> test +verify -> check +verify -Dskip.unit.tests -> integTest +package -DskipTests -> assemble +install -DskipTests -> install diff --git a/README.textile b/README.textile index 63f1841822c..804f46a1811 100644 --- a/README.textile +++ b/README.textile @@ -200,19 +200,22 @@ We have just covered a very small portion of what Elasticsearch is all about. Fo h3. Building from Source -Elasticsearch uses "Maven":http://maven.apache.org for its build system. +Elasticsearch uses "Gradle":http://gradle.org for its build system. You'll need to have a modern version of Gradle installed - 2.8 should do. -In order to create a distribution, simply run the @mvn clean package --DskipTests@ command in the cloned directory. +In order to create a distribution, simply run the @gradle build@ command in the cloned directory. The distribution for each project will be created under the @target/releases@ directory in that project. See the "TESTING":TESTING.asciidoc file for more information about running the Elasticsearch test suite. -h3. Upgrading to Elasticsearch 1.x? +h3. Upgrading from Elasticsearch 1.x? -In order to ensure a smooth upgrade process from earlier versions of Elasticsearch (< 1.0.0), it is recommended to perform a full cluster restart. Please see the "setup reference":https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html for more details on the upgrade process. +In order to ensure a smooth upgrade process from earlier versions of +Elasticsearch (1.x), it is required to perform a full cluster restart. Please +see the "setup reference": +https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html +for more details on the upgrade process. h1. License diff --git a/TESTING.asciidoc b/TESTING.asciidoc index 66eb83fcf20..569c16b0747 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -13,7 +13,7 @@ To create a distribution without running the tests, simply run the following: ----------------------------- -mvn clean package -DskipTests +gradle assemble ----------------------------- == Other test options @@ -35,7 +35,7 @@ Use local transport (default since 1.3): Alternatively, you can set the `ES_TEST_LOCAL` environment variable: ------------------------------------- -export ES_TEST_LOCAL=true && mvn test +export ES_TEST_LOCAL=true && gradle test ------------------------------------- === Running Elasticsearch from a checkout @@ -44,7 +44,7 @@ In order to run Elasticsearch from source without building a package, you can run it using Maven: ------------------------------------- -./run.sh +gradle run ------------------------------------- === Test case filtering. @@ -55,20 +55,20 @@ run it using Maven: Run a single test case (variants) ---------------------------------------------------------- -mvn test -Dtests.class=org.elasticsearch.package.ClassName -mvn test "-Dtests.class=*.ClassName" +gradle test -Dtests.class=org.elasticsearch.package.ClassName +gradle test "-Dtests.class=*.ClassName" ---------------------------------------------------------- Run all tests in a package and sub-packages ---------------------------------------------------- -mvn test "-Dtests.class=org.elasticsearch.package.*" +gradle test "-Dtests.class=org.elasticsearch.package.*" ---------------------------------------------------- Run any test methods that contain 'esi' (like: ...r*esi*ze...). ------------------------------- -mvn test "-Dtests.method=*esi*" +gradle test "-Dtests.method=*esi*" ------------------------------- You can also filter tests by certain annotations ie: @@ -81,7 +81,7 @@ You can also filter tests by certain annotations ie: Those annotation names can be combined into a filter expression like: ------------------------------------------------ -mvn test -Dtests.filter="@nightly and not @backwards" +gradle test -Dtests.filter="@nightly and not @backwards" ------------------------------------------------ to run all nightly test but not the ones that are backwards tests. `tests.filter` supports @@ -89,7 +89,7 @@ the boolean operators `and, or, not` and grouping ie: --------------------------------------------------------------- -mvn test -Dtests.filter="@nightly and not(@badapple or @backwards)" +gradle test -Dtests.filter="@nightly and not(@badapple or @backwards)" --------------------------------------------------------------- === Seed and repetitions. @@ -97,7 +97,7 @@ mvn test -Dtests.filter="@nightly and not(@badapple or @backwards)" Run with a given seed (seed is a hex-encoded long). ------------------------------ -mvn test -Dtests.seed=DEADBEEF +gradle test -Dtests.seed=DEADBEEF ------------------------------ === Repeats _all_ tests of ClassName N times. @@ -106,7 +106,7 @@ Every test repetition will have a different method seed (derived from a single random master seed). -------------------------------------------------- -mvn test -Dtests.iters=N -Dtests.class=*.ClassName +gradle test -Dtests.iters=N -Dtests.class=*.ClassName -------------------------------------------------- === Repeats _all_ tests of ClassName N times. @@ -115,7 +115,7 @@ Every test repetition will have exactly the same master (0xdead) and method-level (0xbeef) seed. ------------------------------------------------------------------------ -mvn test -Dtests.iters=N -Dtests.class=*.ClassName -Dtests.seed=DEAD:BEEF +gradle test -Dtests.iters=N -Dtests.class=*.ClassName -Dtests.seed=DEAD:BEEF ------------------------------------------------------------------------ === Repeats a given test N times @@ -125,14 +125,14 @@ ie: testFoo[0], testFoo[1], etc... so using testmethod or tests.method ending in a glob is necessary to ensure iterations are run). ------------------------------------------------------------------------- -mvn test -Dtests.iters=N -Dtests.class=*.ClassName -Dtests.method=mytest* +gradle test -Dtests.iters=N -Dtests.class=*.ClassName -Dtests.method=mytest* ------------------------------------------------------------------------- Repeats N times but skips any tests after the first failure or M initial failures. ------------------------------------------------------------- -mvn test -Dtests.iters=N -Dtests.failfast=true -Dtestcase=... -mvn test -Dtests.iters=N -Dtests.maxfailures=M -Dtestcase=... +gradle test -Dtests.iters=N -Dtests.failfast=true -Dtestcase=... +gradle test -Dtests.iters=N -Dtests.maxfailures=M -Dtestcase=... ------------------------------------------------------------- === Test groups. @@ -142,24 +142,30 @@ Test groups can be enabled or disabled (true/false). Default value provided below in [brackets]. ------------------------------------------------------------------ -mvn test -Dtests.nightly=[false] - nightly test group (@Nightly) -mvn test -Dtests.weekly=[false] - weekly tests (@Weekly) -mvn test -Dtests.awaitsfix=[false] - known issue (@AwaitsFix) +gradle test -Dtests.nightly=[false] - nightly test group (@Nightly) +gradle test -Dtests.weekly=[false] - weekly tests (@Weekly) +gradle test -Dtests.awaitsfix=[false] - known issue (@AwaitsFix) ------------------------------------------------------------------ === Load balancing and caches. -By default, the tests run sequentially on a single forked JVM. - -To run with more forked JVMs than the default use: +By default the tests run on up to 4 JVMs based on the number of cores. If you +want to explicitly specify the number of JVMs you can do so on the command +line: ---------------------------- -mvn test -Dtests.jvms=8 test +gradle test -Dtests.jvms=8 ---------------------------- -Don't count hypercores for CPU-intense tests and leave some slack -for JVM-internal threads (like the garbage collector). Make sure there is -enough RAM to handle child JVMs. +Or in `~/.gradle/gradle.properties`: + +---------------------------- +systemProp.tests.jvms=8 +---------------------------- + +Its difficult to pick the "right" number here. Hypercores don't count for CPU +intensive tests and you should leave some slack for JVM-interal threads like +the garbage collector. And you have to have enough RAM to handle each JVM. === Test compatibility. @@ -167,7 +173,7 @@ It is possible to provide a version that allows to adapt the tests behaviour to older features or bugs that have been changed or fixed in the meantime. ----------------------------------------- -mvn test -Dtests.compatibility=1.0.0 +gradle test -Dtests.compatibility=1.0.0 ----------------------------------------- @@ -176,45 +182,50 @@ mvn test -Dtests.compatibility=1.0.0 Run all tests without stopping on errors (inspect log files). ----------------------------------------- -mvn test -Dtests.haltonfailure=false test +gradle test -Dtests.haltonfailure=false ----------------------------------------- Run more verbose output (slave JVM parameters, etc.). ---------------------- -mvn test -verbose test +gradle test -verbose ---------------------- Change the default suite timeout to 5 seconds for all tests (note the exclamation mark). --------------------------------------- -mvn test -Dtests.timeoutSuite=5000! ... +gradle test -Dtests.timeoutSuite=5000! ... --------------------------------------- -Change the logging level of ES (not mvn) +Change the logging level of ES (not gradle) -------------------------------- -mvn test -Des.logger.level=DEBUG +gradle test -Des.logger.level=DEBUG -------------------------------- Print all the logging output from the test runs to the commandline even if tests are passing. ------------------------------ -mvn test -Dtests.output=always +gradle test -Dtests.output=always ------------------------------ Configure the heap size. ------------------------------ -mvn test -Dtests.heap.size=512m +gradle test -Dtests.heap.size=512m ------------------------------ Pass arbitrary jvm arguments. ------------------------------ -mvn test -Dtests.jvm.argline="-XX:HeapDumpPath=/path/to/heapdumps" +# specify heap dump path +gradle test -Dtests.jvm.argline="-XX:HeapDumpPath=/path/to/heapdumps" +# enable gc logging +gradle test -Dtests.jvm.argline="-verbose:gc" +# enable security debugging +gradle test -Dtests.jvm.argline="-Djava.security.debug=access,failure" ------------------------------ == Backwards Compatibility Tests @@ -225,7 +236,7 @@ To run backwards compatibilty tests untar or unzip a release and run the tests with the following command: --------------------------------------------------------------------------- -mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch -Dtests.security.manager=false +gradle test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch -Dtests.security.manager=false --------------------------------------------------------------------------- Note that backwards tests must be run with security manager disabled. @@ -233,7 +244,7 @@ If the elasticsearch release is placed under `./backwards/elasticsearch-x.y.z` t can be omitted: --------------------------------------------------------------------------- -mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.security.manager=false +gradle test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.security.manager=false --------------------------------------------------------------------------- To setup the bwc test environment execute the following steps (provided you are @@ -245,19 +256,25 @@ $ curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elastic $ tar -xzf elasticsearch-1.2.1.tar.gz --------------------------------------------------------------------------- -== Running integration tests +== Running verification tasks -To run the integration tests: +To run all verification tasks, including static checks, unit tests, and integration tests: --------------------------------------------------------------------------- -mvn verify +gradle check --------------------------------------------------------------------------- -Note that this will also run the unit tests first. If you want to just -run the integration tests only (because you are debugging them): +Note that this will also run the unit tests and precommit tasks first. If you want to just +run the integration tests (because you are debugging them): --------------------------------------------------------------------------- -mvn verify -Dskip.unit.tests +gradle integTest +--------------------------------------------------------------------------- + +If you want to just run the precommit checks: + +--------------------------------------------------------------------------- +gradle precommit --------------------------------------------------------------------------- == Testing the REST layer @@ -269,11 +286,20 @@ The REST layer is tested through specific tests that are shared between all the elasticsearch official clients and consist of YAML files that describe the operations to be executed and the obtained results that need to be tested. -The REST tests are run automatically when executing the maven test command. To run only the +The REST tests are run automatically when executing the "gradle check" command. To run only the REST tests use the following command: --------------------------------------------------------------------------- -mvn verify -Dtests.filter="@Rest" +gradle :distribution:tar:integTest \ + -Dtests.class=org.elasticsearch.test.rest.RestIT +--------------------------------------------------------------------------- + +A specific test case can be run with + +--------------------------------------------------------------------------- +gradle :distribution:tar:integTest \ + -Dtests.class=org.elasticsearch.test.rest.RestIT \ + -Dtests.method="test {p0=cat.shards/10_basic/Help}" --------------------------------------------------------------------------- `RestNIT` are the executable test classes that runs all the @@ -298,20 +324,6 @@ comma separated list of nodes to connect to (e.g. localhost:9300). A transport c be created based on that and used for all the before|after test operations, and to extract the http addresses of the nodes so that REST requests can be sent to them. -== Skip validate - -To disable validation step (forbidden API or `// NOCOMMIT`) use - ---------------------------------------------------------------------------- -mvn test -Dvalidate.skip=true ---------------------------------------------------------------------------- - -You can also skip this by using the "dev" profile: - ---------------------------------------------------------------------------- -mvn test -Pdev ---------------------------------------------------------------------------- - == Testing scripts The simplest way to test scripts and the packaged distributions is to use @@ -329,152 +341,63 @@ vagrant plugin install vagrant-cachier . Validate your installed dependencies: ------------------------------------- -mvn -Dtests.vagrant -pl qa/vagrant validate +gradle :qa:vagrant:checkVagrantVersion ------------------------------------- -. Download the VMs. Since Maven or ant or something eats the progress reports -from Vagrant when you run it inside mvn its probably best if you run this one -time to setup all the VMs one at a time. Run this to download and setup the VMs -we use for testing by default: +. Download and smoke test the VMs with `gradle vagrantSmokeTest` or +`gradle vagrantSmokeTestAllDistros`. The first time you run this it will +download the base images and provision the boxes and immediately quit. If you +you this again it'll skip the download step. --------------------------------------------------------- -vagrant up --provision trusty --provider virtualbox && vagrant halt trusty -vagrant up --provision centos-7 --provider virtualbox && vagrant halt centos-7 --------------------------------------------------------- +. Run the tests with `gradle checkPackages`. This will cause gradle to build +the tar, zip, and deb packages and all the plugins. It will then run the tests +on ubuntu-1404 and centos-7. We chose those two distributions as the default +because they cover deb and rpm packaging and SyvVinit and systemd. -or run this to download and setup all the VMs: +You can run on all the VMs by running `gradle checkPackagesAllDistros`. You can +run a particular VM with a command like `gradle checkOel7`. See `gradle tasks` +for a list. Its important to know that if you ctrl-c any of these `gradle` +commands then the boxes will remain running and you'll have to terminate them +with `vagrant halt`. -------------------------------------------------------------------------------- -vagrant halt -for box in $(vagrant status | grep 'poweroff\|not created' | cut -f1 -d' '); do - vagrant up --provision $box --provider virtualbox - vagrant halt $box -done -------------------------------------------------------------------------------- - -. Smoke test the maven/ant dance that we use to get vagrant involved in -integration testing is working: - ---------------------------------------------- -mvn -Dtests.vagrant -Psmoke-vms -pl qa/vagrant verify ---------------------------------------------- - -or this to validate all the VMs: - -------------------------------------------------- -mvn -Dtests.vagrant=all -Psmoke-vms -pl qa/vagrant verify -------------------------------------------------- - -That will start up the VMs and then immediate quit. - -. Finally run the tests. The fastest way to get this started is to run: - ------------------------------------ -mvn clean install -DskipTests -mvn -Dtests.vagrant -pl qa/vagrant verify ------------------------------------ - -You could just run: - --------------------- -mvn -Dtests.vagrant verify --------------------- - -but that will run all the tests. Which is probably a good thing, but not always -what you want. - -Whichever snippet you run mvn will build the tar, zip and deb packages. If you -have rpmbuild installed it'll build the rpm package as well. Then mvn will -spin up trusty and verify the tar, zip, and deb package. If you have rpmbuild -installed it'll spin up centos-7 and verify the tar, zip and rpm packages. We -chose those two distributions as the default because they cover deb and rpm -packaging and SyvVinit and systemd. - -You can control the boxes that are used for testing like so. Run just -fedora-22 with: - --------------------------------------------- -mvn -Dtests.vagrant -pl qa/vagrant verify -DboxesToTest=fedora-22 --------------------------------------------- - -or run jessie and trusty: - ------------------------------------------------------------------- -mvn -Dtests.vagrant -pl qa/vagrant verify -DboxesToTest='jessie, trusty' ------------------------------------------------------------------- - -or run all the boxes: - ---------------------------------------- -mvn -Dtests.vagrant=all -pl qa/vagrant verify ---------------------------------------- - -If you want to run a specific test on several boxes you can do: - ---------------------------------------- -mvn -Dtests.vagrant=all -pl qa/vagrant verify -DtestScripts=*tar*.bats ---------------------------------------- - -Its important to know that if you ctrl-c any of these `mvn` runs that you'll -probably leave a VM up. You can terminate it by running: - ------------- -vagrant halt ------------- - -This is just regular vagrant so you can run normal multi box vagrant commands -to test things manually. Just run: - ---------------------------------------- -vagrant up trusty --provider virtualbox && vagrant ssh trusty ---------------------------------------- - -to get an Ubuntu or - -------------------------------------------- -vagrant up centos-7 --provider virtualbox && vagrant ssh centos-7 -------------------------------------------- - -to get a CentOS. Once you are done with them you should halt them: - -------------------- -vagrant halt trusty -------------------- +All the regular vagrant commands should just work so you can get a shell in a +VM running trusty by running +`vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404`. These are the linux flavors the Vagrantfile currently supports: -* precise aka Ubuntu 12.04 -* trusty aka Ubuntu 14.04 -* vivid aka Ubuntun 15.04 -* jessie aka Debian 8, the current debina stable distribution +* ubuntu-1204 aka precise +* ubuntu-1404 aka trusty +* ubuntu-1504 aka vivid +* debian-8 aka jessie, the current debian stable distribution * centos-6 * centos-7 * fedora-22 * oel-7 aka Oracle Enterprise Linux 7 * sles-12 +* opensuse-13 We're missing the following from the support matrix because there aren't high quality boxes available in vagrant atlas: * sles-11 -* opensuse-13 * oel-6 We're missing the follow because our tests are very linux/bash centric: * Windows Server 2012 -Its important to think of VMs like cattle: if they become lame you just shoot +Its important to think of VMs like cattle. If they become lame you just shoot them and let vagrant reprovision them. Say you've hosed your precise VM: ---------------------------------------------------- -vagrant ssh precise -c 'sudo rm -rf /bin'; echo oops +vagrant ssh ubuntu-1404 -c 'sudo rm -rf /bin'; echo oops ---------------------------------------------------- All you've got to do to get another one is ---------------------------------------------- -vagrant destroy -f trusty && vagrant up trusty --provider virtualbox +vagrant destroy -f ubuntu-1404 && vagrant up ubuntu-1404 --provider virtualbox ---------------------------------------------- The whole process takes a minute and a half on a modern laptop, two and a half @@ -492,13 +415,8 @@ vagrant halt vagrant destroy -f ------------------ - ----------- -vagrant up ----------- - -would normally start all the VMs but we've prevented that because that'd -consume a ton of ram. +`vagrant up` would normally start all the VMs but we've prevented that because +that'd consume a ton of ram. == Testing scripts more directly @@ -507,7 +425,7 @@ destructive. When working with a single package its generally faster to run its tests in a tighter loop than maven provides. In one window: -------------------------------- -mvn -pl distribution/rpm package +gradle :distribution:rpm:assemble -------------------------------- and in another window: @@ -521,10 +439,7 @@ sudo bats $BATS/*rpm*.bats If you wanted to retest all the release artifacts on a single VM you could: ------------------------------------------------- -# Build all the distributions fresh but skip recompiling elasticsearch: -mvn -amd -pl distribution install -DskipTests -# Copy them all the testroot -mvn -Dtests.vagrant -pl qa/vagrant pre-integration-test +gradle prepareTestRoot vagrant up trusty --provider virtualbox && vagrant ssh trusty cd $TESTROOT sudo bats $BATS/*.bats @@ -555,5 +470,22 @@ mvn -Dtests.coverage verify jacoco:report == Debugging from an IDE -If you want to run elasticsearch from your IDE, you should execute ./run.sh -It opens a remote debugging port that you can connect with your IDE. +If you want to run elasticsearch from your IDE, the `gradle run` task +supports a remote debugging option: + +--------------------------------------------------------------------------- +gradle run --debug-jvm +--------------------------------------------------------------------------- + +== Building with extra plugins +Additional plugins may be built alongside elasticsearch, where their +dependency on elasticsearch will be substituted with the local elasticsearch +build. To add your plugin, create a directory called x-plugins as a sibling +of elasticsearch. Checkout your plugin underneath x-plugins and the build +will automatically pick it up. You can verify the plugin is included as part +of the build by checking the projects of the build. + +--------------------------------------------------------------------------- +gradle projects +--------------------------------------------------------------------------- + diff --git a/Vagrantfile b/Vagrantfile index 7c76e23df8e..454d114f1a2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -22,15 +22,15 @@ # under the License. Vagrant.configure(2) do |config| - config.vm.define "precise" do |config| + config.vm.define "ubuntu-1204" do |config| config.vm.box = "ubuntu/precise64" ubuntu_common config end - config.vm.define "trusty" do |config| + config.vm.define "ubuntu-1404" do |config| config.vm.box = "ubuntu/trusty64" ubuntu_common config end - config.vm.define "vivid" do |config| + config.vm.define "ubuntu-1504" do |config| config.vm.box = "ubuntu/vivid64" ubuntu_common config, extra: <<-SHELL # Install Jayatana so we can work around it being present. @@ -40,7 +40,7 @@ Vagrant.configure(2) do |config| # Wheezy's backports don't contain Openjdk 8 and the backflips required to # get the sun jdk on there just aren't worth it. We have jessie for testing # debian and it works fine. - config.vm.define "jessie" do |config| + config.vm.define "debian-8" do |config| config.vm.box = "debian/jessie64" deb_common config, 'echo deb http://http.debian.net/debian jessie-backports main > /etc/apt/sources.list.d/backports.list', 'backports' @@ -137,7 +137,7 @@ def deb_common(config, add_openjdk_repository_command, openjdk_list, extra: '') extra: <<-SHELL export DEBIAN_FRONTEND=noninteractive ls /etc/apt/sources.list.d/#{openjdk_list}.list > /dev/null 2>&1 || - (echo "Importing java-8 ppa" && + (echo "==> Importing java-8 ppa" && #{add_openjdk_repository_command} && apt-get update) #{extra} @@ -223,9 +223,11 @@ def provision(config, install() { # Only apt-get update if we haven't in the last day if [ ! -f #{update_tracking_file} ] || [ "x$(find #{update_tracking_file} -mtime +0)" == "x#{update_tracking_file}" ]; then - #{update_command} || true - touch #{update_tracking_file} + echo "==> Updating repository" + #{update_command} || true + touch #{update_tracking_file} fi + echo "==> Installing $1" #{install_command} $1 } ensure() { @@ -242,17 +244,18 @@ def provision(config, installed bats || { # Bats lives in a git repository.... ensure git + echo "==> Installing bats" git clone https://github.com/sstephenson/bats /tmp/bats # Centos doesn't add /usr/local/bin to the path.... /tmp/bats/install.sh /usr rm -rf /tmp/bats } cat \<\ /etc/profile.d/elasticsearch_vars.sh -export ZIP=/elasticsearch/distribution/zip/target/releases -export TAR=/elasticsearch/distribution/tar/target/releases -export RPM=/elasticsearch/distribution/rpm/target/releases -export DEB=/elasticsearch/distribution/deb/target/releases -export TESTROOT=/elasticsearch/qa/vagrant/target/testroot +export ZIP=/elasticsearch/distribution/zip/build/distributions +export TAR=/elasticsearch/distribution/tar/build/distributions +export RPM=/elasticsearch/distribution/rpm/build/distributions +export DEB=/elasticsearch/distribution/deb/build/distributions +export TESTROOT=/elasticsearch/qa/vagrant/build/testroot export BATS=/elasticsearch/qa/vagrant/src/test/resources/packaging/scripts VARS SHELL diff --git a/build.gradle b/build.gradle new file mode 100644 index 00000000000..831db456a19 --- /dev/null +++ b/build.gradle @@ -0,0 +1,250 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.bmuschko.gradle.nexus.NexusPlugin +import org.gradle.plugins.ide.eclipse.model.SourceFolder + +// common maven publishing configuration +subprojects { + group = 'org.elasticsearch' + version = org.elasticsearch.gradle.VersionProperties.elasticsearch + + plugins.withType(NexusPlugin).whenPluginAdded { + modifyPom { + project { + url 'https://github.com/elastic/elasticsearch' + inceptionYear '2009' + + scm { + url 'https://github.com/elastic/elasticsearch' + connection 'scm:https://elastic@github.com/elastic/elasticsearch' + developerConnection 'scm:git://github.com/elastic/elasticsearch.git' + } + + licenses { + license { + name 'The Apache Software License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution 'repo' + } + } + } + } + extraArchive { + javadoc = true + tests = false + } + // we have our own username/password prompts so that they only happen once + // TODO: add gpg signing prompts + project.gradle.taskGraph.whenReady { taskGraph -> + if (taskGraph.allTasks.any { it.name == 'uploadArchives' }) { + Console console = System.console() + if (project.hasProperty('nexusUsername') == false) { + String nexusUsername = console.readLine('\nNexus username: ') + project.rootProject.allprojects.each { + it.ext.nexusUsername = nexusUsername + } + } + if (project.hasProperty('nexusPassword') == false) { + String nexusPassword = new String(console.readPassword('\nNexus password: ')) + project.rootProject.allprojects.each { + it.ext.nexusPassword = nexusPassword + } + } + } + } + } +} + +allprojects { + // injecting groovy property variables into all projects + project.ext { + // for eclipse hacks... + isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse') + } +} + +subprojects { + project.afterEvaluate { + // include license and notice in jars + tasks.withType(Jar) { + into('META-INF') { + from project.rootProject.rootDir + include 'LICENSE.txt' + include 'NOTICE.txt' + } + } + // ignore missing javadocs + tasks.withType(Javadoc) { Javadoc javadoc -> + // the -quiet here is because of a bug in gradle, in that adding a string option + // by itself is not added to the options. By adding quiet, both this option and + // the "value" -quiet is added, separated by a space. This is ok since the javadoc + // command already adds -quiet, so we are just duplicating it + // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959 + javadoc.options.encoding='UTF8' + javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet') + } + } + + /* Sets up the dependencies that we build as part of this project but + register as thought they were external to resolve internally. We register + them as external dependencies so the build plugin that we use can be used + to build elasticsearch plugins outside of the elasticsearch source tree. */ + ext.projectSubstitutions = [ + "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec', + "org.elasticsearch:elasticsearch:${version}": ':core', + "org.elasticsearch:test-framework:${version}": ':test-framework', + "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip', + "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip', + "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar', + "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm', + "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb', + ] + configurations.all { + resolutionStrategy.dependencySubstitution { DependencySubstitutions subs -> + projectSubstitutions.each { k,v -> + subs.substitute(subs.module(k)).with(subs.project(v)) + } + } + } +} + +// Ensure similar tasks in dependent projects run first. The projectsEvaluated here is +// important because, while dependencies.all will pickup future dependencies, +// it is not necessarily true that the task exists in both projects at the time +// the dependency is added. +gradle.projectsEvaluated { + allprojects { + if (project.path == ':test-framework') { + // :test-framework:test cannot run before and after :core:test + return + } + configurations.all { + dependencies.all { Dependency dep -> + Project upstreamProject = null + if (dep instanceof ProjectDependency) { + upstreamProject = dep.dependencyProject + } else { + // gradle doesn't apply substitutions until resolve time, so they won't + // show up as a ProjectDependency above + String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}") + if (substitution != null) { + upstreamProject = findProject(substitution) + } + } + if (upstreamProject != null) { + if (project.path == upstreamProject.path) { + // TODO: distribution integ tests depend on themselves (!), fix that + return + } + for (String taskName : ['test', 'integTest']) { + Task task = project.tasks.findByName(taskName) + Task upstreamTask = upstreamProject.tasks.findByName(taskName) + if (task != null && upstreamTask != null) { + task.mustRunAfter(upstreamTask) + } + } + } + } + } + } +} + +// intellij configuration +allprojects { + apply plugin: 'idea' +} + +idea { + project { + languageLevel = org.elasticsearch.gradle.BuildPlugin.minimumJava.toString() + vcs = 'Git' + } +} +// Make sure gradle idea was run before running anything in intellij (including import). +File ideaMarker = new File(projectDir, '.local-idea-is-configured') +tasks.idea.doLast { + ideaMarker.setText('', 'UTF-8') +} +if (System.getProperty('idea.active') != null && ideaMarker.exists() == false) { + throw new GradleException('You must run gradle idea from the root of elasticsearch before importing into IntelliJ') +} +// add buildSrc itself as a groovy project +task buildSrcIdea(type: GradleBuild) { + buildFile = 'buildSrc/build.gradle' + tasks = ['cleanIdea', 'ideaModule'] +} +tasks.idea.dependsOn(buildSrcIdea) + + +// eclipse configuration +allprojects { + apply plugin: 'eclipse' + + plugins.withType(JavaBasePlugin) { + eclipse.classpath.defaultOutputDir = new File(project.buildDir, 'eclipse') + eclipse.classpath.file.whenMerged { classpath -> + // give each source folder a unique corresponding output folder + int i = 0; + classpath.entries.findAll { it instanceof SourceFolder }.each { folder -> + i++; + // this is *NOT* a path or a file. + folder.output = "build/eclipse/" + i + } + } + } + task copyEclipseSettings(type: Copy) { + // TODO: "package this up" for external builds + from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings') + into '.settings' + } + // otherwise .settings is not nuked entirely + tasks.cleanEclipse { + delete '.settings' + } + // otherwise the eclipse merging is *super confusing* + tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings) +} + +// add buildSrc itself as a groovy project +task buildSrcEclipse(type: GradleBuild) { + buildFile = 'buildSrc/build.gradle' + tasks = ['cleanEclipse', 'eclipse'] +} +tasks.eclipse.dependsOn(buildSrcEclipse) + +// we need to add the same --debug-jvm option as +// the real RunTask has, so we can pass it through +class Run extends DefaultTask { + boolean debug = false + + @org.gradle.api.internal.tasks.options.Option( + option = "debug-jvm", + description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch." + ) + public void setDebug(boolean enabled) { + project.project(':distribution').run.clusterConfig.debug = enabled + } +} +task run(type: Run) { + dependsOn ':distribution:run' + description = 'Runs elasticsearch in the foreground' + group = 'Verification' + impliesSubProjects = true +} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 00000000000..e46f9cb33c0 --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// we must use buildscript + apply so that an external plugin +// can apply this file, since the plugins directive is not +// supported through file includes +buildscript { + repositories { + jcenter() + } + dependencies { + classpath 'com.bmuschko:gradle-nexus-plugin:2.3.1' + } +} +apply plugin: 'groovy' +apply plugin: 'com.bmuschko.nexus' +// TODO: move common IDE configuration to a common file to include +apply plugin: 'idea' +apply plugin: 'eclipse' + +group = 'org.elasticsearch.gradle' +archivesBaseName = 'build-tools' + +Properties props = new Properties() +props.load(project.file('version.properties').newDataInputStream()) +version = props.getProperty('elasticsearch') + +repositories { + mavenCentral() + maven { + name 'sonatype-snapshots' + url "https://oss.sonatype.org/content/repositories/snapshots/" + } + jcenter() +} + +dependencies { + compile gradleApi() + compile localGroovy() + compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}" + compile("junit:junit:${props.getProperty('junit')}") { + transitive = false + } + compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3' + compile 'com.netflix.nebula:gradle-info-plugin:3.0.3' + compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r' + compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE.... + compile 'de.thetaphi:forbiddenapis:2.0' + compile 'com.bmuschko:gradle-nexus-plugin:2.3.1' +} + +processResources { + inputs.file('version.properties') + from 'version.properties' +} + +extraArchive { + javadoc = false + tests = false +} + +eclipse { + classpath { + defaultOutputDir = new File(file('build'), 'eclipse') + } +} + +task copyEclipseSettings(type: Copy) { + from project.file('src/main/resources/eclipse.settings') + into '.settings' +} +// otherwise .settings is not nuked entirely +tasks.cleanEclipse { + delete '.settings' +} +tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings) diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/BalancersConfiguration.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/BalancersConfiguration.groovy new file mode 100644 index 00000000000..91355bf2494 --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/BalancersConfiguration.groovy @@ -0,0 +1,53 @@ +package com.carrotsearch.gradle.junit4 + +import com.carrotsearch.ant.tasks.junit4.SuiteBalancer +import com.carrotsearch.ant.tasks.junit4.balancers.ExecutionTimeBalancer +import com.carrotsearch.ant.tasks.junit4.listeners.ExecutionTimesReport +import org.apache.tools.ant.types.FileSet + +class BalancersConfiguration { + // parent task, so executionTime can register an additional listener + RandomizedTestingTask task + List balancers = new ArrayList<>() + + void executionTime(Map properties) { + ExecutionTimeBalancer balancer = new ExecutionTimeBalancer() + + FileSet fileSet = new FileSet() + Object filename = properties.remove('cacheFilename') + if (filename == null) { + throw new IllegalArgumentException('cacheFilename is required for executionTime balancer') + } + fileSet.setIncludes(filename.toString()) + + File cacheDir = task.project.projectDir + Object dir = properties.remove('cacheDir') + if (dir != null) { + cacheDir = new File(dir.toString()) + } + fileSet.setDir(cacheDir) + balancer.add(fileSet) + + int historySize = 10 + Object size = properties.remove('historySize') + if (size instanceof Integer) { + historySize = (Integer)size + } else if (size != null) { + throw new IllegalArgumentException('historySize must be an integer') + } + ExecutionTimesReport listener = new ExecutionTimesReport() + listener.setFile(new File(cacheDir, filename.toString())) + listener.setHistoryLength(historySize) + + if (properties.isEmpty() == false) { + throw new IllegalArgumentException('Unknown properties for executionTime balancer: ' + properties.keySet()) + } + + task.listenersConfig.listeners.add(listener) + balancers.add(balancer) + } + + void custom(SuiteBalancer balancer) { + balancers.add(balancer) + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/ListenersConfiguration.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/ListenersConfiguration.groovy new file mode 100644 index 00000000000..5fa5baa8ffb --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/ListenersConfiguration.groovy @@ -0,0 +1,25 @@ +package com.carrotsearch.gradle.junit4 + +import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener +import com.carrotsearch.ant.tasks.junit4.listeners.antxml.AntXmlReport + + +class ListenersConfiguration { + RandomizedTestingTask task + List listeners = new ArrayList<>() + + void junitReport(Map props) { + AntXmlReport reportListener = new AntXmlReport() + Object dir = props == null ? null : props.get('dir') + if (dir != null) { + reportListener.setDir(task.project.file(dir)) + } else { + reportListener.setDir(new File(task.project.buildDir, 'reports' + File.separator + "${task.name}Junit")) + } + listeners.add(reportListener) + } + + void custom(AggregatedEventListener listener) { + listeners.add(listener) + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/LoggingOutputStream.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/LoggingOutputStream.groovy new file mode 100644 index 00000000000..ce0995a5a8c --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/LoggingOutputStream.groovy @@ -0,0 +1,64 @@ +package com.carrotsearch.gradle.junit4 + +import org.gradle.api.logging.LogLevel +import org.gradle.api.logging.Logger + +/** + * Writes data passed to this stream as log messages. + * + * The stream will be flushed whenever a newline is detected. + * Allows setting an optional prefix before each line of output. + */ +public class LoggingOutputStream extends OutputStream { + + /** The starting length of the buffer */ + static final int DEFAULT_BUFFER_LENGTH = 4096 + + /** The buffer of bytes sent to the stream */ + byte[] buffer = new byte[DEFAULT_BUFFER_LENGTH] + + /** Offset of the start of unwritten data in the buffer */ + int start = 0 + + /** Offset of the end (semi-open) of unwritten data in the buffer */ + int end = 0 + + /** Logger to write stream data to */ + Logger logger + + /** Prefix to add before each line of output */ + String prefix = "" + + /** Log level to write log messages to */ + LogLevel level + + void write(final int b) throws IOException { + if (b == 0) return; + if (b == (int)'\n' as char) { + // always flush with newlines instead of adding to the buffer + flush() + return + } + + if (end == buffer.length) { + if (start != 0) { + // first try shifting the used buffer back to the beginning to make space + System.arraycopy(buffer, start, buffer, 0, end - start) + } else { + // need more space, extend the buffer + } + final int newBufferLength = buffer.length + DEFAULT_BUFFER_LENGTH; + final byte[] newBuffer = new byte[newBufferLength]; + System.arraycopy(buffer, 0, newBuffer, 0, buffer.length); + buffer = newBuffer; + } + + buffer[end++] = (byte) b; + } + + void flush() { + if (end == start) return + logger.log(level, prefix + new String(buffer, start, end - start)); + start = end + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy new file mode 100644 index 00000000000..e2230b116c7 --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy @@ -0,0 +1,47 @@ +package com.carrotsearch.gradle.junit4 + +import com.carrotsearch.ant.tasks.junit4.JUnit4 +import org.gradle.api.AntBuilder +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.plugins.JavaBasePlugin +import org.gradle.api.tasks.TaskContainer +import org.gradle.api.tasks.testing.Test + +class RandomizedTestingPlugin implements Plugin { + + void apply(Project project) { + replaceTestTask(project.tasks) + configureAnt(project.ant) + } + + static void replaceTestTask(TaskContainer tasks) { + Test oldTestTask = tasks.findByPath('test') + if (oldTestTask == null) { + // no test task, ok, user will use testing task on their own + return + } + tasks.remove(oldTestTask) + + Map properties = [ + name: 'test', + type: RandomizedTestingTask, + dependsOn: oldTestTask.dependsOn, + group: JavaBasePlugin.VERIFICATION_GROUP, + description: 'Runs unit tests with the randomized testing framework' + ] + RandomizedTestingTask newTestTask = tasks.create(properties) + newTestTask.classpath = oldTestTask.classpath + newTestTask.testClassesDir = oldTestTask.testClassesDir + + // hack so check task depends on custom test + Task checkTask = tasks.findByPath('check') + checkTask.dependsOn.remove(oldTestTask) + checkTask.dependsOn.add(newTestTask) + } + + static void configureAnt(AntBuilder ant) { + ant.project.addTaskDefinition('junit4:junit4', JUnit4.class) + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy new file mode 100644 index 00000000000..ccb5d5904bf --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy @@ -0,0 +1,304 @@ +package com.carrotsearch.gradle.junit4 + +import com.carrotsearch.ant.tasks.junit4.ListenersList +import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener +import com.esotericsoftware.kryo.serializers.FieldSerializer +import groovy.xml.NamespaceBuilder +import groovy.xml.NamespaceBuilderSupport +import org.apache.tools.ant.BuildException +import org.apache.tools.ant.DefaultLogger +import org.apache.tools.ant.RuntimeConfigurable +import org.apache.tools.ant.UnknownElement +import org.gradle.api.DefaultTask +import org.gradle.api.file.FileCollection +import org.gradle.api.file.FileTreeElement +import org.gradle.api.internal.tasks.options.Option +import org.gradle.api.specs.Spec +import org.gradle.api.tasks.* +import org.gradle.api.tasks.util.PatternFilterable +import org.gradle.api.tasks.util.PatternSet +import org.gradle.logging.ProgressLoggerFactory +import org.gradle.util.ConfigureUtil + +import javax.inject.Inject + +class RandomizedTestingTask extends DefaultTask { + + // TODO: change to "executable" to match gradle test params? + @Optional + @Input + String jvm = 'java' + + @Optional + @Input + File workingDir = new File(project.buildDir, 'testrun' + File.separator + name) + + @Optional + @Input + FileCollection classpath + + @Input + String parallelism = '1' + + @InputDirectory + File testClassesDir + + @Optional + @Input + boolean haltOnFailure = true + + @Optional + @Input + boolean shuffleOnSlave = true + + @Optional + @Input + boolean enableAssertions = true + + @Optional + @Input + boolean enableSystemAssertions = true + + @Optional + @Input + boolean leaveTemporary = false + + @Optional + @Input + String ifNoTests = 'ignore' + + TestLoggingConfiguration testLoggingConfig = new TestLoggingConfiguration() + + BalancersConfiguration balancersConfig = new BalancersConfiguration(task: this) + ListenersConfiguration listenersConfig = new ListenersConfiguration(task: this) + + List jvmArgs = new ArrayList<>() + + @Optional + @Input + String argLine = null + + Map systemProperties = new HashMap<>() + PatternFilterable patternSet = new PatternSet() + + RandomizedTestingTask() { + outputs.upToDateWhen {false} // randomized tests are never up to date + listenersConfig.listeners.add(new TestProgressLogger(factory: getProgressLoggerFactory())) + listenersConfig.listeners.add(new TestReportLogger(logger: logger, config: testLoggingConfig)) + } + + @Inject + ProgressLoggerFactory getProgressLoggerFactory() { + throw new UnsupportedOperationException(); + } + + void jvmArgs(Iterable arguments) { + jvmArgs.addAll(arguments) + } + + void jvmArg(String argument) { + jvmArgs.add(argument) + } + + void systemProperty(String property, Object value) { + systemProperties.put(property, value) + } + + void include(String... includes) { + this.patternSet.include(includes); + } + + void include(Iterable includes) { + this.patternSet.include(includes); + } + + void include(Spec includeSpec) { + this.patternSet.include(includeSpec); + } + + void include(Closure includeSpec) { + this.patternSet.include(includeSpec); + } + + void exclude(String... excludes) { + this.patternSet.exclude(excludes); + } + + void exclude(Iterable excludes) { + this.patternSet.exclude(excludes); + } + + void exclude(Spec excludeSpec) { + this.patternSet.exclude(excludeSpec); + } + + void exclude(Closure excludeSpec) { + this.patternSet.exclude(excludeSpec); + } + + @Input + void testLogging(Closure closure) { + ConfigureUtil.configure(closure, testLoggingConfig) + } + + @Input + void balancers(Closure closure) { + ConfigureUtil.configure(closure, balancersConfig) + } + + @Input + void listeners(Closure closure) { + ConfigureUtil.configure(closure, listenersConfig) + } + + @Option( + option = "tests", + description = "Sets test class or method name to be included. This is for IDEs. Use -Dtests.class and -Dtests.method" + ) + void setTestNameIncludePattern(String testNamePattern) { + // This is only implemented to give support for IDEs running tests. There are 3 patterns expected: + // * An exact test class and method + // * An exact test class + // * A package name prefix, ending with .* + // There is no way to distinguish the first two without looking at classes, so we use the rule + // that class names start with an uppercase letter... + // TODO: this doesn't work yet, but not sure why...intellij says it is using --tests, and this work from the command line... + String[] parts = testNamePattern.split('\\.') + String lastPart = parts[parts.length - 1] + String classname + String methodname = null + if (lastPart.equals('*') || lastPart.charAt(0).isUpperCase()) { + // package name or class name, just pass through + classname = testNamePattern + } else { + // method name, need to separate + methodname = lastPart + classname = testNamePattern.substring(0, testNamePattern.length() - lastPart.length() - 1) + } + ant.setProperty('tests.class', classname) + if (methodname != null) { + ant.setProperty('tests.method', methodname) + } + } + + @TaskAction + void executeTests() { + Map attributes = [ + jvm: jvm, + parallelism: parallelism, + heartbeat: testLoggingConfig.slowTests.heartbeat, + dir: workingDir, + tempdir: new File(workingDir, 'temp'), + haltOnFailure: true, // we want to capture when a build failed, but will decide whether to rethrow later + shuffleOnSlave: shuffleOnSlave, + leaveTemporary: leaveTemporary, + ifNoTests: ifNoTests + ] + + DefaultLogger listener = null + ByteArrayOutputStream antLoggingBuffer = null + if (logger.isInfoEnabled() == false) { + // in info logging, ant already outputs info level, so we see everything + // but on errors or when debugging, we want to see info level messages + // because junit4 emits jvm output with ant logging + if (testLoggingConfig.outputMode == TestLoggingConfiguration.OutputMode.ALWAYS) { + // we want all output, so just stream directly + listener = new DefaultLogger( + errorPrintStream: System.err, + outputPrintStream: System.out, + messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO) + } else { + // we want to buffer the info, and emit it if the test fails + antLoggingBuffer = new ByteArrayOutputStream() + PrintStream stream = new PrintStream(antLoggingBuffer, true, "UTF-8") + listener = new DefaultLogger( + errorPrintStream: stream, + outputPrintStream: stream, + messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO) + } + project.ant.project.addBuildListener(listener) + } + + NamespaceBuilderSupport junit4 = NamespaceBuilder.newInstance(ant, 'junit4') + try { + junit4.junit4(attributes) { + classpath { + pathElement(path: classpath.asPath) + } + if (enableAssertions) { + jvmarg(value: '-ea') + } + if (enableSystemAssertions) { + jvmarg(value: '-esa') + } + for (String arg : jvmArgs) { + jvmarg(value: arg) + } + if (argLine != null) { + jvmarg(line: argLine) + } + fileset(dir: testClassesDir) { + for (String includePattern : patternSet.getIncludes()) { + include(name: includePattern) + } + for (String excludePattern : patternSet.getExcludes()) { + exclude(name: excludePattern) + } + } + for (Map.Entry prop : systemProperties) { + sysproperty key: prop.getKey(), value: prop.getValue().toString() + } + makeListeners() + } + } catch (BuildException e) { + if (antLoggingBuffer != null) { + logger.error('JUnit4 test failed, ant output was:') + logger.error(antLoggingBuffer.toString('UTF-8')) + } + if (haltOnFailure) { + throw e; + } + } + + if (listener != null) { + // remove the listener we added so other ant tasks dont have verbose logging! + project.ant.project.removeBuildListener(listener) + } + } + + static class ListenersElement extends UnknownElement { + AggregatedEventListener[] listeners + + ListenersElement() { + super('listeners') + setNamespace('junit4') + setQName('listeners') + } + + public void handleChildren(Object realThing, RuntimeConfigurable wrapper) { + assert realThing instanceof ListenersList + ListenersList list = (ListenersList)realThing + + for (AggregatedEventListener listener : listeners) { + list.addConfigured(listener) + } + } + } + + /** + * Makes an ant xml element for 'listeners' just as AntBuilder would, except configuring + * the element adds the already created children. + */ + def makeListeners() { + def context = ant.getAntXmlContext() + def parentWrapper = context.currentWrapper() + def parent = parentWrapper.getProxy() + UnknownElement element = new ListenersElement(listeners: listenersConfig.listeners) + element.setProject(context.getProject()) + element.setRealThing(logger) + ((UnknownElement)parent).addChild(element) + RuntimeConfigurable wrapper = new RuntimeConfigurable(element, element.getQName()) + parentWrapper.addChild(wrapper) + return wrapper.getProxy() + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/SlowTestsConfiguration.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/SlowTestsConfiguration.groovy new file mode 100644 index 00000000000..2705fdeaacb --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/SlowTestsConfiguration.groovy @@ -0,0 +1,14 @@ +package com.carrotsearch.gradle.junit4 + +class SlowTestsConfiguration { + int heartbeat = 0 + int summarySize = 0 + + void heartbeat(int heartbeat) { + this.heartbeat = heartbeat + } + + void summarySize(int summarySize) { + this.summarySize = summarySize + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/StackTraceFiltersConfiguration.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/StackTraceFiltersConfiguration.groovy new file mode 100644 index 00000000000..5e5610ab68e --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/StackTraceFiltersConfiguration.groovy @@ -0,0 +1,14 @@ +package com.carrotsearch.gradle.junit4 + +class StackTraceFiltersConfiguration { + List patterns = new ArrayList<>() + List contains = new ArrayList<>() + + void regex(String pattern) { + patterns.add(pattern) + } + + void contains(String contain) { + contains.add(contain) + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestLoggingConfiguration.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestLoggingConfiguration.groovy new file mode 100644 index 00000000000..97251252f54 --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestLoggingConfiguration.groovy @@ -0,0 +1,43 @@ +package com.carrotsearch.gradle.junit4 + +import org.gradle.api.tasks.Input +import org.gradle.util.ConfigureUtil + +class TestLoggingConfiguration { + /** Display mode for output streams. */ + static enum OutputMode { + /** Always display the output emitted from tests. */ + ALWAYS, + /** + * Display the output only if a test/ suite failed. This requires internal buffering + * so the output will be shown only after a test completes. + */ + ONERROR, + /** Don't display the output, even on test failures. */ + NEVER + } + + OutputMode outputMode = OutputMode.ONERROR + SlowTestsConfiguration slowTests = new SlowTestsConfiguration() + StackTraceFiltersConfiguration stackTraceFilters = new StackTraceFiltersConfiguration() + + /** Summarize the first N failures at the end of the test. */ + @Input + int showNumFailuresAtEnd = 3 // match TextReport default + + void slowTests(Closure closure) { + ConfigureUtil.configure(closure, slowTests) + } + + void stackTraceFilters(Closure closure) { + ConfigureUtil.configure(closure, stackTraceFilters) + } + + void outputMode(String mode) { + outputMode = mode.toUpperCase() as OutputMode + } + + void showNumFailuresAtEnd(int n) { + showNumFailuresAtEnd = n + } +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy new file mode 100644 index 00000000000..5e919ce9ba2 --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy @@ -0,0 +1,187 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.carrotsearch.gradle.junit4 + +import com.carrotsearch.ant.tasks.junit4.JUnit4 +import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.eventbus.Subscribe +import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedStartEvent +import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteResultEvent +import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedTestResultEvent +import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener +import org.gradle.logging.ProgressLogger +import org.gradle.logging.ProgressLoggerFactory +import org.junit.runner.Description + +import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.* +import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.formatDurationInSeconds +import static java.lang.Math.max + +/** + * Adapts junit4's event listeners into gradle's ProgressLogger. Note that + * junit4 guarantees (via guava) that methods on this class won't be called by + * multiple threads simultaneously which is helpful in making it simpler. + * + * Every time a test finishes this class will update the logger. It will log + * the last finished test method on the logger line until the first suite + * finishes. Once the first suite finishes it always logs the last finished + * suite. This means that in test runs with a single suite the logger will be + * updated with the test name the whole time which is useful because these runs + * usually have longer individual tests. For test runs with lots of suites the + * majority of the time is spent showing the last suite that finished which is + * more useful for those test runs because test methods there tend to be very + * quick. + */ +class TestProgressLogger implements AggregatedEventListener { + /** Factory to build a progress logger when testing starts */ + ProgressLoggerFactory factory + ProgressLogger progressLogger + int totalSuites + int totalSlaves + + // sprintf formats used to align the integers we print + String suitesFormat + String slavesFormat + String testsFormat + + // Counters incremented test completion. + volatile int suitesCompleted = 0 + volatile int testsCompleted = 0 + volatile int testsFailed = 0 + volatile int testsIgnored = 0 + + // Information about the last, most interesting event. + volatile String eventDescription + volatile int eventSlave + volatile long eventExecutionTime + + /** Have we finished a whole suite yet? */ + volatile boolean suiteFinished = false + /* Note that we probably overuse volatile here but it isn't hurting us and + lets us move things around without worying about breaking things. */ + + @Subscribe + void onStart(AggregatedStartEvent e) throws IOException { + totalSuites = e.suiteCount + totalSlaves = e.slaveCount + progressLogger = factory.newOperation(TestProgressLogger) + progressLogger.setDescription('Randomized test runner') + progressLogger.started() + progressLogger.progress( + "Starting JUnit4 for ${totalSuites} suites on ${totalSlaves} jvms") + + suitesFormat = "%0${widthForTotal(totalSuites)}d" + slavesFormat = "%-${widthForTotal(totalSlaves)}s" + /* Just guess the number of tests because we can't figure it out from + here and it isn't worth doing anything fancy to prevent the console + from jumping around a little. 200 is a pretty wild guess for the + minimum but it makes REST tests output sanely. */ + int totalNumberOfTestsGuess = max(200, totalSuites * 10) + testsFormat = "%0${widthForTotal(totalNumberOfTestsGuess)}d" + } + + @Subscribe + void onTestResult(AggregatedTestResultEvent e) throws IOException { + testsCompleted++ + switch (e.status) { + case ERROR: + case FAILURE: + testsFailed++ + break + case IGNORED: + case IGNORED_ASSUMPTION: + testsIgnored++ + break + case OK: + break + default: + throw new IllegalArgumentException( + "Unknown test status: [${e.status}]") + } + if (!suiteFinished) { + updateEventInfo(e) + } + + log() + } + + @Subscribe + void onSuiteResult(AggregatedSuiteResultEvent e) throws IOException { + suitesCompleted++ + suiteFinished = true + updateEventInfo(e) + log() + } + + /** + * Update the suite information with a junit4 event. + */ + private void updateEventInfo(Object e) { + eventDescription = simpleName(e.description.className) + if (e.description.methodName != null) { + eventDescription += "#${e.description.methodName}" + } + eventSlave = e.slave.id + eventExecutionTime = e.executionTime + } + + /** + * Extract a Class#getSimpleName style name from Class#getName style + * string. We can't just use Class#getSimpleName because junit descriptions + * don't alway s set the class field but they always set the className + * field. + */ + private static String simpleName(String className) { + return className.substring(className.lastIndexOf('.') + 1) + } + + private void log() { + /* Remember that instances of this class are only ever active on one + thread at a time so there really aren't race conditions here. It'd be + OK if there were because they'd only display an overcount + temporarily. */ + String log = '' + if (totalSuites > 1) { + /* Skip printing the suites to save space when there is only a + single suite. This is nice because when there is only a single + suite we log the method name and those can be long. */ + log += sprintf("Suites [${suitesFormat}/${suitesFormat}], ", + [suitesCompleted, totalSuites]) + } + log += sprintf("Tests [${testsFormat}|%d|%d], ", + [testsCompleted, testsFailed, testsIgnored]) + log += "in ${formatDurationInSeconds(eventExecutionTime)} " + if (totalSlaves > 1) { + /* Skip printing the slaves if there is only one of them. This is + nice because when there is only a single slave there is often + only a single suite and we could use the extra space to log the + test method names. */ + log += "J${sprintf(slavesFormat, eventSlave)} " + } + log += "completed ${eventDescription}" + progressLogger.progress(log) + } + + private static int widthForTotal(int total) { + return ((total - 1) as String).length() + } + + @Override + void setOuter(JUnit4 junit) {} +} diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy new file mode 100644 index 00000000000..0813713353f --- /dev/null +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy @@ -0,0 +1,388 @@ +package com.carrotsearch.gradle.junit4 + +import com.carrotsearch.ant.tasks.junit4.JUnit4 +import com.carrotsearch.ant.tasks.junit4.Pluralize +import com.carrotsearch.ant.tasks.junit4.TestsSummaryEventListener +import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.base.Strings +import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.eventbus.Subscribe +import com.carrotsearch.ant.tasks.junit4.events.* +import com.carrotsearch.ant.tasks.junit4.events.aggregated.* +import com.carrotsearch.ant.tasks.junit4.events.mirrors.FailureMirror +import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener +import com.carrotsearch.ant.tasks.junit4.listeners.StackTraceFilter +import org.apache.tools.ant.filters.TokenFilter +import org.gradle.api.logging.LogLevel +import org.gradle.api.logging.Logger +import org.junit.runner.Description + +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger + +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.Clip; +import javax.sound.sampled.Line; +import javax.sound.sampled.LineEvent; +import javax.sound.sampled.LineListener; + +import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.* +import static com.carrotsearch.gradle.junit4.TestLoggingConfiguration.OutputMode + +class TestReportLogger extends TestsSummaryEventListener implements AggregatedEventListener { + + static final String FAILURE_MARKER = " <<< FAILURES!" + + /** Status names column. */ + static EnumMap statusNames; + static { + statusNames = new EnumMap<>(TestStatus.class); + for (TestStatus s : TestStatus.values()) { + statusNames.put(s, + s == TestStatus.IGNORED_ASSUMPTION + ? "IGNOR/A" : s.toString()); + } + } + + JUnit4 owner + + /** Logger to write the report to */ + Logger logger + + TestLoggingConfiguration config + + /** Forked concurrent JVM count. */ + int forkedJvmCount + + /** Format line for JVM ID string. */ + String jvmIdFormat + + /** Output stream that logs messages to the given logger */ + LoggingOutputStream outStream + LoggingOutputStream errStream + + /** A list of failed tests, if to be displayed at the end. */ + List failedTests = new ArrayList<>() + + /** Stack trace filters. */ + StackTraceFilter stackFilter = new StackTraceFilter() + + Map suiteTimes = new HashMap<>() + boolean slowTestsFound = false + + int totalSuites + AtomicInteger suitesCompleted = new AtomicInteger() + + @Subscribe + void onStart(AggregatedStartEvent e) throws IOException { + this.totalSuites = e.getSuiteCount(); + StringBuilder info = new StringBuilder('==> Test Info: ') + info.append('seed=' + owner.getSeed() + '; ') + info.append(Pluralize.pluralize(e.getSlaveCount(), 'jvm') + '=' + e.getSlaveCount() + '; ') + info.append(Pluralize.pluralize(e.getSuiteCount(), 'suite') + '=' + e.getSuiteCount()) + logger.lifecycle(info.toString()) + + forkedJvmCount = e.getSlaveCount(); + jvmIdFormat = " J%-" + (1 + (int) Math.floor(Math.log10(forkedJvmCount))) + "d"; + + outStream = new LoggingOutputStream(logger: logger, level: LogLevel.LIFECYCLE, prefix: " 1> ") + errStream = new LoggingOutputStream(logger: logger, level: LogLevel.ERROR, prefix: " 2> ") + + for (String contains : config.stackTraceFilters.contains) { + TokenFilter.ContainsString containsFilter = new TokenFilter.ContainsString() + containsFilter.setContains(contains) + stackFilter.addContainsString(containsFilter) + } + for (String pattern : config.stackTraceFilters.patterns) { + TokenFilter.ContainsRegex regexFilter = new TokenFilter.ContainsRegex() + regexFilter.setPattern(pattern) + stackFilter.addContainsRegex(regexFilter) + } + } + + @Subscribe + void onChildBootstrap(ChildBootstrap e) throws IOException { + logger.info("Started J" + e.getSlave().id + " PID(" + e.getSlave().getPidString() + ")."); + } + + @Subscribe + void onHeartbeat(HeartBeatEvent e) throws IOException { + logger.warn("HEARTBEAT J" + e.getSlave().id + " PID(" + e.getSlave().getPidString() + "): " + + formatTime(e.getCurrentTime()) + ", stalled for " + + formatDurationInSeconds(e.getNoEventDuration()) + " at: " + + (e.getDescription() == null ? "" : formatDescription(e.getDescription()))) + try { + playBeat(); + } catch (Exception nosound) { /* handling exceptions with style */ } + slowTestsFound = true + } + + void playBeat() throws Exception { + Clip clip = (Clip)AudioSystem.getLine(new Line.Info(Clip.class)); + final AtomicBoolean stop = new AtomicBoolean(); + clip.addLineListener(new LineListener() { + @Override + public void update(LineEvent event) { + if (event.getType() == LineEvent.Type.STOP) { + stop.set(true); + } + } + }); + InputStream stream = getClass().getResourceAsStream("/beat.wav"); + try { + clip.open(AudioSystem.getAudioInputStream(stream)); + clip.start(); + while (!stop.get()) { + Thread.sleep(20); + } + clip.close(); + } finally { + stream.close(); + } + } + + @Subscribe + void onQuit(AggregatedQuitEvent e) throws IOException { + if (config.showNumFailuresAtEnd > 0 && !failedTests.isEmpty()) { + List sublist = this.failedTests + StringBuilder b = new StringBuilder() + b.append('Tests with failures') + if (sublist.size() > config.showNumFailuresAtEnd) { + sublist = sublist.subList(0, config.showNumFailuresAtEnd) + b.append(" (first " + config.showNumFailuresAtEnd + " out of " + failedTests.size() + ")") + } + b.append(':\n') + for (Description description : sublist) { + b.append(" - ").append(formatDescription(description, true)).append('\n') + } + logger.warn(b.toString()) + } + if (config.slowTests.summarySize > 0) { + List> sortedSuiteTimes = new ArrayList<>(suiteTimes.entrySet()) + Collections.sort(sortedSuiteTimes, new Comparator>() { + @Override + int compare(Map.Entry o1, Map.Entry o2) { + return o2.value - o1.value // sort descending + } + }) + LogLevel level = slowTestsFound ? LogLevel.WARN : LogLevel.INFO + int numToLog = Math.min(config.slowTests.summarySize, sortedSuiteTimes.size()) + logger.log(level, 'Slow Tests Summary:') + for (int i = 0; i < numToLog; ++i) { + logger.log(level, String.format(Locale.ENGLISH, '%6.2fs | %s', + sortedSuiteTimes.get(i).value / 1000.0, + sortedSuiteTimes.get(i).key)); + } + logger.log(level, '') // extra vertical separation + } + if (failedTests.isEmpty()) { + // summary is already printed for failures + logger.lifecycle('==> Test Summary: ' + getResult().toString()) + } + } + + @Subscribe + void onSuiteStart(AggregatedSuiteStartedEvent e) throws IOException { + if (isPassthrough()) { + SuiteStartedEvent evt = e.getSuiteStartedEvent(); + emitSuiteStart(LogLevel.LIFECYCLE, evt.getDescription()); + } + } + + @Subscribe + void onOutput(PartialOutputEvent e) throws IOException { + if (isPassthrough()) { + // We only allow passthrough output if there is one JVM. + switch (e.getEvent().getType()) { + case EventType.APPEND_STDERR: + ((IStreamEvent) e.getEvent()).copyTo(errStream); + break; + case EventType.APPEND_STDOUT: + ((IStreamEvent) e.getEvent()).copyTo(outStream); + break; + default: + break; + } + } + } + + @Subscribe + void onTestResult(AggregatedTestResultEvent e) throws IOException { + if (isPassthrough() && e.getStatus() != TestStatus.OK) { + flushOutput(); + emitStatusLine(LogLevel.ERROR, e, e.getStatus(), e.getExecutionTime()); + } + + if (!e.isSuccessful()) { + failedTests.add(e.getDescription()); + } + } + + @Subscribe + void onSuiteResult(AggregatedSuiteResultEvent e) throws IOException { + final int completed = suitesCompleted.incrementAndGet(); + + if (e.isSuccessful() && e.getTests().isEmpty()) { + return; + } + if (config.slowTests.summarySize > 0) { + suiteTimes.put(e.getDescription().getDisplayName(), e.getExecutionTime()) + } + + LogLevel level = e.isSuccessful() && config.outputMode != OutputMode.ALWAYS ? LogLevel.INFO : LogLevel.LIFECYCLE + + // We must emit buffered test and stream events (in case of failures). + if (!isPassthrough()) { + emitSuiteStart(level, e.getDescription()) + emitBufferedEvents(level, e) + } + + // Emit a synthetic failure for suite-level errors, if any. + if (!e.getFailures().isEmpty()) { + emitStatusLine(level, e, TestStatus.ERROR, 0) + } + + if (!e.getFailures().isEmpty()) { + failedTests.add(e.getDescription()) + } + + emitSuiteEnd(level, e, completed) + } + + /** Suite prologue. */ + void emitSuiteStart(LogLevel level, Description description) throws IOException { + logger.log(level, 'Suite: ' + description.getDisplayName()); + } + + void emitBufferedEvents(LogLevel level, AggregatedSuiteResultEvent e) throws IOException { + if (config.outputMode == OutputMode.NEVER) { + return + } + + final IdentityHashMap eventMap = new IdentityHashMap<>(); + for (AggregatedTestResultEvent tre : e.getTests()) { + eventMap.put(tre.getTestFinishedEvent(), tre) + } + + final boolean emitOutput = config.outputMode == OutputMode.ALWAYS && isPassthrough() == false || + config.outputMode == OutputMode.ONERROR && e.isSuccessful() == false + + for (IEvent event : e.getEventStream()) { + switch (event.getType()) { + case EventType.APPEND_STDOUT: + if (emitOutput) ((IStreamEvent) event).copyTo(outStream); + break; + + case EventType.APPEND_STDERR: + if (emitOutput) ((IStreamEvent) event).copyTo(errStream); + break; + + case EventType.TEST_FINISHED: + assert eventMap.containsKey(event) + final AggregatedTestResultEvent aggregated = eventMap.get(event); + if (aggregated.getStatus() != TestStatus.OK) { + flushOutput(); + emitStatusLine(level, aggregated, aggregated.getStatus(), aggregated.getExecutionTime()); + } + + default: + break; + } + } + + if (emitOutput) { + flushOutput() + } + } + + void emitSuiteEnd(LogLevel level, AggregatedSuiteResultEvent e, int suitesCompleted) throws IOException { + + final StringBuilder b = new StringBuilder(); + b.append(String.format(Locale.ENGLISH, 'Completed [%d/%d]%s in %.2fs, ', + suitesCompleted, + totalSuites, + e.getSlave().slaves > 1 ? ' on J' + e.getSlave().id : '', + e.getExecutionTime() / 1000.0d)); + b.append(e.getTests().size()).append(Pluralize.pluralize(e.getTests().size(), ' test')); + + int failures = e.getFailureCount(); + if (failures > 0) { + b.append(', ').append(failures).append(Pluralize.pluralize(failures, ' failure')); + } + + int errors = e.getErrorCount(); + if (errors > 0) { + b.append(', ').append(errors).append(Pluralize.pluralize(errors, ' error')); + } + + int ignored = e.getIgnoredCount(); + if (ignored > 0) { + b.append(', ').append(ignored).append(' skipped'); + } + + if (!e.isSuccessful()) { + b.append(' <<< FAILURES!'); + } + + b.append('\n') + logger.log(level, b.toString()); + } + + /** Emit status line for an aggregated event. */ + void emitStatusLine(LogLevel level, AggregatedResultEvent result, TestStatus status, long timeMillis) throws IOException { + final StringBuilder line = new StringBuilder(); + + line.append(Strings.padEnd(statusNames.get(status), 8, ' ' as char)) + line.append(formatDurationInSeconds(timeMillis)) + if (forkedJvmCount > 1) { + line.append(String.format(Locale.ENGLISH, jvmIdFormat, result.getSlave().id)) + } + line.append(' | ') + + line.append(formatDescription(result.getDescription())) + if (!result.isSuccessful()) { + line.append(FAILURE_MARKER) + } + logger.log(level, line.toString()) + + PrintWriter writer = new PrintWriter(new LoggingOutputStream(logger: logger, level: level, prefix: ' > ')) + + if (status == TestStatus.IGNORED && result instanceof AggregatedTestResultEvent) { + writer.write('Cause: ') + writer.write(((AggregatedTestResultEvent) result).getCauseForIgnored()) + writer.flush() + } + + final List failures = result.getFailures(); + if (!failures.isEmpty()) { + int count = 0; + for (FailureMirror fm : failures) { + count++; + if (fm.isAssumptionViolation()) { + writer.write(String.format(Locale.ENGLISH, + 'Assumption #%d: %s', + count, fm.getMessage() == null ? '(no message)' : fm.getMessage())); + } else { + writer.write(String.format(Locale.ENGLISH, + 'Throwable #%d: %s', + count, + stackFilter.apply(fm.getTrace()))); + } + } + writer.flush() + } + } + + void flushOutput() throws IOException { + outStream.flush() + errStream.flush() + } + + /** Returns true if output should be logged immediately. */ + boolean isPassthrough() { + return forkedJvmCount == 1 && config.outputMode == OutputMode.ALWAYS + } + + @Override + void setOuter(JUnit4 task) { + owner = task + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy new file mode 100644 index 00000000000..c4d0ced6b5c --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -0,0 +1,426 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle + +import org.gradle.process.ExecResult + +import java.time.ZonedDateTime +import java.time.ZoneOffset + +import nebula.plugin.extraconfigurations.ProvidedBasePlugin +import org.elasticsearch.gradle.precommit.PrecommitTasks +import org.gradle.api.* +import org.gradle.api.artifacts.* +import org.gradle.api.artifacts.dsl.RepositoryHandler +import org.gradle.api.artifacts.maven.MavenPom +import org.gradle.api.tasks.bundling.Jar +import org.gradle.api.tasks.compile.JavaCompile +import org.gradle.internal.jvm.Jvm +import org.gradle.util.GradleVersion + +/** + * Encapsulates build configuration for elasticsearch projects. + */ +class BuildPlugin implements Plugin { + + static final JavaVersion minimumJava = JavaVersion.VERSION_1_8 + + @Override + void apply(Project project) { + project.pluginManager.apply('java') + project.pluginManager.apply('carrotsearch.randomized-testing') + // these plugins add lots of info to our jars + configureJarManifest(project) // jar config must be added before info broker + project.pluginManager.apply('nebula.info-broker') + project.pluginManager.apply('nebula.info-basic') + project.pluginManager.apply('nebula.info-java') + project.pluginManager.apply('nebula.info-scm') + project.pluginManager.apply('nebula.info-jar') + project.pluginManager.apply('com.bmuschko.nexus') + project.pluginManager.apply(ProvidedBasePlugin) + + globalBuildInfo(project) + configureRepositories(project) + configureConfigurations(project) + project.ext.versions = VersionProperties.versions + configureCompile(project) + + configureTest(project) + configurePrecommit(project) + } + + /** Performs checks on the build environment and prints information about the build environment. */ + static void globalBuildInfo(Project project) { + if (project.rootProject.ext.has('buildChecksDone') == false) { + String javaHome = findJavaHome() + File gradleJavaHome = Jvm.current().javaHome + String gradleJavaVersionDetails = "${System.getProperty('java.vendor')} ${System.getProperty('java.version')}" + + " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]" + + String javaVersionDetails = gradleJavaVersionDetails + String javaVersion = System.getProperty('java.version') + JavaVersion javaVersionEnum = JavaVersion.current() + if (new File(javaHome).canonicalPath != gradleJavaHome.canonicalPath) { + javaVersionDetails = findJavaVersionDetails(project, javaHome) + javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome)) + javaVersion = findJavaVersion(project, javaHome) + } + + // Build debugging info + println '=======================================' + println 'Elasticsearch Build Hamster says Hello!' + println '=======================================' + println " Gradle Version : ${project.gradle.gradleVersion}" + println " OS Info : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})" + if (gradleJavaVersionDetails != javaVersionDetails) { + println " JDK Version (gradle) : ${gradleJavaVersionDetails}" + println " JDK Version (compile) : ${javaVersionDetails}" + } else { + println " JDK Version : ${gradleJavaVersionDetails}" + } + + // enforce gradle version + GradleVersion minGradle = GradleVersion.version('2.8') + if (GradleVersion.current() < minGradle) { + throw new GradleException("${minGradle} or above is required to build elasticsearch") + } + + // enforce Java version + if (javaVersionEnum < minimumJava) { + throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch") + } + + project.rootProject.ext.javaHome = javaHome + project.rootProject.ext.javaVersion = javaVersion + project.rootProject.ext.buildChecksDone = true + } + project.targetCompatibility = minimumJava + project.sourceCompatibility = minimumJava + // set java home for each project, so they dont have to find it in the root project + project.ext.javaHome = project.rootProject.ext.javaHome + project.ext.javaVersion = project.rootProject.ext.javaVersion + } + + /** Finds and enforces JAVA_HOME is set */ + private static String findJavaHome() { + String javaHome = System.getenv('JAVA_HOME') + if (javaHome == null) { + if (System.getProperty("idea.active") != null) { + // intellij doesn't set JAVA_HOME, so we use the jdk gradle was run with + javaHome = Jvm.current().javaHome + } else { + throw new GradleException('JAVA_HOME must be set to build Elasticsearch') + } + } + return javaHome + } + + /** Finds printable java version of the given JAVA_HOME */ + private static String findJavaVersionDetails(Project project, String javaHome) { + String versionInfoScript = 'print(' + + 'java.lang.System.getProperty("java.vendor") + " " + java.lang.System.getProperty("java.version") + ' + + '" [" + java.lang.System.getProperty("java.vm.name") + " " + java.lang.System.getProperty("java.vm.version") + "]");' + return runJavascript(project, javaHome, versionInfoScript).trim() + } + + /** Finds the parsable java specification version */ + private static String findJavaSpecificationVersion(Project project, String javaHome) { + String versionScript = 'print(java.lang.System.getProperty("java.specification.version"));' + return runJavascript(project, javaHome, versionScript) + } + + /** Finds the parsable java specification version */ + private static String findJavaVersion(Project project, String javaHome) { + String versionScript = 'print(java.lang.System.getProperty("java.version"));' + return runJavascript(project, javaHome, versionScript) + } + + /** Runs the given javascript using jjs from the jdk, and returns the output */ + private static String runJavascript(Project project, String javaHome, String script) { + File tmpScript = File.createTempFile('es-gradle-tmp', '.js') + tmpScript.setText(script, 'UTF-8') + ByteArrayOutputStream output = new ByteArrayOutputStream() + ExecResult result = project.exec { + executable = new File(javaHome, 'bin/jjs') + args tmpScript.toString() + standardOutput = output + errorOutput = new ByteArrayOutputStream() + ignoreExitValue = true // we do not fail so we can first cleanup the tmp file + } + java.nio.file.Files.delete(tmpScript.toPath()) + result.assertNormalExitValue() + return output.toString('UTF-8').trim() + } + + /** Return the configuration name used for finding transitive deps of the given dependency. */ + private static String transitiveDepConfigName(String groupId, String artifactId, String version) { + return "_transitive_${groupId}:${artifactId}:${version}" + } + + /** + * Makes dependencies non-transitive. + * + * Gradle allows setting all dependencies as non-transitive very easily. + * Sadly this mechanism does not translate into maven pom generation. In order + * to effectively make the pom act as if it has no transitive dependencies, + * we must exclude each transitive dependency of each direct dependency. + * + * Determining the transitive deps of a dependency which has been resolved as + * non-transitive is difficult because the process of resolving removes the + * transitive deps. To sidestep this issue, we create a configuration per + * direct dependency version. This specially named and unique configuration + * will contain all of the transitive dependencies of this particular + * dependency. We can then use this configuration during pom generation + * to iterate the transitive dependencies and add excludes. + */ + static void configureConfigurations(Project project) { + // fail on any conflicting dependency versions + project.configurations.all({ Configuration configuration -> + if (configuration.name.startsWith('_transitive_')) { + // don't force transitive configurations to not conflict with themselves, since + // we just have them to find *what* transitive deps exist + return + } + configuration.resolutionStrategy.failOnVersionConflict() + }) + + // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself + Closure disableTransitiveDeps = { ModuleDependency dep -> + if (!(dep instanceof ProjectDependency) && dep.getGroup() != 'org.elasticsearch') { + dep.transitive = false + + // also create a configuration just for this dependency version, so that later + // we can determine which transitive dependencies it has + String depConfig = transitiveDepConfigName(dep.group, dep.name, dep.version) + if (project.configurations.findByName(depConfig) == null) { + project.configurations.create(depConfig) + project.dependencies.add(depConfig, "${dep.group}:${dep.name}:${dep.version}") + } + } + } + + project.configurations.compile.dependencies.all(disableTransitiveDeps) + project.configurations.testCompile.dependencies.all(disableTransitiveDeps) + project.configurations.provided.dependencies.all(disableTransitiveDeps) + + // add exclusions to the pom directly, for each of the transitive deps of this project's deps + project.modifyPom { MavenPom pom -> + pom.withXml { XmlProvider xml -> + // first find if we have dependencies at all, and grab the node + NodeList depsNodes = xml.asNode().get('dependencies') + if (depsNodes.isEmpty()) { + return + } + + // check each dependency for any transitive deps + for (Node depNode : depsNodes.get(0).children()) { + String groupId = depNode.get('groupId').get(0).text() + String artifactId = depNode.get('artifactId').get(0).text() + String version = depNode.get('version').get(0).text() + + // collect the transitive deps now that we know what this dependency is + String depConfig = transitiveDepConfigName(groupId, artifactId, version) + Configuration configuration = project.configurations.findByName(depConfig) + if (configuration == null) { + continue // we did not make this dep non-transitive + } + Set artifacts = configuration.resolvedConfiguration.resolvedArtifacts + if (artifacts.size() <= 1) { + // this dep has no transitive deps (or the only artifact is itself) + continue + } + + // we now know we have something to exclude, so add the exclusion elements + Node exclusions = depNode.appendNode('exclusions') + for (ResolvedArtifact transitiveArtifact : artifacts) { + ModuleVersionIdentifier transitiveDep = transitiveArtifact.moduleVersion.id + if (transitiveDep.group == groupId && transitiveDep.name == artifactId) { + continue; // don't exclude the dependency itself! + } + Node exclusion = exclusions.appendNode('exclusion') + exclusion.appendNode('groupId', transitiveDep.group) + exclusion.appendNode('artifactId', transitiveDep.name) + } + } + } + } + } + + /** Adds repositores used by ES dependencies */ + static void configureRepositories(Project project) { + RepositoryHandler repos = project.repositories + repos.mavenCentral() + repos.maven { + name 'sonatype-snapshots' + url 'http://oss.sonatype.org/content/repositories/snapshots/' + } + String luceneVersion = VersionProperties.lucene + if (luceneVersion.contains('-snapshot')) { + // extract the revision number from the version with a regex matcher + String revision = (luceneVersion =~ /\w+-snapshot-(\d+)/)[0][1] + repos.maven { + name 'lucene-snapshots' + url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" + } + } + } + + /** Adds compiler settings to the project */ + static void configureCompile(Project project) { + project.ext.compactProfile = 'compact3' + project.afterEvaluate { + // fail on all javac warnings + project.tasks.withType(JavaCompile) { + options.fork = true + options.forkOptions.executable = new File(project.javaHome, 'bin/javac') + options.forkOptions.memoryMaximumSize = "1g" + /* + * -path because gradle will send in paths that don't always exist. + * -missing because we have tons of missing @returns and @param. + */ + // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :) + options.compilerArgs << '-Werror' << '-Xlint:all,-path' << '-Xdoclint:all' << '-Xdoclint:-missing' + // compile with compact 3 profile by default + // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE + if (project.compactProfile != 'full') { + options.compilerArgs << '-profile' << project.compactProfile + } + options.encoding = 'UTF-8' + } + } + } + + /** Adds additional manifest info to jars */ + static void configureJarManifest(Project project) { + project.tasks.withType(Jar) { Jar jarTask -> + jarTask.doFirst { + // this doFirst is added before the info plugin, therefore it will run + // after the doFirst added by the info plugin, and we can override attributes + jarTask.manifest.attributes( + 'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch, + 'X-Compile-Lucene-Version': VersionProperties.lucene, + 'Build-Date': ZonedDateTime.now(ZoneOffset.UTC), + 'Build-Java-Version': project.javaVersion) + if (jarTask.manifest.attributes.containsKey('Change') == false) { + logger.warn('Building without git revision id.') + jarTask.manifest.attributes('Change': 'N/A') + } + } + } + } + + /** Returns a closure of common configuration shared by unit and integration tests. */ + static Closure commonTestConfig(Project project) { + return { + jvm "${project.javaHome}/bin/java" + parallelism System.getProperty('tests.jvms', 'auto') + ifNoTests 'fail' + leaveTemporary true + + // TODO: why are we not passing maxmemory to junit4? + jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m') + jvmArg '-Xms' + System.getProperty('tests.heap.size', '512m') + if (JavaVersion.current().isJava7()) { + // some tests need a large permgen, but that only exists on java 7 + jvmArg '-XX:MaxPermSize=128m' + } + jvmArg '-XX:MaxDirectMemorySize=512m' + jvmArg '-XX:+HeapDumpOnOutOfMemoryError' + File heapdumpDir = new File(project.buildDir, 'heapdump') + heapdumpDir.mkdirs() + jvmArg '-XX:HeapDumpPath=' + heapdumpDir + argLine System.getProperty('tests.jvm.argline') + + // we use './temp' since this is per JVM and tests are forbidden from writing to CWD + systemProperty 'java.io.tmpdir', './temp' + systemProperty 'java.awt.headless', 'true' + systemProperty 'tests.maven', 'true' // TODO: rename this once we've switched to gradle! + systemProperty 'tests.artifact', project.name + systemProperty 'tests.task', path + systemProperty 'tests.security.manager', 'true' + // default test sysprop values + systemProperty 'tests.ifNoTests', 'fail' + systemProperty 'es.logger.level', 'WARN' + for (Map.Entry property : System.properties.entrySet()) { + if (property.getKey().startsWith('tests.') || + property.getKey().startsWith('es.')) { + systemProperty property.getKey(), property.getValue() + } + } + + // System assertions (-esa) are disabled for now because of what looks like a + // JDK bug triggered by Groovy on JDK7. We should look at re-enabling system + // assertions when we upgrade to a new version of Groovy (currently 2.4.4) or + // require JDK8. See https://issues.apache.org/jira/browse/GROOVY-7528. + enableSystemAssertions false + + testLogging { + showNumFailuresAtEnd 25 + slowTests { + heartbeat 10 + summarySize 5 + } + stackTraceFilters { + // custom filters: we carefully only omit test infra noise here + contains '.SlaveMain.' + regex(/^(\s+at )(org\.junit\.)/) + // also includes anonymous classes inside these two: + regex(/^(\s+at )(com\.carrotsearch\.randomizedtesting\.RandomizedRunner)/) + regex(/^(\s+at )(com\.carrotsearch\.randomizedtesting\.ThreadLeakControl)/) + regex(/^(\s+at )(com\.carrotsearch\.randomizedtesting\.rules\.)/) + regex(/^(\s+at )(org\.apache\.lucene\.util\.TestRule)/) + regex(/^(\s+at )(org\.apache\.lucene\.util\.AbstractBeforeAfterRule)/) + } + if (System.getProperty('tests.class') != null && System.getProperty('tests.output') == null) { + // if you are debugging, you want to see the output! + outputMode 'always' + } else { + outputMode System.getProperty('tests.output', 'onerror') + } + } + + balancers { + executionTime cacheFilename: ".local-${project.version}-${name}-execution-times.log" + } + + listeners { + junitReport() + } + + exclude '**/*$*.class' + } + } + + /** Configures the test task */ + static Task configureTest(Project project) { + Task test = project.tasks.getByName('test') + test.configure(commonTestConfig(project)) + test.configure { + include '**/*Tests.class' + } + return test + } + + private static configurePrecommit(Project project) { + Task precommit = PrecommitTasks.create(project, true) + project.check.dependsOn(precommit) + project.test.mustRunAfter(precommit) + project.dependencyLicenses.dependencies = project.configurations.runtime - project.configurations.provided + } +} diff --git a/core/src/main/java/org/elasticsearch/index/settings/IndexSettings.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy similarity index 60% rename from core/src/main/java/org/elasticsearch/index/settings/IndexSettings.java rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy index b4305644762..d2059bc4719 100644 --- a/core/src/main/java/org/elasticsearch/index/settings/IndexSettings.java +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy @@ -16,26 +16,33 @@ * specific language governing permissions and limitations * under the License. */ +package org.elasticsearch.gradle -package org.elasticsearch.index.settings; - -import org.elasticsearch.common.inject.BindingAnnotation; - -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.* +import org.gradle.internal.nativeintegration.filesystem.Chmod +import java.io.File +import javax.inject.Inject /** - * + * Creates an empty directory. */ +class EmptyDirTask extends DefaultTask { + @Input + Object dir -@BindingAnnotation -@Target({FIELD, PARAMETER}) -@Retention(RUNTIME) -@Documented -public @interface IndexSettings { + @Input + int dirMode = 0755 + + @TaskAction + void create() { + dir = dir as File + dir.mkdirs() + getChmod().chmod(dir, dirMode) + } + + @Inject + Chmod getChmod() { + throw new UnsupportedOperationException() + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy new file mode 100644 index 00000000000..4df6d1b32df --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/FileContentsTask.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.* +import java.io.File + +/** + * Creates a file and sets it contents to something. + */ +class FileContentsTask extends DefaultTask { + /** + * The file to be built. Must be of type File to make @OutputFile happy. + */ + @OutputFile + File file + + @Input + Object contents + + /** + * The file to be built. Takes any objecct and coerces to a file. + */ + void setFile(Object file) { + this.file = file as File + } + + @TaskAction + void setContents() { + file = file as File + file.text = contents.toString() + } +} diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MissingFieldQueryExtension.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy similarity index 52% rename from core/src/main/java/org/apache/lucene/queryparser/classic/MissingFieldQueryExtension.java rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy index f9fc8c9d5dc..1896cdf1b67 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MissingFieldQueryExtension.java +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy @@ -17,26 +17,26 @@ * under the License. */ -package org.apache.lucene.queryparser.classic; +package org.elasticsearch.gradle -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Query; -import org.elasticsearch.index.query.MissingQueryBuilder; -import org.elasticsearch.index.query.QueryShardContext; +import org.gradle.api.GradleException +import org.gradle.api.tasks.Exec /** - * + * A wrapper around gradle's Exec task to capture output and log on error. */ -public class MissingFieldQueryExtension implements FieldQueryExtension { - - public static final String NAME = "_missing_"; - - @Override - public Query query(QueryShardContext context, String queryText) { - Query query = MissingQueryBuilder.newFilter(context, queryText, MissingQueryBuilder.DEFAULT_EXISTENCE_VALUE, MissingQueryBuilder.DEFAULT_NULL_VALUE); - if (query != null) { - return new ConstantScoreQuery(query); +class LoggedExec extends Exec { + LoggedExec() { + if (logger.isInfoEnabled() == false) { + standardOutput = new ByteArrayOutputStream() + errorOutput = standardOutput + ignoreExitValue = true + doLast { + if (execResult.exitValue != 0) { + standardOutput.toString('UTF-8').eachLine { line -> logger.error(line) } + throw new GradleException("Process '${executable} ${args.join(' ')}' finished with non-zero exit value ${execResult.exitValue}") + } + } } - return null; } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/MavenFilteringHack.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/MavenFilteringHack.groovy new file mode 100644 index 00000000000..b459003e7a8 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/MavenFilteringHack.groovy @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle + +import org.apache.tools.ant.filters.ReplaceTokens +import org.gradle.api.file.CopySpec + +/** + * Gradle provides "expansion" functionality using groovy's SimpleTemplatingEngine (TODO: check name). + * However, it allows substitutions of the form {@code $foo} (no curlies). Rest tests provide + * some substitution from the test runner, which this form is used for. + * + * This class provides a helper to do maven filtering, where only the form {@code $\{foo\}} is supported. + * + * TODO: we should get rid of this hack, and make the rest tests use some other identifier + * for builtin vars + */ +class MavenFilteringHack { + /** + * Adds a filter to the given copy spec that will substitute maven variables. + * @param CopySpec + */ + static void filter(CopySpec copySpec, Map substitutions) { + Map mavenSubstitutions = substitutions.collectEntries() { + key, value -> ["{${key}".toString(), value.toString()] + } + copySpec.filter(ReplaceTokens, tokens: mavenSubstitutions, beginToken: '$', endToken: '}') + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy new file mode 100644 index 00000000000..c24431b4cbc --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle + +/** + * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions. + */ +class VersionProperties { + static final String elasticsearch + static final String lucene + static final Map versions = new HashMap<>() + static { + Properties props = new Properties() + InputStream propsStream = VersionProperties.class.getResourceAsStream('/version.properties') + if (propsStream == null) { + throw new RuntimeException('/version.properties resource missing') + } + props.load(propsStream) + elasticsearch = props.getProperty('elasticsearch') + lucene = props.getProperty('lucene') + for (String property : props.stringPropertyNames()) { + versions.put(property, props.getProperty(property)) + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy new file mode 100644 index 00000000000..0d936ab0e15 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.plugin + +import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.test.RunTask +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.tasks.SourceSet +import org.gradle.api.tasks.bundling.Zip + +/** + * Encapsulates build configuration for an Elasticsearch plugin. + */ +public class PluginBuildPlugin extends BuildPlugin { + + @Override + public void apply(Project project) { + super.apply(project) + configureDependencies(project) + // this afterEvaluate must happen before the afterEvaluate added by integTest creation, + // so that the file name resolution for installing the plugin will be setup + project.afterEvaluate { + String name = project.pluginProperties.extension.name + project.jar.baseName = name + project.bundlePlugin.baseName = name + + project.integTest.dependsOn(project.bundlePlugin) + project.tasks.run.dependsOn(project.bundlePlugin) + if (project.path.startsWith(':modules:')) { + project.integTest.clusterConfig.module(project) + project.tasks.run.clusterConfig.module(project) + } else { + project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) + project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) + } + } + createIntegTestTask(project) + createBundleTask(project) + project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build + } + + private static void configureDependencies(Project project) { + project.dependencies { + provided "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}" + testCompile "org.elasticsearch:test-framework:${project.versions.elasticsearch}" + // we "upgrade" these optional deps to provided for plugins, since they will run + // with a full elasticsearch server that includes optional deps + provided "com.spatial4j:spatial4j:${project.versions.spatial4j}" + provided "com.vividsolutions:jts:${project.versions.jts}" + provided "log4j:log4j:${project.versions.log4j}" + provided "log4j:apache-log4j-extras:${project.versions.log4j}" + provided "org.slf4j:slf4j-api:${project.versions.slf4j}" + provided "net.java.dev.jna:jna:${project.versions.jna}" + } + } + + /** Adds an integTest task which runs rest tests */ + private static void createIntegTestTask(Project project) { + RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class) + integTest.mustRunAfter(project.precommit, project.test) + project.check.dependsOn(integTest) + } + + /** + * Adds a bundlePlugin task which builds the zip containing the plugin jars, + * metadata, properties, and packaging files + */ + private static void createBundleTask(Project project) { + File pluginMetadata = project.file('src/main/plugin-metadata') + + // create a task to build the properties file for this plugin + PluginPropertiesTask buildProperties = project.tasks.create('pluginProperties', PluginPropertiesTask.class) + + // add the plugin properties and metadata to test resources, so unit tests can + // know about the plugin (used by test security code to statically initialize the plugin in unit tests) + SourceSet testSourceSet = project.sourceSets.test + testSourceSet.output.dir(buildProperties.generatedResourcesDir, builtBy: 'pluginProperties') + testSourceSet.resources.srcDir(pluginMetadata) + + // create the actual bundle task, which zips up all the files for the plugin + Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [project.jar, buildProperties]) { + from buildProperties // plugin properties file + from pluginMetadata // metadata (eg custom security policy) + from project.jar // this plugin's jar + from project.configurations.runtime - project.configurations.provided // the dep jars + // extra files for the plugin to go into the zip + from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging + from('src/main') { + include 'config/**' + include 'bin/**' + } + from('src/site') { + include '_site/**' + } + } + project.assemble.dependsOn(bundle) + + // remove jar from the archives (things that will be published), and set it to the zip + project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar } + project.artifacts.add('archives', bundle) + + // also make the zip the default artifact (used when depending on this project) + project.configurations.getByName('default').extendsFrom = [] + project.artifacts.add('default', bundle) + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy new file mode 100644 index 00000000000..ce6b3958eca --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.plugin + +import org.gradle.api.Project +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Optional + +/** + * A container for plugin properties that will be written to the plugin descriptor, for easy + * manipulation in the gradle DSL. + */ +class PluginPropertiesExtension { + + @Input + String name + + @Input + String version + + @Input + String description + + @Input + boolean jvm = true + + @Input + String classname + + @Input + boolean site = false + + @Input + boolean isolated = true + + PluginPropertiesExtension(Project project) { + name = project.name + version = project.version + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy new file mode 100644 index 00000000000..51853f85e00 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.plugin + +import org.elasticsearch.gradle.VersionProperties +import org.gradle.api.InvalidUserDataException +import org.gradle.api.Task +import org.gradle.api.tasks.Copy + +/** + * Creates a plugin descriptor. + */ +class PluginPropertiesTask extends Copy { + + PluginPropertiesExtension extension + File generatedResourcesDir = new File(project.projectDir, 'generated-resources') + + PluginPropertiesTask() { + File templateFile = new File(project.buildDir, 'templates/plugin-descriptor.properties') + Task copyPluginPropertiesTemplate = project.tasks.create('copyPluginPropertiesTemplate') { + doLast { + InputStream resourceTemplate = PluginPropertiesTask.getResourceAsStream('/plugin-descriptor.properties') + templateFile.parentFile.mkdirs() + templateFile.setText(resourceTemplate.getText('UTF-8'), 'UTF-8') + } + } + dependsOn(copyPluginPropertiesTemplate) + extension = project.extensions.create('esplugin', PluginPropertiesExtension, project) + project.clean.delete(generatedResourcesDir) + project.afterEvaluate { + // check require properties are set + if (extension.name == null) { + throw new InvalidUserDataException('name is a required setting for esplugin') + } + if (extension.description == null) { + throw new InvalidUserDataException('description is a required setting for esplugin') + } + if (extension.jvm && extension.classname == null) { + throw new InvalidUserDataException('classname is a required setting for esplugin with jvm=true') + } + doFirst { + if (extension.jvm && extension.isolated == false) { + String warning = "WARNING: Disabling plugin isolation in ${project.path} is deprecated and will be removed in the future" + logger.warn("${'=' * warning.length()}\n${warning}\n${'=' * warning.length()}") + } + } + // configure property substitution + from(templateFile) + into(generatedResourcesDir) + expand(generateSubstitutions()) + } + } + + Map generateSubstitutions() { + return [ + 'name': extension.name, + 'description': extension.description, + 'version': extension.version, + 'elasticsearchVersion': VersionProperties.elasticsearch, + 'javaVersion': project.targetCompatibility as String, + 'jvm': extension.jvm as String, + 'site': extension.site as String, + 'isolated': extension.isolated as String, + 'classname': extension.jvm ? extension.classname : 'NA' + ] + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy new file mode 100644 index 00000000000..e2f10100269 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy @@ -0,0 +1,231 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.precommit + +import org.gradle.api.* +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.TaskAction + +import java.nio.file.Files +import java.security.MessageDigest +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * A task to check licenses for dependencies. + * + * There are two parts to the check: + *
    + *
  • LICENSE and NOTICE files
    • + *
  • SHA checksums for each dependency jar
    • + *
+ * + * The directory to find the license and sha files in defaults to the dir @{code licenses} + * in the project directory for this task. You can override this directory: + * 
+ *   dependencyLicenses {
+ *     licensesDir = project.file('mybetterlicensedir')
+ *   }
+ *
+ * + * The jar files to check default to the dependencies from the default configuration. You + * can override this, for example, to only check compile dependencies: + * 
+ *   dependencyLicenses {
+ *     dependencies = project.configurations.compile
+ *   }
+ *
+ * + * Every jar must have a {@code .sha1} file in the licenses dir. These can be managed + * automatically using the {@code updateShas} helper task that is created along + * with this task. It will add {@code .sha1} files for new jars that are in dependencies + * and remove old {@code .sha1} files that are no longer needed. + * + * Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share + * LICENSE and NOTICE files by mapping a pattern to the same name. + * 
+ *   dependencyLicenses {
+ *     mapping from: /lucene-.*/, to: 'lucene'
+ *   }
+ *
+ */ +public class DependencyLicensesTask extends DefaultTask { + static final String SHA_EXTENSION = '.sha1' + + // TODO: we should be able to default this to eg compile deps, but we need to move the licenses + // check from distribution to core (ie this should only be run on java projects) + /** A collection of jar files that should be checked. */ + @InputFiles + public FileCollection dependencies + + /** The directory to find the license and sha files in. */ + @InputDirectory + public File licensesDir = new File(project.projectDir, 'licenses') + + /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */ + private LinkedHashMap mappings = new LinkedHashMap<>() + + /** + * Add a mapping from a regex pattern for the jar name, to a prefix to find + * the LICENSE and NOTICE file for that jar. + */ + @Input + public void mapping(Map props) { + String from = props.remove('from') + if (from == null) { + throw new InvalidUserDataException('Missing "from" setting for license name mapping') + } + String to = props.remove('to') + if (to == null) { + throw new InvalidUserDataException('Missing "to" setting for license name mapping') + } + if (props.isEmpty() == false) { + throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: ${props.keySet()}") + } + mappings.put(from, to) + } + + @TaskAction + public void checkDependencies() { + if (dependencies.isEmpty()) { + if (licensesDir.exists()) { + throw new GradleException("Licenses dir ${licensesDir} exists, but there are no dependencies") + } + return // no dependencies to check + } else if (licensesDir.exists() == false) { + throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies") + } + + + // order is the same for keys and values iteration since we use a linked hashmap + List mapped = new ArrayList<>(mappings.values()) + Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')') + Map licenses = new HashMap<>() + Map notices = new HashMap<>() + Set shaFiles = new HashSet() + + licensesDir.eachFile { + String name = it.getName() + if (name.endsWith(SHA_EXTENSION)) { + shaFiles.add(it) + } else if (name.endsWith('-LICENSE') || name.endsWith('-LICENSE.txt')) { + // TODO: why do we support suffix of LICENSE *and* LICENSE.txt?? + licenses.put(name, 0) + } else if (name.contains('-NOTICE') || name.contains('-NOTICE.txt')) { + notices.put(name, 0) + } + } + + for (File dependency : dependencies) { + String jarName = dependency.getName() + logger.info("Checking license/notice/sha for " + jarName) + checkSha(dependency, jarName, shaFiles) + + String name = jarName - ~/\-\d+.*/ + Matcher match = mappingsPattern.matcher(name) + if (match.matches()) { + int i = 0 + while (i < match.groupCount() && match.group(i + 1) == null) ++i; + logger.info("Mapped dependency name ${name} to ${mapped.get(i)} for license check") + name = mapped.get(i) + } + checkFile(name, jarName, licenses, 'LICENSE') + checkFile(name, jarName, notices, 'NOTICE') + } + + licenses.each { license, count -> + if (count == 0) { + throw new GradleException("Unused license ${license}") + } + } + notices.each { notice, count -> + if (count == 0) { + throw new GradleException("Unused notice ${notice}") + } + } + if (shaFiles.isEmpty() == false) { + throw new GradleException("Unused sha files found: \n${shaFiles.join('\n')}") + } + } + + private void checkSha(File jar, String jarName, Set shaFiles) { + File shaFile = new File(licensesDir, jarName + SHA_EXTENSION) + if (shaFile.exists() == false) { + throw new GradleException("Missing SHA for ${jarName}. Run 'gradle updateSHAs' to create") + } + // TODO: shouldn't have to trim, sha files should not have trailing newline + String expectedSha = shaFile.getText('UTF-8').trim() + String sha = MessageDigest.getInstance("SHA-1").digest(jar.getBytes()).encodeHex().toString() + if (expectedSha.equals(sha) == false) { + throw new GradleException("SHA has changed! Expected ${expectedSha} for ${jarName} but got ${sha}. " + + "\nThis usually indicates a corrupt dependency cache or artifacts changed upstream." + + "\nEither wipe your cache, fix the upstream artifact, or delete ${shaFile} and run updateShas") + } + shaFiles.remove(shaFile) + } + + private void checkFile(String name, String jarName, Map counters, String type) { + String fileName = "${name}-${type}" + Integer count = counters.get(fileName) + if (count == null) { + // try the other suffix...TODO: get rid of this, just support ending in .txt + fileName = "${fileName}.txt" + counters.get(fileName) + } + count = counters.get(fileName) + if (count == null) { + throw new GradleException("Missing ${type} for ${jarName}, expected in ${fileName}") + } + counters.put(fileName, count + 1) + } + + /** A helper task to update the sha files in the license dir. */ + public static class UpdateShasTask extends DefaultTask { + private DependencyLicensesTask parentTask + + @TaskAction + public void updateShas() { + Set shaFiles = new HashSet() + parentTask.licensesDir.eachFile { + String name = it.getName() + if (name.endsWith(SHA_EXTENSION)) { + shaFiles.add(it) + } + } + for (File dependency : parentTask.dependencies) { + String jarName = dependency.getName() + File shaFile = new File(parentTask.licensesDir, jarName + SHA_EXTENSION) + if (shaFile.exists() == false) { + logger.lifecycle("Adding sha for ${jarName}") + String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString() + shaFile.setText(sha, 'UTF-8') + } else { + shaFiles.remove(shaFile) + } + } + shaFiles.each { shaFile -> + logger.lifecycle("Removing unused sha ${shaFile.getName()}") + Files.delete(shaFile.toPath()) + } + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy new file mode 100644 index 00000000000..6809adca946 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.precommit + +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.InvalidUserDataException +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.SourceSet +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.util.PatternFilterable +import org.gradle.api.tasks.util.PatternSet + +import java.util.regex.Pattern + +/** + * Checks for patterns in source files for the project which are forbidden. + */ +public class ForbiddenPatternsTask extends DefaultTask { + + /** The rules: a map from the rule name, to a rule regex pattern. */ + private Map patterns = new LinkedHashMap<>() + /** A pattern set of which files should be checked. */ + private PatternFilterable filesFilter = new PatternSet() + + @OutputFile + File outputMarker = new File(project.buildDir, "markers/forbiddenPatterns") + + public ForbiddenPatternsTask() { + description = 'Checks source files for invalid patterns like nocommits or tabs' + + // we always include all source files, and exclude what should not be checked + filesFilter.include('**') + // exclude known binary extensions + filesFilter.exclude('**/*.gz') + filesFilter.exclude('**/*.ico') + filesFilter.exclude('**/*.jar') + filesFilter.exclude('**/*.zip') + filesFilter.exclude('**/*.jks') + filesFilter.exclude('**/*.crt') + filesFilter.exclude('**/*.png') + + // add mandatory rules + patterns.put('nocommit', /nocommit/) + patterns.put('tab', /\t/) + + inputs.property("excludes", filesFilter.excludes) + inputs.property("rules", patterns) + } + + /** Adds a file glob pattern to be excluded */ + public void exclude(String... excludes) { + filesFilter.exclude(excludes) + } + + /** Adds a pattern to forbid. T */ + void rule(Map props) { + String name = props.remove('name') + if (name == null) { + throw new InvalidUserDataException('Missing [name] for invalid pattern rule') + } + String pattern = props.remove('pattern') + if (pattern == null) { + throw new InvalidUserDataException('Missing [pattern] for invalid pattern rule') + } + if (props.isEmpty() == false) { + throw new InvalidUserDataException("Unknown arguments for ForbiddenPatterns rule mapping: ${props.keySet()}") + } + // TODO: fail if pattern contains a newline, it won't work (currently) + patterns.put(name, pattern) + } + + /** Returns the files this task will check */ + @InputFiles + FileCollection files() { + List collections = new ArrayList<>() + for (SourceSet sourceSet : project.sourceSets) { + collections.add(sourceSet.allSource.matching(filesFilter)) + } + return project.files(collections.toArray()) + } + + @TaskAction + void checkInvalidPatterns() { + Pattern allPatterns = Pattern.compile('(' + patterns.values().join(')|(') + ')') + List failures = new ArrayList<>() + for (File f : files()) { + f.eachLine('UTF-8') { String line, int lineNumber -> + if (allPatterns.matcher(line).find()) { + addErrorMessages(failures, f, line, lineNumber) + } + } + } + if (failures.isEmpty() == false) { + throw new GradleException('Found invalid patterns:\n' + failures.join('\n')) + } + outputMarker.setText('done', 'UTF-8') + } + + // iterate through patterns to find the right ones for nice error messages + void addErrorMessages(List failures, File f, String line, int lineNumber) { + String path = project.getRootProject().projectDir.toURI().relativize(f.toURI()).toString() + for (Map.Entry pattern : patterns.entrySet()) { + if (Pattern.compile(pattern.value).matcher(line).find()) { + failures.add('- ' + pattern.key + ' on line ' + lineNumber + ' of ' + path) + } + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy new file mode 100644 index 00000000000..2873fbd4df5 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.precommit + +import org.elasticsearch.gradle.LoggedExec +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.OutputFile + +/** + * Runs CheckJarHell on a classpath. + */ +public class JarHellTask extends LoggedExec { + + /** + * We use a simple "marker" file that we touch when the task succeeds + * as the task output. This is compared against the modified time of the + * inputs (ie the jars/class files). + */ + @OutputFile + public File successMarker = new File(project.buildDir, 'markers/jarHell') + + /** The classpath to run jarhell check on, defaults to the test runtime classpath */ + @InputFile + public FileCollection classpath = project.sourceSets.test.runtimeClasspath + + public JarHellTask() { + project.afterEvaluate { + dependsOn(classpath) + description = "Runs CheckJarHell on ${classpath}" + executable = new File(project.javaHome, 'bin/java') + doFirst({ + /* JarHell doesn't like getting directories that don't exist but + gradle isn't especially careful about that. So we have to do it + filter it ourselves. */ + FileCollection taskClasspath = classpath.filter { it.exists() } + args('-cp', taskClasspath.asPath, 'org.elasticsearch.bootstrap.JarHell') + }) + doLast({ + successMarker.parentFile.mkdirs() + successMarker.setText("", 'UTF-8') + }) + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy new file mode 100644 index 00000000000..04878d979e9 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.precommit + +import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.plugins.JavaBasePlugin + +/** + * Validation tasks which should be run before committing. These run before tests. + */ +class PrecommitTasks { + + /** Adds a precommit task, which depends on non-test verification tasks. */ + public static Task create(Project project, boolean includeDependencyLicenses) { + + List precommitTasks = [ + configureForbiddenApis(project), + project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class), + project.tasks.create('jarHell', JarHellTask.class)] + + // tasks with just tests don't need dependency licenses, so this flag makes adding + // the task optional + if (includeDependencyLicenses) { + DependencyLicensesTask dependencyLicenses = project.tasks.create('dependencyLicenses', DependencyLicensesTask.class) + precommitTasks.add(dependencyLicenses) + // we also create the updateShas helper task that is associated with dependencyLicenses + UpdateShasTask updateShas = project.tasks.create('updateShas', UpdateShasTask.class) + updateShas.parentTask = dependencyLicenses + } + + Map precommitOptions = [ + name: 'precommit', + group: JavaBasePlugin.VERIFICATION_GROUP, + description: 'Runs all non-test checks.', + dependsOn: precommitTasks + ] + return project.tasks.create(precommitOptions) + } + + private static Task configureForbiddenApis(Project project) { + project.pluginManager.apply(ForbiddenApisPlugin.class) + project.forbiddenApis { + internalRuntimeForbidden = true + failOnUnsupportedJava = false + bundledSignatures = ['jdk-unsafe', 'jdk-deprecated'] + signaturesURLs = [getClass().getResource('/forbidden/all-signatures.txt')] + suppressAnnotations = ['**.SuppressForbidden'] + } + Task mainForbidden = project.tasks.findByName('forbiddenApisMain') + if (mainForbidden != null) { + mainForbidden.configure { + bundledSignatures += 'jdk-system-out' + signaturesURLs += getClass().getResource('/forbidden/core-signatures.txt') + } + } + Task testForbidden = project.tasks.findByName('forbiddenApisTest') + if (testForbidden != null) { + testForbidden.configure { + signaturesURLs += getClass().getResource('/forbidden/test-signatures.txt') + } + } + Task forbiddenApis = project.tasks.findByName('forbiddenApis') + forbiddenApis.group = "" // clear group, so this does not show up under verification tasks + return forbiddenApis + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy new file mode 100644 index 00000000000..4a174688aa1 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.precommit + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.TaskAction + +import java.nio.file.Files +import java.security.MessageDigest + +/** + * A task to update shas used by {@code DependencyLicensesCheck} + */ +public class UpdateShasTask extends DefaultTask { + + /** The parent dependency licenses task to use configuration from */ + public DependencyLicensesTask parentTask + + public UpdateShasTask() { + description = 'Updates the sha files for the dependencyLicenses check' + onlyIf { parentTask.licensesDir.exists() } + } + + @TaskAction + public void updateShas() { + Set shaFiles = new HashSet() + parentTask.licensesDir.eachFile { + String name = it.getName() + if (name.endsWith(DependencyLicensesTask.SHA_EXTENSION)) { + shaFiles.add(it) + } + } + for (File dependency : parentTask.dependencies) { + String jarName = dependency.getName() + File shaFile = new File(parentTask.licensesDir, jarName + DependencyLicensesTask.SHA_EXTENSION) + if (shaFile.exists() == false) { + logger.lifecycle("Adding sha for ${jarName}") + String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString() + shaFile.setText(sha, 'UTF-8') + } else { + shaFiles.remove(shaFile) + } + } + shaFiles.each { shaFile -> + logger.lifecycle("Removing unused sha ${shaFile.getName()}") + Files.delete(shaFile.toPath()) + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy new file mode 100644 index 00000000000..fa23299cee4 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test + +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.Input + +/** Configuration for an elasticsearch cluster, used for integration tests. */ +class ClusterConfiguration { + + @Input + String distribution = 'integ-test-zip' + + @Input + int numNodes = 1 + + @Input + int httpPort = 0 + + @Input + int transportPort = 0 + + @Input + boolean daemonize = true + + @Input + boolean debug = false + + @Input + String jvmArgs = System.getProperty('tests.jvm.argline', '') + + /** + * A closure to call before the cluster is considered ready. The closure is passed the node info, + * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait + * condition is for http on the http port. + */ + @Input + Closure waitCondition = { NodeInfo node, AntBuilder ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://${node.httpUri()}", + dest: tmpFile.toString(), + ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task + retries: 10) + return tmpFile.exists() + } + + Map systemProperties = new HashMap<>() + + Map settings = new HashMap<>() + + // map from destination path, to source file + Map extraConfigFiles = new HashMap<>() + + LinkedHashMap plugins = new LinkedHashMap<>() + + List modules = new ArrayList<>() + + LinkedHashMap setupCommands = new LinkedHashMap<>() + + @Input + void systemProperty(String property, String value) { + systemProperties.put(property, value) + } + + @Input + void setting(String name, String value) { + settings.put(name, value) + } + + @Input + void plugin(String name, FileCollection file) { + plugins.put(name, file) + } + + @Input + void plugin(String name, Project pluginProject) { + plugins.put(name, pluginProject) + } + + /** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */ + @Input + void module(Project moduleProject) { + modules.add(moduleProject) + } + + @Input + void setupCommand(String name, Object... args) { + setupCommands.put(name, args) + } + + /** + * Add an extra configuration file. The path is relative to the config dir, and the sourceFile + * is anything accepted by project.file() + */ + @Input + void extraConfigFile(String path, Object sourceFile) { + if (path == 'elasticsearch.yml') { + throw new GradleException('Overwriting elasticsearch.yml is not allowed, add additional settings using cluster { setting "foo", "bar" }') + } + extraConfigFiles.put(path, sourceFile) + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy new file mode 100644 index 00000000000..08976dbdb39 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -0,0 +1,565 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test + +import org.apache.tools.ant.DefaultLogger +import org.apache.tools.ant.taskdefs.condition.Os +import org.elasticsearch.gradle.LoggedExec +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.plugin.PluginBuildPlugin +import org.gradle.api.* +import org.gradle.api.artifacts.Configuration +import org.gradle.api.file.FileCollection +import org.gradle.api.logging.Logger +import org.gradle.api.tasks.* + +import java.nio.file.Paths + +/** + * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished. + */ +class ClusterFormationTasks { + + /** + * Adds dependent tasks to the given task to start and stop a cluster with the given configuration. + * + * Returns an object that will resolve at execution time of the given task to a uri for the cluster. + */ + static Object setup(Project project, Task task, ClusterConfiguration config) { + if (task.getEnabled() == false) { + // no need to add cluster formation tasks if the task won't run! + return + } + configureDistributionDependency(project, config.distribution) + List startTasks = [] + List nodes = [] + for (int i = 0; i < config.numNodes; ++i) { + NodeInfo node = new NodeInfo(config, i, project, task) + nodes.add(node) + startTasks.add(configureNode(project, task, node)) + } + + Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks) + task.dependsOn(wait) + + // delay the resolution of the uri by wrapping in a closure, so it is not used until read for tests + return "${-> nodes[0].transportUri()}" + } + + /** Adds a dependency on the given distribution */ + static void configureDistributionDependency(Project project, String distro) { + String elasticsearchVersion = VersionProperties.elasticsearch + String packaging = distro + if (distro == 'tar') { + packaging = 'tar.gz' + } else if (distro == 'integ-test-zip') { + packaging = 'zip' + } + project.configurations { + elasticsearchDistro + } + project.dependencies { + elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}" + } + } + + /** + * Adds dependent tasks to start an elasticsearch cluster before the given task is executed, + * and stop it after it has finished executing. + * + * The setup of the cluster involves the following: + *
    + *
  1. Cleanup the extraction directory
    2. + *
  2. Extract a fresh copy of elasticsearch
    3. + *
  3. Write an elasticsearch.yml config file
    4. + *
  4. Copy plugins that will be installed to a temporary dir (which contains spaces)
    5. + *
  5. Install plugins
    6. + *
  6. Run additional setup commands
    7. + *
  7. Start elasticsearch
  8. + *
+ * + * @return a task which starts the node. + */ + static Task configureNode(Project project, Task task, NodeInfo node) { + + // tasks are chained so their execution order is maintained + Task setup = project.tasks.create(name: taskName(task, node, 'clean'), type: Delete, dependsOn: task.dependsOn.collect()) { + delete node.homeDir + delete node.cwd + doLast { + node.cwd.mkdirs() + } + } + setup = configureCheckPreviousTask(taskName(task, node, 'checkPrevious'), project, setup, node) + setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node) + setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node) + setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node) + setup = configureExtraConfigFilesTask(taskName(task, node, 'extraConfig'), project, setup, node) + setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node) + + // install modules + for (Project module : node.config.modules) { + String actionName = pluginTaskName('install', module.name, 'Module') + setup = configureInstallModuleTask(taskName(task, node, actionName), project, setup, node, module) + } + + // install plugins + for (Map.Entry plugin : node.config.plugins.entrySet()) { + String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin') + setup = configureInstallPluginTask(taskName(task, node, actionName), project, setup, node, plugin.getValue()) + } + + // extra setup commands + for (Map.Entry command : node.config.setupCommands.entrySet()) { + // the first argument is the actual script name, relative to home + Object[] args = command.getValue().clone() + args[0] = new File(node.homeDir, args[0].toString()) + setup = configureExecTask(taskName(task, node, command.getKey()), project, setup, node, args) + } + + Task start = configureStartTask(taskName(task, node, 'start'), project, setup, node) + + if (node.config.daemonize) { + // if we are running in the background, make sure to stop the server when the task completes + Task stop = configureStopTask(taskName(task, node, 'stop'), project, [], node) + task.finalizedBy(stop) + } + return start + } + + /** Adds a task to extract the elasticsearch distribution */ + static Task configureExtractTask(String name, Project project, Task setup, NodeInfo node) { + List extractDependsOn = [project.configurations.elasticsearchDistro, setup] + /* project.configurations.elasticsearchDistro.singleFile will be an + external artifact if this is being run by a plugin not living in the + elasticsearch source tree. If this is a plugin built in the + elasticsearch source tree or this is a distro in the elasticsearch + source tree then this should be the version of elasticsearch built + by the source tree. If it isn't then Bad Things(TM) will happen. */ + Task extract + switch (node.config.distribution) { + case 'integ-test-zip': + case 'zip': + extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { + from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) } + into node.baseDir + } + break; + case 'tar': + extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { + from { + project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile)) + } + into node.baseDir + } + break; + case 'rpm': + File rpmDatabase = new File(node.baseDir, 'rpm-database') + File rpmExtracted = new File(node.baseDir, 'rpm-extracted') + /* Delay reading the location of the rpm file until task execution */ + Object rpm = "${ -> project.configurations.elasticsearchDistro.singleFile}" + extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { + commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers', + '--dbpath', rpmDatabase, + '--relocate', "/=${rpmExtracted}", + '-i', rpm + doFirst { + rpmDatabase.deleteDir() + rpmExtracted.deleteDir() + } + } + break; + case 'deb': + /* Delay reading the location of the deb file until task execution */ + File debExtracted = new File(node.baseDir, 'deb-extracted') + Object deb = "${ -> project.configurations.elasticsearchDistro.singleFile}" + extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { + commandLine 'dpkg-deb', '-x', deb, debExtracted + doFirst { + debExtracted.deleteDir() + } + } + break; + default: + throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}") + } + return extract + } + + /** Adds a task to write elasticsearch.yml for the given node configuration */ + static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node) { + Map esConfig = [ + 'cluster.name' : node.clusterName, + 'pidfile' : node.pidFile, + 'path.repo' : "${node.homeDir}/repo", + 'path.shared_data' : "${node.homeDir}/../", + // Define a node attribute so we can test that it exists + 'node.testattr' : 'test', + 'repositories.url.allowed_urls': 'http://snapshot.test*' + ] + if (node.config.numNodes == 1) { + esConfig['http.port'] = node.config.httpPort + esConfig['transport.tcp.port'] = node.config.transportPort + } else { + // TODO: fix multi node so it doesn't use hardcoded prots + esConfig['http.port'] = 9400 + node.nodeNum + esConfig['transport.tcp.port'] = 9500 + node.nodeNum + esConfig['discovery.zen.ping.unicast.hosts'] = (0.. "${key}: ${value}" }.join('\n'), 'UTF-8') + } + } + + static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) { + if (node.config.extraConfigFiles.isEmpty()) { + return setup + } + Copy copyConfig = project.tasks.create(name: name, type: Copy, dependsOn: setup) + copyConfig.into(new File(node.homeDir, 'config')) // copy must always have a general dest dir, even though we don't use it + for (Map.Entry extraConfigFile : node.config.extraConfigFiles.entrySet()) { + copyConfig.doFirst { + // make sure the copy won't be a no-op or act on a directory + File srcConfigFile = project.file(extraConfigFile.getValue()) + if (srcConfigFile.isDirectory()) { + throw new GradleException("Source for extraConfigFile must be a file: ${srcConfigFile}") + } + if (srcConfigFile.exists() == false) { + throw new GradleException("Source file for extraConfigFile does not exist: ${srcConfigFile}") + } + } + File destConfigFile = new File(node.homeDir, 'config/' + extraConfigFile.getKey()) + copyConfig.into(destConfigFile.canonicalFile.parentFile) + .from({ extraConfigFile.getValue() }) // wrap in closure to delay resolution to execution time + .rename { destConfigFile.name } + } + return copyConfig + } + + /** + * Adds a task to copy plugins to a temp dir, which they will later be installed from. + * + * For each plugin, if the plugin has rest spec apis in its tests, those api files are also copied + * to the test resources for this project. + */ + static Task configureCopyPluginsTask(String name, Project project, Task setup, NodeInfo node) { + if (node.config.plugins.isEmpty()) { + return setup + } + Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) + + List pluginFiles = [] + for (Map.Entry plugin : node.config.plugins.entrySet()) { + FileCollection pluginZip + if (plugin.getValue() instanceof Project) { + Project pluginProject = plugin.getValue() + if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) { + throw new GradleException("Task ${name} cannot project ${pluginProject.path} which is not an esplugin") + } + String configurationName = "_plugin_${pluginProject.path}" + Configuration configuration = project.configurations.findByName(configurationName) + if (configuration == null) { + configuration = project.configurations.create(configurationName) + } + project.dependencies.add(configurationName, pluginProject) + setup.dependsOn(pluginProject.tasks.bundlePlugin) + pluginZip = configuration + + // also allow rest tests to use the rest spec from the plugin + Copy copyRestSpec = null + for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) { + File restApiDir = new File(resourceDir, 'rest-api-spec/api') + if (restApiDir.exists() == false) continue + if (copyRestSpec == null) { + copyRestSpec = project.tasks.create(name: pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec'), type: Copy) + copyPlugins.dependsOn(copyRestSpec) + copyRestSpec.into(project.sourceSets.test.output.resourcesDir) + } + copyRestSpec.from(resourceDir).include('rest-api-spec/api/**') + } + } else { + pluginZip = plugin.getValue() + } + pluginFiles.add(pluginZip) + } + + copyPlugins.into(node.pluginsTmpDir) + copyPlugins.from(pluginFiles) + return copyPlugins + } + + static Task configureInstallModuleTask(String name, Project project, Task setup, NodeInfo node, Project module) { + if (node.config.distribution != 'integ-test-zip') { + throw new GradleException("Module ${module.path} not allowed be installed distributions other than integ-test-zip because they should already have all modules bundled!") + } + if (module.plugins.hasPlugin(PluginBuildPlugin) == false) { + throw new GradleException("Task ${name} cannot include module ${module.path} which is not an esplugin") + } + Copy installModule = project.tasks.create(name, Copy.class) + installModule.dependsOn(setup) + installModule.into(new File(node.homeDir, "modules/${module.name}")) + installModule.from({ project.zipTree(module.tasks.bundlePlugin.outputs.files.singleFile) }) + return installModule + } + + static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Object plugin) { + FileCollection pluginZip + if (plugin instanceof Project) { + pluginZip = project.configurations.getByName("_plugin_${plugin.path}") + } else { + pluginZip = plugin + } + // delay reading the file location until execution time by wrapping in a closure within a GString + String file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}" + Object[] args = [new File(node.homeDir, 'bin/plugin'), 'install', file] + return configureExecTask(name, project, setup, node, args) + } + + /** Adds a task to execute a command to help setup the cluster */ + static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) { + return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { + workingDir node.cwd + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + executable 'cmd' + args '/C', 'call' + } else { + executable 'sh' + } + args execArgs + } + } + + /** Adds a task to start an elasticsearch node with the given configuration */ + static Task configureStartTask(String name, Project project, Task setup, NodeInfo node) { + + // this closure is converted into ant nodes by groovy's AntBuilder + Closure antRunner = { AntBuilder ant -> + ant.exec(executable: node.executable, spawn: node.config.daemonize, dir: node.cwd, taskname: 'elasticsearch') { + node.env.each { key, value -> env(key: key, value: value) } + node.args.each { arg(value: it) } + } + } + + // this closure is the actual code to run elasticsearch + Closure elasticsearchRunner = { + // Due to how ant exec works with the spawn option, we lose all stdout/stderr from the + // process executed. To work around this, when spawning, we wrap the elasticsearch start + // command inside another shell script, which simply internally redirects the output + // of the real elasticsearch script. This allows ant to keep the streams open with the + // dummy process, but us to have the output available if there is an error in the + // elasticsearch start script + if (node.config.daemonize) { + node.writeWrapperScript() + } + + // we must add debug options inside the closure so the config is read at execution time, as + // gradle task options are not processed until the end of the configuration phase + if (node.config.debug) { + println 'Running elasticsearch in debug mode, suspending until connected on port 8000' + node.env['JAVA_OPTS'] = '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000' + } + + node.getCommandString().eachLine { line -> logger.info(line) } + + if (logger.isInfoEnabled() || node.config.daemonize == false) { + runAntCommand(project, antRunner, System.out, System.err) + } else { + // buffer the output, we may not need to print it + PrintStream captureStream = new PrintStream(node.buffer, true, "UTF-8") + runAntCommand(project, antRunner, captureStream, captureStream) + } + } + + Task start = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) + start.doLast(elasticsearchRunner) + return start + } + + static Task configureWaitTask(String name, Project project, List nodes, List startTasks) { + Task wait = project.tasks.create(name: name, dependsOn: startTasks) + wait.doLast { + ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") { + or { + for (NodeInfo node : nodes) { + resourceexists { + file(file: node.failedMarker.toString()) + } + } + and { + for (NodeInfo node : nodes) { + resourceexists { + file(file: node.pidFile.toString()) + } + resourceexists { + file(file: node.httpPortsFile.toString()) + } + resourceexists { + file(file: node.transportPortsFile.toString()) + } + } + } + } + } + boolean anyNodeFailed = false + for (NodeInfo node : nodes) { + anyNodeFailed |= node.failedMarker.exists() + } + if (ant.properties.containsKey("failed${name}".toString()) || anyNodeFailed) { + waitFailed(nodes, logger, 'Failed to start elasticsearch') + } + + // go through each node checking the wait condition + for (NodeInfo node : nodes) { + // first bind node info to the closure, then pass to the ant runner so we can get good logging + Closure antRunner = node.config.waitCondition.curry(node) + + boolean success + if (logger.isInfoEnabled()) { + success = runAntCommand(project, antRunner, System.out, System.err) + } else { + PrintStream captureStream = new PrintStream(node.buffer, true, "UTF-8") + success = runAntCommand(project, antRunner, captureStream, captureStream) + } + + if (success == false) { + waitFailed(nodes, logger, 'Elasticsearch cluster failed to pass wait condition') + } + } + } + return wait + } + + static void waitFailed(List nodes, Logger logger, String msg) { + for (NodeInfo node : nodes) { + if (logger.isInfoEnabled() == false) { + // We already log the command at info level. No need to do it twice. + node.getCommandString().eachLine { line -> logger.error(line) } + } + logger.error("Node ${node.nodeNum} output:") + logger.error("|-----------------------------------------") + logger.error("| failure marker exists: ${node.failedMarker.exists()}") + logger.error("| pid file exists: ${node.pidFile.exists()}") + logger.error("| http ports file exists: ${node.httpPortsFile.exists()}") + logger.error("| transport ports file exists: ${node.transportPortsFile.exists()}") + // the waitfor failed, so dump any output we got (if info logging this goes directly to stdout) + logger.error("|\n| [ant output]") + node.buffer.toString('UTF-8').eachLine { line -> logger.error("| ${line}") } + // also dump the log file for the startup script (which will include ES logging output to stdout) + if (node.startLog.exists()) { + logger.error("|\n| [log]") + node.startLog.eachLine { line -> logger.error("| ${line}") } + } + logger.error("|-----------------------------------------") + } + throw new GradleException(msg) + } + + /** Adds a task to check if the process with the given pidfile is actually elasticsearch */ + static Task configureCheckPreviousTask(String name, Project project, Object depends, NodeInfo node) { + return project.tasks.create(name: name, type: Exec, dependsOn: depends) { + onlyIf { node.pidFile.exists() } + // the pid file won't actually be read until execution time, since the read is wrapped within an inner closure of the GString + ext.pid = "${ -> node.pidFile.getText('UTF-8').trim()}" + File jps + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + jps = getJpsExecutableByName(project, "jps.exe") + } else { + jps = getJpsExecutableByName(project, "jps") + } + if (!jps.exists()) { + throw new GradleException("jps executable not found; ensure that you're running Gradle with the JDK rather than the JRE") + } + commandLine jps, '-l' + standardOutput = new ByteArrayOutputStream() + doLast { + String out = standardOutput.toString() + if (out.contains("${pid} org.elasticsearch.bootstrap.Elasticsearch") == false) { + logger.error('jps -l') + logger.error(out) + logger.error("pid file: ${pidFile}") + logger.error("pid: ${pid}") + throw new GradleException("jps -l did not report any process with org.elasticsearch.bootstrap.Elasticsearch\n" + + "Did you run gradle clean? Maybe an old pid file is still lying around.") + } else { + logger.info(out) + } + } + } + } + + private static File getJpsExecutableByName(Project project, String jpsExecutableName) { + return Paths.get(project.javaHome.toString(), "bin/" + jpsExecutableName).toFile() + } + + /** Adds a task to kill an elasticsearch node with the given pidfile */ + static Task configureStopTask(String name, Project project, Object depends, NodeInfo node) { + return project.tasks.create(name: name, type: LoggedExec, dependsOn: depends) { + onlyIf { node.pidFile.exists() } + // the pid file won't actually be read until execution time, since the read is wrapped within an inner closure of the GString + ext.pid = "${ -> node.pidFile.getText('UTF-8').trim()}" + doFirst { + logger.info("Shutting down external node with pid ${pid}") + } + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + executable 'Taskkill' + args '/PID', pid, '/F' + } else { + executable 'kill' + args '-9', pid + } + doLast { + project.delete(node.pidFile) + } + } + } + + /** Returns a unique task name for this task and node configuration */ + static String taskName(Task parentTask, NodeInfo node, String action) { + if (node.config.numNodes > 1) { + return "${parentTask.name}#node${node.nodeNum}.${action}" + } else { + return "${parentTask.name}#${action}" + } + } + + public static String pluginTaskName(String action, String name, String suffix) { + // replace every dash followed by a character with just the uppercase character + String camelName = name.replaceAll(/-(\w)/) { _, c -> c.toUpperCase(Locale.ROOT) } + return action + camelName[0].toUpperCase(Locale.ROOT) + camelName.substring(1) + suffix + } + + /** Runs an ant command, sending output to the given out and error streams */ + static Object runAntCommand(Project project, Closure command, PrintStream outputStream, PrintStream errorStream) { + DefaultLogger listener = new DefaultLogger( + errorPrintStream: errorStream, + outputPrintStream: outputStream, + messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO) + + project.ant.project.addBuildListener(listener) + Object retVal = command(project.ant) + project.ant.project.removeBuildListener(listener) + return retVal + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy new file mode 100644 index 00000000000..1cca2c5aa49 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test + +import org.elasticsearch.gradle.plugin.PluginBuildPlugin +import org.gradle.api.Project +import org.gradle.api.artifacts.Dependency +import org.gradle.api.artifacts.ProjectDependency +import org.gradle.api.tasks.Copy + +/** + * A plugin to run messy tests, which are generally tests that depend on plugins. + * + * This plugin will add the same test configuration as standalone tests, except + * also add the plugin-metadata and properties files for each plugin project + * dependency. + */ +class MessyTestPlugin extends StandaloneTestPlugin { + @Override + public void apply(Project project) { + super.apply(project) + + project.configurations.testCompile.dependencies.all { Dependency dep -> + // this closure is run every time a compile dependency is added + if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) { + project.gradle.projectsEvaluated { + addPluginResources(project, dep.dependencyProject) + } + } + } + } + + private static addPluginResources(Project project, Project pluginProject) { + String outputDir = "generated-resources/${pluginProject.name}" + String taskName = ClusterFormationTasks.pluginTaskName("copy", pluginProject.name, "Metadata") + Copy copyPluginMetadata = project.tasks.create(taskName, Copy.class) + copyPluginMetadata.into(outputDir) + copyPluginMetadata.from(pluginProject.tasks.pluginProperties) + copyPluginMetadata.from(pluginProject.file('src/main/plugin-metadata')) + project.sourceSets.test.output.dir(outputDir, builtBy: taskName) + + // add each generated dir to the test classpath in IDEs + //project.eclipse.classpath.sourceSets = [project.sourceSets.test] + project.idea.module.singleEntryLibraries= ['TEST': [project.file(outputDir)]] + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy new file mode 100644 index 00000000000..b369d35c03a --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -0,0 +1,215 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test + +import org.apache.tools.ant.taskdefs.condition.Os +import org.elasticsearch.gradle.VersionProperties +import org.gradle.api.InvalidUserDataException +import org.gradle.api.Project +import org.gradle.api.Task + +/** + * A container for the files and configuration associated with a single node in a test cluster. + */ +class NodeInfo { + /** common configuration for all nodes, including this one */ + ClusterConfiguration config + + /** node number within the cluster, for creating unique names and paths */ + int nodeNum + + /** name of the cluster this node is part of */ + String clusterName + + /** root directory all node files and operations happen under */ + File baseDir + + /** the pid file the node will use */ + File pidFile + + /** a file written by elasticsearch containing the ports of each bound address for http */ + File httpPortsFile + + /** a file written by elasticsearch containing the ports of each bound address for transport */ + File transportPortsFile + + /** elasticsearch home dir */ + File homeDir + + /** config directory */ + File confDir + + /** THE config file */ + File configFile + + /** working directory for the node process */ + File cwd + + /** file that if it exists, indicates the node failed to start */ + File failedMarker + + /** stdout/stderr log of the elasticsearch process for this node */ + File startLog + + /** directory to install plugins from */ + File pluginsTmpDir + + /** environment variables to start the node with */ + Map env + + /** arguments to start the node with */ + List args + + /** Executable to run the bin/elasticsearch with, either cmd or sh */ + String executable + + /** Path to the elasticsearch start script */ + File esScript + + /** script to run when running in the background */ + File wrapperScript + + /** buffer for ant output when starting this node */ + ByteArrayOutputStream buffer = new ByteArrayOutputStream() + + /** Creates a node to run as part of a cluster for the given task */ + NodeInfo(ClusterConfiguration config, int nodeNum, Project project, Task task) { + this.config = config + this.nodeNum = nodeNum + clusterName = "${task.path.replace(':', '_').substring(1)}" + baseDir = new File(project.buildDir, "cluster/${task.name} node${nodeNum}") + pidFile = new File(baseDir, 'es.pid') + homeDir = homeDir(baseDir, config.distribution) + confDir = confDir(baseDir, config.distribution) + configFile = new File(confDir, 'elasticsearch.yml') + // even for rpm/deb, the logs are under home because we dont start with real services + File logsDir = new File(homeDir, 'logs') + httpPortsFile = new File(logsDir, 'http.ports') + transportPortsFile = new File(logsDir, 'transport.ports') + cwd = new File(baseDir, "cwd") + failedMarker = new File(cwd, 'run.failed') + startLog = new File(cwd, 'run.log') + pluginsTmpDir = new File(baseDir, "plugins tmp") + + args = [] + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + executable = 'cmd' + args.add('/C') + args.add('"') // quote the entire command + wrapperScript = new File(cwd, "run.bat") + esScript = new File(homeDir, 'bin/elasticsearch.bat') + } else { + executable = 'sh' + wrapperScript = new File(cwd, "run") + esScript = new File(homeDir, 'bin/elasticsearch') + } + if (config.daemonize) { + args.add("${wrapperScript}") + } else { + args.add("${esScript}") + } + + env = [ + 'JAVA_HOME' : project.javaHome, + 'ES_GC_OPTS': config.jvmArgs // we pass these with the undocumented gc opts so the argline can set gc, etc + ] + args.add("-Des.tests.portsfile=true") + args.addAll(config.systemProperties.collect { key, value -> "-D${key}=${value}" }) + for (Map.Entry property : System.properties.entrySet()) { + if (property.getKey().startsWith('es.')) { + args.add("-D${property.getKey()}=${property.getValue()}") + } + } + args.add("-Des.path.conf=${confDir}") + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + args.add('"') // end the entire command, quoted + } + } + + /** Returns debug string for the command that started this node. */ + String getCommandString() { + String esCommandString = "\nNode ${nodeNum} configuration:\n" + esCommandString += "|-----------------------------------------\n" + esCommandString += "| cwd: ${cwd}\n" + esCommandString += "| command: ${executable} ${args.join(' ')}\n" + esCommandString += '| environment:\n' + env.each { k, v -> esCommandString += "| ${k}: ${v}\n" } + if (config.daemonize) { + esCommandString += "|\n| [${wrapperScript.name}]\n" + wrapperScript.eachLine('UTF-8', { line -> esCommandString += " ${line}\n"}) + } + esCommandString += '|\n| [elasticsearch.yml]\n' + configFile.eachLine('UTF-8', { line -> esCommandString += "| ${line}\n" }) + esCommandString += "|-----------------------------------------" + return esCommandString + } + + void writeWrapperScript() { + String argsPasser = '"$@"' + String exitMarker = "; if [ \$? != 0 ]; then touch run.failed; fi" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + argsPasser = '%*' + exitMarker = "\r\n if \"%errorlevel%\" neq \"0\" ( type nul >> run.failed )" + } + wrapperScript.setText("\"${esScript}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8') + } + + /** Returns an address and port suitable for a uri to connect to this node over http */ + String httpUri() { + return httpPortsFile.readLines("UTF-8").get(0) + } + + /** Returns an address and port suitable for a uri to connect to this node over transport protocol */ + String transportUri() { + return transportPortsFile.readLines("UTF-8").get(0) + } + + /** Returns the directory elasticsearch home is contained in for the given distribution */ + static File homeDir(File baseDir, String distro) { + String path + switch (distro) { + case 'integ-test-zip': + case 'zip': + case 'tar': + path = "elasticsearch-${VersionProperties.elasticsearch}" + break + case 'rpm': + case 'deb': + path = "${distro}-extracted/usr/share/elasticsearch" + break + default: + throw new InvalidUserDataException("Unknown distribution: ${distro}") + } + return new File(baseDir, path) + } + + static File confDir(File baseDir, String distro) { + switch (distro) { + case 'integ-test-zip': + case 'zip': + case 'tar': + return new File(homeDir(baseDir, distro), 'config') + case 'rpm': + case 'deb': + return new File(baseDir, "${distro}-extracted/etc/elasticsearch") + default: + throw new InvalidUserDataException("Unkown distribution: ${distro}") + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy new file mode 100644 index 00000000000..24bd57a3a59 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test + +import com.carrotsearch.gradle.junit4.RandomizedTestingTask +import org.elasticsearch.gradle.BuildPlugin +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.internal.tasks.options.Option +import org.gradle.api.plugins.JavaBasePlugin +import org.gradle.api.tasks.Input +import org.gradle.util.ConfigureUtil + +/** + * Runs integration tests, but first starts an ES cluster, + * and passes the ES cluster info as parameters to the tests. + */ +public class RestIntegTestTask extends RandomizedTestingTask { + + ClusterConfiguration clusterConfig = new ClusterConfiguration() + + /** Flag indicating whether the rest tests in the rest spec should be run. */ + @Input + boolean includePackaged = false + + public RestIntegTestTask() { + description = 'Runs rest tests against an elasticsearch cluster.' + group = JavaBasePlugin.VERIFICATION_GROUP + dependsOn(project.testClasses) + classpath = project.sourceSets.test.runtimeClasspath + testClassesDir = project.sourceSets.test.output.classesDir + + // start with the common test configuration + configure(BuildPlugin.commonTestConfig(project)) + // override/add more for rest tests + parallelism = '1' + include('**/*IT.class') + systemProperty('tests.rest.load_packaged', 'false') + + // copy the rest spec/tests into the test resources + RestSpecHack.configureDependencies(project) + project.afterEvaluate { + dependsOn(RestSpecHack.configureTask(project, includePackaged)) + } + // this must run after all projects have been configured, so we know any project + // references can be accessed as a fully configured + project.gradle.projectsEvaluated { + Object clusterUri = ClusterFormationTasks.setup(project, this, clusterConfig) + systemProperty('tests.cluster', clusterUri) + } + } + + @Option( + option = "debug-jvm", + description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch." + ) + public void setDebug(boolean enabled) { + clusterConfig.debug = enabled; + } + + @Input + public void cluster(Closure closure) { + ConfigureUtil.configure(closure, clusterConfig) + } + + public ClusterConfiguration getCluster() { + return clusterConfig + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy new file mode 100644 index 00000000000..43b5c2f6f38 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test + +import org.elasticsearch.gradle.VersionProperties +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.tasks.Copy + +/** + * The rest-api-spec tests are loaded from the classpath. However, they + * currently must be available on the local filesystem. This class encapsulates + * setting up tasks to copy the rest spec api to test resources. + */ +public class RestSpecHack { + /** + * Sets dependencies needed to copy the rest spec. + * @param project The project to add rest spec dependency to + */ + public static void configureDependencies(Project project) { + project.configurations { + restSpec + } + project.dependencies { + restSpec "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}" + } + } + + /** + * Creates a task to copy the rest spec files. + * + * @param project The project to add the copy task to + * @param includePackagedTests true if the packaged tests should be copied, false otherwise + */ + public static Task configureTask(Project project, boolean includePackagedTests) { + Map copyRestSpecProps = [ + name : 'copyRestSpec', + type : Copy, + dependsOn: [project.configurations.restSpec, 'processTestResources'] + ] + Task copyRestSpec = project.tasks.create(copyRestSpecProps) { + from { project.zipTree(project.configurations.restSpec.singleFile) } + include 'rest-api-spec/api/**' + if (includePackagedTests) { + include 'rest-api-spec/test/**' + } + into project.sourceSets.test.output.resourcesDir + } + project.idea { + module { + if (scopes.TEST != null) { + scopes.TEST.plus.add(project.configurations.restSpec) + } + } + } + return copyRestSpec + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy new file mode 100644 index 00000000000..dc9aa769388 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test + +import org.gradle.api.Plugin +import org.gradle.api.Project + +/** A plugin to add rest integration tests. Used for qa projects. */ +public class RestTestPlugin implements Plugin { + + @Override + public void apply(Project project) { + project.pluginManager.apply(StandaloneTestBasePlugin) + + RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class) + integTest.cluster.distribution = 'zip' // rest tests should run with the real zip + integTest.mustRunAfter(project.precommit) + project.check.dependsOn(integTest) + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy new file mode 100644 index 00000000000..842ef8c35cd --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy @@ -0,0 +1,35 @@ +package org.elasticsearch.gradle.test + +import org.gradle.api.DefaultTask +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.internal.tasks.options.Option +import org.gradle.util.ConfigureUtil + +public class RunTask extends DefaultTask { + + ClusterConfiguration clusterConfig = new ClusterConfiguration(httpPort: 9200, transportPort: 9300, daemonize: false) + + public RunTask() { + description = "Runs elasticsearch with '${project.path}'" + group = 'Verification' + project.afterEvaluate { + ClusterFormationTasks.setup(project, this, clusterConfig) + } + } + + @Option( + option = "debug-jvm", + description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch." + ) + public void setDebug(boolean enabled) { + clusterConfig.debug = enabled; + } + + /** Configure the cluster that will be run. */ + @Override + public Task configure(Closure closure) { + ConfigureUtil.configure(closure, clusterConfig) + return this + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy new file mode 100644 index 00000000000..f317254cd45 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.elasticsearch.gradle.test + +import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin +import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.precommit.PrecommitTasks +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.plugins.JavaBasePlugin +import org.gradle.plugins.ide.eclipse.model.EclipseClasspath + +/** Configures the build to have a rest integration test. */ +public class StandaloneTestBasePlugin implements Plugin { + + @Override + public void apply(Project project) { + project.pluginManager.apply(JavaBasePlugin) + project.pluginManager.apply(RandomizedTestingPlugin) + + BuildPlugin.globalBuildInfo(project) + BuildPlugin.configureRepositories(project) + + // only setup tests to build + project.sourceSets.create('test') + project.dependencies.add('testCompile', "org.elasticsearch:test-framework:${VersionProperties.elasticsearch}") + + project.eclipse.classpath.sourceSets = [project.sourceSets.test] + project.eclipse.classpath.plusConfigurations = [project.configurations.testRuntime] + project.idea.module.testSourceDirs += project.sourceSets.test.java.srcDirs + project.idea.module.scopes['TEST'] = [plus: [project.configurations.testRuntime]] + + PrecommitTasks.create(project, false) + project.check.dependsOn(project.precommit) + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy new file mode 100644 index 00000000000..0a2cc841282 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test + +import com.carrotsearch.gradle.junit4.RandomizedTestingTask +import org.elasticsearch.gradle.BuildPlugin +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.plugins.JavaBasePlugin + +/** A plugin to add tests only. Used for QA tests that run arbitrary unit tests. */ +public class StandaloneTestPlugin implements Plugin { + + @Override + public void apply(Project project) { + project.pluginManager.apply(StandaloneTestBasePlugin) + + Map testOptions = [ + name: 'test', + type: RandomizedTestingTask, + dependsOn: 'testClasses', + group: JavaBasePlugin.VERIFICATION_GROUP, + description: 'Runs unit tests that are separate' + ] + RandomizedTestingTask test = project.tasks.create(testOptions) + test.configure(BuildPlugin.commonTestConfig(project)) + test.classpath = project.sourceSets.test.runtimeClasspath + test.testClassesDir project.sourceSets.test.output.classesDir + test.mustRunAfter(project.precommit) + project.check.dependsOn(test) + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy new file mode 100644 index 00000000000..6af9edd119c --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.vagrant + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.* +import org.gradle.logging.ProgressLogger +import org.gradle.logging.ProgressLoggerFactory +import org.gradle.process.internal.ExecAction +import org.gradle.process.internal.ExecActionFactory + +import javax.inject.Inject + +/** + * Runs bats over vagrant. Pretty much like running it using Exec but with a + * nicer output formatter. + */ +class BatsOverVagrantTask extends DefaultTask { + String command + String boxName + ExecAction execAction + + BatsOverVagrantTask() { + execAction = getExecActionFactory().newExecAction() + } + + @Inject + ProgressLoggerFactory getProgressLoggerFactory() { + throw new UnsupportedOperationException(); + } + + @Inject + ExecActionFactory getExecActionFactory() { + throw new UnsupportedOperationException(); + } + + void boxName(String boxName) { + this.boxName = boxName + } + + void command(String command) { + this.command = command + } + + @TaskAction + void exec() { + // It'd be nice if --machine-readable were, well, nice + execAction.commandLine(['vagrant', 'ssh', boxName, '--command', command]) + execAction.setStandardOutput(new TapLoggerOutputStream( + command: command, + factory: getProgressLoggerFactory(), + logger: logger)) + execAction.execute(); + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy new file mode 100644 index 00000000000..5f4a5e0a0c4 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.vagrant + +import com.carrotsearch.gradle.junit4.LoggingOutputStream +import org.gradle.api.GradleScriptException +import org.gradle.api.logging.Logger +import org.gradle.logging.ProgressLogger + +import java.util.regex.Matcher + +/** + * Adapts an OutputStream containing output from bats into a ProgressLogger + * and a Logger. Every test output goes to the ProgressLogger and all failures + * and non-test output goes to the Logger. That means you can always glance + * at the result of the last test and the cumulative pass/fail/skip stats and + * the failures are all logged. + * + * There is a Tap4j project but we can't use it because it wants to parse the + * entire TAP stream at once and won't parse it stream-wise. + */ +class TapLoggerOutputStream extends LoggingOutputStream { + ProgressLogger progressLogger + Logger logger + int testsCompleted = 0 + int testsFailed = 0 + int testsSkipped = 0 + Integer testCount + String countsFormat + + TapLoggerOutputStream(Map args) { + logger = args.logger + progressLogger = args.factory.newOperation(VagrantLoggerOutputStream) + progressLogger.setDescription("TAP output for $args.command") + progressLogger.started() + progressLogger.progress("Starting $args.command...") + } + + void flush() { + if (end == start) return + line(new String(buffer, start, end - start)) + start = end + } + + void line(String line) { + // System.out.print "===> $line\n" + if (testCount == null) { + try { + testCount = line.split('\\.').last().toInteger() + def length = (testCount as String).length() + countsFormat = "%0${length}d" + countsFormat = "[$countsFormat|$countsFormat|$countsFormat/$countsFormat]" + return + } catch (Exception e) { + throw new GradleScriptException( + 'Error parsing first line of TAP stream!!', e) + } + } + Matcher m = line =~ /(?ok|not ok) \d+(? # skip (?\(.+\))?)? \[(?.+)\] (?.+)/ + if (!m.matches()) { + /* These might be failure report lines or comments or whatever. Its hard + to tell and it doesn't matter. */ + logger.warn(line) + return + } + boolean skipped = m.group('skip') != null + boolean success = !skipped && m.group('status') == 'ok' + String skipReason = m.group('skipReason') + String suiteName = m.group('suite') + String testName = m.group('test') + + String status + if (skipped) { + status = "SKIPPED" + testsSkipped++ + } else if (success) { + status = " OK" + testsCompleted++ + } else { + status = " FAILED" + testsFailed++ + } + + String counts = sprintf(countsFormat, + [testsCompleted, testsFailed, testsSkipped, testCount]) + progressLogger.progress("Tests $counts, $status [$suiteName] $testName") + if (!success) { + logger.warn(line) + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy new file mode 100644 index 00000000000..351c34a893b --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.vagrant + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.* +import org.gradle.logging.ProgressLogger +import org.gradle.logging.ProgressLoggerFactory +import org.gradle.process.internal.ExecAction +import org.gradle.process.internal.ExecActionFactory + +import javax.inject.Inject + +/** + * Runs a vagrant command. Pretty much like Exec task but with a nicer output + * formatter and defaults to `vagrant` as first part of commandLine. + */ +class VagrantCommandTask extends DefaultTask { + List commandLine + String boxName + ExecAction execAction + + VagrantCommandTask() { + execAction = getExecActionFactory().newExecAction() + } + + @Inject + ProgressLoggerFactory getProgressLoggerFactory() { + throw new UnsupportedOperationException(); + } + + @Inject + ExecActionFactory getExecActionFactory() { + throw new UnsupportedOperationException(); + } + + void boxName(String boxName) { + this.boxName = boxName + } + + void commandLine(Object... commandLine) { + this.commandLine = commandLine + } + + @TaskAction + void exec() { + // It'd be nice if --machine-readable were, well, nice + execAction.commandLine(['vagrant'] + commandLine) + execAction.setStandardOutput(new VagrantLoggerOutputStream( + command: commandLine.join(' '), + factory: getProgressLoggerFactory(), + /* Vagrant tends to output a lot of stuff, but most of the important + stuff starts with ==> $box */ + squashedPrefix: "==> $boxName: ")) + execAction.execute(); + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy new file mode 100644 index 00000000000..488c4511b1f --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.vagrant + +import com.carrotsearch.gradle.junit4.LoggingOutputStream +import org.gradle.logging.ProgressLogger + +/** + * Adapts an OutputStream being written to by vagrant into a ProcessLogger. It + * has three hacks to make the output nice: + * + * 1. Attempt to filter out the "unimportant" output from vagrant. Usually + * vagrant prefixes its more important output with "==> $boxname: ". The stuff + * that isn't prefixed that way can just be thrown out. + * + * 2. It also attempts to detect when vagrant does tricks assuming its writing + * to a terminal emulator and renders the output more like gradle users expect. + * This means that progress indicators for things like box downloading work and + * box importing look pretty good. + * + * 3. It catches lines that look like "==> $boxName ==> Heading text" and stores + * the text after the second arrow as a "heading" for use in annotating + * provisioning. It does this because provisioning can spit out _lots_ of text + * and its very easy to lose context when there isn't a scrollback. So we've + * sprinkled `echo "==> Heading text"` into the provisioning scripts for this + * to catch so it can render the output like + * "Heading text > stdout from the provisioner". + */ +class VagrantLoggerOutputStream extends LoggingOutputStream { + static final String HEADING_PREFIX = '==> ' + + ProgressLogger progressLogger + String squashedPrefix + String lastLine = '' + boolean inProgressReport = false + String heading = '' + + VagrantLoggerOutputStream(Map args) { + progressLogger = args.factory.newOperation(VagrantLoggerOutputStream) + progressLogger.setDescription("Vagrant $args.command") + progressLogger.started() + progressLogger.progress("Starting vagrant $args.command...") + squashedPrefix = args.squashedPrefix + } + + void flush() { + if (end == start) return + line(new String(buffer, start, end - start)) + start = end + } + + void line(String line) { + // debugPrintLine(line) // Uncomment me to log every incoming line + if (line.startsWith('\r\u001b')) { + /* We don't want to try to be a full terminal emulator but we want to + keep the escape sequences from leaking and catch _some_ of the + meaning. */ + line = line.substring(2) + if ('[K' == line) { + inProgressReport = true + } + return + } + if (line.startsWith(squashedPrefix)) { + line = line.substring(squashedPrefix.length()) + inProgressReport = false + lastLine = line + if (line.startsWith(HEADING_PREFIX)) { + line = line.substring(HEADING_PREFIX.length()) + heading = line + ' > ' + } else { + line = heading + line + } + } else if (inProgressReport) { + inProgressReport = false + line = lastLine + line + } else { + return + } + // debugLogLine(line) // Uncomment me to log every line we add to the logger + progressLogger.progress(line) + } + + void debugPrintLine(line) { + System.out.print '----------> ' + for (int i = start; i < end; i++) { + switch (buffer[i] as char) { + case ' '..'~': + System.out.print buffer[i] as char + break + default: + System.out.print '%' + System.out.print Integer.toHexString(buffer[i]) + } + } + System.out.print '\n' + } + + void debugLogLine(line) { + System.out.print '>>>>>>>>>>> ' + System.out.print line + System.out.print '\n' + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/carrotsearch.randomized-testing.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/carrotsearch.randomized-testing.properties new file mode 100644 index 00000000000..e1a1b8297c8 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/carrotsearch.randomized-testing.properties @@ -0,0 +1 @@ +implementation-class=com.carrotsearch.gradle.junit4.RandomizedTestingPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.build.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.build.properties new file mode 100644 index 00000000000..c80e45ff2e9 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.build.properties @@ -0,0 +1 @@ +implementation-class=org.elasticsearch.gradle.BuildPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.esplugin.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.esplugin.properties new file mode 100644 index 00000000000..3116e0415f5 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.esplugin.properties @@ -0,0 +1 @@ +implementation-class=org.elasticsearch.gradle.plugin.PluginBuildPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.messy-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.messy-test.properties new file mode 100644 index 00000000000..507a0f85a04 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.messy-test.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.MessyTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-test.properties new file mode 100644 index 00000000000..7d5c63c35c3 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-test.properties @@ -0,0 +1 @@ +implementation-class=org.elasticsearch.gradle.test.RestTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-test.properties new file mode 100644 index 00000000000..5bf4cffe35b --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-test.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.StandaloneTestPlugin diff --git a/buildSrc/src/main/resources/beat.wav b/buildSrc/src/main/resources/beat.wav new file mode 100644 index 00000000000..4083a4ce618 Binary files /dev/null and b/buildSrc/src/main/resources/beat.wav differ diff --git a/buildSrc/src/main/resources/deb/README b/buildSrc/src/main/resources/deb/README new file mode 100644 index 00000000000..cdecf00756d --- /dev/null +++ b/buildSrc/src/main/resources/deb/README @@ -0,0 +1,4 @@ +This directory contains templates that work around gradle-ospackage-plugin +trying to be helpful and adding templates for your os packaging scripts. We +have relatively nice scripts already so we just override the templates to be +mostly noops. diff --git a/buildSrc/src/main/resources/deb/conffiles.ftl b/buildSrc/src/main/resources/deb/conffiles.ftl new file mode 100644 index 00000000000..db7788a4e44 --- /dev/null +++ b/buildSrc/src/main/resources/deb/conffiles.ftl @@ -0,0 +1,3 @@ +<% files.each {file -> %><%= file +%> +<% } %> diff --git a/buildSrc/src/main/resources/deb/postinst.ftl b/buildSrc/src/main/resources/deb/postinst.ftl new file mode 100644 index 00000000000..5f67242c265 --- /dev/null +++ b/buildSrc/src/main/resources/deb/postinst.ftl @@ -0,0 +1,2 @@ +#!/bin/sh -e +<% commands.each {command -> %><%= command %><% } %> diff --git a/buildSrc/src/main/resources/deb/preinst.ftl b/buildSrc/src/main/resources/deb/preinst.ftl new file mode 100644 index 00000000000..5f67242c265 --- /dev/null +++ b/buildSrc/src/main/resources/deb/preinst.ftl @@ -0,0 +1,2 @@ +#!/bin/sh -e +<% commands.each {command -> %><%= command %><% } %> diff --git a/.settings/org.eclipse.core.resources.prefs b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.core.resources.prefs similarity index 70% rename from .settings/org.eclipse.core.resources.prefs rename to buildSrc/src/main/resources/eclipse.settings/org.eclipse.core.resources.prefs index 5731b2f8244..6fd0a9aab13 100644 --- a/.settings/org.eclipse.core.resources.prefs +++ b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.core.resources.prefs @@ -1,6 +1,6 @@ eclipse.preferences.version=1 encoding//src/main/java=UTF-8 encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 encoding//src/test/resources=UTF-8 -encoding/=UTF-8 -encoding/rest-api-spec=UTF-8 +encoding/=UTF-8 \ No newline at end of file diff --git a/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000000..9bee5e587b0 --- /dev/null +++ b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,22 @@ +eclipse.preferences.version=1 + +# previous configuration from maven build +# this is merged with gradle's generated properties during 'gradle eclipse' + +# NOTE: null pointer analysis etc is not enabled currently, it seems very unstable +# (e.g. crashing eclipse etc) +# org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=enabled +# org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +# org.eclipse.jdt.core.compiler.annotation.nullable=org.elasticsearch.common.Nullable +# org.eclipse.jdt.core.compiler.annotation.nullanalysis=enabled +# org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +# org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=warning +# org.eclipse.jdt.core.compiler.problem.nullReference=warning +# org.eclipse.jdt.core.compiler.problem.nullSpecViolation=warning +# org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +# org.eclipse.jdt.core.compiler.problem.potentialNullReference=warning + +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.formatter.lineSplit=140 +org.eclipse.jdt.core.formatter.tabulation.char=space +org.eclipse.jdt.core.formatter.tabulation.size=4 diff --git a/.settings/org.eclipse.jdt.ui.prefs b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.ui.prefs similarity index 81% rename from .settings/org.eclipse.jdt.ui.prefs rename to buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.ui.prefs index 4a9959fc9fc..391a8715868 100644 --- a/.settings/org.eclipse.jdt.ui.prefs +++ b/buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.ui.prefs @@ -3,4 +3,4 @@ formatter_settings_version=12 # Intellij IDEA import order org.eclipse.jdt.ui.importorder=;com;org;java;javax;\#; # License header -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= diff --git a/buildSrc/src/main/resources/elasticsearch.properties b/buildSrc/src/main/resources/elasticsearch.properties new file mode 100644 index 00000000000..773b5439a2a --- /dev/null +++ b/buildSrc/src/main/resources/elasticsearch.properties @@ -0,0 +1,2 @@ +version=@version@ +luceneVersion=@luceneVersion@ diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/buildSrc/src/main/resources/forbidden/all-signatures.txt similarity index 68% rename from dev-tools/src/main/resources/forbidden/all-signatures.txt rename to buildSrc/src/main/resources/forbidden/all-signatures.txt index 447e994f507..c1e65cbaf22 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/all-signatures.txt @@ -38,9 +38,13 @@ org.apache.lucene.index.DocsEnum org.apache.lucene.index.DocsAndPositionsEnum org.apache.lucene.queries.TermFilter org.apache.lucene.queries.TermsFilter +org.apache.lucene.search.Filter +org.apache.lucene.search.FilteredQuery org.apache.lucene.search.TermRangeFilter org.apache.lucene.search.NumericRangeFilter org.apache.lucene.search.PrefixFilter +org.apache.lucene.search.QueryWrapperFilter +org.apache.lucene.search.join.BitDocIdSetCachingWrapperFilter java.nio.file.Paths @ Use org.elasticsearch.common.io.PathUtils.get() instead. java.nio.file.FileSystems#getDefault() @ use org.elasticsearch.common.io.PathUtils.getDefaultFileSystem() instead. @@ -88,8 +92,37 @@ java.net.InetAddress#getCanonicalHostName() java.net.InetSocketAddress#getHostName() @ Use getHostString() instead, which avoids a DNS lookup @defaultMessage Do not violate java's access system +java.lang.Class#getDeclaredClasses() @ Do not violate java's access system: Use getClasses() instead +java.lang.Class#getDeclaredConstructor(java.lang.Class[]) @ Do not violate java's access system: Use getConstructor() instead +java.lang.Class#getDeclaredConstructors() @ Do not violate java's access system: Use getConstructors() instead +java.lang.Class#getDeclaredField(java.lang.String) @ Do not violate java's access system: Use getField() instead +java.lang.Class#getDeclaredFields() @ Do not violate java's access system: Use getFields() instead +java.lang.Class#getDeclaredMethod(java.lang.String, java.lang.Class[]) @ Do not violate java's access system: Use getMethod() instead +java.lang.Class#getDeclaredMethods() @ Do not violate java's access system: Use getMethods() instead java.lang.reflect.AccessibleObject#setAccessible(boolean) java.lang.reflect.AccessibleObject#setAccessible(java.lang.reflect.AccessibleObject[], boolean) @defaultMessage this should not have been added to lucene in the first place -org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey() \ No newline at end of file +org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey() + +@defaultMessage this method needs special permission +java.lang.Thread#getAllStackTraces() + +@defaultMessage Please do not terminate the application +java.lang.System#exit(int) +java.lang.Runtime#exit(int) +java.lang.Runtime#halt(int) + +@defaultMessage Treat system properties as immutable +java.lang.System#setProperties(java.util.Properties) +java.lang.System#setProperty(java.lang.String,java.lang.String) +java.lang.System#clearProperty(java.lang.String) +java.lang.System#getProperties() @ Use BootstrapInfo.getSystemProperties for a read-only view + +@defaultMessage Avoid unchecked warnings by using Collections#empty(List|Map|Set) methods +java.util.Collections#EMPTY_LIST +java.util.Collections#EMPTY_MAP +java.util.Collections#EMPTY_SET + +java.util.Collections#shuffle(java.util.List) @ Use java.util.Collections#shuffle(java.util.List, java.util.Random) with a reproducible source of randomness +java.util.Random#() @ Use org.elasticsearch.common.random.Randomness#create for reproducible sources of randomness diff --git a/dev-tools/src/main/resources/forbidden/core-signatures.txt b/buildSrc/src/main/resources/forbidden/core-signatures.txt similarity index 90% rename from dev-tools/src/main/resources/forbidden/core-signatures.txt rename to buildSrc/src/main/resources/forbidden/core-signatures.txt index 08c548f1dcc..c6ab430595c 100644 --- a/dev-tools/src/main/resources/forbidden/core-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/core-signatures.txt @@ -87,3 +87,15 @@ java.util.concurrent.Future#cancel(boolean) @defaultMessage Don't try reading from paths that are not configured in Environment, resolve from Environment instead org.elasticsearch.common.io.PathUtils#get(java.lang.String, java.lang.String[]) org.elasticsearch.common.io.PathUtils#get(java.net.URI) + +@defaultMessage Don't use deprecated Query#setBoost, wrap the query into a BoostQuery instead +org.apache.lucene.search.Query#setBoost(float) + +@defaultMessage Constructing a DateTime without a time zone is dangerous +org.joda.time.DateTime#() +org.joda.time.DateTime#(long) +org.joda.time.DateTime#(int, int, int, int, int) +org.joda.time.DateTime#(int, int, int, int, int, int) +org.joda.time.DateTime#(int, int, int, int, int, int, int) +org.joda.time.DateTime#now() +org.joda.time.DateTimeZone#getDefault() diff --git a/dev-tools/src/main/resources/forbidden/test-signatures.txt b/buildSrc/src/main/resources/forbidden/test-signatures.txt similarity index 94% rename from dev-tools/src/main/resources/forbidden/test-signatures.txt rename to buildSrc/src/main/resources/forbidden/test-signatures.txt index 3cde7352618..bd6744ee05f 100644 --- a/dev-tools/src/main/resources/forbidden/test-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/test-signatures.txt @@ -21,3 +21,5 @@ com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded r org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead org.apache.lucene.util.LuceneTestCase$Slow @ Don't write slow tests org.junit.Ignore @ Use AwaitsFix instead + +org.junit.Test @defaultMessage Just name your test method testFooBar diff --git a/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties b/buildSrc/src/main/resources/plugin-descriptor.properties similarity index 77% rename from dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties rename to buildSrc/src/main/resources/plugin-descriptor.properties index 1588e113d86..4c676c26cad 100644 --- a/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties +++ b/buildSrc/src/main/resources/plugin-descriptor.properties @@ -24,26 +24,26 @@ # jvm=true # classname=foo.bar.BazPlugin # description=My cool plugin -# version=2.0.0-rc1 +# version=2.0 # elasticsearch.version=2.0 # java.version=1.7 # ### mandatory elements for all plugins: # # 'description': simple summary of the plugin -description=${project.description} +description=${description} # # 'version': plugin's version -version=${project.version} +version=${version} # # 'name': the plugin name -name=${elasticsearch.plugin.name} +name=${name} ### mandatory elements for site plugins: # # 'site': set to true to indicate contents of the _site/ # directory in the root of the plugin should be served. -site=${elasticsearch.plugin.site} +site=${site} # ### mandatory elements for jvm plugins : # @@ -52,29 +52,25 @@ site=${elasticsearch.plugin.site} # Note that only jar files in the root directory are # added to the classpath for the plugin! If you need # other resources, package them into a resources jar. -jvm=${elasticsearch.plugin.jvm} +jvm=${jvm} # # 'classname': the name of the class to load, fully-qualified. -classname=${elasticsearch.plugin.classname} +classname=${classname} # # 'java.version' version of java the code is built against # use the system property java.specification.version # version string must be a sequence of nonnegative decimal integers # separated by "."'s and may have leading zeros -java.version=${maven.compiler.target} +java.version=${javaVersion} # # 'elasticsearch.version' version of elasticsearch compiled against -# You will have to release a new version of the plugin for each new -# elasticsearch release. This version is checked when the plugin -# is loaded so Elasticsearch will refuse to start in the presence of -# plugins with the incorrect elasticsearch.version. -elasticsearch.version=${elasticsearch.version} +elasticsearch.version=${elasticsearchVersion} # ### deprecated elements for jvm plugins : # # 'isolated': true if the plugin should have its own classloader. -# passing false is deprecated, and only intended to support plugins +# passing false is deprecated, and only intended to support plugins # that have hard dependencies against each other. If this is # not specified, then the plugin is isolated by default. -isolated=${elasticsearch.plugin.isolated} -# +isolated=${isolated} +# \ No newline at end of file diff --git a/buildSrc/version.properties b/buildSrc/version.properties new file mode 100644 index 00000000000..e33383afa23 --- /dev/null +++ b/buildSrc/version.properties @@ -0,0 +1,19 @@ +elasticsearch = 3.0.0-SNAPSHOT +lucene = 5.5.0-snapshot-1719088 + +# optional dependencies +spatial4j = 0.5 +jts = 1.13 +jackson = 2.6.2 +log4j = 1.2.17 +slf4j = 1.6.2 +jna = 4.1.0 + + +# test dependencies +randomizedrunner = 2.3.2 +junit = 4.11 +httpclient = 4.3.6 +httpcore = 4.3.3 +commonslogging = 1.1.3 +commonscodec = 1.10 diff --git a/core/.local-3.0.0-SNAPSHOT-test-execution-times.log b/core/.local-3.0.0-SNAPSHOT-test-execution-times.log deleted file mode 100644 index 8b79a35e102..00000000000 --- a/core/.local-3.0.0-SNAPSHOT-test-execution-times.log +++ /dev/null @@ -1,559 +0,0 @@ -org.apache.lucene.analysis.miscellaneous.TruncateTokenFilterTests=190 -org.apache.lucene.analysis.miscellaneous.UniqueTokenFilterTests=187 -org.apache.lucene.queries.BlendedTermQueryTests=696 -org.apache.lucene.queries.MinDocQueryTests=503 -org.apache.lucene.search.postingshighlight.CustomPassageFormatterTests=69 -org.apache.lucene.search.postingshighlight.CustomPostingsHighlighterTests=599 -org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIteratorTests=99 -org.apache.lucene.util.SloppyMathTests=734 -org.elasticsearch.ESExceptionTests=701 -org.elasticsearch.ExceptionSerializationTests=3740 -org.elasticsearch.NamingConventionTests=1061 -org.elasticsearch.SpecialPermissionTests=90 -org.elasticsearch.VersionTests=179 -org.elasticsearch.action.OriginalIndicesTests=66 -org.elasticsearch.action.admin.cluster.health.ClusterHealthResponsesTests=120 -org.elasticsearch.action.admin.cluster.state.ClusterStateRequestTests=32 -org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilderTests=51 -org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestTests=60 -org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestTests=2294 -org.elasticsearch.action.admin.indices.shards.IndicesShardStoreResponseTests=61 -org.elasticsearch.action.admin.indices.stats.IndicesStatsTests=2832 -org.elasticsearch.action.admin.indices.template.put.MetaDataIndexTemplateServiceTests=72 -org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequestTests=98 -org.elasticsearch.action.bulk.BulkRequestTests=578 -org.elasticsearch.action.count.CountRequestBuilderTests=495 -org.elasticsearch.action.count.CountRequestTests=21 -org.elasticsearch.action.count.CountResponseTests=63 -org.elasticsearch.action.fieldstats.FieldStatsRequestTests=45 -org.elasticsearch.action.get.MultiGetShardRequestTests=81 -org.elasticsearch.action.index.IndexRequestBuilderTests=372 -org.elasticsearch.action.index.IndexRequestTests=78 -org.elasticsearch.action.indexedscripts.get.GetIndexedScriptRequestTests=58 -org.elasticsearch.action.percolate.MultiPercolatorRequestTests=144 -org.elasticsearch.action.search.MultiSearchRequestTests=57 -org.elasticsearch.action.search.SearchRequestBuilderTests=291 -org.elasticsearch.action.support.IndicesOptionsTests=83 -org.elasticsearch.action.support.ListenableActionFutureTests=55 -org.elasticsearch.action.support.TransportActionFilterChainTests=52 -org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeActionTests=110 -org.elasticsearch.action.support.replication.BroadcastReplicationTests=151 -org.elasticsearch.action.support.replication.ShardReplicationTests=236 -org.elasticsearch.action.termvectors.TermVectorsUnitTests=293 -org.elasticsearch.action.update.UpdateRequestTests=67 -org.elasticsearch.bootstrap.BootstrapCliParserTests=73 -org.elasticsearch.bootstrap.ESPolicyTests=55 -org.elasticsearch.bootstrap.JNANativesTests=77 -org.elasticsearch.bootstrap.JarHellTests=171 -org.elasticsearch.bootstrap.JavaVersionTests=65 -org.elasticsearch.bootstrap.SeccompTests=123 -org.elasticsearch.bootstrap.SecurityTests=238 -org.elasticsearch.client.node.NodeClientHeadersTests=355 -org.elasticsearch.client.transport.TransportClientHeadersTests=640 -org.elasticsearch.client.transport.TransportClientNodesServiceTests=3307 -org.elasticsearch.cluster.ClusterModuleTests=73 -org.elasticsearch.cluster.ClusterStateTests=17 -org.elasticsearch.cluster.DiskUsageTests=107 -org.elasticsearch.cluster.block.ClusterBlockTests=41 -org.elasticsearch.cluster.metadata.DateMathExpressionResolverTests=158 -org.elasticsearch.cluster.metadata.HumanReadableIndexSettingsTests=41 -org.elasticsearch.cluster.metadata.IndexNameExpressionResolverTests=370 -org.elasticsearch.cluster.metadata.MappingMetaDataParserTests=103 -org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeServiceTests=26 -org.elasticsearch.cluster.metadata.ToAndFromJsonMetaDataTests=122 -org.elasticsearch.cluster.metadata.WildcardExpressionResolverTests=80 -org.elasticsearch.cluster.node.DiscoveryNodeFiltersTests=62 -org.elasticsearch.cluster.routing.AllocationIdTests=79 -org.elasticsearch.cluster.routing.RoutingBackwardCompatibilityTests=3477 -org.elasticsearch.cluster.routing.RoutingServiceTests=368 -org.elasticsearch.cluster.routing.RoutingTableTests=123 -org.elasticsearch.cluster.routing.ShardRoutingTests=179 -org.elasticsearch.cluster.routing.UnassignedInfoTests=146 -org.elasticsearch.cluster.routing.allocation.AddIncrementallyTests=97 -org.elasticsearch.cluster.routing.allocation.AllocationCommandsTests=137 -org.elasticsearch.cluster.routing.allocation.AllocationPriorityTests=34 -org.elasticsearch.cluster.routing.allocation.AwarenessAllocationTests=334 -org.elasticsearch.cluster.routing.allocation.BalanceConfigurationTests=426 -org.elasticsearch.cluster.routing.allocation.BalanceUnbalancedClusterTests=9557 -org.elasticsearch.cluster.routing.allocation.ClusterRebalanceRoutingTests=908 -org.elasticsearch.cluster.routing.allocation.ConcurrentRebalanceRoutingTests=157 -org.elasticsearch.cluster.routing.allocation.DeadNodesAllocationTests=72 -org.elasticsearch.cluster.routing.allocation.ElectReplicaAsPrimaryDuringRelocationTests=50 -org.elasticsearch.cluster.routing.allocation.ExpectedShardSizeAllocationTests=127 -org.elasticsearch.cluster.routing.allocation.FailedNodeRoutingTests=48 -org.elasticsearch.cluster.routing.allocation.FailedShardsRoutingTests=151 -org.elasticsearch.cluster.routing.allocation.FilterRoutingTests=53 -org.elasticsearch.cluster.routing.allocation.IndexBalanceTests=118 -org.elasticsearch.cluster.routing.allocation.NodeVersionAllocationDeciderTests=424 -org.elasticsearch.cluster.routing.allocation.PreferLocalPrimariesToRelocatingPrimariesTests=75 -org.elasticsearch.cluster.routing.allocation.PreferPrimaryAllocationTests=97 -org.elasticsearch.cluster.routing.allocation.PrimaryElectionRoutingTests=337 -org.elasticsearch.cluster.routing.allocation.PrimaryNotRelocatedWhileBeingRecoveredTests=2581 -org.elasticsearch.cluster.routing.allocation.RandomAllocationDeciderTests=53 -org.elasticsearch.cluster.routing.allocation.RebalanceAfterActiveTests=79 -org.elasticsearch.cluster.routing.allocation.ReplicaAllocatedAfterPrimaryTests=98 -org.elasticsearch.cluster.routing.allocation.RoutingNodesIntegrityTests=47 -org.elasticsearch.cluster.routing.allocation.SameShardRoutingTests=99 -org.elasticsearch.cluster.routing.allocation.ShardVersioningTests=54 -org.elasticsearch.cluster.routing.allocation.ShardsLimitAllocationTests=69 -org.elasticsearch.cluster.routing.allocation.SingleShardNoReplicasRoutingTests=209 -org.elasticsearch.cluster.routing.allocation.SingleShardOneReplicaRoutingTests=57 -org.elasticsearch.cluster.routing.allocation.StartedShardsRoutingTests=83 -org.elasticsearch.cluster.routing.allocation.TenShardsOneReplicaRoutingTests=39 -org.elasticsearch.cluster.routing.allocation.ThrottlingAllocationTests=14 -org.elasticsearch.cluster.routing.allocation.UpdateNumberOfReplicasTests=30 -org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDeciderTests=75 -org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDeciderUnitTests=107 -org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationTests=108 -org.elasticsearch.cluster.routing.operation.hash.murmur3.Murmur3HashFunctionTests=71 -org.elasticsearch.cluster.serialization.ClusterSerializationTests=25 -org.elasticsearch.cluster.serialization.ClusterStateToStringTests=42 -org.elasticsearch.cluster.serialization.DiffableTests=95 -org.elasticsearch.cluster.settings.SettingsValidatorTests=53 -org.elasticsearch.cluster.structure.RoutingIteratorTests=1016 -org.elasticsearch.codecs.CodecTests=9816 -org.elasticsearch.common.Base64Tests=2127 -org.elasticsearch.common.BooleansTests=54 -org.elasticsearch.common.ChannelsTests=222 -org.elasticsearch.common.ParseFieldTests=92 -org.elasticsearch.common.PidFileTests=2205 -org.elasticsearch.common.StringsTests=96 -org.elasticsearch.common.TableTests=90 -org.elasticsearch.common.UUIDTests=1844 -org.elasticsearch.common.blobstore.BlobStoreTests=44 -org.elasticsearch.common.breaker.MemoryCircuitBreakerTests=187 -org.elasticsearch.common.bytes.BytesReferenceTests=42 -org.elasticsearch.common.bytes.PagedBytesReferenceTests=890 -org.elasticsearch.common.cli.CheckFileCommandTests=462 -org.elasticsearch.common.cli.CliToolTests=195 -org.elasticsearch.common.cli.TerminalTests=111 -org.elasticsearch.common.collect.CopyOnWriteHashMapTests=138 -org.elasticsearch.common.compress.deflate.DeflateCompressedStreamTests=3050 -org.elasticsearch.common.compress.deflate.DeflateXContentTests=1022 -org.elasticsearch.common.compress.lzf.CorruptedCompressorTests=47 -org.elasticsearch.common.compress.lzf.LZFCompressedStreamTests=3845 -org.elasticsearch.common.compress.lzf.LZFXContentTests=738 -org.elasticsearch.common.geo.GeoDistanceTests=183 -org.elasticsearch.common.geo.GeoHashTests=603 -org.elasticsearch.common.geo.GeoJSONShapeParserTests=271 -org.elasticsearch.common.geo.ShapeBuilderTests=649 -org.elasticsearch.common.geo.ShapeRelationTests=63 -org.elasticsearch.common.geo.SpatialStrategyTests=141 -org.elasticsearch.common.hash.MessageDigestsTests=6973 -org.elasticsearch.common.hashing.MurmurHash3Tests=55 -org.elasticsearch.common.hppc.HppcMapsTests=46 -org.elasticsearch.common.io.FileSystemUtilsTests=2105 -org.elasticsearch.common.io.StreamsTests=134 -org.elasticsearch.common.io.stream.BytesStreamsTests=552 -org.elasticsearch.common.joda.DateMathParserTests=127 -org.elasticsearch.common.logging.jdk.JDKESLoggerTests=43 -org.elasticsearch.common.logging.log4j.Log4jESLoggerTests=135 -org.elasticsearch.common.logging.log4j.LoggingConfigurationTests=2414 -org.elasticsearch.common.lucene.IndexCacheableQueryTests=124 -org.elasticsearch.common.lucene.LuceneTests=1704 -org.elasticsearch.common.lucene.ShardCoreKeyMapTests=177 -org.elasticsearch.common.lucene.all.SimpleAllTests=2588 -org.elasticsearch.common.lucene.index.ESDirectoryReaderTests=323 -org.elasticsearch.common.lucene.index.FreqTermsEnumTests=682 -org.elasticsearch.common.lucene.search.MultiPhrasePrefixQueryTests=58 -org.elasticsearch.common.lucene.search.function.ScriptScoreFunctionTests=54 -org.elasticsearch.common.lucene.search.morelikethis.MoreLikeThisQueryTests=69 -org.elasticsearch.common.lucene.search.morelikethis.XMoreLikeThisTests=262 -org.elasticsearch.common.lucene.store.ByteArrayIndexInputTests=112 -org.elasticsearch.common.lucene.store.InputStreamIndexInputTests=89 -org.elasticsearch.common.lucene.uid.VersionsTests=500 -org.elasticsearch.common.math.MathUtilsTests=133 -org.elasticsearch.common.network.NetworkAddressTests=2234 -org.elasticsearch.common.network.NetworkServiceTests=103 -org.elasticsearch.common.network.NetworkUtilsTests=37 -org.elasticsearch.common.path.PathTrieTests=67 -org.elasticsearch.common.property.PropertyPlaceholderTests=22 -org.elasticsearch.common.recycler.ConcurrentRecyclerTests=57 -org.elasticsearch.common.recycler.LockedRecyclerTests=105 -org.elasticsearch.common.recycler.NoneRecyclerTests=15 -org.elasticsearch.common.recycler.QueueRecyclerTests=76 -org.elasticsearch.common.regex.RegexTests=103 -org.elasticsearch.common.rounding.RoundingTests=59 -org.elasticsearch.common.rounding.TimeZoneRoundingTests=180 -org.elasticsearch.common.settings.SettingsFilterTests=9 -org.elasticsearch.common.settings.SettingsTests=137 -org.elasticsearch.common.settings.loader.JsonSettingsLoaderTests=29 -org.elasticsearch.common.settings.loader.PropertiesSettingsLoaderTests=64 -org.elasticsearch.common.settings.loader.YamlSettingsLoaderTests=214 -org.elasticsearch.common.transport.BoundTransportAddressTests=139 -org.elasticsearch.common.unit.ByteSizeUnitTests=77 -org.elasticsearch.common.unit.ByteSizeValueTests=106 -org.elasticsearch.common.unit.DistanceUnitTests=58 -org.elasticsearch.common.unit.FuzzinessTests=55 -org.elasticsearch.common.unit.RatioValueTests=49 -org.elasticsearch.common.unit.SizeValueTests=88 -org.elasticsearch.common.unit.TimeValueTests=120 -org.elasticsearch.common.util.ArrayUtilsTests=43 -org.elasticsearch.common.util.BigArraysTests=4095 -org.elasticsearch.common.util.ByteUtilsTests=103 -org.elasticsearch.common.util.BytesRefHashTests=1372 -org.elasticsearch.common.util.CancellableThreadsTests=37 -org.elasticsearch.common.util.CollectionUtilsTests=219 -org.elasticsearch.common.util.LongHashTests=501 -org.elasticsearch.common.util.LongObjectHashMapTests=820 -org.elasticsearch.common.util.MultiDataPathUpgraderTests=984 -org.elasticsearch.common.util.SingleObjectCacheTests=107 -org.elasticsearch.common.util.URIPatternTests=128 -org.elasticsearch.common.util.concurrent.CountDownTests=266 -org.elasticsearch.common.util.concurrent.EsExecutorsTests=351 -org.elasticsearch.common.util.concurrent.PrioritizedExecutorsTests=436 -org.elasticsearch.common.util.concurrent.RefCountedTests=111 -org.elasticsearch.common.util.iterable.IterablesTests=16 -org.elasticsearch.common.xcontent.ObjectParserTests=110 -org.elasticsearch.common.xcontent.XContentFactoryTests=16 -org.elasticsearch.common.xcontent.builder.BuilderRawFieldTests=93 -org.elasticsearch.common.xcontent.builder.XContentBuilderTests=90 -org.elasticsearch.common.xcontent.cbor.CborXContentParserTests=65 -org.elasticsearch.common.xcontent.cbor.JsonVsCborTests=33 -org.elasticsearch.common.xcontent.smile.JsonVsSmileTests=38 -org.elasticsearch.common.xcontent.support.XContentHelperTests=44 -org.elasticsearch.common.xcontent.support.XContentMapValuesTests=102 -org.elasticsearch.common.xcontent.support.filtering.CborFilteringGeneratorTests=100 -org.elasticsearch.common.xcontent.support.filtering.JsonFilteringGeneratorTests=156 -org.elasticsearch.common.xcontent.support.filtering.SmileFilteringGeneratorTests=177 -org.elasticsearch.common.xcontent.support.filtering.YamlFilteringGeneratorTests=121 -org.elasticsearch.deps.jackson.JacksonLocationTests=20 -org.elasticsearch.deps.joda.SimpleJodaTests=223 -org.elasticsearch.deps.lucene.SimpleLuceneTests=432 -org.elasticsearch.deps.lucene.VectorHighlighterTests=354 -org.elasticsearch.discovery.BlockingClusterStatePublishResponseHandlerTests=21 -org.elasticsearch.discovery.DiscoveryModuleTests=27 -org.elasticsearch.discovery.ZenFaultDetectionTests=270 -org.elasticsearch.discovery.zen.ElectMasterServiceTests=50 -org.elasticsearch.discovery.zen.NodeJoinControllerTests=251 -org.elasticsearch.discovery.zen.ZenDiscoveryUnitTests=65 -org.elasticsearch.discovery.zen.ZenPingTests=16 -org.elasticsearch.discovery.zen.publish.PendingClusterStatesQueueTests=179 -org.elasticsearch.discovery.zen.publish.PublishClusterStateActionTests=887 -org.elasticsearch.env.EnvironmentTests=85 -org.elasticsearch.env.NodeEnvironmentTests=678 -org.elasticsearch.fieldstats.FieldStatsTests=1846 -org.elasticsearch.gateway.AsyncShardFetchTests=400 -org.elasticsearch.gateway.DanglingIndicesStateTests=43 -org.elasticsearch.gateway.GatewayMetaStateTests=202 -org.elasticsearch.gateway.GatewayModuleTests=14 -org.elasticsearch.gateway.GatewayServiceTests=62 -org.elasticsearch.gateway.GatewayTests=66 -org.elasticsearch.gateway.MetaDataStateFormatTests=232 -org.elasticsearch.gateway.MetaStateServiceTests=255 -org.elasticsearch.gateway.PrimaryShardAllocatorTests=85 -org.elasticsearch.gateway.PriorityComparatorTests=49 -org.elasticsearch.gateway.ReplicaShardAllocatorTests=64 -org.elasticsearch.http.netty.NettyHttpChannelTests=100 -org.elasticsearch.http.netty.NettyHttpServerPipeliningTests=3173 -org.elasticsearch.http.netty.pipelining.HttpPipeliningHandlerTests=335 -org.elasticsearch.index.IndexModuleTests=66 -org.elasticsearch.index.IndexServiceTests=15 -org.elasticsearch.index.VersionTypeTests=26 -org.elasticsearch.index.aliases.IndexAliasesServiceTests=318 -org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactoryTests=765 -org.elasticsearch.index.analysis.AnalysisFactoryTests=150 -org.elasticsearch.index.analysis.AnalysisModuleTests=371 -org.elasticsearch.index.analysis.AnalysisTests=34 -org.elasticsearch.index.analysis.AnalyzerBackwardsCompatTests=1446 -org.elasticsearch.index.analysis.CJKFilterFactoryTests=39 -org.elasticsearch.index.analysis.CharFilterTests=94 -org.elasticsearch.index.analysis.CompoundAnalysisTests=171 -org.elasticsearch.index.analysis.HunspellTokenFilterFactoryTests=2896 -org.elasticsearch.index.analysis.KeepFilterFactoryTests=53 -org.elasticsearch.index.analysis.KeepTypesFilterFactoryTests=82 -org.elasticsearch.index.analysis.LimitTokenCountFilterFactoryTests=54 -org.elasticsearch.index.analysis.NGramTokenizerFactoryTests=63 -org.elasticsearch.index.analysis.NumericAnalyzerTests=13 -org.elasticsearch.index.analysis.PatternAnalyzerTests=1636 -org.elasticsearch.index.analysis.PatternCaptureTokenFilterTests=147 -org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactoryTests=48 -org.elasticsearch.index.analysis.PreBuiltAnalyzerTests=293 -org.elasticsearch.index.analysis.PreBuiltCharFilterFactoryFactoryTests=86 -org.elasticsearch.index.analysis.PreBuiltTokenFilterFactoryFactoryTests=128 -org.elasticsearch.index.analysis.PreBuiltTokenizerFactoryFactoryTests=26 -org.elasticsearch.index.analysis.ShingleTokenFilterFactoryTests=136 -org.elasticsearch.index.analysis.SnowballAnalyzerTests=52 -org.elasticsearch.index.analysis.StemmerTokenFilterFactoryTests=2163 -org.elasticsearch.index.analysis.StopAnalyzerTests=3016 -org.elasticsearch.index.analysis.StopTokenFilterTests=375 -org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactoryTests=336 -org.elasticsearch.index.analysis.commongrams.CommonGramsTokenFilterFactoryTests=359 -org.elasticsearch.index.analysis.synonyms.SynonymsAnalysisTests=166 -org.elasticsearch.index.cache.IndexCacheModuleTests=33 -org.elasticsearch.index.cache.bitset.BitSetFilterCacheTests=657 -org.elasticsearch.index.codec.CodecTests=479 -org.elasticsearch.index.codec.postingformat.PostingsFormatTests=6900 -org.elasticsearch.index.engine.CommitStatsTests=76 -org.elasticsearch.index.engine.InternalEngineSettingsTests=323 -org.elasticsearch.index.engine.InternalEngineTests=13034 -org.elasticsearch.index.engine.ShadowEngineTests=3902 -org.elasticsearch.index.fielddata.BinaryDVFieldDataTests=503 -org.elasticsearch.index.fielddata.DisabledFieldDataFormatTests=1035 -org.elasticsearch.index.fielddata.DoubleFieldDataTests=943 -org.elasticsearch.index.fielddata.DuelFieldDataTests=6036 -org.elasticsearch.index.fielddata.FieldDataTests=152 -org.elasticsearch.index.fielddata.FilterFieldDataTests=650 -org.elasticsearch.index.fielddata.FloatFieldDataTests=1246 -org.elasticsearch.index.fielddata.IndexFieldDataServiceTests=2723 -org.elasticsearch.index.fielddata.LongFieldDataTests=4912 -org.elasticsearch.index.fielddata.NoOrdinalsStringFieldDataTests=5655 -org.elasticsearch.index.fielddata.PagedBytesStringFieldDataTests=5923 -org.elasticsearch.index.fielddata.ParentChildFieldDataTests=1012 -org.elasticsearch.index.fielddata.ScriptDocValuesTests=224 -org.elasticsearch.index.fielddata.SortedSetDVStringFieldDataTests=5307 -org.elasticsearch.index.fielddata.fieldcomparator.ReplaceMissingTests=127 -org.elasticsearch.index.fielddata.ordinals.MultiOrdinalsTests=132 -org.elasticsearch.index.fielddata.ordinals.SingleOrdinalsTests=436 -org.elasticsearch.index.indexing.IndexingSlowLogTests=49 -org.elasticsearch.index.mapper.DocumentParserTests=371 -org.elasticsearch.index.mapper.DynamicMappingTests=1335 -org.elasticsearch.index.mapper.FieldTypeLookupTests=29 -org.elasticsearch.index.mapper.MapperServiceTests=230 -org.elasticsearch.index.mapper.UidTests=57 -org.elasticsearch.index.mapper.all.SimpleAllMapperTests=1376 -org.elasticsearch.index.mapper.binary.BinaryMappingTests=3554 -org.elasticsearch.index.mapper.boost.CustomBoostMappingTests=243 -org.elasticsearch.index.mapper.boost.FieldLevelBoostTests=2704 -org.elasticsearch.index.mapper.camelcase.CamelCaseFieldNameTests=358 -org.elasticsearch.index.mapper.completion.CompletionFieldMapperTests=429 -org.elasticsearch.index.mapper.compound.CompoundTypesTests=332 -org.elasticsearch.index.mapper.copyto.CopyToMapperTests=940 -org.elasticsearch.index.mapper.core.BinaryFieldTypeTests=95 -org.elasticsearch.index.mapper.core.BooleanFieldMapperTests=414 -org.elasticsearch.index.mapper.core.BooleanFieldTypeTests=133 -org.elasticsearch.index.mapper.core.ByteFieldTypeTests=86 -org.elasticsearch.index.mapper.core.CompletionFieldTypeTests=113 -org.elasticsearch.index.mapper.core.DateFieldTypeTests=73 -org.elasticsearch.index.mapper.core.DoubleFieldTypeTests=54 -org.elasticsearch.index.mapper.core.FloatFieldTypeTests=82 -org.elasticsearch.index.mapper.core.IntegerFieldTypeTests=56 -org.elasticsearch.index.mapper.core.LongFieldTypeTests=66 -org.elasticsearch.index.mapper.core.ShortFieldTypeTests=66 -org.elasticsearch.index.mapper.core.StringFieldTypeTests=79 -org.elasticsearch.index.mapper.core.TokenCountFieldMapperTests=300 -org.elasticsearch.index.mapper.date.DateBackwardsCompatibilityTests=1544 -org.elasticsearch.index.mapper.date.SimpleDateMappingTests=378 -org.elasticsearch.index.mapper.dynamictemplate.genericstore.GenericStoreDynamicTemplateTests=3256 -org.elasticsearch.index.mapper.dynamictemplate.pathmatch.PathMatchDynamicTemplateTests=303 -org.elasticsearch.index.mapper.dynamictemplate.simple.SimpleDynamicTemplatesTests=259 -org.elasticsearch.index.mapper.externalvalues.SimpleExternalMappingTests=539 -org.elasticsearch.index.mapper.geo.GeoEncodingTests=113 -org.elasticsearch.index.mapper.geo.GeoPointFieldMapperTests=2345 -org.elasticsearch.index.mapper.geo.GeoPointFieldTypeTests=113 -org.elasticsearch.index.mapper.geo.GeoShapeFieldMapperTests=768 -org.elasticsearch.index.mapper.geo.GeoShapeFieldTypeTests=92 -org.elasticsearch.index.mapper.geo.GeohashMappingGeoPointTests=308 -org.elasticsearch.index.mapper.id.IdMappingTests=712 -org.elasticsearch.index.mapper.index.IndexTypeMapperTests=330 -org.elasticsearch.index.mapper.internal.AllFieldTypeTests=95 -org.elasticsearch.index.mapper.internal.FieldNamesFieldMapperTests=636 -org.elasticsearch.index.mapper.internal.FieldNamesFieldTypeTests=119 -org.elasticsearch.index.mapper.internal.IdFieldTypeTests=54 -org.elasticsearch.index.mapper.internal.IndexFieldTypeTests=49 -org.elasticsearch.index.mapper.internal.ParentFieldMapperTests=83 -org.elasticsearch.index.mapper.internal.ParentFieldTypeTests=75 -org.elasticsearch.index.mapper.internal.RoutingFieldTypeTests=72 -org.elasticsearch.index.mapper.internal.SourceFieldTypeTests=129 -org.elasticsearch.index.mapper.internal.TimestampFieldTypeTests=61 -org.elasticsearch.index.mapper.internal.TypeFieldTypeTests=53 -org.elasticsearch.index.mapper.internal.UidFieldTypeTests=30 -org.elasticsearch.index.mapper.internal.VersionFieldTypeTests=39 -org.elasticsearch.index.mapper.ip.SimpleIpMappingTests=592 -org.elasticsearch.index.mapper.lucene.DoubleIndexingDocTests=142 -org.elasticsearch.index.mapper.lucene.StoredNumericValuesTests=328 -org.elasticsearch.index.mapper.merge.TestMergeMapperTests=1501 -org.elasticsearch.index.mapper.multifield.MultiFieldTests=633 -org.elasticsearch.index.mapper.multifield.merge.JavaMultiFieldMergeTests=218 -org.elasticsearch.index.mapper.nested.NestedMappingTests=749 -org.elasticsearch.index.mapper.null_value.NullValueTests=5152 -org.elasticsearch.index.mapper.numeric.SimpleNumericTests=884 -org.elasticsearch.index.mapper.object.NullValueObjectMappingTests=299 -org.elasticsearch.index.mapper.object.SimpleObjectMappingTests=728 -org.elasticsearch.index.mapper.parent.ParentMappingTests=294 -org.elasticsearch.index.mapper.path.PathMapperTests=515 -org.elasticsearch.index.mapper.routing.RoutingTypeMapperTests=497 -org.elasticsearch.index.mapper.simple.SimpleMapperTests=258 -org.elasticsearch.index.mapper.source.CompressSourceMappingTests=2902 -org.elasticsearch.index.mapper.source.DefaultSourceMappingTests=1323 -org.elasticsearch.index.mapper.string.SimpleStringMappingTests=177 -org.elasticsearch.index.mapper.string.StringFieldMapperPositionIncrementGapTests=702 -org.elasticsearch.index.mapper.timestamp.TimestampMappingTests=2991 -org.elasticsearch.index.mapper.ttl.TTLMappingTests=1822 -org.elasticsearch.index.mapper.typelevels.ParseDocumentTypeLevelsTests=440 -org.elasticsearch.index.mapper.typelevels.ParseMappingTypeLevelTests=187 -org.elasticsearch.index.mapper.update.UpdateMappingTests=843 -org.elasticsearch.index.query.BoolQueryBuilderTests=154 -org.elasticsearch.index.query.BoostingQueryBuilderTests=183 -org.elasticsearch.index.query.CombineFunctionTests=53 -org.elasticsearch.index.query.CommonTermsQueryBuilderTests=95 -org.elasticsearch.index.query.CommonTermsQueryParserTests=709 -org.elasticsearch.index.query.ConstantScoreQueryBuilderTests=285 -org.elasticsearch.index.query.DisMaxQueryBuilderTests=330 -org.elasticsearch.index.query.ExistsQueryBuilderTests=139 -org.elasticsearch.index.query.FieldMaskingSpanQueryBuilderTests=152 -org.elasticsearch.index.query.FuzzyQueryBuilderTests=210 -org.elasticsearch.index.query.GeoBoundingBoxQueryBuilderTests=315 -org.elasticsearch.index.query.GeoDistanceQueryBuilderTests=192 -org.elasticsearch.index.query.GeoDistanceRangeQueryTests=156 -org.elasticsearch.index.query.GeoPolygonQueryBuilderTests=445 -org.elasticsearch.index.query.GeoShapeQueryBuilderTests=246 -org.elasticsearch.index.query.GeohashCellQueryBuilderTests=85 -org.elasticsearch.index.query.HasChildQueryBuilderTests=255 -org.elasticsearch.index.query.HasChildQueryParserTests=82 -org.elasticsearch.index.query.HasParentQueryBuilderTests=336 -org.elasticsearch.index.query.IdsQueryBuilderTests=197 -org.elasticsearch.index.query.IndicesQueryBuilderTests=279 -org.elasticsearch.index.query.MatchAllQueryBuilderTests=188 -org.elasticsearch.index.query.MatchNoneQueryBuilderTests=257 -org.elasticsearch.index.query.MatchQueryBuilderTests=2712 -org.elasticsearch.index.query.MissingQueryBuilderTests=180 -org.elasticsearch.index.query.MoreLikeThisQueryBuilderTests=3351 -org.elasticsearch.index.query.MultiMatchQueryBuilderTests=59 -org.elasticsearch.index.query.NestedQueryBuilderTests=193 -org.elasticsearch.index.query.NotQueryBuilderTests=3071 -org.elasticsearch.index.query.OperatorTests=90 -org.elasticsearch.index.query.PrefixQueryBuilderTests=149 -org.elasticsearch.index.query.QueryFilterBuilderTests=100 -org.elasticsearch.index.query.QueryStringQueryBuilderTests=490 -org.elasticsearch.index.query.RangeQueryBuilderTests=577 -org.elasticsearch.index.query.RegexpQueryBuilderTests=235 -org.elasticsearch.index.query.ScoreModeTests=52 -org.elasticsearch.index.query.ScriptQueryBuilderTests=108 -org.elasticsearch.index.query.SimpleQueryStringBuilderTests=158 -org.elasticsearch.index.query.SpanContainingQueryBuilderTests=213 -org.elasticsearch.index.query.SpanFirstQueryBuilderTests=105 -org.elasticsearch.index.query.SpanMultiTermQueryBuilderTests=1847 -org.elasticsearch.index.query.SpanNearQueryBuilderTests=91 -org.elasticsearch.index.query.SpanNotQueryBuilderTests=589 -org.elasticsearch.index.query.SpanOrQueryBuilderTests=2712 -org.elasticsearch.index.query.SpanTermQueryBuilderTests=85 -org.elasticsearch.index.query.SpanWithinQueryBuilderTests=61 -org.elasticsearch.index.query.TemplateQueryBuilderTests=417 -org.elasticsearch.index.query.TemplateQueryParserTests=288 -org.elasticsearch.index.query.TermQueryBuilderTests=49 -org.elasticsearch.index.query.TermsQueryBuilderTests=69 -org.elasticsearch.index.query.TypeQueryBuilderTests=258 -org.elasticsearch.index.query.WildcardQueryBuilderTests=38 -org.elasticsearch.index.query.WrapperQueryBuilderTests=138 -org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionModifierTests=69 -org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilderTests=227 -org.elasticsearch.index.query.functionscore.ScoreFunctionBuilderTests=53 -org.elasticsearch.index.query.support.QueryInnerHitsTests=53 -org.elasticsearch.index.search.MultiMatchQueryTests=135 -org.elasticsearch.index.search.geo.GeoPointParsingTests=90 -org.elasticsearch.index.search.geo.GeoUtilsTests=142 -org.elasticsearch.index.search.nested.DoubleNestedSortingTests=269 -org.elasticsearch.index.search.nested.FloatNestedSortingTests=575 -org.elasticsearch.index.search.nested.LongNestedSortingTests=305 -org.elasticsearch.index.search.nested.NestedSortingTests=264 -org.elasticsearch.index.shard.CommitPointsTests=58 -org.elasticsearch.index.shard.IndexShardTests=2664 -org.elasticsearch.index.shard.MergePolicySettingsTests=85 -org.elasticsearch.index.shard.NewPathForShardTests=87 -org.elasticsearch.index.shard.ShardPathTests=123 -org.elasticsearch.index.shard.ShardUtilsTests=69 -org.elasticsearch.index.shard.VersionFieldUpgraderTests=27 -org.elasticsearch.index.similarity.SimilarityTests=1282 -org.elasticsearch.index.snapshots.blobstore.FileInfoTests=350 -org.elasticsearch.index.snapshots.blobstore.SlicedInputStreamTests=28 -org.elasticsearch.index.store.DirectoryUtilsTests=179 -org.elasticsearch.index.store.IndexStoreBWCTests=397 -org.elasticsearch.index.store.IndexStoreTests=13 -org.elasticsearch.index.store.LegacyVerificationTests=44 -org.elasticsearch.index.store.StoreTests=433 -org.elasticsearch.index.translog.BufferedTranslogTests=4946 -org.elasticsearch.index.translog.TranslogTests=4070 -org.elasticsearch.index.translog.TranslogVersionTests=42 -org.elasticsearch.indices.IndicesLifecycleListenerSingleNodeTests=611 -org.elasticsearch.indices.IndicesModuleTests=1493 -org.elasticsearch.indices.IndicesServiceTests=5140 -org.elasticsearch.indices.cache.query.terms.TermsLookupTests=22 -org.elasticsearch.indices.flush.SyncedFlushSingleNodeTests=1243 -org.elasticsearch.indices.flush.SyncedFlushUnitTests=64 -org.elasticsearch.indices.memory.IndexingMemoryControllerTests=65 -org.elasticsearch.indices.memory.breaker.CircuitBreakerUnitTests=101 -org.elasticsearch.indices.recovery.RecoverySourceHandlerTests=691 -org.elasticsearch.indices.recovery.RecoveryStateTests=153 -org.elasticsearch.indices.recovery.RecoveryStatusTests=62 -org.elasticsearch.indices.recovery.StartRecoveryRequestTests=84 -org.elasticsearch.indices.store.IndicesStoreTests=83 -org.elasticsearch.monitor.fs.FsProbeTests=41 -org.elasticsearch.monitor.jvm.JvmStatsTests=43 -org.elasticsearch.monitor.os.OsProbeTests=45 -org.elasticsearch.monitor.process.ProcessProbeTests=42 -org.elasticsearch.node.internal.InternalSettingsPreparerTests=140 -org.elasticsearch.plugins.PluginInfoTests=372 -org.elasticsearch.plugins.PluginManagerCliTests=153 -org.elasticsearch.plugins.PluginManagerUnitTests=51 -org.elasticsearch.plugins.PluginsServiceTests=68 -org.elasticsearch.recovery.RecoveriesCollectionTests=430 -org.elasticsearch.recovery.RecoverySettingsTests=569 -org.elasticsearch.rest.BytesRestResponseTests=83 -org.elasticsearch.rest.HeadersAndContextCopyClientTests=194 -org.elasticsearch.rest.RestFilterChainTests=77 -org.elasticsearch.rest.RestRequestTests=39 -org.elasticsearch.rest.action.support.RestTableTests=88 -org.elasticsearch.rest.util.RestUtilsTests=85 -org.elasticsearch.script.FileScriptTests=39 -org.elasticsearch.script.NativeScriptTests=111 -org.elasticsearch.script.ScriptContextRegistryTests=27 -org.elasticsearch.script.ScriptContextTests=85 -org.elasticsearch.script.ScriptModesTests=115 -org.elasticsearch.script.ScriptParameterParserTests=173 -org.elasticsearch.script.ScriptServiceTests=421 -org.elasticsearch.script.mustache.MustacheScriptEngineTests=115 -org.elasticsearch.script.mustache.MustacheTests=65 -org.elasticsearch.search.MultiValueModeTests=149 -org.elasticsearch.search.SearchModuleTests=89 -org.elasticsearch.search.SearchServiceTests=1170 -org.elasticsearch.search.aggregations.AggregationCollectorTests=644 -org.elasticsearch.search.aggregations.bucket.nested.NestedAggregatorTests=419 -org.elasticsearch.search.aggregations.bucket.significant.SignificanceHeuristicTests=120 -org.elasticsearch.search.aggregations.metrics.cardinality.HyperLogLogPlusPlusTests=695 -org.elasticsearch.search.aggregations.pipeline.PipelineAggregationHelperTests=27 -org.elasticsearch.search.aggregations.pipeline.moving.avg.MovAvgUnitTests=122 -org.elasticsearch.search.aggregations.support.MissingValuesTests=52 -org.elasticsearch.search.aggregations.support.PathTests=90 -org.elasticsearch.search.aggregations.support.ScriptValuesTests=67 -org.elasticsearch.search.builder.SearchSourceBuilderTests=49 -org.elasticsearch.search.compress.SearchSourceCompressTests=3136 -org.elasticsearch.search.fetch.innerhits.NestedChildrenFilterTests=128 -org.elasticsearch.search.internal.InternalSearchHitTests=46 -org.elasticsearch.search.query.QueryPhaseTests=185 -org.elasticsearch.search.sort.SortParserTests=319 -org.elasticsearch.search.stats.SearchStatsUnitTests=50 -org.elasticsearch.search.suggest.CompletionTokenStreamTests=160 -org.elasticsearch.search.suggest.completion.CompletionPostingsFormatTests=1319 -org.elasticsearch.search.suggest.context.GeoLocationContextMappingTests=109 -org.elasticsearch.search.suggest.phrase.NoisyChannelSpellCheckerTests=1409 -org.elasticsearch.snapshots.SnapshotRequestsTests=30 -org.elasticsearch.snapshots.SnapshotUtilsTests=30 -org.elasticsearch.test.rest.test.AssertionParsersTests=30 -org.elasticsearch.test.rest.test.DoSectionParserTests=198 -org.elasticsearch.test.rest.test.FileUtilsTests=60 -org.elasticsearch.test.rest.test.JsonPathTests=83 -org.elasticsearch.test.rest.test.RestApiParserFailingTests=102 -org.elasticsearch.test.rest.test.RestApiParserTests=73 -org.elasticsearch.test.rest.test.RestTestParserTests=145 -org.elasticsearch.test.rest.test.SetSectionParserTests=116 -org.elasticsearch.test.rest.test.SetupSectionParserTests=248 -org.elasticsearch.test.rest.test.SkipSectionParserTests=95 -org.elasticsearch.test.rest.test.TestSectionParserTests=134 -org.elasticsearch.test.test.InternalTestClusterTests=40 -org.elasticsearch.test.test.LoggingListenerTests=72 -org.elasticsearch.test.test.VersionUtilsTests=28 -org.elasticsearch.threadpool.ThreadPoolSerializationTests=56 -org.elasticsearch.threadpool.ThreadPoolStatsTests=99 -org.elasticsearch.threadpool.UpdateThreadPoolSettingsTests=24 -org.elasticsearch.transport.NettySizeHeaderFrameDecoderTests=183 -org.elasticsearch.transport.TransportMessageTests=51 -org.elasticsearch.transport.local.SimpleLocalTransportTests=1174 -org.elasticsearch.transport.netty.KeyedLockTests=414 -org.elasticsearch.transport.netty.NettyScheduledPingTests=1662 -org.elasticsearch.transport.netty.NettyTransportMultiPortTests=382 -org.elasticsearch.transport.netty.NettyTransportTests=137 -org.elasticsearch.transport.netty.SimpleNettyTransportTests=5528 -org.elasticsearch.tribe.TribeUnitTests=2098 -org.elasticsearch.watcher.FileWatcherTests=203 -org.elasticsearch.watcher.ResourceWatcherServiceTests=101 diff --git a/core/build.gradle b/core/build.gradle new file mode 100644 index 00000000000..fd8a0c10f5a --- /dev/null +++ b/core/build.gradle @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.carrotsearch.gradle.junit4.RandomizedTestingTask +import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.test.RestSpecHack + +apply plugin: 'elasticsearch.build' +apply plugin: 'com.bmuschko.nexus' +apply plugin: 'nebula.optional-base' + +archivesBaseName = 'elasticsearch' + +dependencies { + + // lucene + compile "org.apache.lucene:lucene-core:${versions.lucene}" + compile "org.apache.lucene:lucene-analyzers-common:${versions.lucene}" + compile "org.apache.lucene:lucene-backward-codecs:${versions.lucene}" + compile "org.apache.lucene:lucene-grouping:${versions.lucene}" + compile "org.apache.lucene:lucene-highlighter:${versions.lucene}" + compile "org.apache.lucene:lucene-join:${versions.lucene}" + compile "org.apache.lucene:lucene-memory:${versions.lucene}" + compile "org.apache.lucene:lucene-misc:${versions.lucene}" + compile "org.apache.lucene:lucene-queries:${versions.lucene}" + compile "org.apache.lucene:lucene-queryparser:${versions.lucene}" + compile "org.apache.lucene:lucene-sandbox:${versions.lucene}" + compile "org.apache.lucene:lucene-spatial:${versions.lucene}" + compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}" + compile "org.apache.lucene:lucene-suggest:${versions.lucene}" + + compile 'org.elasticsearch:securesm:1.0' + + // utilities + compile 'commons-cli:commons-cli:1.3.1' + compile 'com.carrotsearch:hppc:0.7.1' + + // time handling, remove with java 8 time + compile 'joda-time:joda-time:2.8.2' + // joda 2.0 moved to using volatile fields for datetime + // When updating to a new version, make sure to update our copy of BaseDateTime + compile 'org.joda:joda-convert:1.2' + + // json and yaml + compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + + // network stack + compile 'io.netty:netty:3.10.5.Final' + // percentiles aggregation + compile 'com.tdunning:t-digest:3.0' + // precentil ranks aggregation + compile 'org.hdrhistogram:HdrHistogram:2.1.6' + + // lucene spatial + compile "com.spatial4j:spatial4j:${versions.spatial4j}", optional + compile "com.vividsolutions:jts:${versions.jts}", optional + + // logging + compile "log4j:log4j:${versions.log4j}", optional + compile "log4j:apache-log4j-extras:${versions.log4j}", optional + compile "org.slf4j:slf4j-api:${versions.slf4j}", optional + + compile "net.java.dev.jna:jna:${versions.jna}", optional + + if (isEclipse == false || project.path == ":core-tests") { + testCompile("org.elasticsearch:test-framework:${version}") { + // tests use the locally compiled version of core + exclude group: 'org.elasticsearch', module: 'elasticsearch' + } + } +} + +if (isEclipse) { + // in eclipse the project is under a fake root, we need to change around the source sets + sourceSets { + if (project.path == ":core") { + main.java.srcDirs = ['java'] + main.resources.srcDirs = ['resources'] + } else { + test.java.srcDirs = ['java'] + test.resources.srcDirs = ['resources'] + } + } +} + +compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-fallthrough,-overrides,-rawtypes,-serial,-try,-unchecked" +compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-fallthrough,-overrides,-rawtypes,-serial,-try,-unchecked" + +forbiddenPatterns { + exclude '**/*.json' + exclude '**/*.jmx' + exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt' +} + +// dependency license are currently checked in distribution +dependencyLicenses.enabled = false + +if (isEclipse == false || project.path == ":core-tests") { + task integTest(type: RandomizedTestingTask, + group: JavaBasePlugin.VERIFICATION_GROUP, + description: 'Multi-node tests', + dependsOn: test.dependsOn) { + configure(BuildPlugin.commonTestConfig(project)) + classpath = project.test.classpath + testClassesDir = project.test.testClassesDir + include '**/*IT.class' + } + check.dependsOn integTest + integTest.mustRunAfter test +} diff --git a/core/pom.xml b/core/pom.xml deleted file mode 100644 index 023d9296fa6..00000000000 --- a/core/pom.xml +++ /dev/null @@ -1,362 +0,0 @@ - - - 4.0.0 - - org.elasticsearch - parent - 3.0.0-SNAPSHOT - - - org.elasticsearch - elasticsearch - - Elasticsearch: Core - Elasticsearch - Open Source, Distributed, RESTful Search Engine - - - -Xlint:-cast,-deprecation,-fallthrough,-overrides,-rawtypes,-serial,-try,-unchecked - - - - - org.hamcrest - hamcrest-all - test - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - test - - - org.apache.lucene - lucene-test-framework - test - - - org.apache.httpcomponents - httpclient - test - - - com.google.jimfs - jimfs - test - - - - org.apache.lucene - lucene-core - - - org.apache.lucene - lucene-backward-codecs - - - org.apache.lucene - lucene-analyzers-common - - - org.apache.lucene - lucene-queries - - - org.apache.lucene - lucene-memory - - - org.apache.lucene - lucene-highlighter - - - org.apache.lucene - lucene-queryparser - - - org.apache.lucene - lucene-suggest - - - org.apache.lucene - lucene-join - - - org.apache.lucene - lucene-spatial - - - com.spatial4j - spatial4j - true - - - com.vividsolutions - jts - true - - - - com.github.spullara.mustache.java - compiler - true - - - - - com.carrotsearch - hppc - - - joda-time - joda-time - - - org.joda - joda-convert - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.fasterxml.jackson.core - jackson-databind - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - - - io.netty - netty - - - com.ning - compress-lzf - - - com.tdunning - t-digest - - - org.hdrhistogram - HdrHistogram - - - commons-cli - commons-cli - - - - log4j - log4j - true - - - log4j - apache-log4j-extras - true - - - org.slf4j - slf4j-api - true - - - net.java.dev.jna - jna - true - - - - - - - - - ${project.basedir}/src/main/resources - - es-build.properties - - true - - - ${project.basedir}/src/main/resources - - **/*.* - - - - - - - ${project.basedir}/src/test/resources - - **/*.* - - - - ${elasticsearch.tools.directory}/rest-api-spec - rest-api-spec - - api/*.json - test/**/*.yaml - - - - - ${elasticsearch.tools.directory}/shared-test-resources - false - - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-test-sources - - test-jar - - - - org/elasticsearch/test/**/* - org/elasticsearch/bootstrap/BootstrapForTesting.class - org/elasticsearch/bootstrap/MockPluginPolicy.class - org/elasticsearch/common/cli/CliToolTestCase.class - org/elasticsearch/common/cli/CliToolTestCase$*.class - - - - org/elasticsearch/test/rest/test/**/* - - org/elasticsearch/test/test/**/* - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - prepare-package - - test-jar - - - - rest-api-spec/**/* - org/elasticsearch/test/**/* - org/elasticsearch/bootstrap/BootstrapForTesting.class - org/elasticsearch/bootstrap/MockPluginPolicy.class - org/elasticsearch/common/cli/CliToolTestCase.class - org/elasticsearch/common/cli/CliToolTestCase$*.class - org/elasticsearch/cluster/MockInternalClusterInfoService.class - org/elasticsearch/cluster/MockInternalClusterInfoService$*.class - org/elasticsearch/index/MockEngineFactoryPlugin.class - org/elasticsearch/search/MockSearchService.class - org/elasticsearch/search/MockSearchService$*.class - org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.class - org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.class - org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams$*.class - org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.class - org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams$*.class - org/elasticsearch/search/aggregations/bucket/script/TestScript.class - org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.class - org/elasticsearch/percolator/PercolatorTestUtil.class - org/elasticsearch/cache/recycler/MockPageCacheRecycler.class - org/elasticsearch/cache/recycler/MockPageCacheRecycler$*.class - org/elasticsearch/common/util/MockBigArrays.class - org/elasticsearch/common/util/MockBigArrays$*.class - org/elasticsearch/node/NodeMocksPlugin.class - org/elasticsearch/node/MockNode.class - org/elasticsearch/common/io/PathUtilsForTesting.class - - - - org/elasticsearch/test/rest/test/**/* - - org/elasticsearch/test/test/**/* - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - check-license - none - - - - - - - - org.jacoco - jacoco-maven-plugin - - - org/apache/lucene/** - - - - - com.mycila - license-maven-plugin - - - - src/main/java/org/elasticsearch/common/inject/** - - src/main/java/org/apache/lucene/**/X*.java - - src/main/java/org/elasticsearch/http/netty/pipelining/** - - src/main/java/org/elasticsearch/common/network/InetAddresses.java - src/test/java/org/elasticsearch/common/network/InetAddressesTests.java - src/test/java/org/elasticsearch/common/collect/EvictingQueueTests.java - - src/main/java/org/joda/time/base/BaseDateTime.java - src/main/java/org/joda/time/format/StrictISODateTimeFormat.java - - - - - - - - - - license - - - license.generation - true - - - - - - diff --git a/core/src/main/eclipse-build.gradle b/core/src/main/eclipse-build.gradle new file mode 100644 index 00000000000..e46e7134456 --- /dev/null +++ b/core/src/main/eclipse-build.gradle @@ -0,0 +1,3 @@ + +// this is just shell gradle file for eclipse to have separate projects for core src and tests +apply from: '../../build.gradle' diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index e411139bab0..81f49055223 100644 --- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -18,18 +18,9 @@ */ package org.apache.lucene.queries; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.TermState; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.index.*; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DisjunctionMaxQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.InPlaceMergeSorter; import org.apache.lucene.util.ToStringUtils; @@ -37,6 +28,7 @@ import org.apache.lucene.util.ToStringUtils; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Objects; /** * BlendedTermQuery can be used to unify term statistics across @@ -77,6 +69,10 @@ public abstract class BlendedTermQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; + } IndexReaderContext context = reader.getContext(); TermContext[] ctx = new TermContext[terms.length]; int[] docFreqs = new int[ctx.length]; @@ -87,9 +83,7 @@ public abstract class BlendedTermQuery extends Query { final int maxDoc = reader.maxDoc(); blend(ctx, maxDoc, reader); - Query query = topLevelQuery(terms, ctx, docFreqs, maxDoc); - query.setBoost(getBoost()); - return query; + return topLevelQuery(terms, ctx, docFreqs, maxDoc); } protected abstract Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc); @@ -274,20 +268,15 @@ public abstract class BlendedTermQuery extends Query { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; BlendedTermQuery that = (BlendedTermQuery) o; - if (!Arrays.equals(equalsTerms(), that.equalsTerms())) return false; - - return true; + return Arrays.equals(equalsTerms(), that.equalsTerms()); } @Override public int hashCode() { - int result = super.hashCode(); - result = 31 * result + Arrays.hashCode(equalsTerms()); - return result; + return Objects.hash(super.hashCode(), Arrays.hashCode(equalsTerms())); } public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final boolean disableCoord) { @@ -298,16 +287,16 @@ public abstract class BlendedTermQuery extends Query { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { - BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(disableCoord); + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + booleanQueryBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { - TermQuery termQuery = new TermQuery(terms[i], ctx[i]); - if (boosts != null) { - termQuery.setBoost(boosts[i]); + Query query = new TermQuery(terms[i], ctx[i]); + if (boosts != null && boosts[i] != 1f) { + query = new BoostQuery(query, boosts[i]); } - query.add(termQuery, BooleanClause.Occur.SHOULD); + booleanQueryBuilder.add(query, BooleanClause.Occur.SHOULD); } - return query.build(); + return booleanQueryBuilder.build(); } }; } @@ -321,16 +310,16 @@ public abstract class BlendedTermQuery extends Query { BooleanQuery.Builder lowBuilder = new BooleanQuery.Builder(); lowBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { - TermQuery termQuery = new TermQuery(terms[i], ctx[i]); - if (boosts != null) { - termQuery.setBoost(boosts[i]); + Query query = new TermQuery(terms[i], ctx[i]); + if (boosts != null && boosts[i] != 1f) { + query = new BoostQuery(query, boosts[i]); } if ((maxTermFrequency >= 1f && docFreqs[i] > maxTermFrequency) || (docFreqs[i] > (int) Math.ceil(maxTermFrequency * (float) maxDoc))) { - highBuilder.add(termQuery, BooleanClause.Occur.SHOULD); + highBuilder.add(query, BooleanClause.Occur.SHOULD); } else { - lowBuilder.add(termQuery, BooleanClause.Occur.SHOULD); + lowBuilder.add(query, BooleanClause.Occur.SHOULD); } } BooleanQuery high = highBuilder.build(); @@ -363,15 +352,15 @@ public abstract class BlendedTermQuery extends Query { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { - DisjunctionMaxQuery query = new DisjunctionMaxQuery(tieBreakerMultiplier); + DisjunctionMaxQuery disMaxQuery = new DisjunctionMaxQuery(tieBreakerMultiplier); for (int i = 0; i < terms.length; i++) { - TermQuery termQuery = new TermQuery(terms[i], ctx[i]); - if (boosts != null) { - termQuery.setBoost(boosts[i]); + Query query = new TermQuery(terms[i], ctx[i]); + if (boosts != null && boosts[i] != 1f) { + query = new BoostQuery(query, boosts[i]); } - query.add(termQuery); + disMaxQuery.add(query); } - return query; + return disMaxQuery; } }; } diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java index 1e9ecf7ae6f..86982bfc949 100644 --- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import java.io.IOException; +import java.util.Objects; /** A {@link Query} that only matches documents that are greater than or equal * to a configured doc ID. */ @@ -43,7 +44,7 @@ public final class MinDocQuery extends Query { @Override public int hashCode() { - return 31 * super.hashCode() + minDoc; + return Objects.hash(super.hashCode(), minDoc); } @Override diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index 3ef6e5a2c6d..9f2b1b66221 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -23,13 +23,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.DisjunctionMaxQuery; -import org.apache.lucene.search.FuzzyQuery; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.MultiPhraseQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; +import org.apache.lucene.search.*; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.lucene.search.Queries; @@ -41,12 +35,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded; @@ -65,7 +54,6 @@ public class MapperQueryParser extends QueryParser { static { Map fieldQueryExtensions = new HashMap<>(); fieldQueryExtensions.put(ExistsFieldQueryExtension.NAME, new ExistsFieldQueryExtension()); - fieldQueryExtensions.put(MissingFieldQueryExtension.NAME, new MissingFieldQueryExtension()); FIELD_QUERY_EXTENSIONS = unmodifiableMap(fieldQueryExtensions); } @@ -148,8 +136,7 @@ public class MapperQueryParser extends QueryParser { Query q = getFieldQuerySingle(mField, queryText, quoted); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -161,8 +148,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getFieldQuerySingle(mField, queryText, quoted); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -250,9 +236,8 @@ public class MapperQueryParser extends QueryParser { Query q = super.getFieldQuery(mField, queryText, slop); if (q != null) { added = true; - applyBoost(mField, q); q = applySlop(q, slop); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -264,9 +249,8 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = super.getFieldQuery(mField, queryText, slop); if (q != null) { - applyBoost(mField, q); q = applySlop(q, slop); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -305,8 +289,7 @@ public class MapperQueryParser extends QueryParser { Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -318,8 +301,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -371,8 +353,7 @@ public class MapperQueryParser extends QueryParser { Query q = getFuzzyQuerySingle(mField, termStr, minSimilarity); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -383,8 +364,9 @@ public class MapperQueryParser extends QueryParser { List clauses = new ArrayList<>(); for (String mField : fields) { Query q = getFuzzyQuerySingle(mField, termStr, minSimilarity); - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + if (q != null) { + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); + } } return getBooleanQuery(clauses, true); } @@ -434,8 +416,7 @@ public class MapperQueryParser extends QueryParser { Query q = getPrefixQuerySingle(mField, termStr); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -447,8 +428,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getPrefixQuerySingle(mField, termStr); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -566,8 +546,7 @@ public class MapperQueryParser extends QueryParser { Query q = getWildcardQuerySingle(mField, termStr); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -579,8 +558,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getWildcardQuerySingle(mField, termStr); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -697,8 +675,7 @@ public class MapperQueryParser extends QueryParser { Query q = getRegexpQuerySingle(mField, termStr); if (q != null) { added = true; - applyBoost(mField, q); - disMaxQuery.add(q); + disMaxQuery.add(applyBoost(mField, q)); } } if (!added) { @@ -710,8 +687,7 @@ public class MapperQueryParser extends QueryParser { for (String mField : fields) { Query q = getRegexpQuerySingle(mField, termStr); if (q != null) { - applyBoost(mField, q); - clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD)); + clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } if (clauses.size() == 0) // happens for stopwords @@ -761,11 +737,12 @@ public class MapperQueryParser extends QueryParser { return fixNegativeQueryIfNeeded(q); } - private void applyBoost(String field, Query q) { + private Query applyBoost(String field, Query q) { Float fieldBoost = settings.fieldsAndWeights().get(field); - if (fieldBoost != null) { - q.setBoost(fieldBoost); + if (fieldBoost != null && fieldBoost != 1f) { + return new BoostQuery(q, fieldBoost); } + return q; } private Query applySlop(Query q, int slop) { @@ -779,7 +756,9 @@ public class MapperQueryParser extends QueryParser { builder.add(terms[i], positions[i]); } pq = builder.build(); - pq.setBoost(q.getBoost()); + //make sure that the boost hasn't been set beforehand, otherwise we'd lose it + assert q.getBoost() == 1f; + assert q instanceof BoostQuery == false; return pq; } else if (q instanceof MultiPhraseQuery) { ((MultiPhraseQuery) q).setSlop(slop); diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java deleted file mode 100644 index b2b23a29981..00000000000 --- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java +++ /dev/null @@ -1,1136 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lucene.search.suggest.analyzing; - -import com.carrotsearch.hppc.ObjectIntHashMap; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.TokenStreamToAutomaton; -import org.apache.lucene.search.suggest.InputIterator; -import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.store.*; -import org.apache.lucene.util.*; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.LimitedFiniteStringsIterator; -import org.apache.lucene.util.automaton.Operations; -import org.apache.lucene.util.automaton.Transition; -import org.apache.lucene.util.fst.*; -import org.apache.lucene.util.fst.FST.BytesReader; -import org.apache.lucene.util.fst.PairOutputs.Pair; -import org.apache.lucene.util.fst.Util.Result; -import org.apache.lucene.util.fst.Util.TopResults; -import org.elasticsearch.common.collect.HppcMaps; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; - -/** - * Suggester that first analyzes the surface form, adds the - * analyzed form to a weighted FST, and then does the same - * thing at lookup time. This means lookup is based on the - * analyzed form while suggestions are still the surface - * form(s). - * - *

- * This can result in powerful suggester functionality. For - * example, if you use an analyzer removing stop words, - * then the partial text "ghost chr..." could see the - * suggestion "The Ghost of Christmas Past". Note that - * position increments MUST NOT be preserved for this example - * to work, so you should call the constructor with - * preservePositionIncrements parameter set to - * false - * - *

- * If SynonymFilter is used to map wifi and wireless network to - * hotspot then the partial text "wirele..." could suggest - * "wifi router". Token normalization like stemmers, accent - * removal, etc., would allow suggestions to ignore such - * variations. - * - *

- * When two matching suggestions have the same weight, they - * are tie-broken by the analyzed form. If their analyzed - * form is the same then the order is undefined. - * - *

- * There are some limitations: - *

    - * - *
  • A lookup from a query like "net" in English won't - * be any different than "net " (ie, user added a - * trailing space) because analyzers don't reflect - * when they've seen a token separator and when they - * haven't. - * - *
  • If you're using {@code StopFilter}, and the user will - * type "fast apple", but so far all they've typed is - * "fast a", again because the analyzer doesn't convey whether - * it's seen a token separator after the "a", - * {@code StopFilter} will remove that "a" causing - * far more matches than you'd expect. - * - *
  • Lookups with the empty string return no results - * instead of all results. - *
- * - * @lucene.experimental - */ -public class XAnalyzingSuggester extends Lookup { - - /** - * FST<Weight,Surface>: - * input is the analyzed form, with a null byte between terms - * weights are encoded as costs: (Integer.MAX_VALUE-weight) - * surface is the original, unanalyzed form. - */ - private FST> fst = null; - - /** - * Analyzer that will be used for analyzing suggestions at - * index time. - */ - private final Analyzer indexAnalyzer; - - /** - * Analyzer that will be used for analyzing suggestions at - * query time. - */ - private final Analyzer queryAnalyzer; - - /** - * True if exact match suggestions should always be returned first. - */ - private final boolean exactFirst; - - /** - * True if separator between tokens should be preserved. - */ - private final boolean preserveSep; - - /** Include this flag in the options parameter to {@code - * #XAnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean,FST,boolean,int,int,int,int,int)} to always - * return the exact match first, regardless of score. This - * has no performance impact but could result in - * low-quality suggestions. */ - public static final int EXACT_FIRST = 1; - - /** Include this flag in the options parameter to {@code - * #XAnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean,FST,boolean,int,int,int,int,int)} to preserve - * token separators when matching. */ - public static final int PRESERVE_SEP = 2; - - /** Represents the separation between tokens, if - * PRESERVE_SEP was specified */ - public static final int SEP_LABEL = '\u001F'; - - /** Marks end of the analyzed input and start of dedup - * byte. */ - public static final int END_BYTE = 0x0; - - /** Maximum number of dup surface forms (different surface - * forms for the same analyzed form). */ - private final int maxSurfaceFormsPerAnalyzedForm; - - /** Maximum graph paths to index for a single analyzed - * surface form. This only matters if your analyzer - * makes lots of alternate paths (e.g. contains - * SynonymFilter). */ - private final int maxGraphExpansions; - - /** Highest number of analyzed paths we saw for any single - * input surface form. For analyzers that never create - * graphs this will always be 1. */ - private int maxAnalyzedPathsForOneInput; - - private boolean hasPayloads; - - private final int sepLabel; - private final int payloadSep; - private final int endByte; - private final int holeCharacter; - - public static final int PAYLOAD_SEP = '\u001F'; - public static final int HOLE_CHARACTER = '\u001E'; - - private final Automaton queryPrefix; - - /** Whether position holes should appear in the automaton. */ - private boolean preservePositionIncrements; - - /** Number of entries the lookup was built with */ - private long count = 0; - - /** - * Calls {@code #XAnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean,FST,boolean,int,int,int,int,int) - * AnalyzingSuggester(analyzer, analyzer, EXACT_FIRST | - * PRESERVE_SEP, 256, -1)} - */ - public XAnalyzingSuggester(Analyzer analyzer) { - this(analyzer, null, analyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, true, null, false, 0, SEP_LABEL, PAYLOAD_SEP, END_BYTE, HOLE_CHARACTER); - } - - /** - * Calls {@code #XAnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean,FST,boolean,int,int,int,int,int) - * AnalyzingSuggester(indexAnalyzer, queryAnalyzer, EXACT_FIRST | - * PRESERVE_SEP, 256, -1)} - */ - public XAnalyzingSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer) { - this(indexAnalyzer, null, queryAnalyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, true, null, false, 0, SEP_LABEL, PAYLOAD_SEP, END_BYTE, HOLE_CHARACTER); - } - - /** - * Creates a new suggester. - * - * @param indexAnalyzer Analyzer that will be used for - * analyzing suggestions while building the index. - * @param queryAnalyzer Analyzer that will be used for - * analyzing query text during lookup - * @param options see {@link #EXACT_FIRST}, {@link #PRESERVE_SEP} - * @param maxSurfaceFormsPerAnalyzedForm Maximum number of - * surface forms to keep for a single analyzed form. - * When there are too many surface forms we discard the - * lowest weighted ones. - * @param maxGraphExpansions Maximum number of graph paths - * to expand from the analyzed form. Set this to -1 for - * no limit. - */ - public XAnalyzingSuggester(Analyzer indexAnalyzer, Automaton queryPrefix, Analyzer queryAnalyzer, int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions, - boolean preservePositionIncrements, FST> fst, boolean hasPayloads, int maxAnalyzedPathsForOneInput, - int sepLabel, int payloadSep, int endByte, int holeCharacter) { - // SIMON EDIT: I added fst, hasPayloads and maxAnalyzedPathsForOneInput - this.indexAnalyzer = indexAnalyzer; - this.queryAnalyzer = queryAnalyzer; - this.fst = fst; - this.hasPayloads = hasPayloads; - if ((options & ~(EXACT_FIRST | PRESERVE_SEP)) != 0) { - throw new IllegalArgumentException("options should only contain EXACT_FIRST and PRESERVE_SEP; got " + options); - } - this.exactFirst = (options & EXACT_FIRST) != 0; - this.preserveSep = (options & PRESERVE_SEP) != 0; - - // FLORIAN EDIT: I added queryPrefix for context dependent suggestions - this.queryPrefix = queryPrefix; - - // NOTE: this is just an implementation limitation; if - // somehow this is a problem we could fix it by using - // more than one byte to disambiguate ... but 256 seems - // like it should be way more then enough. - if (maxSurfaceFormsPerAnalyzedForm <= 0 || maxSurfaceFormsPerAnalyzedForm > 256) { - throw new IllegalArgumentException("maxSurfaceFormsPerAnalyzedForm must be > 0 and < 256 (got: " + maxSurfaceFormsPerAnalyzedForm + ")"); - } - this.maxSurfaceFormsPerAnalyzedForm = maxSurfaceFormsPerAnalyzedForm; - - if (maxGraphExpansions < 1 && maxGraphExpansions != -1) { - throw new IllegalArgumentException("maxGraphExpansions must -1 (no limit) or > 0 (got: " + maxGraphExpansions + ")"); - } - this.maxGraphExpansions = maxGraphExpansions; - this.maxAnalyzedPathsForOneInput = maxAnalyzedPathsForOneInput; - this.preservePositionIncrements = preservePositionIncrements; - this.sepLabel = sepLabel; - this.payloadSep = payloadSep; - this.endByte = endByte; - this.holeCharacter = holeCharacter; - } - - /** Returns byte size of the underlying FST. */ - @Override -public long ramBytesUsed() { - return fst == null ? 0 : fst.ramBytesUsed(); - } - - public int getMaxAnalyzedPathsForOneInput() { - return maxAnalyzedPathsForOneInput; - } - - // Replaces SEP with epsilon or remaps them if - // we were asked to preserve them: - private Automaton replaceSep(Automaton a) { - - Automaton result = new Automaton(); - - // Copy all states over - int numStates = a.getNumStates(); - for(int s=0;s visited = new HashSet<>(); - final LinkedList worklist = new LinkedList<>(); - worklist.add(0); - visited.add(0); - int upto = 0; - states[upto] = 0; - upto++; - Transition t = new Transition(); - while (worklist.size() > 0) { - int s = worklist.removeFirst(); - int count = a.initTransition(s, t); - for (int i=0;i { - - private final boolean hasPayloads; - - public AnalyzingComparator(boolean hasPayloads) { - this.hasPayloads = hasPayloads; - } - - private final ByteArrayDataInput readerA = new ByteArrayDataInput(); - private final ByteArrayDataInput readerB = new ByteArrayDataInput(); - private final BytesRef scratchA = new BytesRef(); - private final BytesRef scratchB = new BytesRef(); - - @Override - public int compare(BytesRef a, BytesRef b) { - - // First by analyzed form: - readerA.reset(a.bytes, a.offset, a.length); - scratchA.length = readerA.readShort(); - scratchA.bytes = a.bytes; - scratchA.offset = readerA.getPosition(); - - readerB.reset(b.bytes, b.offset, b.length); - scratchB.bytes = b.bytes; - scratchB.length = readerB.readShort(); - scratchB.offset = readerB.getPosition(); - - int cmp = scratchA.compareTo(scratchB); - if (cmp != 0) { - return cmp; - } - readerA.skipBytes(scratchA.length); - readerB.skipBytes(scratchB.length); - // Next by cost: - long aCost = readerA.readInt(); - long bCost = readerB.readInt(); - if (aCost < bCost) { - return -1; - } else if (aCost > bCost) { - return 1; - } - - // Finally by surface form: - if (hasPayloads) { - scratchA.length = readerA.readShort(); - scratchA.offset = readerA.getPosition(); - scratchB.length = readerB.readShort(); - scratchB.offset = readerB.getPosition(); - } else { - scratchA.offset = readerA.getPosition(); - scratchA.length = a.length - scratchA.offset; - scratchB.offset = readerB.getPosition(); - scratchB.length = b.length - scratchB.offset; - } - return scratchA.compareTo(scratchB); - } - } - - @Override - public void build(InputIterator iterator) throws IOException { - String prefix = getClass().getSimpleName(); - Path directory = OfflineSorter.defaultTempDir(); - Path tempInput = Files.createTempFile(directory, prefix, ".input"); - Path tempSorted = Files.createTempFile(directory, prefix, ".sorted"); - - hasPayloads = iterator.hasPayloads(); - - OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput); - OfflineSorter.ByteSequencesReader reader = null; - BytesRefBuilder scratch = new BytesRefBuilder(); - - TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton(); - - boolean success = false; - count = 0; - byte buffer[] = new byte[8]; - try { - ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); - - for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { - LimitedFiniteStringsIterator finiteStrings = - new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); - for (IntsRef string; (string = finiteStrings.next()) != null; count++) { - Util.toBytesRef(string, scratch); - - // length of the analyzed text (FST input) - if (scratch.length() > Short.MAX_VALUE-2) { - throw new IllegalArgumentException("cannot handle analyzed forms > " + (Short.MAX_VALUE-2) + " in length (got " + scratch.length() + ")"); - } - short analyzedLength = (short) scratch.length(); - - // compute the required length: - // analyzed sequence + weight (4) + surface + analyzedLength (short) - int requiredLength = analyzedLength + 4 + surfaceForm.length + 2; - - BytesRef payload; - - if (hasPayloads) { - if (surfaceForm.length > (Short.MAX_VALUE-2)) { - throw new IllegalArgumentException("cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); - } - payload = iterator.payload(); - // payload + surfaceLength (short) - requiredLength += payload.length + 2; - } else { - payload = null; - } - - buffer = ArrayUtil.grow(buffer, requiredLength); - - output.reset(buffer); - - output.writeShort(analyzedLength); - - output.writeBytes(scratch.bytes(), 0, scratch.length()); - - output.writeInt(encodeWeight(iterator.weight())); - - if (hasPayloads) { - for(int i=0;i outputs = new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()); - Builder> builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs); - - // Build FST: - BytesRefBuilder previousAnalyzed = null; - BytesRefBuilder analyzed = new BytesRefBuilder(); - BytesRef surface = new BytesRef(); - IntsRefBuilder scratchInts = new IntsRefBuilder(); - ByteArrayDataInput input = new ByteArrayDataInput(); - - // Used to remove duplicate surface forms (but we - // still index the hightest-weight one). We clear - // this when we see a new analyzed form, so it cannot - // grow unbounded (at most 256 entries): - Set seenSurfaceForms = new HashSet<>(); - - int dedup = 0; - while (reader.read(scratch)) { - input.reset(scratch.bytes(), 0, scratch.length()); - short analyzedLength = input.readShort(); - analyzed.grow(analyzedLength+2); - input.readBytes(analyzed.bytes(), 0, analyzedLength); - analyzed.setLength(analyzedLength); - - long cost = input.readInt(); - - surface.bytes = scratch.bytes(); - if (hasPayloads) { - surface.length = input.readShort(); - surface.offset = input.getPosition(); - } else { - surface.offset = input.getPosition(); - surface.length = scratch.length() - surface.offset; - } - - if (previousAnalyzed == null) { - previousAnalyzed = new BytesRefBuilder(); - previousAnalyzed.copyBytes(analyzed); - seenSurfaceForms.add(BytesRef.deepCopyOf(surface)); - } else if (analyzed.get().equals(previousAnalyzed.get())) { - dedup++; - if (dedup >= maxSurfaceFormsPerAnalyzedForm) { - // More than maxSurfaceFormsPerAnalyzedForm - // dups: skip the rest: - continue; - } - if (seenSurfaceForms.contains(surface)) { - continue; - } - seenSurfaceForms.add(BytesRef.deepCopyOf(surface)); - } else { - dedup = 0; - previousAnalyzed.copyBytes(analyzed); - seenSurfaceForms.clear(); - seenSurfaceForms.add(BytesRef.deepCopyOf(surface)); - } - - // TODO: I think we can avoid the extra 2 bytes when - // there is no dup (dedup==0), but we'd have to fix - // the exactFirst logic ... which would be sort of - // hairy because we'd need to special case the two - // (dup/not dup)... - - // NOTE: must be byte 0 so we sort before whatever - // is next - analyzed.append((byte) 0); - analyzed.append((byte) dedup); - - Util.toIntsRef(analyzed.get(), scratchInts); - //System.out.println("ADD: " + scratchInts + " -> " + cost + ": " + surface.utf8ToString()); - if (!hasPayloads) { - builder.add(scratchInts.get(), outputs.newPair(cost, BytesRef.deepCopyOf(surface))); - } else { - int payloadOffset = input.getPosition() + surface.length; - int payloadLength = scratch.length() - payloadOffset; - BytesRef br = new BytesRef(surface.length + 1 + payloadLength); - System.arraycopy(surface.bytes, surface.offset, br.bytes, 0, surface.length); - br.bytes[surface.length] = (byte) payloadSep; - System.arraycopy(scratch.bytes(), payloadOffset, br.bytes, surface.length+1, payloadLength); - br.length = br.bytes.length; - builder.add(scratchInts.get(), outputs.newPair(cost, br)); - } - } - fst = builder.finish(); - - //PrintWriter pw = new PrintWriter("/tmp/out.dot"); - //Util.toDot(fst, pw, true, true); - //pw.close(); - - success = true; - } finally { - IOUtils.closeWhileHandlingException(reader, writer); - - if (success) { - IOUtils.deleteFilesIfExist(tempInput, tempSorted); - } else { - IOUtils.deleteFilesIgnoringExceptions(tempInput, tempSorted); - } - } - } - - @Override - public boolean store(OutputStream output) throws IOException { - DataOutput dataOut = new OutputStreamDataOutput(output); - try { - if (fst == null) { - return false; - } - - fst.save(dataOut); - dataOut.writeVInt(maxAnalyzedPathsForOneInput); - dataOut.writeByte((byte) (hasPayloads ? 1 : 0)); - } finally { - IOUtils.close(output); - } - return true; - } - - @Override - public long getCount() { - return count; - } - - @Override - public boolean load(InputStream input) throws IOException { - DataInput dataIn = new InputStreamDataInput(input); - try { - this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton())); - maxAnalyzedPathsForOneInput = dataIn.readVInt(); - hasPayloads = dataIn.readByte() == 1; - } finally { - IOUtils.close(input); - } - return true; - } - - private LookupResult getLookupResult(Long output1, BytesRef output2, CharsRefBuilder spare) { - LookupResult result; - if (hasPayloads) { - int sepIndex = -1; - for(int i=0;i= output2.length) { - return false; - } - for(int i=0;i lookup(final CharSequence key, Set contexts, boolean onlyMorePopular, int num) { - assert num > 0; - - if (onlyMorePopular) { - throw new IllegalArgumentException("this suggester only works with onlyMorePopular=false"); - } - if (fst == null) { - return Collections.emptyList(); - } - - //System.out.println("lookup key=" + key + " num=" + num); - for (int i = 0; i < key.length(); i++) { - if (key.charAt(i) == holeCharacter) { - throw new IllegalArgumentException("lookup key cannot contain HOLE character U+001E; this character is reserved"); - } - if (key.charAt(i) == sepLabel) { - throw new IllegalArgumentException("lookup key cannot contain unit separator character U+001F; this character is reserved"); - } - } - final BytesRef utf8Key = new BytesRef(key); - try { - - Automaton lookupAutomaton = toLookupAutomaton(key); - - final CharsRefBuilder spare = new CharsRefBuilder(); - - //System.out.println(" now intersect exactFirst=" + exactFirst); - - // Intersect automaton w/ suggest wFST and get all - // prefix starting nodes & their outputs: - //final PathIntersector intersector = getPathIntersector(lookupAutomaton, fst); - - //System.out.println(" prefixPaths: " + prefixPaths.size()); - - BytesReader bytesReader = fst.getBytesReader(); - - FST.Arc> scratchArc = new FST.Arc<>(); - - final List results = new ArrayList<>(); - - List>> prefixPaths = FSTUtil.intersectPrefixPaths(convertAutomaton(lookupAutomaton), fst); - - if (exactFirst) { - - int count = 0; - for (FSTUtil.Path> path : prefixPaths) { - if (fst.findTargetArc(endByte, path.fstNode, scratchArc, bytesReader) != null) { - // This node has END_BYTE arc leaving, meaning it's an - // "exact" match: - count++; - } - } - - // Searcher just to find the single exact only - // match, if present: - Util.TopNSearcher> searcher; - searcher = new Util.TopNSearcher<>(fst, count * maxSurfaceFormsPerAnalyzedForm, count * maxSurfaceFormsPerAnalyzedForm, weightComparator); - - // NOTE: we could almost get away with only using - // the first start node. The only catch is if - // maxSurfaceFormsPerAnalyzedForm had kicked in and - // pruned our exact match from one of these nodes - // ...: - for (FSTUtil.Path> path : prefixPaths) { - if (fst.findTargetArc(endByte, path.fstNode, scratchArc, bytesReader) != null) { - // This node has END_BYTE arc leaving, meaning it's an - // "exact" match: - searcher.addStartPaths(scratchArc, fst.outputs.add(path.output, scratchArc.output), false, path.input); - } - } - - Util.TopResults> completions = searcher.search(); - - // NOTE: this is rather inefficient: we enumerate - // every matching "exactly the same analyzed form" - // path, and then do linear scan to see if one of - // these exactly matches the input. It should be - // possible (though hairy) to do something similar - // to getByOutput, since the surface form is encoded - // into the FST output, so we more efficiently hone - // in on the exact surface-form match. Still, I - // suspect very little time is spent in this linear - // seach: it's bounded by how many prefix start - // nodes we have and the - // maxSurfaceFormsPerAnalyzedForm: - for(Result> completion : completions) { - BytesRef output2 = completion.output.output2; - if (sameSurfaceForm(utf8Key, output2)) { - results.add(getLookupResult(completion.output.output1, output2, spare)); - break; - } - } - - if (results.size() == num) { - // That was quick: - return results; - } - } - - Util.TopNSearcher> searcher; - searcher = new Util.TopNSearcher>(fst, - num - results.size(), - num * maxAnalyzedPathsForOneInput, - weightComparator) { - private final Set seen = new HashSet<>(); - - @Override - protected boolean acceptResult(IntsRef input, Pair output) { - - // Dedup: when the input analyzes to a graph we - // can get duplicate surface forms: - if (seen.contains(output.output2)) { - return false; - } - seen.add(output.output2); - - if (!exactFirst) { - return true; - } else { - // In exactFirst mode, don't accept any paths - // matching the surface form since that will - // create duplicate results: - if (sameSurfaceForm(utf8Key, output.output2)) { - // We found exact match, which means we should - // have already found it in the first search: - assert results.size() == 1; - return false; - } else { - return true; - } - } - } - }; - - prefixPaths = getFullPrefixPaths(prefixPaths, lookupAutomaton, fst); - - for (FSTUtil.Path> path : prefixPaths) { - searcher.addStartPaths(path.fstNode, path.output, true, path.input); - } - - TopResults> completions = searcher.search(); - - for(Result> completion : completions) { - - LookupResult result = getLookupResult(completion.output.output1, completion.output.output2, spare); - - // TODO: for fuzzy case would be nice to return - // how many edits were required - - //System.out.println(" result=" + result); - results.add(result); - - if (results.size() == num) { - // In the exactFirst=true case the search may - // produce one extra path - break; - } - } - - return results; - } catch (IOException bogus) { - throw new RuntimeException(bogus); - } - } - - @Override - public boolean store(DataOutput output) throws IOException { - output.writeVLong(count); - if (fst == null) { - return false; - } - - fst.save(output); - output.writeVInt(maxAnalyzedPathsForOneInput); - output.writeByte((byte) (hasPayloads ? 1 : 0)); - return true; - } - - @Override - public boolean load(DataInput input) throws IOException { - count = input.readVLong(); - this.fst = new FST<>(input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton())); - maxAnalyzedPathsForOneInput = input.readVInt(); - hasPayloads = input.readByte() == 1; - return true; - } - - /** Returns all completion paths to initialize the search. */ - protected List>> getFullPrefixPaths(List>> prefixPaths, - Automaton lookupAutomaton, - FST> fst) - throws IOException { - return prefixPaths; - } - - final Automaton toAutomaton(final BytesRef surfaceForm, final TokenStreamToAutomaton ts2a) throws IOException { - try (TokenStream ts = indexAnalyzer.tokenStream("", surfaceForm.utf8ToString())) { - return toAutomaton(ts, ts2a); - } - } - - final Automaton toAutomaton(TokenStream ts, final TokenStreamToAutomaton ts2a) throws IOException { - // Create corresponding automaton: labels are bytes - // from each analyzed token, with byte 0 used as - // separator between tokens: - Automaton automaton = ts2a.toAutomaton(ts); - - automaton = replaceSep(automaton); - automaton = convertAutomaton(automaton); - - // TODO: LUCENE-5660 re-enable this once we disallow massive suggestion strings - // assert SpecialOperations.isFinite(automaton); - - // Get all paths from the automaton (there can be - // more than one path, eg if the analyzer created a - // graph using SynFilter or WDF): - - return automaton; - } - - // EDIT: Adrien, needed by lookup providers - // NOTE: these XForks are unmaintainable, we need to get rid of them... - public Set toFiniteStrings(TokenStream stream) throws IOException { - final TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton(); - Automaton automaton; - try (TokenStream ts = stream) { - automaton = toAutomaton(ts, ts2a); - } - LimitedFiniteStringsIterator finiteStrings = - new LimitedFiniteStringsIterator(automaton, maxGraphExpansions); - Set set = new HashSet<>(); - for (IntsRef string = finiteStrings.next(); string != null; string = finiteStrings.next()) { - set.add(IntsRef.deepCopyOf(string)); - } - return Collections.unmodifiableSet(set); - } - - final Automaton toLookupAutomaton(final CharSequence key) throws IOException { - // TODO: is there a Reader from a CharSequence? - // Turn tokenstream into automaton: - Automaton automaton = null; - - try (TokenStream ts = queryAnalyzer.tokenStream("", key.toString())) { - automaton = getTokenStreamToAutomaton().toAutomaton(ts); - } - - automaton = replaceSep(automaton); - - // TODO: we can optimize this somewhat by determinizing - // while we convert - - // This automaton should not blow up during determinize: - automaton = Operations.determinize(automaton, Integer.MAX_VALUE); - return automaton; - } - - - - /** - * Returns the weight associated with an input string, - * or null if it does not exist. - */ - public Object get(CharSequence key) { - throw new UnsupportedOperationException(); - } - - /** cost -> weight */ - public static int decodeWeight(long encoded) { - return (int)(Integer.MAX_VALUE - encoded); - } - - /** weight -> cost */ - public static int encodeWeight(long value) { - if (value < 0 || value > Integer.MAX_VALUE) { - throw new UnsupportedOperationException("cannot encode value: " + value); - } - return Integer.MAX_VALUE - (int)value; - } - - static final Comparator> weightComparator = new Comparator> () { - @Override - public int compare(Pair left, Pair right) { - return left.output1.compareTo(right.output1); - } - }; - - - public static class XBuilder { - private Builder> builder; - private int maxSurfaceFormsPerAnalyzedForm; - private IntsRefBuilder scratchInts = new IntsRefBuilder(); - private final PairOutputs outputs; - private boolean hasPayloads; - private BytesRefBuilder analyzed = new BytesRefBuilder(); - private final SurfaceFormAndPayload[] surfaceFormsAndPayload; - private int count; - private ObjectIntHashMap seenSurfaceForms = HppcMaps.Object.Integer.ensureNoNullKeys(256, 0.75f); - private int payloadSep; - - public XBuilder(int maxSurfaceFormsPerAnalyzedForm, boolean hasPayloads, int payloadSep) { - this.payloadSep = payloadSep; - this.outputs = new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()); - this.builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs); - this.maxSurfaceFormsPerAnalyzedForm = maxSurfaceFormsPerAnalyzedForm; - this.hasPayloads = hasPayloads; - surfaceFormsAndPayload = new SurfaceFormAndPayload[maxSurfaceFormsPerAnalyzedForm]; - - } - public void startTerm(BytesRef analyzed) { - this.analyzed.grow(analyzed.length+2); - this.analyzed.copyBytes(analyzed); - } - - private final static class SurfaceFormAndPayload implements Comparable { - BytesRef payload; - long weight; - - public SurfaceFormAndPayload(BytesRef payload, long cost) { - super(); - this.payload = payload; - this.weight = cost; - } - - @Override - public int compareTo(SurfaceFormAndPayload o) { - int res = compare(weight, o.weight); - if (res == 0 ){ - return payload.compareTo(o.payload); - } - return res; - } - public static int compare(long x, long y) { - return (x < y) ? -1 : ((x == y) ? 0 : 1); - } - } - - public void addSurface(BytesRef surface, BytesRef payload, long cost) throws IOException { - int surfaceIndex = -1; - long encodedWeight = cost == -1 ? cost : encodeWeight(cost); - /* - * we need to check if we have seen this surface form, if so only use the - * the surface form with the highest weight and drop the rest no matter if - * the payload differs. - */ - if (count >= maxSurfaceFormsPerAnalyzedForm) { - // More than maxSurfaceFormsPerAnalyzedForm - // dups: skip the rest: - return; - } - - BytesRef surfaceCopy; - final int keySlot; - if (count > 0 && (keySlot = seenSurfaceForms.indexOf(surface)) >= 0) { - surfaceIndex = seenSurfaceForms.indexGet(keySlot); - SurfaceFormAndPayload surfaceFormAndPayload = surfaceFormsAndPayload[surfaceIndex]; - if (encodedWeight >= surfaceFormAndPayload.weight) { - return; - } - surfaceCopy = BytesRef.deepCopyOf(surface); - } else { - surfaceIndex = count++; - surfaceCopy = BytesRef.deepCopyOf(surface); - seenSurfaceForms.put(surfaceCopy, surfaceIndex); - } - - BytesRef payloadRef; - if (!hasPayloads) { - payloadRef = surfaceCopy; - } else { - int len = surface.length + 1 + payload.length; - final BytesRef br = new BytesRef(len); - System.arraycopy(surface.bytes, surface.offset, br.bytes, 0, surface.length); - br.bytes[surface.length] = (byte) payloadSep; - System.arraycopy(payload.bytes, payload.offset, br.bytes, surface.length + 1, payload.length); - br.length = len; - payloadRef = br; - } - if (surfaceFormsAndPayload[surfaceIndex] == null) { - surfaceFormsAndPayload[surfaceIndex] = new SurfaceFormAndPayload(payloadRef, encodedWeight); - } else { - surfaceFormsAndPayload[surfaceIndex].payload = payloadRef; - surfaceFormsAndPayload[surfaceIndex].weight = encodedWeight; - } - } - - public void finishTerm(long defaultWeight) throws IOException { - ArrayUtil.timSort(surfaceFormsAndPayload, 0, count); - int deduplicator = 0; - analyzed.append((byte) 0); - analyzed.setLength(analyzed.length() + 1); - analyzed.grow(analyzed.length()); - for (int i = 0; i < count; i++) { - analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++); - Util.toIntsRef(analyzed.get(), scratchInts); - SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i]; - long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight; - builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload)); - } - seenSurfaceForms.clear(); - count = 0; - } - - public FST> build() throws IOException { - return builder.finish(); - } - - public boolean hasPayloads() { - return hasPayloads; - } - - public int maxSurfaceFormsPerAnalyzedForm() { - return maxSurfaceFormsPerAnalyzedForm; - } - - } -} diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XFuzzySuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XFuzzySuggester.java deleted file mode 100644 index a4338f8a65a..00000000000 --- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XFuzzySuggester.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lucene.search.suggest.analyzing; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStreamToAutomaton; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IntsRef; -import org.apache.lucene.util.UnicodeUtil; -import org.apache.lucene.util.automaton.*; -import org.apache.lucene.util.fst.FST; -import org.apache.lucene.util.fst.PairOutputs; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES; - -/** - * Implements a fuzzy {@link AnalyzingSuggester}. The similarity measurement is - * based on the Damerau-Levenshtein (optimal string alignment) algorithm, though - * you can explicitly choose classic Levenshtein by passing false - * for the transpositions parameter. - *

- * At most, this query will match terms up to - * {@value org.apache.lucene.util.automaton.LevenshteinAutomata#MAXIMUM_SUPPORTED_DISTANCE} - * edits. Higher distances are not supported. Note that the - * fuzzy distance is measured in "byte space" on the bytes - * returned by the {@link org.apache.lucene.analysis.TokenStream}'s {@link - * org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute}, usually UTF8. By default - * the analyzed bytes must be at least 3 {@link - * #DEFAULT_MIN_FUZZY_LENGTH} bytes before any edits are - * considered. Furthermore, the first 1 {@link - * #DEFAULT_NON_FUZZY_PREFIX} byte is not allowed to be - * edited. We allow up to 1 (@link - * #DEFAULT_MAX_EDITS} edit. - * If {@link #unicodeAware} parameter in the constructor is set to true, maxEdits, - * minFuzzyLength, transpositions and nonFuzzyPrefix are measured in Unicode code - * points (actual letters) instead of bytes.* - * - *

- * NOTE: This suggester does not boost suggestions that - * required no edits over suggestions that did require - * edits. This is a known limitation. - * - *

- * Note: complex query analyzers can have a significant impact on the lookup - * performance. It's recommended to not use analyzers that drop or inject terms - * like synonyms to keep the complexity of the prefix intersection low for good - * lookup performance. At index time, complex analyzers can safely be used. - *

- * - * @lucene.experimental - */ -public final class XFuzzySuggester extends XAnalyzingSuggester { - private final int maxEdits; - private final boolean transpositions; - private final int nonFuzzyPrefix; - private final int minFuzzyLength; - private final boolean unicodeAware; - - /** - * Measure maxEdits, minFuzzyLength, transpositions and nonFuzzyPrefix - * parameters in Unicode code points (actual letters) - * instead of bytes. - */ - public static final boolean DEFAULT_UNICODE_AWARE = false; - - /** - * The default minimum length of the key passed to {@link - * #lookup} before any edits are allowed. - */ - public static final int DEFAULT_MIN_FUZZY_LENGTH = 3; - - /** - * The default prefix length where edits are not allowed. - */ - public static final int DEFAULT_NON_FUZZY_PREFIX = 1; - - /** - * The default maximum number of edits for fuzzy - * suggestions. - */ - public static final int DEFAULT_MAX_EDITS = 1; - - /** - * The default transposition value passed to {@link org.apache.lucene.util.automaton.LevenshteinAutomata} - */ - public static final boolean DEFAULT_TRANSPOSITIONS = true; - - /** - * Creates a {@link FuzzySuggester} instance initialized with default values. - * - * @param analyzer the analyzer used for this suggester - */ - public XFuzzySuggester(Analyzer analyzer) { - this(analyzer, analyzer); - } - - /** - * Creates a {@link FuzzySuggester} instance with an index & a query analyzer initialized with default values. - * - * @param indexAnalyzer - * Analyzer that will be used for analyzing suggestions while building the index. - * @param queryAnalyzer - * Analyzer that will be used for analyzing query text during lookup - */ - public XFuzzySuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer) { - this(indexAnalyzer, null, queryAnalyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, DEFAULT_MAX_EDITS, DEFAULT_TRANSPOSITIONS, - DEFAULT_NON_FUZZY_PREFIX, DEFAULT_MIN_FUZZY_LENGTH, DEFAULT_UNICODE_AWARE, null, false, 0, SEP_LABEL, PAYLOAD_SEP, END_BYTE, HOLE_CHARACTER); - - } - - /** - * Creates a {@link FuzzySuggester} instance. - * - * @param indexAnalyzer Analyzer that will be used for - * analyzing suggestions while building the index. - * @param queryAnalyzer Analyzer that will be used for - * analyzing query text during lookup - * @param options see {@link #EXACT_FIRST}, {@link #PRESERVE_SEP} - * @param maxSurfaceFormsPerAnalyzedForm Maximum number of - * surface forms to keep for a single analyzed form. - * When there are too many surface forms we discard the - * lowest weighted ones. - * @param maxGraphExpansions Maximum number of graph paths - * to expand from the analyzed form. Set this to -1 for - * no limit. - * @param maxEdits must be >= 0 and <= {@link org.apache.lucene.util.automaton.LevenshteinAutomata#MAXIMUM_SUPPORTED_DISTANCE} . - * @param transpositions true if transpositions should be treated as a primitive - * edit operation. If this is false, comparisons will implement the classic - * Levenshtein algorithm. - * @param nonFuzzyPrefix length of common (non-fuzzy) prefix (see default {@link #DEFAULT_NON_FUZZY_PREFIX} - * @param minFuzzyLength minimum length of lookup key before any edits are allowed (see default {@link #DEFAULT_MIN_FUZZY_LENGTH}) - * @param sepLabel separation label - * @param payloadSep payload separator byte - * @param endByte end byte marker byte - */ - public XFuzzySuggester(Analyzer indexAnalyzer, Automaton queryPrefix, Analyzer queryAnalyzer, int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions, - int maxEdits, boolean transpositions, int nonFuzzyPrefix, int minFuzzyLength, boolean unicodeAware, - FST> fst, boolean hasPayloads, int maxAnalyzedPathsForOneInput, - int sepLabel, int payloadSep, int endByte, int holeCharacter) { - super(indexAnalyzer, queryPrefix, queryAnalyzer, options, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, true, fst, hasPayloads, maxAnalyzedPathsForOneInput, sepLabel, payloadSep, endByte, holeCharacter); - if (maxEdits < 0 || maxEdits > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) { - throw new IllegalArgumentException("maxEdits must be between 0 and " + LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE); - } - if (nonFuzzyPrefix < 0) { - throw new IllegalArgumentException("nonFuzzyPrefix must not be >= 0 (got " + nonFuzzyPrefix + ")"); - } - if (minFuzzyLength < 0) { - throw new IllegalArgumentException("minFuzzyLength must not be >= 0 (got " + minFuzzyLength + ")"); - } - - this.maxEdits = maxEdits; - this.transpositions = transpositions; - this.nonFuzzyPrefix = nonFuzzyPrefix; - this.minFuzzyLength = minFuzzyLength; - this.unicodeAware = unicodeAware; - } - - @Override - protected List>> getFullPrefixPaths(List>> prefixPaths, - Automaton lookupAutomaton, - FST> fst) - throws IOException { - - // TODO: right now there's no penalty for fuzzy/edits, - // ie a completion whose prefix matched exactly what the - // user typed gets no boost over completions that - // required an edit, which get no boost over completions - // requiring two edits. I suspect a multiplicative - // factor is appropriate (eg, say a fuzzy match must be at - // least 2X better weight than the non-fuzzy match to - // "compete") ... in which case I think the wFST needs - // to be log weights or something ... - - Automaton levA = convertAutomaton(toLevenshteinAutomata(lookupAutomaton)); - /* - Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8"); - w.write(levA.toDot()); - w.close(); - System.out.println("Wrote LevA to out.dot"); - */ - return FSTUtil.intersectPrefixPaths(levA, fst); - } - - @Override - protected Automaton convertAutomaton(Automaton a) { - if (unicodeAware) { - // FLORIAN EDIT: get converted Automaton from superclass - Automaton utf8automaton = new UTF32ToUTF8().convert(super.convertAutomaton(a)); - // This automaton should not blow up during determinize: - utf8automaton = Operations.determinize(utf8automaton, Integer.MAX_VALUE); - return utf8automaton; - } else { - return super.convertAutomaton(a); - } - } - - @Override - public TokenStreamToAutomaton getTokenStreamToAutomaton() { - final TokenStreamToAutomaton tsta = super.getTokenStreamToAutomaton(); - tsta.setUnicodeArcs(unicodeAware); - return tsta; - } - - Automaton toLevenshteinAutomata(Automaton automaton) { - List subs = new ArrayList<>(); - FiniteStringsIterator finiteStrings = new FiniteStringsIterator(automaton); - for (IntsRef string; (string = finiteStrings.next()) != null;) { - if (string.length <= nonFuzzyPrefix || string.length < minFuzzyLength) { - subs.add(Automata.makeString(string.ints, string.offset, string.length)); - } else { - int ints[] = new int[string.length-nonFuzzyPrefix]; - System.arraycopy(string.ints, string.offset+nonFuzzyPrefix, ints, 0, ints.length); - // TODO: maybe add alphaMin to LevenshteinAutomata, - // and pass 1 instead of 0? We probably don't want - // to allow the trailing dedup bytes to be - // edited... but then 0 byte is "in general" allowed - // on input (but not in UTF8). - LevenshteinAutomata lev = new LevenshteinAutomata(ints, unicodeAware ? Character.MAX_CODE_POINT : 255, transpositions); - subs.add(lev.toAutomaton(maxEdits, UnicodeUtil.newString(string.ints, string.offset, nonFuzzyPrefix))); - } - } - - if (subs.isEmpty()) { - // automaton is empty, there is no accepted paths through it - return Automata.makeEmpty(); // matches nothing - } else if (subs.size() == 1) { - // no synonyms or anything: just a single path through the tokenstream - return subs.get(0); - } else { - // multiple paths: this is really scary! is it slow? - // maybe we should not do this and throw UOE? - Automaton a = Operations.union(subs); - // TODO: we could call toLevenshteinAutomata() before det? - // this only happens if you have multiple paths anyway (e.g. synonyms) - return Operations.determinize(a, DEFAULT_MAX_DETERMINIZED_STATES); - } - } -} diff --git a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java index e1a8d81b7a0..95657de5158 100644 --- a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java +++ b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java @@ -22,14 +22,7 @@ package org.apache.lucene.search.vectorhighlight; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.BlendedTermQuery; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; -import org.apache.lucene.search.MultiPhraseQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryWrapperFilter; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.*; import org.apache.lucene.search.spans.SpanTermQuery; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; @@ -65,9 +58,6 @@ public class CustomFieldQuery extends FieldQuery { flatten(((ConstantScoreQuery) sourceQuery).getQuery(), reader, flatQueries, boost); } else if (sourceQuery instanceof FunctionScoreQuery) { flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries, boost); - } else if (sourceQuery instanceof FilteredQuery) { - flatten(((FilteredQuery) sourceQuery).getQuery(), reader, flatQueries, boost); - flatten(((FilteredQuery) sourceQuery).getFilter(), reader, flatQueries); } else if (sourceQuery instanceof MultiPhrasePrefixQuery) { flatten(sourceQuery.rewrite(reader), reader, flatQueries, boost); } else if (sourceQuery instanceof FiltersFunctionScoreQuery) { @@ -109,8 +99,7 @@ public class CustomFieldQuery extends FieldQuery { for (int i = 0; i < termsIdx.length; i++) { queryBuilder.add(terms.get(i)[termsIdx[i]], pos[i]); } - PhraseQuery query = queryBuilder.build(); - query.setBoost(orig.getBoost()); + Query query = queryBuilder.build(); this.flatten(query, reader, flatQueries, orig.getBoost()); } else { Term[] t = terms.get(currentPos); @@ -120,14 +109,4 @@ public class CustomFieldQuery extends FieldQuery { } } } - - void flatten(Filter sourceFilter, IndexReader reader, Collection flatQueries) throws IOException { - Boolean highlight = highlightFilters.get(); - if (highlight == null || highlight.equals(Boolean.FALSE)) { - return; - } - if (sourceFilter instanceof QueryWrapperFilter) { - flatten(((QueryWrapperFilter) sourceFilter).getQuery(), reader, flatQueries, 1.0F); - } - } } diff --git a/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java b/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java index e50c92065e2..ea504f7688c 100644 --- a/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java +++ b/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java @@ -36,7 +36,7 @@ public class StoreRateLimiting { void onPause(long nanos); } - public static enum Type { + public enum Type { NONE, MERGE, ALL; diff --git a/core/src/main/java/org/apache/lucene/util/XGeoHashUtils.java b/core/src/main/java/org/apache/lucene/util/XGeoHashUtils.java deleted file mode 100644 index 2b9841ea6a7..00000000000 --- a/core/src/main/java/org/apache/lucene/util/XGeoHashUtils.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lucene.util; - -import java.util.ArrayList; -import java.util.Collection; - -/** - * Utilities for converting to/from the GeoHash standard - * - * The geohash long format is represented as lon/lat (x/y) interleaved with the 4 least significant bits - * representing the level (1-12) [xyxy...xyxyllll] - * - * This differs from a morton encoded value which interleaves lat/lon (y/x). - * - * @lucene.experimental - */ -public class XGeoHashUtils { - public static final char[] BASE_32 = {'0', '1', '2', '3', '4', '5', '6', - '7', '8', '9', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; - - public static final String BASE_32_STRING = new String(BASE_32); - - public static final int PRECISION = 12; - private static final short MORTON_OFFSET = (XGeoUtils.BITS<<1) - (PRECISION*5); - - /** - * Encode lon/lat to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) - */ - public static final long longEncode(final double lon, final double lat, final int level) { - // shift to appropriate level - final short msf = (short)(((12 - level) * 5) + MORTON_OFFSET); - return ((BitUtil.flipFlop(XGeoUtils.mortonHash(lon, lat)) >>> msf) << 4) | level; - } - - /** - * Encode from geohash string to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) - */ - public static final long longEncode(final String hash) { - int level = hash.length()-1; - long b; - long l = 0L; - for(char c : hash.toCharArray()) { - b = (long)(BASE_32_STRING.indexOf(c)); - l |= (b<<(level--*5)); - } - return (l<<4)|hash.length(); - } - - /** - * Encode an existing geohash long to the provided precision - */ - public static long longEncode(long geohash, int level) { - final short precision = (short)(geohash & 15); - if (precision == level) { - return geohash; - } else if (precision > level) { - return ((geohash >>> (((precision - level) * 5) + 4)) << 4) | level; - } - return ((geohash >>> 4) << (((level - precision) * 5) + 4) | level); - } - - /** - * Encode to a geohash string from the geohash based long format - */ - public static final String stringEncode(long geoHashLong) { - int level = (int)geoHashLong&15; - geoHashLong >>>= 4; - char[] chars = new char[level]; - do { - chars[--level] = BASE_32[(int)(geoHashLong&31L)]; - geoHashLong>>>=5; - } while(level > 0); - - return new String(chars); - } - - /** - * Encode to a geohash string from full resolution longitude, latitude) - */ - public static final String stringEncode(final double lon, final double lat) { - return stringEncode(lon, lat, 12); - } - - /** - * Encode to a level specific geohash string from full resolution longitude, latitude - */ - public static final String stringEncode(final double lon, final double lat, final int level) { - // bit twiddle to geohash (since geohash is a swapped (lon/lat) encoding) - final long hashedVal = BitUtil.flipFlop(XGeoUtils.mortonHash(lon, lat)); - - StringBuilder geoHash = new StringBuilder(); - short precision = 0; - final short msf = (XGeoUtils.BITS<<1)-5; - long mask = 31L<>>(msf-(precision*5)))]); - // next 5 bits - mask >>>= 5; - } while (++precision < level); - return geoHash.toString(); - } - - /** - * Encode to a full precision geohash string from a given morton encoded long value - */ - public static final String stringEncodeFromMortonLong(final long hashedVal) throws Exception { - return stringEncode(hashedVal, PRECISION); - } - - /** - * Encode to a geohash string at a given level from a morton long - */ - public static final String stringEncodeFromMortonLong(long hashedVal, final int level) { - // bit twiddle to geohash (since geohash is a swapped (lon/lat) encoding) - hashedVal = BitUtil.flipFlop(hashedVal); - - StringBuilder geoHash = new StringBuilder(); - short precision = 0; - final short msf = (XGeoUtils.BITS<<1)-5; - long mask = 31L<>>(msf-(precision*5)))]); - // next 5 bits - mask >>>= 5; - } while (++precision < level); - return geoHash.toString(); - } - - /** - * Encode to a morton long value from a given geohash string - */ - public static final long mortonEncode(final String hash) { - int level = 11; - long b; - long l = 0L; - for(char c : hash.toCharArray()) { - b = (long)(BASE_32_STRING.indexOf(c)); - l |= (b<<((level--*5) + MORTON_OFFSET)); - } - return BitUtil.flipFlop(l); - } - - /** - * Encode to a morton long value from a given geohash long value - */ - public static final long mortonEncode(final long geoHashLong) { - final int level = (int)(geoHashLong&15); - final short odd = (short)(level & 1); - - return BitUtil.flipFlop((geoHashLong >>> 4) << odd) << (((12 - level) * 5) + (MORTON_OFFSET - odd)); - } - - private static final char encode(int x, int y) { - return BASE_32[((x & 1) + ((y & 1) * 2) + ((x & 2) * 2) + ((y & 2) * 4) + ((x & 4) * 4)) % 32]; - } - - /** - * Calculate all neighbors of a given geohash cell. - * - * @param geohash Geohash of the defined cell - * @return geohashes of all neighbor cells - */ - public static Collection neighbors(String geohash) { - return addNeighbors(geohash, geohash.length(), new ArrayList(8)); - } - - /** - * Calculate the geohash of a neighbor of a geohash - * - * @param geohash the geohash of a cell - * @param level level of the geohash - * @param dx delta of the first grid coordinate (must be -1, 0 or +1) - * @param dy delta of the second grid coordinate (must be -1, 0 or +1) - * @return geohash of the defined cell - */ - private final static String neighbor(String geohash, int level, int dx, int dy) { - int cell = BASE_32_STRING.indexOf(geohash.charAt(level -1)); - - // Decoding the Geohash bit pattern to determine grid coordinates - int x0 = cell & 1; // first bit of x - int y0 = cell & 2; // first bit of y - int x1 = cell & 4; // second bit of x - int y1 = cell & 8; // second bit of y - int x2 = cell & 16; // third bit of x - - // combine the bitpattern to grid coordinates. - // note that the semantics of x and y are swapping - // on each level - int x = x0 + (x1 / 2) + (x2 / 4); - int y = (y0 / 2) + (y1 / 4); - - if (level == 1) { - // Root cells at north (namely "bcfguvyz") or at - // south (namely "0145hjnp") do not have neighbors - // in north/south direction - if ((dy < 0 && y == 0) || (dy > 0 && y == 3)) { - return null; - } else { - return Character.toString(encode(x + dx, y + dy)); - } - } else { - // define grid coordinates for next level - final int nx = ((level % 2) == 1) ? (x + dx) : (x + dy); - final int ny = ((level % 2) == 1) ? (y + dy) : (y + dx); - - // if the defined neighbor has the same parent a the current cell - // encode the cell directly. Otherwise find the cell next to this - // cell recursively. Since encoding wraps around within a cell - // it can be encoded here. - // xLimit and YLimit must always be respectively 7 and 3 - // since x and y semantics are swapping on each level. - if (nx >= 0 && nx <= 7 && ny >= 0 && ny <= 3) { - return geohash.substring(0, level - 1) + encode(nx, ny); - } else { - String neighbor = neighbor(geohash, level - 1, dx, dy); - return (neighbor != null) ? neighbor + encode(nx, ny) : neighbor; - } - } - } - - /** - * Add all geohashes of the cells next to a given geohash to a list. - * - * @param geohash Geohash of a specified cell - * @param neighbors list to add the neighbors to - * @return the given list - */ - public static final > E addNeighbors(String geohash, E neighbors) { - return addNeighbors(geohash, geohash.length(), neighbors); - } - - /** - * Add all geohashes of the cells next to a given geohash to a list. - * - * @param geohash Geohash of a specified cell - * @param length level of the given geohash - * @param neighbors list to add the neighbors to - * @return the given list - */ - public static final > E addNeighbors(String geohash, int length, E neighbors) { - String south = neighbor(geohash, length, 0, -1); - String north = neighbor(geohash, length, 0, +1); - if (north != null) { - neighbors.add(neighbor(north, length, -1, 0)); - neighbors.add(north); - neighbors.add(neighbor(north, length, +1, 0)); - } - - neighbors.add(neighbor(geohash, length, -1, 0)); - neighbors.add(neighbor(geohash, length, +1, 0)); - - if (south != null) { - neighbors.add(neighbor(south, length, -1, 0)); - neighbors.add(south); - neighbors.add(neighbor(south, length, +1, 0)); - } - - return neighbors; - } -} \ No newline at end of file diff --git a/core/src/main/java/org/apache/lucene/util/XGeoProjectionUtils.java b/core/src/main/java/org/apache/lucene/util/XGeoProjectionUtils.java deleted file mode 100644 index 5d13c2fef9e..00000000000 --- a/core/src/main/java/org/apache/lucene/util/XGeoProjectionUtils.java +++ /dev/null @@ -1,383 +0,0 @@ -package org.apache.lucene.util; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Reusable geo-spatial projection utility methods. - * - * @lucene.experimental - */ -public class XGeoProjectionUtils { - // WGS84 earth-ellipsoid major (a) minor (b) radius, (f) flattening and eccentricity (e) - static final double SEMIMAJOR_AXIS = 6_378_137; // [m] - static final double FLATTENING = 1.0/298.257223563; - static final double SEMIMINOR_AXIS = SEMIMAJOR_AXIS * (1.0 - FLATTENING); //6_356_752.31420; // [m] - static final double ECCENTRICITY = StrictMath.sqrt((2.0 - FLATTENING) * FLATTENING); - static final double PI_OVER_2 = StrictMath.PI / 2.0D; - static final double SEMIMAJOR_AXIS2 = SEMIMAJOR_AXIS * SEMIMAJOR_AXIS; - static final double SEMIMINOR_AXIS2 = SEMIMINOR_AXIS * SEMIMINOR_AXIS; - - /** - * Converts from geocentric earth-centered earth-fixed to geodesic lat/lon/alt - * @param x Cartesian x coordinate - * @param y Cartesian y coordinate - * @param z Cartesian z coordinate - * @param lla 0: longitude 1: latitude: 2: altitude - * @return double array as 0: longitude 1: latitude 2: altitude - */ - public static final double[] ecfToLLA(final double x, final double y, final double z, double[] lla) { - boolean atPole = false; - final double ad_c = 1.0026000D; - final double e2 = (SEMIMAJOR_AXIS2 - SEMIMINOR_AXIS2)/(SEMIMAJOR_AXIS2); - final double ep2 = (SEMIMAJOR_AXIS2 - SEMIMINOR_AXIS2)/(SEMIMINOR_AXIS2); - final double cos67P5 = 0.38268343236508977D; - - if (lla == null) { - lla = new double[3]; - } - - if (x != 0.0) { - lla[0] = StrictMath.atan2(y,x); - } else { - if (y > 0) { - lla[0] = PI_OVER_2; - } else if (y < 0) { - lla[0] = -PI_OVER_2; - } else { - atPole = true; - lla[0] = 0.0D; - if (z > 0.0) { - lla[1] = PI_OVER_2; - } else if (z < 0.0) { - lla[1] = -PI_OVER_2; - } else { - lla[1] = PI_OVER_2; - lla[2] = -SEMIMINOR_AXIS; - return lla; - } - } - } - - final double w2 = x*x + y*y; - final double w = StrictMath.sqrt(w2); - final double t0 = z * ad_c; - final double s0 = StrictMath.sqrt(t0 * t0 + w2); - final double sinB0 = t0 / s0; - final double cosB0 = w / s0; - final double sin3B0 = sinB0 * sinB0 * sinB0; - final double t1 = z + SEMIMINOR_AXIS * ep2 * sin3B0; - final double sum = w - SEMIMAJOR_AXIS * e2 * cosB0 * cosB0 * cosB0; - final double s1 = StrictMath.sqrt(t1 * t1 + sum * sum); - final double sinP1 = t1 / s1; - final double cosP1 = sum / s1; - final double rn = SEMIMAJOR_AXIS / StrictMath.sqrt(1.0D - e2 * sinP1 * sinP1); - - if (cosP1 >= cos67P5) { - lla[2] = w / cosP1 - rn; - } else if (cosP1 <= -cos67P5) { - lla[2] = w / -cosP1 - rn; - } else { - lla[2] = z / sinP1 + rn * (e2 - 1.0); - } - if (!atPole) { - lla[1] = StrictMath.atan(sinP1/cosP1); - } - lla[0] = StrictMath.toDegrees(lla[0]); - lla[1] = StrictMath.toDegrees(lla[1]); - - return lla; - } - - /** - * Converts from geodesic lon lat alt to geocentric earth-centered earth-fixed - * @param lon geodesic longitude - * @param lat geodesic latitude - * @param alt geodesic altitude - * @param ecf reusable earth-centered earth-fixed result - * @return either a new ecef array or the reusable ecf parameter - */ - public static final double[] llaToECF(double lon, double lat, double alt, double[] ecf) { - lon = StrictMath.toRadians(lon); - lat = StrictMath.toRadians(lat); - - final double sl = StrictMath.sin(lat); - final double s2 = sl*sl; - final double cl = StrictMath.cos(lat); - final double ge2 = (SEMIMAJOR_AXIS2 - SEMIMINOR_AXIS2)/(SEMIMAJOR_AXIS2); - - if (ecf == null) { - ecf = new double[3]; - } - - if (lat < -PI_OVER_2 && lat > -1.001D * PI_OVER_2) { - lat = -PI_OVER_2; - } else if (lat > PI_OVER_2 && lat < 1.001D * PI_OVER_2) { - lat = PI_OVER_2; - } - assert (lat >= -PI_OVER_2) || (lat <= PI_OVER_2); - - if (lon > StrictMath.PI) { - lon -= (2*StrictMath.PI); - } - - final double rn = SEMIMAJOR_AXIS / StrictMath.sqrt(1.0D - ge2 * s2); - ecf[0] = (rn+alt) * cl * StrictMath.cos(lon); - ecf[1] = (rn+alt) * cl * StrictMath.sin(lon); - ecf[2] = ((rn*(1.0-ge2))+alt)*sl; - - return ecf; - } - - /** - * Converts from lat lon alt (in degrees) to East North Up right-hand coordinate system - * @param lon longitude in degrees - * @param lat latitude in degrees - * @param alt altitude in meters - * @param centerLon reference point longitude in degrees - * @param centerLat reference point latitude in degrees - * @param centerAlt reference point altitude in meters - * @param enu result east, north, up coordinate - * @return east, north, up coordinate - */ - public static double[] llaToENU(final double lon, final double lat, final double alt, double centerLon, - double centerLat, final double centerAlt, double[] enu) { - if (enu == null) { - enu = new double[3]; - } - - // convert point to ecf coordinates - final double[] ecf = llaToECF(lon, lat, alt, null); - - // convert from ecf to enu - return ecfToENU(ecf[0], ecf[1], ecf[2], centerLon, centerLat, centerAlt, enu); - } - - /** - * Converts from East North Up right-hand rule to lat lon alt in degrees - * @param x easting (in meters) - * @param y northing (in meters) - * @param z up (in meters) - * @param centerLon reference point longitude (in degrees) - * @param centerLat reference point latitude (in degrees) - * @param centerAlt reference point altitude (in meters) - * @param lla resulting lat, lon, alt point (in degrees) - * @return lat, lon, alt point (in degrees) - */ - public static double[] enuToLLA(final double x, final double y, final double z, final double centerLon, - final double centerLat, final double centerAlt, double[] lla) { - // convert enuToECF - if (lla == null) { - lla = new double[3]; - } - - // convert enuToECF, storing intermediate result in lla - lla = enuToECF(x, y, z, centerLon, centerLat, centerAlt, lla); - - // convert ecf to LLA - return ecfToLLA(lla[0], lla[1], lla[2], lla); - } - - /** - * Convert from Earth-Centered-Fixed to Easting, Northing, Up Right Hand System - * @param x ECF X coordinate (in meters) - * @param y ECF Y coordinate (in meters) - * @param z ECF Z coordinate (in meters) - * @param centerLon ENU origin longitude (in degrees) - * @param centerLat ENU origin latitude (in degrees) - * @param centerAlt ENU altitude (in meters) - * @param enu reusable enu result - * @return Easting, Northing, Up coordinate - */ - public static double[] ecfToENU(double x, double y, double z, final double centerLon, - final double centerLat, final double centerAlt, double[] enu) { - if (enu == null) { - enu = new double[3]; - } - - // create rotation matrix and rotate to enu orientation - final double[][] phi = createPhiTransform(centerLon, centerLat, null); - - // convert origin to ENU - final double[] originECF = llaToECF(centerLon, centerLat, centerAlt, null); - final double[] originENU = new double[3]; - originENU[0] = ((phi[0][0] * originECF[0]) + (phi[0][1] * originECF[1]) + (phi[0][2] * originECF[2])); - originENU[1] = ((phi[1][0] * originECF[0]) + (phi[1][1] * originECF[1]) + (phi[1][2] * originECF[2])); - originENU[2] = ((phi[2][0] * originECF[0]) + (phi[2][1] * originECF[1]) + (phi[2][2] * originECF[2])); - - // rotate then translate - enu[0] = ((phi[0][0] * x) + (phi[0][1] * y) + (phi[0][2] * z)) - originENU[0]; - enu[1] = ((phi[1][0] * x) + (phi[1][1] * y) + (phi[1][2] * z)) - originENU[1]; - enu[2] = ((phi[2][0] * x) + (phi[2][1] * y) + (phi[2][2] * z)) - originENU[2]; - - return enu; - } - - /** - * Convert from Easting, Northing, Up Right-Handed system to Earth Centered Fixed system - * @param x ENU x coordinate (in meters) - * @param y ENU y coordinate (in meters) - * @param z ENU z coordinate (in meters) - * @param centerLon ENU origin longitude (in degrees) - * @param centerLat ENU origin latitude (in degrees) - * @param centerAlt ENU origin altitude (in meters) - * @param ecf reusable ecf result - * @return ecf result coordinate - */ - public static double[] enuToECF(final double x, final double y, final double z, double centerLon, - double centerLat, final double centerAlt, double[] ecf) { - if (ecf == null) { - ecf = new double[3]; - } - - double[][] phi = createTransposedPhiTransform(centerLon, centerLat, null); - double[] ecfOrigin = llaToECF(centerLon, centerLat, centerAlt, null); - - // rotate and translate - ecf[0] = (phi[0][0]*x + phi[0][1]*y + phi[0][2]*z) + ecfOrigin[0]; - ecf[1] = (phi[1][0]*x + phi[1][1]*y + phi[1][2]*z) + ecfOrigin[1]; - ecf[2] = (phi[2][0]*x + phi[2][1]*y + phi[2][2]*z) + ecfOrigin[2]; - - return ecf; - } - - /** - * Create the rotation matrix for converting Earth Centered Fixed to Easting Northing Up - * @param originLon ENU origin longitude (in degrees) - * @param originLat ENU origin latitude (in degrees) - * @param phiMatrix reusable phi matrix result - * @return phi rotation matrix - */ - private static double[][] createPhiTransform(double originLon, double originLat, double[][] phiMatrix) { - - if (phiMatrix == null) { - phiMatrix = new double[3][3]; - } - - originLon = StrictMath.toRadians(originLon); - originLat = StrictMath.toRadians(originLat); - - final double sLon = StrictMath.sin(originLon); - final double cLon = StrictMath.cos(originLon); - final double sLat = StrictMath.sin(originLat); - final double cLat = StrictMath.cos(originLat); - - phiMatrix[0][0] = -sLon; - phiMatrix[0][1] = cLon; - phiMatrix[0][2] = 0.0D; - phiMatrix[1][0] = -sLat * cLon; - phiMatrix[1][1] = -sLat * sLon; - phiMatrix[1][2] = cLat; - phiMatrix[2][0] = cLat * cLon; - phiMatrix[2][1] = cLat * sLon; - phiMatrix[2][2] = sLat; - - return phiMatrix; - } - - /** - * Create the transposed rotation matrix for converting Easting Northing Up coordinates to Earth Centered Fixed - * @param originLon ENU origin longitude (in degrees) - * @param originLat ENU origin latitude (in degrees) - * @param phiMatrix reusable phi rotation matrix result - * @return transposed phi rotation matrix - */ - private static double[][] createTransposedPhiTransform(double originLon, double originLat, double[][] phiMatrix) { - - if (phiMatrix == null) { - phiMatrix = new double[3][3]; - } - - originLon = StrictMath.toRadians(originLon); - originLat = StrictMath.toRadians(originLat); - - final double sLat = StrictMath.sin(originLat); - final double cLat = StrictMath.cos(originLat); - final double sLon = StrictMath.sin(originLon); - final double cLon = StrictMath.cos(originLon); - - phiMatrix[0][0] = -sLon; - phiMatrix[1][0] = cLon; - phiMatrix[2][0] = 0.0D; - phiMatrix[0][1] = -sLat * cLon; - phiMatrix[1][1] = -sLat * sLon; - phiMatrix[2][1] = cLat; - phiMatrix[0][2] = cLat * cLon; - phiMatrix[1][2] = cLat * sLon; - phiMatrix[2][2] = sLat; - - return phiMatrix; - } - - /** - * Finds a point along a bearing from a given lon,lat geolocation using vincenty's distance formula - * - * @param lon origin longitude in degrees - * @param lat origin latitude in degrees - * @param bearing azimuthal bearing in degrees - * @param dist distance in meters - * @param pt resulting point - * @return the point along a bearing at a given distance in meters - */ - public static final double[] pointFromLonLatBearing(double lon, double lat, double bearing, double dist, double[] pt) { - - if (pt == null) { - pt = new double[2]; - } - - final double alpha1 = StrictMath.toRadians(bearing); - final double cosA1 = StrictMath.cos(alpha1); - final double sinA1 = StrictMath.sin(alpha1); - final double tanU1 = (1-FLATTENING) * StrictMath.tan(StrictMath.toRadians(lat)); - final double cosU1 = 1 / StrictMath.sqrt((1+tanU1*tanU1)); - final double sinU1 = tanU1*cosU1; - final double sig1 = StrictMath.atan2(tanU1, cosA1); - final double sinAlpha = cosU1 * sinA1; - final double cosSqAlpha = 1 - sinAlpha*sinAlpha; - final double uSq = cosSqAlpha * (SEMIMAJOR_AXIS2 - SEMIMINOR_AXIS2) / SEMIMINOR_AXIS2; - final double A = 1 + uSq/16384D*(4096D + uSq * (-768D + uSq * (320D - 175D*uSq))); - final double B = uSq/1024D * (256D + uSq * (-128D + uSq * (74D - 47D * uSq))); - - double sigma = dist / (SEMIMINOR_AXIS*A); - double sigmaP; - double sinSigma, cosSigma, cos2SigmaM, deltaSigma; - - do { - cos2SigmaM = StrictMath.cos(2*sig1 + sigma); - sinSigma = StrictMath.sin(sigma); - cosSigma = StrictMath.cos(sigma); - - deltaSigma = B * sinSigma * (cos2SigmaM + (B/4D) * (cosSigma*(-1+2*cos2SigmaM*cos2SigmaM)- - (B/6) * cos2SigmaM*(-3+4*sinSigma*sinSigma)*(-3+4*cos2SigmaM*cos2SigmaM))); - sigmaP = sigma; - sigma = dist / (SEMIMINOR_AXIS*A) + deltaSigma; - } while (StrictMath.abs(sigma-sigmaP) > 1E-12); - - final double tmp = sinU1*sinSigma - cosU1*cosSigma*cosA1; - final double lat2 = StrictMath.atan2(sinU1*cosSigma + cosU1*sinSigma*cosA1, - (1-FLATTENING) * StrictMath.sqrt(sinAlpha*sinAlpha + tmp*tmp)); - final double lambda = StrictMath.atan2(sinSigma*sinA1, cosU1*cosSigma - sinU1*sinSigma*cosA1); - final double c = FLATTENING/16 * cosSqAlpha * (4 + FLATTENING * (4 - 3 * cosSqAlpha)); - - final double lam = lambda - (1-c) * FLATTENING * sinAlpha * - (sigma + c * sinSigma * (cos2SigmaM + c * cosSigma * (-1 + 2* cos2SigmaM*cos2SigmaM))); - pt[0] = lon + StrictMath.toDegrees(lam); - pt[1] = StrictMath.toDegrees(lat2); - - return pt; - } -} \ No newline at end of file diff --git a/core/src/main/java/org/apache/lucene/util/XGeoUtils.java b/core/src/main/java/org/apache/lucene/util/XGeoUtils.java deleted file mode 100644 index df22e377949..00000000000 --- a/core/src/main/java/org/apache/lucene/util/XGeoUtils.java +++ /dev/null @@ -1,429 +0,0 @@ -package org.apache.lucene.util; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.ArrayList; - -/** - * Basic reusable geo-spatial utility methods - * - * @lucene.experimental - */ -public final class XGeoUtils { - private static final short MIN_LON = -180; - private static final short MIN_LAT = -90; - public static final short BITS = 31; - private static final double LON_SCALE = (0x1L<>> 1)); - } - - private static long scaleLon(final double val) { - return (long) ((val-MIN_LON) * LON_SCALE); - } - - private static long scaleLat(final double val) { - return (long) ((val-MIN_LAT) * LAT_SCALE); - } - - private static double unscaleLon(final long val) { - return (val / LON_SCALE) + MIN_LON; - } - - private static double unscaleLat(final long val) { - return (val / LAT_SCALE) + MIN_LAT; - } - - /** - * Interleaves the first 32 bits of each long value - * - * Adapted from: http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN - */ - public static long interleave(long v1, long v2) { - v1 = (v1 | (v1 << SHIFT[4])) & MAGIC[4]; - v1 = (v1 | (v1 << SHIFT[3])) & MAGIC[3]; - v1 = (v1 | (v1 << SHIFT[2])) & MAGIC[2]; - v1 = (v1 | (v1 << SHIFT[1])) & MAGIC[1]; - v1 = (v1 | (v1 << SHIFT[0])) & MAGIC[0]; - v2 = (v2 | (v2 << SHIFT[4])) & MAGIC[4]; - v2 = (v2 | (v2 << SHIFT[3])) & MAGIC[3]; - v2 = (v2 | (v2 << SHIFT[2])) & MAGIC[2]; - v2 = (v2 | (v2 << SHIFT[1])) & MAGIC[1]; - v2 = (v2 | (v2 << SHIFT[0])) & MAGIC[0]; - - return (v2<<1) | v1; - } - - /** - * Deinterleaves long value back to two concatenated 32bit values - */ - public static long deinterleave(long b) { - b &= MAGIC[0]; - b = (b ^ (b >>> SHIFT[0])) & MAGIC[1]; - b = (b ^ (b >>> SHIFT[1])) & MAGIC[2]; - b = (b ^ (b >>> SHIFT[2])) & MAGIC[3]; - b = (b ^ (b >>> SHIFT[3])) & MAGIC[4]; - b = (b ^ (b >>> SHIFT[4])) & MAGIC[5]; - return b; - } - - public static double compare(final double v1, final double v2) { - final double compare = v1-v2; - return Math.abs(compare) <= TOLERANCE ? 0 : compare; - } - - /** - * Puts longitude in range of -180 to +180. - */ - public static double normalizeLon(double lon_deg) { - if (lon_deg >= -180 && lon_deg <= 180) { - return lon_deg; //common case, and avoids slight double precision shifting - } - double off = (lon_deg + 180) % 360; - if (off < 0) { - return 180 + off; - } else if (off == 0 && lon_deg > 0) { - return 180; - } else { - return -180 + off; - } - } - - /** - * Puts latitude in range of -90 to 90. - */ - public static double normalizeLat(double lat_deg) { - if (lat_deg >= -90 && lat_deg <= 90) { - return lat_deg; //common case, and avoids slight double precision shifting - } - double off = Math.abs((lat_deg + 90) % 360); - return (off <= 180 ? off : 360-off) - 90; - } - - public static final boolean bboxContains(final double lon, final double lat, final double minLon, - final double minLat, final double maxLon, final double maxLat) { - return (compare(lon, minLon) >= 0 && compare(lon, maxLon) <= 0 - && compare(lat, minLat) >= 0 && compare(lat, maxLat) <= 0); - } - - /** - * simple even-odd point in polygon computation - * 1. Determine if point is contained in the longitudinal range - * 2. Determine whether point crosses the edge by computing the latitudinal delta - * between the end-point of a parallel vector (originating at the point) and the - * y-component of the edge sink - * - * NOTE: Requires polygon point (x,y) order either clockwise or counter-clockwise - */ - public static boolean pointInPolygon(double[] x, double[] y, double lat, double lon) { - assert x.length == y.length; - boolean inPoly = false; - /** - * Note: This is using a euclidean coordinate system which could result in - * upwards of 110KM error at the equator. - * TODO convert coordinates to cylindrical projection (e.g. mercator) - */ - for (int i = 1; i < x.length; i++) { - if (x[i] < lon && x[i-1] >= lon || x[i-1] < lon && x[i] >= lon) { - if (y[i] + (lon - x[i]) / (x[i-1] - x[i]) * (y[i-1] - y[i]) < lat) { - inPoly = !inPoly; - } - } - } - return inPoly; - } - - public static String geoTermToString(long term) { - StringBuilder s = new StringBuilder(64); - final int numberOfLeadingZeros = Long.numberOfLeadingZeros(term); - for (int i = 0; i < numberOfLeadingZeros; i++) { - s.append('0'); - } - if (term != 0) { - s.append(Long.toBinaryString(term)); - } - return s.toString(); - } - - - public static boolean rectDisjoint(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return (aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY); - } - - /** - * Computes whether a rectangle is wholly within another rectangle (shared boundaries allowed) - */ - public static boolean rectWithin(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY); - } - - public static boolean rectCrosses(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !(rectDisjoint(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY) || - rectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY)); - } - - /** - * Computes whether rectangle a contains rectangle b (touching allowed) - */ - public static boolean rectContains(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !(bMinX < aMinX || bMinY < aMinY || bMaxX > aMaxX || bMaxY > aMaxY); - } - - /** - * Computes whether a rectangle intersects another rectangle (crosses, within, touching, etc) - */ - public static boolean rectIntersects(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !((aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY) ); - } - - /** - * Computes whether a rectangle crosses a shape. (touching not allowed) - */ - public static boolean rectCrossesPoly(final double rMinX, final double rMinY, final double rMaxX, - final double rMaxY, final double[] shapeX, final double[] shapeY, - final double sMinX, final double sMinY, final double sMaxX, - final double sMaxY) { - // short-circuit: if the bounding boxes are disjoint then the shape does not cross - if (rectDisjoint(rMinX, rMinY, rMaxX, rMaxY, sMinX, sMinY, sMaxX, sMaxY)) { - return false; - } - - final double[][] bbox = new double[][] { {rMinX, rMinY}, {rMaxX, rMinY}, {rMaxX, rMaxY}, {rMinX, rMaxY}, {rMinX, rMinY} }; - final int polyLength = shapeX.length-1; - double d, s, t, a1, b1, c1, a2, b2, c2; - double x00, y00, x01, y01, x10, y10, x11, y11; - - // computes the intersection point between each bbox edge and the polygon edge - for (short b=0; b<4; ++b) { - a1 = bbox[b+1][1]-bbox[b][1]; - b1 = bbox[b][0]-bbox[b+1][0]; - c1 = a1*bbox[b+1][0] + b1*bbox[b+1][1]; - for (int p=0; p s || x01 < s || y00 > t || y01 < t || x10 > s || x11 < s || y10 > t || y11 < t)) { - return true; - } - } - } // for each poly edge - } // for each bbox edge - return false; - } - - /** - * Converts a given circle (defined as a point/radius) to an approximated line-segment polygon - * - * @param lon longitudinal center of circle (in degrees) - * @param lat latitudinal center of circle (in degrees) - * @param radius distance radius of circle (in meters) - * @return a list of lon/lat points representing the circle - */ - @SuppressWarnings({"unchecked","rawtypes"}) - public static ArrayList circleToPoly(final double lon, final double lat, final double radius) { - double angle; - // a little under-sampling (to limit the number of polygonal points): using archimedes estimation of pi - final int sides = 25; - ArrayList geometry = new ArrayList(); - double[] lons = new double[sides]; - double[] lats = new double[sides]; - - double[] pt = new double[2]; - final int sidesLen = sides-1; - for (int i=0; i radius - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMinX)*1000.0 > radius - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMaxX)*1000.0 > radius - || SloppyMath.haversin(centerLat, centerLon, rMinY, rMaxX)*1000.0 > radius); - } - - private static boolean rectAnyCornersInCircle(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radius) { - return (SloppyMath.haversin(centerLat, centerLon, rMinY, rMinX)*1000.0 <= radius - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMinX)*1000.0 <= radius - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMaxX)*1000.0 <= radius - || SloppyMath.haversin(centerLat, centerLon, rMinY, rMaxX)*1000.0 <= radius); - } - - public static boolean rectWithinCircle(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radius) { - return !(rectAnyCornersOutsideCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radius)); - } - - /** - * Computes whether a rectangle crosses a circle - */ - public static boolean rectCrossesCircle(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radius) { - return rectAnyCornersInCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radius) - || lineCrossesSphere(rMinX, rMinY, 0, rMaxX, rMinY, 0, centerLon, centerLat, 0, radius) - || lineCrossesSphere(rMaxX, rMinY, 0, rMaxX, rMaxY, 0, centerLon, centerLat, 0, radius) - || lineCrossesSphere(rMaxX, rMaxY, 0, rMinX, rMaxY, 0, centerLon, centerLat, 0, radius) - || lineCrossesSphere(rMinX, rMaxY, 0, rMinX, rMinY, 0, centerLon, centerLat, 0, radius); - } - - /** - * Computes whether or a 3dimensional line segment intersects or crosses a sphere - * - * @param lon1 longitudinal location of the line segment start point (in degrees) - * @param lat1 latitudinal location of the line segment start point (in degrees) - * @param alt1 altitude of the line segment start point (in degrees) - * @param lon2 longitudinal location of the line segment end point (in degrees) - * @param lat2 latitudinal location of the line segment end point (in degrees) - * @param alt2 altitude of the line segment end point (in degrees) - * @param centerLon longitudinal location of center search point (in degrees) - * @param centerLat latitudinal location of center search point (in degrees) - * @param centerAlt altitude of the center point (in meters) - * @param radius search sphere radius (in meters) - * @return whether the provided line segment is a secant of the - */ - private static boolean lineCrossesSphere(double lon1, double lat1, double alt1, double lon2, - double lat2, double alt2, double centerLon, double centerLat, - double centerAlt, double radius) { - // convert to cartesian 3d (in meters) - double[] ecf1 = XGeoProjectionUtils.llaToECF(lon1, lat1, alt1, null); - double[] ecf2 = XGeoProjectionUtils.llaToECF(lon2, lat2, alt2, null); - double[] cntr = XGeoProjectionUtils.llaToECF(centerLon, centerLat, centerAlt, null); - - final double dX = ecf2[0] - ecf1[0]; - final double dY = ecf2[1] - ecf1[1]; - final double dZ = ecf2[2] - ecf1[2]; - final double fX = ecf1[0] - cntr[0]; - final double fY = ecf1[1] - cntr[1]; - final double fZ = ecf1[2] - cntr[2]; - - final double a = dX*dX + dY*dY + dZ*dZ; - final double b = 2 * (fX*dX + fY*dY + fZ*dZ); - final double c = (fX*fX + fY*fY + fZ*fZ) - (radius*radius); - - double discrim = (b*b)-(4*a*c); - if (discrim < 0) { - return false; - } - - discrim = StrictMath.sqrt(discrim); - final double a2 = 2*a; - final double t1 = (-b - discrim)/a2; - final double t2 = (-b + discrim)/a2; - - if ( (t1 < 0 || t1 > 1) ) { - return !(t2 < 0 || t2 > 1); - } - - return true; - } - - public static boolean isValidLat(double lat) { - return Double.isNaN(lat) == false && lat >= MIN_LAT_INCL && lat <= MAX_LAT_INCL; - } - - public static boolean isValidLon(double lon) { - return Double.isNaN(lon) == false && lon >= MIN_LON_INCL && lon <= MAX_LON_INCL; - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/Build.java b/core/src/main/java/org/elasticsearch/Build.java index 508b4dc4375..248040de0a8 100644 --- a/core/src/main/java/org/elasticsearch/Build.java +++ b/core/src/main/java/org/elasticsearch/Build.java @@ -19,81 +19,101 @@ package org.elasticsearch; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.joda.time.DateTimeZone; -import org.joda.time.format.ISODateTimeFormat; import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.jar.JarInputStream; +import java.util.jar.Manifest; /** + * Information about a build of Elasticsearch. */ public class Build { - + /** + * The current build of Elasticsearch. Filled with information scanned at + * startup from the jar. + */ public static final Build CURRENT; static { - String hash = "NA"; - String hashShort = "NA"; - String timestamp = "NA"; + final String shortHash; + final String date; - try (InputStream is = Build.class.getResourceAsStream("/es-build.properties")){ - Properties props = new Properties(); - props.load(is); - hash = props.getProperty("hash", hash); - if (!hash.equals("NA")) { - hashShort = hash.substring(0, 7); + Path path = getElasticsearchCodebase(); + if (path.toString().endsWith(".jar")) { + try (JarInputStream jar = new JarInputStream(Files.newInputStream(path))) { + Manifest manifest = jar.getManifest(); + shortHash = manifest.getMainAttributes().getValue("Change"); + date = manifest.getMainAttributes().getValue("Build-Date"); + } catch (IOException e) { + throw new RuntimeException(e); } - String gitTimestampRaw = props.getProperty("timestamp"); - if (gitTimestampRaw != null) { - timestamp = ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.UTC).print(Long.parseLong(gitTimestampRaw)); - } - } catch (Exception e) { - // just ignore... + } else { + // not running from a jar (unit tests, IDE) + shortHash = "Unknown"; + date = "Unknown"; + } + if (shortHash == null) { + throw new IllegalStateException("Error finding the build shortHash. " + + "Stopping Elasticsearch now so it doesn't run in subtly broken ways. This is likely a build bug."); + } + if (date == null) { + throw new IllegalStateException("Error finding the build date. " + + "Stopping Elasticsearch now so it doesn't run in subtly broken ways. This is likely a build bug."); } - CURRENT = new Build(hash, hashShort, timestamp); + CURRENT = new Build(shortHash, date); } - private String hash; - private String hashShort; - private String timestamp; - - Build(String hash, String hashShort, String timestamp) { - this.hash = hash; - this.hashShort = hashShort; - this.timestamp = timestamp; + /** + * Returns path to elasticsearch codebase path + */ + @SuppressForbidden(reason = "looks up path of elasticsearch.jar directly") + static Path getElasticsearchCodebase() { + URL url = Build.class.getProtectionDomain().getCodeSource().getLocation(); + try { + return PathUtils.get(url.toURI()); + } catch (URISyntaxException bogus) { + throw new RuntimeException(bogus); + } } - public String hash() { - return hash; + private String shortHash; + private String date; + + Build(String shortHash, String date) { + this.shortHash = shortHash; + this.date = date; } - public String hashShort() { - return hashShort; + public String shortHash() { + return shortHash; } - public String timestamp() { - return timestamp; + public String date() { + return date; } public static Build readBuild(StreamInput in) throws IOException { String hash = in.readString(); - String hashShort = in.readString(); - String timestamp = in.readString(); - return new Build(hash, hashShort, timestamp); + String date = in.readString(); + return new Build(hash, date); } public static void writeBuild(Build build, StreamOutput out) throws IOException { - out.writeString(build.hash()); - out.writeString(build.hashShort()); - out.writeString(build.timestamp()); + out.writeString(build.shortHash()); + out.writeString(build.date()); } @Override public String toString() { - return "[" + hash + "][" + timestamp + "]"; + return "[" + shortHash + "][" + date + "]"; } } diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java b/core/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java index 350bbc31121..d56f580ff9e 100644 --- a/core/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java +++ b/core/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java @@ -42,15 +42,6 @@ public class ElasticsearchCorruptionException extends IOException { * @param ex the exception cause */ public ElasticsearchCorruptionException(Throwable ex) { - this(ex.getMessage()); - if (ex != null) { - this.setStackTrace(ex.getStackTrace()); - } - Throwable[] suppressed = ex.getSuppressed(); - if (suppressed != null) { - for (Throwable supressedExc : suppressed) { - addSuppressed(supressedExc); - } - } + super(ex); } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java index 62eb374733e..18376aff88f 100644 --- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -51,6 +51,13 @@ public class ElasticsearchException extends RuntimeException implements ToXConte private static final Map, ElasticsearchExceptionHandle> CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE; private final Map> headers = new HashMap<>(); + /** + * Construct a ElasticsearchException with the specified cause exception. + */ + public ElasticsearchException(Throwable cause) { + super(cause); + } + /** * Construct a ElasticsearchException with the specified detail message. * @@ -547,7 +554,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte NODE_DISCONNECTED_EXCEPTION(org.elasticsearch.transport.NodeDisconnectedException.class, org.elasticsearch.transport.NodeDisconnectedException::new, 84), ALREADY_EXPIRED_EXCEPTION(org.elasticsearch.index.AlreadyExpiredException.class, org.elasticsearch.index.AlreadyExpiredException::new, 85), AGGREGATION_EXECUTION_EXCEPTION(org.elasticsearch.search.aggregations.AggregationExecutionException.class, org.elasticsearch.search.aggregations.AggregationExecutionException::new, 86), - MERGE_MAPPING_EXCEPTION(org.elasticsearch.index.mapper.MergeMappingException.class, org.elasticsearch.index.mapper.MergeMappingException::new, 87), + // 87 used to be for MergeMappingException INVALID_INDEX_TEMPLATE_EXCEPTION(org.elasticsearch.indices.InvalidIndexTemplateException.class, org.elasticsearch.indices.InvalidIndexTemplateException::new, 88), PERCOLATE_EXCEPTION(org.elasticsearch.percolator.PercolateException.class, org.elasticsearch.percolator.PercolateException::new, 89), REFRESH_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.RefreshFailedEngineException.class, org.elasticsearch.index.engine.RefreshFailedEngineException::new, 90), diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java b/core/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java index 3cc4e7a477a..2b53a3beaae 100644 --- a/core/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java +++ b/core/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java @@ -33,8 +33,12 @@ public class ElasticsearchTimeoutException extends ElasticsearchException { super(in); } + public ElasticsearchTimeoutException(Throwable cause) { + super(cause); + } + public ElasticsearchTimeoutException(String message, Object... args) { - super(message); + super(message, args); } public ElasticsearchTimeoutException(String message, Throwable cause, Object... args) { diff --git a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java index df7be834ebe..555a172c0d8 100644 --- a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java +++ b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java @@ -47,14 +47,14 @@ public final class ExceptionsHelper { if (t instanceof RuntimeException) { return (RuntimeException) t; } - return new ElasticsearchException(t.getMessage(), t); + return new ElasticsearchException(t); } public static ElasticsearchException convertToElastic(Throwable t) { if (t instanceof ElasticsearchException) { return (ElasticsearchException) t; } - return new ElasticsearchException(t.getMessage(), t); + return new ElasticsearchException(t); } public static RestStatus status(Throwable t) { @@ -160,7 +160,7 @@ public final class ExceptionsHelper { main = useOrSuppress(main, ex); } if (main != null) { - throw new ElasticsearchException(main.getMessage(), main); + throw new ElasticsearchException(main); } } diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index a610d8ddccb..a5e2e38ca26 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -253,7 +253,9 @@ public class Version { public static final int V_1_7_2_ID = 1070299; public static final Version V_1_7_2 = new Version(V_1_7_2_ID, false, org.apache.lucene.util.Version.LUCENE_4_10_4); public static final int V_1_7_3_ID = 1070399; - public static final Version V_1_7_3 = new Version(V_1_7_3_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4); + public static final Version V_1_7_3 = new Version(V_1_7_3_ID, false, org.apache.lucene.util.Version.LUCENE_4_10_4); + public static final int V_1_7_4_ID = 1070499; + public static final Version V_1_7_4 = new Version(V_1_7_4_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4); public static final int V_2_0_0_beta1_ID = 2000001; public static final Version V_2_0_0_beta1 = new Version(V_2_0_0_beta1_ID, false, org.apache.lucene.util.Version.LUCENE_5_2_1); @@ -262,13 +264,19 @@ public class Version { public static final int V_2_0_0_rc1_ID = 2000051; public static final Version V_2_0_0_rc1 = new Version(V_2_0_0_rc1_ID, false, org.apache.lucene.util.Version.LUCENE_5_2_1); public static final int V_2_0_0_ID = 2000099; - public static final Version V_2_0_0 = new Version(V_2_0_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_2_1); + public static final Version V_2_0_0 = new Version(V_2_0_0_ID, false, org.apache.lucene.util.Version.LUCENE_5_2_1); + public static final int V_2_0_1_ID = 2000199; + public static final Version V_2_0_1 = new Version(V_2_0_1_ID, false, org.apache.lucene.util.Version.LUCENE_5_2_1); + public static final int V_2_0_2_ID = 2000299; + public static final Version V_2_0_2 = new Version(V_2_0_2_ID, true, org.apache.lucene.util.Version.LUCENE_5_2_1); public static final int V_2_1_0_ID = 2010099; - public static final Version V_2_1_0 = new Version(V_2_1_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_3_0); + public static final Version V_2_1_0 = new Version(V_2_1_0_ID, false, org.apache.lucene.util.Version.LUCENE_5_3_1); + public static final int V_2_1_1_ID = 2010199; + public static final Version V_2_1_1 = new Version(V_2_1_1_ID, true, org.apache.lucene.util.Version.LUCENE_5_3_1); public static final int V_2_2_0_ID = 2020099; - public static final Version V_2_2_0 = new Version(V_2_2_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_3_0); + public static final Version V_2_2_0 = new Version(V_2_2_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_4_0); public static final int V_3_0_0_ID = 3000099; - public static final Version V_3_0_0 = new Version(V_3_0_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_4_0); + public static final Version V_3_0_0 = new Version(V_3_0_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_5_0); public static final Version CURRENT = V_3_0_0; static { @@ -285,8 +293,14 @@ public class Version { return V_3_0_0; case V_2_2_0_ID: return V_2_2_0; + case V_2_1_1_ID: + return V_2_1_1; case V_2_1_0_ID: return V_2_1_0; + case V_2_0_2_ID: + return V_2_0_2; + case V_2_0_1_ID: + return V_2_0_1; case V_2_0_0_ID: return V_2_0_0; case V_2_0_0_rc1_ID: @@ -295,6 +309,8 @@ public class Version { return V_2_0_0_beta2; case V_2_0_0_beta1_ID: return V_2_0_0_beta1; + case V_1_7_4_ID: + return V_1_7_4; case V_1_7_3_ID: return V_1_7_3; case V_1_7_2_ID: @@ -545,7 +561,7 @@ public class Version { } String[] parts = version.split("\\.|\\-"); if (parts.length < 3 || parts.length > 4) { - throw new IllegalArgumentException("the version needs to contain major, minor and revision, and optionally the build: " + version); + throw new IllegalArgumentException("the version needs to contain major, minor, and revision, and optionally the build: " + version); } try { @@ -653,7 +669,7 @@ public class Version { @SuppressForbidden(reason = "System.out.*") public static void main(String[] args) { - System.out.println("Version: " + Version.CURRENT + ", Build: " + Build.CURRENT.hashShort() + "/" + Build.CURRENT.timestamp() + ", JVM: " + JvmInfo.jvmInfo().version()); + System.out.println("Version: " + Version.CURRENT + ", Build: " + Build.CURRENT.shortHash() + "/" + Build.CURRENT.date() + ", JVM: " + JvmInfo.jvmInfo().version()); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/ActionFuture.java b/core/src/main/java/org/elasticsearch/action/ActionFuture.java index 26a9260b710..2d5f6781d71 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionFuture.java +++ b/core/src/main/java/org/elasticsearch/action/ActionFuture.java @@ -37,10 +37,6 @@ public interface ActionFuture extends Future { * Similar to {@link #get()}, just catching the {@link InterruptedException} and throwing * an {@link IllegalStateException} instead. Also catches * {@link java.util.concurrent.ExecutionException} and throws the actual cause instead. - *

- * Note, the actual cause is unwrapped to the actual failure (for example, unwrapped - * from {@link org.elasticsearch.transport.RemoteTransportException}. The root failure is - * still accessible using {@link #getRootFailure()}. */ T actionGet(); @@ -48,10 +44,6 @@ public interface ActionFuture extends Future { * Similar to {@link #get(long, java.util.concurrent.TimeUnit)}, just catching the {@link InterruptedException} and throwing * an {@link IllegalStateException} instead. Also catches * {@link java.util.concurrent.ExecutionException} and throws the actual cause instead. - *

- * Note, the actual cause is unwrapped to the actual failure (for example, unwrapped - * from {@link org.elasticsearch.transport.RemoteTransportException}. The root failure is - * still accessible using {@link #getRootFailure()}. */ T actionGet(String timeout); @@ -59,10 +51,6 @@ public interface ActionFuture extends Future { * Similar to {@link #get(long, java.util.concurrent.TimeUnit)}, just catching the {@link InterruptedException} and throwing * an {@link IllegalStateException} instead. Also catches * {@link java.util.concurrent.ExecutionException} and throws the actual cause instead. - *

- * Note, the actual cause is unwrapped to the actual failure (for example, unwrapped - * from {@link org.elasticsearch.transport.RemoteTransportException}. The root failure is - * still accessible using {@link #getRootFailure()}. * * @param timeoutMillis Timeout in millis */ @@ -72,10 +60,6 @@ public interface ActionFuture extends Future { * Similar to {@link #get(long, java.util.concurrent.TimeUnit)}, just catching the {@link InterruptedException} and throwing * an {@link IllegalStateException} instead. Also catches * {@link java.util.concurrent.ExecutionException} and throws the actual cause instead. - *

- * Note, the actual cause is unwrapped to the actual failure (for example, unwrapped - * from {@link org.elasticsearch.transport.RemoteTransportException}. The root failure is - * still accessible using {@link #getRootFailure()}. */ T actionGet(long timeout, TimeUnit unit); @@ -83,16 +67,6 @@ public interface ActionFuture extends Future { * Similar to {@link #get(long, java.util.concurrent.TimeUnit)}, just catching the {@link InterruptedException} and throwing * an {@link IllegalStateException} instead. Also catches * {@link java.util.concurrent.ExecutionException} and throws the actual cause instead. - *

- * Note, the actual cause is unwrapped to the actual failure (for example, unwrapped - * from {@link org.elasticsearch.transport.RemoteTransportException}. The root failure is - * still accessible using {@link #getRootFailure()}. */ T actionGet(TimeValue timeout); - - /** - * The root (possibly) wrapped failure. - */ - @Nullable - Throwable getRootFailure(); } diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index f8634b1c618..88ccb809712 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -80,6 +80,8 @@ import org.elasticsearch.action.admin.indices.exists.types.TransportTypesExistsA import org.elasticsearch.action.admin.indices.exists.types.TypesExistsAction; import org.elasticsearch.action.admin.indices.flush.FlushAction; import org.elasticsearch.action.admin.indices.flush.TransportFlushAction; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; +import org.elasticsearch.action.admin.indices.forcemerge.TransportForceMergeAction; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.TransportGetIndexAction; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsAction; @@ -91,8 +93,6 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; import org.elasticsearch.action.admin.indices.open.OpenIndexAction; import org.elasticsearch.action.admin.indices.open.TransportOpenIndexAction; -import org.elasticsearch.action.admin.indices.optimize.OptimizeAction; -import org.elasticsearch.action.admin.indices.optimize.TransportOptimizeAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; import org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; @@ -134,8 +134,6 @@ import org.elasticsearch.action.bulk.TransportBulkAction; import org.elasticsearch.action.bulk.TransportShardBulkAction; import org.elasticsearch.action.delete.DeleteAction; import org.elasticsearch.action.delete.TransportDeleteAction; -import org.elasticsearch.action.exists.ExistsAction; -import org.elasticsearch.action.exists.TransportExistsAction; import org.elasticsearch.action.explain.ExplainAction; import org.elasticsearch.action.explain.TransportExplainAction; import org.elasticsearch.action.fieldstats.FieldStatsAction; @@ -295,7 +293,7 @@ public class ActionModule extends AbstractModule { registerAction(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class); registerAction(RefreshAction.INSTANCE, TransportRefreshAction.class); registerAction(FlushAction.INSTANCE, TransportFlushAction.class); - registerAction(OptimizeAction.INSTANCE, TransportOptimizeAction.class); + registerAction(ForceMergeAction.INSTANCE, TransportForceMergeAction.class); registerAction(UpgradeAction.INSTANCE, TransportUpgradeAction.class); registerAction(UpgradeStatusAction.INSTANCE, TransportUpgradeStatusAction.class); registerAction(UpgradeSettingsAction.INSTANCE, TransportUpgradeSettingsAction.class); @@ -314,7 +312,6 @@ public class ActionModule extends AbstractModule { registerAction(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class, TransportShardMultiTermsVectorAction.class); registerAction(DeleteAction.INSTANCE, TransportDeleteAction.class); - registerAction(ExistsAction.INSTANCE, TransportExistsAction.class); registerAction(SuggestAction.INSTANCE, TransportSuggestAction.class); registerAction(UpdateAction.INSTANCE, TransportUpdateAction.class); registerAction(MultiGetAction.INSTANCE, TransportMultiGetAction.class, diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java new file mode 100644 index 00000000000..009d3fc47a9 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; + +/** + * A base class for the response of a write operation that involves a single doc + */ +public abstract class DocWriteResponse extends ReplicationResponse implements StatusToXContent { + + private ShardId shardId; + private String id; + private String type; + private long version; + + public DocWriteResponse(ShardId shardId, String type, String id, long version) { + this.shardId = shardId; + this.type = type; + this.id = id; + this.version = version; + } + + // needed for deserialization + protected DocWriteResponse() { + } + + /** + * The index the document was changed in. + */ + public String getIndex() { + return this.shardId.getIndex(); + } + + + /** + * The exact shard the document was changed in. + */ + public ShardId getShardId() { + return this.shardId; + } + + /** + * The type of the document changed. + */ + public String getType() { + return this.type; + } + + /** + * The id of the document changed. + */ + public String getId() { + return this.id; + } + + /** + * Returns the current version of the doc. + */ + public long getVersion() { + return this.version; + } + + /** returns the rest status for this response (based on {@link ShardInfo#status()} */ + public RestStatus status() { + return getShardInfo().status(); + } + + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + shardId = ShardId.readShardId(in); + type = in.readString(); + id = in.readString(); + version = in.readZLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + shardId.writeTo(out); + out.writeString(type); + out.writeString(id); + out.writeZLong(version); + } + + static final class Fields { + static final XContentBuilderString _INDEX = new XContentBuilderString("_index"); + static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); + static final XContentBuilderString _ID = new XContentBuilderString("_id"); + static final XContentBuilderString _VERSION = new XContentBuilderString("_version"); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + ReplicationResponse.ShardInfo shardInfo = getShardInfo(); + builder.field(Fields._INDEX, shardId.getIndex()) + .field(Fields._TYPE, type) + .field(Fields._ID, id) + .field(Fields._VERSION, version); + shardInfo.toXContent(builder, params); + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/action/ActionWriteResponse.java b/core/src/main/java/org/elasticsearch/action/ReplicationResponse.java similarity index 96% rename from core/src/main/java/org/elasticsearch/action/ActionWriteResponse.java rename to core/src/main/java/org/elasticsearch/action/ReplicationResponse.java index f4152ac85e4..4e358c8d42a 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionWriteResponse.java +++ b/core/src/main/java/org/elasticsearch/action/ReplicationResponse.java @@ -21,7 +21,6 @@ package org.elasticsearch.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.bootstrap.Elasticsearch; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -30,25 +29,23 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.util.Collections; /** * Base class for write action responses. */ -public class ActionWriteResponse extends ActionResponse { +public class ReplicationResponse extends ActionResponse { - public final static ActionWriteResponse.ShardInfo.Failure[] EMPTY = new ActionWriteResponse.ShardInfo.Failure[0]; + public final static ReplicationResponse.ShardInfo.Failure[] EMPTY = new ReplicationResponse.ShardInfo.Failure[0]; private ShardInfo shardInfo; @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - shardInfo = ActionWriteResponse.ShardInfo.readShardInfo(in); + shardInfo = ReplicationResponse.ShardInfo.readShardInfo(in); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java index ba1e73311ec..d603886d924 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java index f12ab123009..feacbb9511f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.cluster.health; import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.common.unit.TimeValue; diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java index cc300a91cf2..27add930ea8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java @@ -22,10 +22,9 @@ package org.elasticsearch.action.admin.cluster.health; import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTableValidation; -import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.health.ClusterStateHealth; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; @@ -36,38 +35,22 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; -import static org.elasticsearch.action.admin.cluster.health.ClusterIndexHealth.readClusterIndexHealth; - /** * */ -public class ClusterHealthResponse extends ActionResponse implements Iterable, StatusToXContent { - +public class ClusterHealthResponse extends ActionResponse implements StatusToXContent { private String clusterName; - int numberOfNodes = 0; - int numberOfDataNodes = 0; - int activeShards = 0; - int relocatingShards = 0; - int activePrimaryShards = 0; - int initializingShards = 0; - int unassignedShards = 0; - int numberOfPendingTasks = 0; - int numberOfInFlightFetch = 0; - int delayedUnassignedShards = 0; - TimeValue taskMaxWaitingTime = TimeValue.timeValueMillis(0); - double activeShardsPercent = 100; - boolean timedOut = false; - ClusterHealthStatus status = ClusterHealthStatus.RED; - private List validationFailures; - Map indices = new HashMap<>(); + private int numberOfPendingTasks = 0; + private int numberOfInFlightFetch = 0; + private int delayedUnassignedShards = 0; + private TimeValue taskMaxWaitingTime = TimeValue.timeValueMillis(0); + private boolean timedOut = false; + private ClusterStateHealth clusterStateHealth; + private ClusterHealthStatus clusterHealthStatus; ClusterHealthResponse() { } @@ -87,107 +70,53 @@ public class ClusterHealthResponse extends ActionResponse implements Iterable shardRoutings = clusterState.getRoutingTable().allShards(); - int activeShardCount = 0; - int totalShardCount = 0; - for (ShardRouting shardRouting : shardRoutings) { - if (shardRouting.active()) activeShardCount++; - totalShardCount++; - } - this.activeShardsPercent = (((double) activeShardCount) / totalShardCount) * 100; - } + this.clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices); + this.clusterHealthStatus = clusterStateHealth.getStatus(); } public String getClusterName() { return clusterName; } + //package private for testing + ClusterStateHealth getClusterStateHealth() { + return clusterStateHealth; + } + /** * The validation failures on the cluster level (without index validation failures). */ public List getValidationFailures() { - return this.validationFailures; + return clusterStateHealth.getValidationFailures(); } - /** - * All the validation failures, including index level validation failures. - */ - public List getAllValidationFailures() { - List allFailures = new ArrayList<>(getValidationFailures()); - for (ClusterIndexHealth indexHealth : indices.values()) { - allFailures.addAll(indexHealth.getValidationFailures()); - } - return allFailures; - } public int getActiveShards() { - return activeShards; + return clusterStateHealth.getActiveShards(); } public int getRelocatingShards() { - return relocatingShards; + return clusterStateHealth.getRelocatingShards(); } public int getActivePrimaryShards() { - return activePrimaryShards; + return clusterStateHealth.getActivePrimaryShards(); } public int getInitializingShards() { - return initializingShards; + return clusterStateHealth.getInitializingShards(); } public int getUnassignedShards() { - return unassignedShards; + return clusterStateHealth.getUnassignedShards(); } public int getNumberOfNodes() { - return this.numberOfNodes; + return clusterStateHealth.getNumberOfNodes(); } public int getNumberOfDataNodes() { - return this.numberOfDataNodes; + return clusterStateHealth.getNumberOfDataNodes(); } public int getNumberOfPendingTasks() { @@ -214,12 +143,28 @@ public class ClusterHealthResponse extends ActionResponse implements Iterable getIndices() { - return indices; + return clusterStateHealth.getIndices(); } /** @@ -234,15 +179,9 @@ public class ClusterHealthResponse extends ActionResponse implements Iterable iterator() { - return indices.values().iterator(); - } - - public static ClusterHealthResponse readResponseFrom(StreamInput in) throws IOException { ClusterHealthResponse response = new ClusterHealthResponse(); response.readFrom(in); @@ -253,36 +192,14 @@ public class ClusterHealthResponse extends ActionResponse implements Iterable listener) { if (request.waitForEvents() != null) { final long endTimeMS = TimeValue.nsecToMSec(System.nanoTime()) + request.timeout().millis(); - clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", request.waitForEvents(), new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", new ClusterStateUpdateTask(request.waitForEvents()) { @Override public ClusterState execute(ClusterState currentState) { return currentState; @@ -184,7 +185,7 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction< // if the state is sufficient for what we where waiting for we don't need to mark this as timedOut. // We spend too much time in waiting for events such that we might already reached a valid state. // this should not mark the request as timed out - response.timedOut = timedOut && valid == false; + response.setTimedOut(timedOut && valid == false); return response; } @@ -204,7 +205,7 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction< indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), request.indices()); waitForCounter++; } catch (IndexNotFoundException e) { - response.status = ClusterHealthStatus.RED; // no indices, make sure its RED + response.setStatus(ClusterHealthStatus.RED); // no indices, make sure its RED // missing indices, wait a bit more... } } @@ -272,13 +273,13 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction< } catch (IndexNotFoundException e) { // one of the specified indices is not there - treat it as RED. ClusterHealthResponse response = new ClusterHealthResponse(clusterName.value(), Strings.EMPTY_ARRAY, clusterState, - numberOfPendingTasks, numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(System.currentTimeMillis(), settings, clusterState), + numberOfPendingTasks, numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue); - response.status = ClusterHealthStatus.RED; + response.setStatus(ClusterHealthStatus.RED); return response; } return new ClusterHealthResponse(clusterName.value(), concreteIndices, clusterState, numberOfPendingTasks, - numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(System.currentTimeMillis(), settings, clusterState), pendingTaskTimeInQueue); + numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 2d683852012..1fa64d5e7b7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -72,14 +72,14 @@ public class NodeInfo extends BaseNodeResponse { private HttpInfo http; @Nullable - private PluginsInfo plugins; + private PluginsAndModules plugins; NodeInfo() { } public NodeInfo(Version version, Build build, DiscoveryNode node, @Nullable Map serviceAttributes, @Nullable Settings settings, @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable ThreadPoolInfo threadPool, - @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsInfo plugins) { + @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsAndModules plugins) { super(node); this.version = version; this.build = build; @@ -172,7 +172,7 @@ public class NodeInfo extends BaseNodeResponse { } @Nullable - public PluginsInfo getPlugins() { + public PluginsAndModules getPlugins() { return this.plugins; } @@ -217,7 +217,8 @@ public class NodeInfo extends BaseNodeResponse { http = HttpInfo.readHttpInfo(in); } if (in.readBoolean()) { - plugins = PluginsInfo.readPluginsInfo(in); + plugins = new PluginsAndModules(); + plugins.readFrom(in); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index 108bb314d4a..329be6c7614 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -77,7 +77,7 @@ public class NodesInfoResponse extends BaseNodesResponse implements To builder.field("ip", nodeInfo.getNode().getHostAddress(), XContentBuilder.FieldCaseConversion.NONE); builder.field("version", nodeInfo.getVersion()); - builder.field("build", nodeInfo.getBuild().hashShort()); + builder.field("build_hash", nodeInfo.getBuild().shortHash()); if (nodeInfo.getServiceAttributes() != null) { for (Map.Entry nodeAttribute : nodeInfo.getServiceAttributes().entrySet()) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java new file mode 100644 index 00000000000..3831fd24f3e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.info; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.plugins.PluginInfo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Information about plugins and modules + */ +public class PluginsAndModules implements Streamable, ToXContent { + private List plugins; + private List modules; + + public PluginsAndModules() { + plugins = new ArrayList<>(); + modules = new ArrayList<>(); + } + + /** + * Returns an ordered list based on plugins name + */ + public List getPluginInfos() { + List plugins = new ArrayList<>(this.plugins); + Collections.sort(plugins, (p1, p2) -> p1.getName().compareTo(p2.getName())); + return plugins; + } + + /** + * Returns an ordered list based on modules name + */ + public List getModuleInfos() { + List modules = new ArrayList<>(this.modules); + Collections.sort(modules, (p1, p2) -> p1.getName().compareTo(p2.getName())); + return modules; + } + + public void addPlugin(PluginInfo info) { + plugins.add(info); + } + + public void addModule(PluginInfo info) { + modules.add(info); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + if (plugins.isEmpty() == false || modules.isEmpty() == false) { + throw new IllegalStateException("instance is already populated"); + } + int plugins_size = in.readInt(); + for (int i = 0; i < plugins_size; i++) { + plugins.add(PluginInfo.readFromStream(in)); + } + int modules_size = in.readInt(); + for (int i = 0; i < modules_size; i++) { + modules.add(PluginInfo.readFromStream(in)); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(plugins.size()); + for (PluginInfo plugin : getPluginInfos()) { + plugin.writeTo(out); + } + out.writeInt(modules.size()); + for (PluginInfo module : getModuleInfos()) { + module.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("plugins"); + for (PluginInfo pluginInfo : getPluginInfos()) { + pluginInfo.toXContent(builder, params); + } + builder.endArray(); + // TODO: not ideal, make a better api for this (e.g. with jar metadata, and so on) + builder.startArray("modules"); + for (PluginInfo moduleInfo : getModuleInfos()) { + moduleInfo.toXContent(builder, params); + } + builder.endArray(); + + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java deleted file mode 100644 index 927a79b6639..00000000000 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.node.info; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; -import org.elasticsearch.plugins.PluginInfo; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -public class PluginsInfo implements Streamable, ToXContent { - static final class Fields { - static final XContentBuilderString PLUGINS = new XContentBuilderString("plugins"); - } - - private List infos; - - public PluginsInfo() { - infos = new ArrayList<>(); - } - - public PluginsInfo(int size) { - infos = new ArrayList<>(size); - } - - /** - * @return an ordered list based on plugins name - */ - public List getInfos() { - Collections.sort(infos, new Comparator() { - @Override - public int compare(final PluginInfo o1, final PluginInfo o2) { - return o1.getName().compareTo(o2.getName()); - } - }); - - return infos; - } - - public void add(PluginInfo info) { - infos.add(info); - } - - public static PluginsInfo readPluginsInfo(StreamInput in) throws IOException { - PluginsInfo infos = new PluginsInfo(); - infos.readFrom(in); - return infos; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - int plugins_size = in.readInt(); - for (int i = 0; i < plugins_size; i++) { - infos.add(PluginInfo.readFromStream(in)); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeInt(infos.size()); - for (PluginInfo plugin : getInfos()) { - plugin.writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(Fields.PLUGINS); - for (PluginInfo pluginInfo : getInfos()) { - pluginInfo.toXContent(builder, params); - } - builder.endArray(); - - return builder; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 4cd050c7dda..1b4b7b06790 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.discovery.DiscoveryStats; import org.elasticsearch.http.HttpStats; import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.indices.breaker.AllCircuitBreakerStats; @@ -78,6 +79,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContent { @Nullable private ScriptStats scriptStats; + @Nullable + private DiscoveryStats discoveryStats; + NodeStats() { } @@ -85,7 +89,8 @@ public class NodeStats extends BaseNodeResponse implements ToXContent { @Nullable OsStats os, @Nullable ProcessStats process, @Nullable JvmStats jvm, @Nullable ThreadPoolStats threadPool, @Nullable FsInfo fs, @Nullable TransportStats transport, @Nullable HttpStats http, @Nullable AllCircuitBreakerStats breaker, - @Nullable ScriptStats scriptStats) { + @Nullable ScriptStats scriptStats, + @Nullable DiscoveryStats discoveryStats) { super(node); this.timestamp = timestamp; this.indices = indices; @@ -98,6 +103,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContent { this.http = http; this.breaker = breaker; this.scriptStats = scriptStats; + this.discoveryStats = discoveryStats; } public long getTimestamp() { @@ -177,6 +183,11 @@ public class NodeStats extends BaseNodeResponse implements ToXContent { return this.scriptStats; } + @Nullable + public DiscoveryStats getDiscoveryStats() { + return this.discoveryStats; + } + public static NodeStats readNodeStats(StreamInput in) throws IOException { NodeStats nodeInfo = new NodeStats(); nodeInfo.readFrom(in); @@ -212,7 +223,8 @@ public class NodeStats extends BaseNodeResponse implements ToXContent { http = HttpStats.readHttpStats(in); } breaker = AllCircuitBreakerStats.readOptionalAllCircuitBreakerStats(in); - scriptStats = in.readOptionalStreamable(new ScriptStats()); + scriptStats = in.readOptionalStreamable(ScriptStats::new); + discoveryStats = in.readOptionalStreamable(() -> new DiscoveryStats(null)); } @@ -270,6 +282,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContent { } out.writeOptionalStreamable(breaker); out.writeOptionalStreamable(scriptStats); + out.writeOptionalStreamable(discoveryStats); } @Override @@ -321,6 +334,10 @@ public class NodeStats extends BaseNodeResponse implements ToXContent { getScriptStats().toXContent(builder, params); } + if (getDiscoveryStats() != null) { + getDiscoveryStats().toXContent(builder, params); + } + return builder; } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index b0d7d7632fb..5916421c1ed 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -41,6 +41,7 @@ public class NodesStatsRequest extends BaseNodesRequest { private boolean http; private boolean breaker; private boolean script; + private boolean discovery; public NodesStatsRequest() { } @@ -67,6 +68,7 @@ public class NodesStatsRequest extends BaseNodesRequest { this.http = true; this.breaker = true; this.script = true; + this.discovery = true; return this; } @@ -84,6 +86,7 @@ public class NodesStatsRequest extends BaseNodesRequest { this.http = false; this.breaker = false; this.script = false; + this.discovery = false; return this; } @@ -234,6 +237,20 @@ public class NodesStatsRequest extends BaseNodesRequest { return this; } + + public boolean discovery() { + return this.discovery; + } + + /** + * Should the node's discovery stats be returned. + */ + public NodesStatsRequest discovery(boolean discovery) { + this.discovery = discovery; + return this; + } + + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -247,6 +264,7 @@ public class NodesStatsRequest extends BaseNodesRequest { http = in.readBoolean(); breaker = in.readBoolean(); script = in.readBoolean(); + discovery = in.readBoolean(); } @Override @@ -262,6 +280,7 @@ public class NodesStatsRequest extends BaseNodesRequest { out.writeBoolean(http); out.writeBoolean(breaker); out.writeBoolean(script); + out.writeBoolean(discovery); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index dfa8007f7cf..dc35eefee7d 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.admin.cluster.node.stats; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; @@ -130,4 +129,12 @@ public class NodesStatsRequestBuilder extends NodesOperationRequestBuilder repositoriesToGet = new LinkedHashSet<>(); // to keep insertion order + for (String repositoryOrPattern : request.repositories()) { + if (Regex.isSimpleMatchPattern(repositoryOrPattern) == false) { + repositoriesToGet.add(repositoryOrPattern); + } else { + for (RepositoryMetaData repository : repositories.repositories()) { + if (Regex.simpleMatch(repositoryOrPattern, repository.name())) { + repositoriesToGet.add(repository.name()); + } + } + } + } List repositoryListBuilder = new ArrayList<>(); - for (String repository : request.repositories()) { + for (String repository : repositoriesToGet) { RepositoryMetaData repositoryMetaData = repositories.repository(repository); if (repositoryMetaData == null) { listener.onFailure(new RepositoryMissingException(repository)); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java index f916c37aec2..d7ec84fb7a5 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java @@ -68,7 +68,7 @@ public class TransportClusterRerouteAction extends TransportMasterNodeAction listener) { - clusterService.submitStateUpdateTask("cluster_reroute (api)", Priority.IMMEDIATE, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("cluster_reroute (api)", new AckedClusterStateUpdateTask(Priority.IMMEDIATE, request, listener) { private volatile ClusterState clusterStateToSend; private volatile RoutingExplanations explanations; diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 983ef37e2b7..73d14a2bb11 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -91,7 +91,8 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct final Settings.Builder transientUpdates = Settings.settingsBuilder(); final Settings.Builder persistentUpdates = Settings.settingsBuilder(); - clusterService.submitStateUpdateTask("cluster_update_settings", Priority.IMMEDIATE, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("cluster_update_settings", + new AckedClusterStateUpdateTask(Priority.IMMEDIATE, request, listener) { private volatile boolean changed = false; @@ -132,7 +133,8 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct // in the components (e.g. FilterAllocationDecider), so the changes made by the first call aren't visible // to the components until the ClusterStateListener instances have been invoked, but are visible after // the first update task has been completed. - clusterService.submitStateUpdateTask("reroute_after_cluster_update_settings", Priority.URGENT, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("reroute_after_cluster_update_settings", + new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { @Override public boolean mustAck(DiscoveryNode discoveryNode) { @@ -162,7 +164,7 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct @Override public ClusterState execute(final ClusterState currentState) { // now, reroute in case things that require it changed (e.g. number of replicas) - RoutingAllocation.Result routingResult = allocationService.reroute(currentState); + RoutingAllocation.Result routingResult = allocationService.reroute(currentState, "reroute after cluster update settings"); if (!routingResult.changed()) { return currentState; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index a0d27979a4c..fd2c97ed5d4 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -41,6 +41,8 @@ public class GetSnapshotsRequest extends MasterNodeRequest private String[] snapshots = Strings.EMPTY_ARRAY; + private boolean ignoreUnavailable; + public GetSnapshotsRequest() { } @@ -112,11 +114,28 @@ public class GetSnapshotsRequest extends MasterNodeRequest return this; } + /** + * Set to true to ignore unavailable snapshots + * + * @return this request + */ + public GetSnapshotsRequest ignoreUnavailable(boolean ignoreUnavailable) { + this.ignoreUnavailable = ignoreUnavailable; + return this; + } + /** + * @return Whether snapshots should be ignored when unavailable (corrupt or temporarily not fetchable) + */ + public boolean ignoreUnavailable() { + return ignoreUnavailable; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); repository = in.readString(); snapshots = in.readStringArray(); + ignoreUnavailable = in.readBoolean(); } @Override @@ -124,5 +143,6 @@ public class GetSnapshotsRequest extends MasterNodeRequest super.writeTo(out); out.writeString(repository); out.writeStringArray(snapshots); + out.writeBoolean(ignoreUnavailable); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java index d989ac3d95c..3b0ac47c69f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java @@ -84,4 +84,16 @@ public class GetSnapshotsRequestBuilder extends MasterNodeOperationRequestBuilde request.snapshots(ArrayUtils.concat(request.snapshots(), snapshots)); return this; } + + /** + * Makes the request ignore unavailable snapshots + * + * @param ignoreUnavailable true to ignore unavailable snapshots. + * @return this builder + */ + public GetSnapshotsRequestBuilder setIgnoreUnavailable(boolean ignoreUnavailable) { + request.ignoreUnavailable(ignoreUnavailable); + return this; + } + } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 6ba93dc3e57..478146de357 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotInfo; @@ -38,7 +39,9 @@ import org.elasticsearch.transport.TransportService; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; /** * Transport Action for get snapshots operation @@ -74,7 +77,7 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction snapshotInfoBuilder = new ArrayList<>(); if (isAllSnapshots(request.snapshots())) { - List snapshots = snapshotsService.snapshots(request.repository()); + List snapshots = snapshotsService.snapshots(request.repository(), request.ignoreUnavailable()); for (Snapshot snapshot : snapshots) { snapshotInfoBuilder.add(new SnapshotInfo(snapshot)); } @@ -84,8 +87,24 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction snapshotsToGet = new LinkedHashSet<>(); // to keep insertion order + List snapshots = null; + for (String snapshotOrPattern : request.snapshots()) { + if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) { + snapshotsToGet.add(snapshotOrPattern); + } else { + if (snapshots == null) { // lazily load snapshots + snapshots = snapshotsService.snapshots(request.repository(), request.ignoreUnavailable()); + } + for (Snapshot snapshot : snapshots) { + if (Regex.simpleMatch(snapshotOrPattern, snapshot.name())) { + snapshotsToGet.add(snapshot.name()); + } + } + } + } + for (String snapshot : snapshotsToGet) { + SnapshotId snapshotId = new SnapshotId(request.repository(), snapshot); snapshotInfoBuilder.add(new SnapshotInfo(snapshotsService.snapshot(snapshotId))); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index d0f91f7e9b9..78c217d3621 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -19,7 +19,7 @@ package org.elasticsearch.action.admin.cluster.stats; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index 0e77e3d6e3a..d8f2a5bbd20 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -74,7 +74,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { versions.add(nodeResponse.nodeInfo().getVersion()); process.addNodeStats(nodeResponse.nodeStats()); jvm.addNodeInfoStats(nodeResponse.nodeInfo(), nodeResponse.nodeStats()); - plugins.addAll(nodeResponse.nodeInfo().getPlugins().getInfos()); + plugins.addAll(nodeResponse.nodeInfo().getPlugins().getPluginInfos()); // now do the stats that should be deduped by hardware (implemented by ip deduping) TransportAddress publishAddress = nodeResponse.nodeInfo().getTransport().address().publishAddress(); @@ -301,6 +301,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { public static class OsStats implements ToXContent, Streamable { int availableProcessors; + int allocatedProcessors; long availableMemory; final ObjectIntHashMap names; @@ -310,6 +311,8 @@ public class ClusterStatsNodes implements ToXContent, Streamable { public void addNodeInfo(NodeInfo nodeInfo) { availableProcessors += nodeInfo.getOs().getAvailableProcessors(); + allocatedProcessors += nodeInfo.getOs().getAllocatedProcessors(); + if (nodeInfo.getOs().getName() != null) { names.addTo(nodeInfo.getOs().getName(), 1); } @@ -319,6 +322,10 @@ public class ClusterStatsNodes implements ToXContent, Streamable { return availableProcessors; } + public int getAllocatedProcessors() { + return allocatedProcessors; + } + public ByteSizeValue getAvailableMemory() { return new ByteSizeValue(availableMemory); } @@ -326,6 +333,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { @Override public void readFrom(StreamInput in) throws IOException { availableProcessors = in.readVInt(); + allocatedProcessors = in.readVInt(); availableMemory = in.readLong(); int size = in.readVInt(); names.clear(); @@ -337,6 +345,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(availableProcessors); + out.writeVInt(allocatedProcessors); out.writeLong(availableMemory); out.writeVInt(names.size()); for (ObjectIntCursor name : names) { @@ -353,6 +362,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { static final class Fields { static final XContentBuilderString AVAILABLE_PROCESSORS = new XContentBuilderString("available_processors"); + static final XContentBuilderString ALLOCATED_PROCESSORS = new XContentBuilderString("allocated_processors"); static final XContentBuilderString NAME = new XContentBuilderString("name"); static final XContentBuilderString NAMES = new XContentBuilderString("names"); static final XContentBuilderString MEM = new XContentBuilderString("mem"); @@ -364,6 +374,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.AVAILABLE_PROCESSORS, availableProcessors); + builder.field(Fields.ALLOCATED_PROCESSORS, allocatedProcessors); builder.startObject(Fields.MEM); builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, availableMemory); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java index aebdf6c31c3..5deb1fd7378 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -19,7 +19,7 @@ package org.elasticsearch.action.admin.cluster.stats; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.support.nodes.BaseNodesResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index cc2f17c433e..3fba14e72bc 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -19,8 +19,7 @@ package org.elasticsearch.action.admin.cluster.stats; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.admin.cluster.health.ClusterIndexHealth; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStats; @@ -31,9 +30,8 @@ import org.elasticsearch.action.support.nodes.BaseNodeRequest; import org.elasticsearch.action.support.nodes.TransportNodesAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -101,7 +99,7 @@ public class TransportClusterStatsAction extends TransportNodesAction shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { @@ -114,26 +112,7 @@ public class TransportClusterStatsAction extends TransportNodesAction { private String field; + private boolean explain = false; + + private String[] attributes = Strings.EMPTY_ARRAY; + public AnalyzeRequest() { } @@ -86,6 +91,9 @@ public class AnalyzeRequest extends SingleShardRequest { } public AnalyzeRequest tokenFilters(String... tokenFilters) { + if (tokenFilters == null) { + throw new IllegalArgumentException("token filters must not be null"); + } this.tokenFilters = tokenFilters; return this; } @@ -95,6 +103,9 @@ public class AnalyzeRequest extends SingleShardRequest { } public AnalyzeRequest charFilters(String... charFilters) { + if (charFilters == null) { + throw new IllegalArgumentException("char filters must not be null"); + } this.charFilters = charFilters; return this; } @@ -112,18 +123,33 @@ public class AnalyzeRequest extends SingleShardRequest { return this.field; } + public AnalyzeRequest explain(boolean explain) { + this.explain = explain; + return this; + } + + public boolean explain() { + return this.explain; + } + + public AnalyzeRequest attributes(String... attributes) { + if (attributes == null) { + throw new IllegalArgumentException("attributes must not be null"); + } + this.attributes = attributes; + return this; + } + + public String[] attributes() { + return this.attributes; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (text == null || text.length == 0) { validationException = addValidationError("text is missing", validationException); } - if (tokenFilters == null) { - validationException = addValidationError("token filters must not be null", validationException); - } - if (charFilters == null) { - validationException = addValidationError("char filters must not be null", validationException); - } return validationException; } @@ -136,6 +162,10 @@ public class AnalyzeRequest extends SingleShardRequest { tokenFilters = in.readStringArray(); charFilters = in.readStringArray(); field = in.readOptionalString(); + if (in.getVersion().onOrAfter(Version.V_2_2_0)) { + explain = in.readBoolean(); + attributes = in.readStringArray(); + } } @Override @@ -147,5 +177,9 @@ public class AnalyzeRequest extends SingleShardRequest { out.writeStringArray(tokenFilters); out.writeStringArray(charFilters); out.writeOptionalString(field); + if (out.getVersion().onOrAfter(Version.V_2_2_0)) { + out.writeBoolean(explain); + out.writeStringArray(attributes); + } } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java index 9ed02e6be1c..23c1739d771 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java @@ -78,6 +78,22 @@ public class AnalyzeRequestBuilder extends SingleShardOperationRequestBuilder, ToXContent { - public static class AnalyzeToken implements Streamable { + public static class AnalyzeToken implements Streamable, ToXContent { private String term; private int startOffset; private int endOffset; private int position; + private Map attributes; private String type; AnalyzeToken() { } - public AnalyzeToken(String term, int position, int startOffset, int endOffset, String type) { + public AnalyzeToken(String term, int position, int startOffset, int endOffset, String type, + Map attributes) { this.term = term; this.position = position; this.startOffset = startOffset; this.endOffset = endOffset; this.type = type; + this.attributes = attributes; } public String getTerm() { @@ -74,6 +79,27 @@ public class AnalyzeResponse extends ActionResponse implements Iterable getAttributes(){ + return this.attributes; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Fields.TOKEN, term); + builder.field(Fields.START_OFFSET, startOffset); + builder.field(Fields.END_OFFSET, endOffset); + builder.field(Fields.TYPE, type); + builder.field(Fields.POSITION, position); + if (attributes != null && !attributes.isEmpty()) { + for (Map.Entry entity : attributes.entrySet()) { + builder.field(entity.getKey(), entity.getValue()); + } + } + builder.endObject(); + return builder; + } + public static AnalyzeToken readAnalyzeToken(StreamInput in) throws IOException { AnalyzeToken analyzeToken = new AnalyzeToken(); analyzeToken.readFrom(in); @@ -87,6 +113,9 @@ public class AnalyzeResponse extends ActionResponse implements Iterable) in.readGenericValue(); + } } @Override @@ -96,22 +125,32 @@ public class AnalyzeResponse extends ActionResponse implements Iterable tokens; AnalyzeResponse() { } - public AnalyzeResponse(List tokens) { + public AnalyzeResponse(List tokens, DetailAnalyzeResponse detail) { this.tokens = tokens; + this.detail = detail; } public List getTokens() { return this.tokens; } + public DetailAnalyzeResponse detail() { + return this.detail; + } + @Override public Iterator iterator() { return tokens.iterator(); @@ -119,17 +158,19 @@ public class AnalyzeResponse extends ActionResponse implements Iterable 0) { + charfilters = new CharFilteredText[size]; + for (int i = 0; i < size; i++) { + charfilters[i] = CharFilteredText.readCharFilteredText(in); + } + } + size = in.readVInt(); + if (size > 0) { + tokenfilters = new AnalyzeTokenList[size]; + for (int i = 0; i < size; i++) { + tokenfilters[i] = AnalyzeTokenList.readAnalyzeTokenList(in); + } + } + } else { + analyzer = AnalyzeTokenList.readAnalyzeTokenList(in); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(customAnalyzer); + if (customAnalyzer) { + tokenizer.writeTo(out); + if (charfilters != null) { + out.writeVInt(charfilters.length); + for (CharFilteredText charfilter : charfilters) { + charfilter.writeTo(out); + } + } else { + out.writeVInt(0); + } + if (tokenfilters != null) { + out.writeVInt(tokenfilters.length); + for (AnalyzeTokenList tokenfilter : tokenfilters) { + tokenfilter.writeTo(out); + } + } else { + out.writeVInt(0); + } + } else { + analyzer.writeTo(out); + } + } + + public static class AnalyzeTokenList implements Streamable, ToXContent { + private String name; + private AnalyzeResponse.AnalyzeToken[] tokens; + + AnalyzeTokenList() { + } + + public AnalyzeTokenList(String name, AnalyzeResponse.AnalyzeToken[] tokens) { + this.name = name; + this.tokens = tokens; + } + + public String getName() { + return name; + } + + public AnalyzeResponse.AnalyzeToken[] getTokens() { + return tokens; + } + + public static AnalyzeTokenList readAnalyzeTokenList(StreamInput in) throws IOException { + AnalyzeTokenList list = new AnalyzeTokenList(); + list.readFrom(in); + return list; + } + + public XContentBuilder toXContentWithoutObject(XContentBuilder builder, Params params) throws IOException { + builder.field(Fields.NAME, this.name); + builder.startArray(AnalyzeResponse.Fields.TOKENS); + for (AnalyzeResponse.AnalyzeToken token : tokens) { + token.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Fields.NAME, this.name); + builder.startArray(AnalyzeResponse.Fields.TOKENS); + for (AnalyzeResponse.AnalyzeToken token : tokens) { + token.toXContent(builder, params); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + name = in.readString(); + int size = in.readVInt(); + if (size > 0) { + tokens = new AnalyzeResponse.AnalyzeToken[size]; + for (int i = 0; i < size; i++) { + tokens[i] = AnalyzeResponse.AnalyzeToken.readAnalyzeToken(in); + } + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + if (tokens != null) { + out.writeVInt(tokens.length); + for (AnalyzeResponse.AnalyzeToken token : tokens) { + token.writeTo(out); + } + } else { + out.writeVInt(0); + } + } + } + + public static class CharFilteredText implements Streamable, ToXContent { + private String name; + private String[] texts; + CharFilteredText() { + } + + public CharFilteredText(String name, String[] texts) { + this.name = name; + if (texts != null) { + this.texts = texts; + } else { + this.texts = Strings.EMPTY_ARRAY; + } + } + + public String getName() { + return name; + } + + public String[] getTexts() { + return texts; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Fields.NAME, name); + builder.field(Fields.FILTERED_TEXT, texts); + builder.endObject(); + return builder; + } + + public static CharFilteredText readCharFilteredText(StreamInput in) throws IOException { + CharFilteredText text = new CharFilteredText(); + text.readFrom(in); + return text; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + name = in.readString(); + texts = in.readStringArray(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeStringArray(texts); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 4f7a605341e..ecdf977b923 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -20,42 +20,40 @@ package org.elasticsearch.action.admin.indices.analyze; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.AttributeReflector; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.ShardsIterator; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FastStringReader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.analysis.CharFilterFactory; -import org.elasticsearch.index.analysis.CharFilterFactoryFactory; -import org.elasticsearch.index.analysis.CustomAnalyzer; -import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.index.analysis.TokenFilterFactoryFactory; -import org.elasticsearch.index.analysis.TokenizerFactory; -import org.elasticsearch.index.analysis.TokenizerFactoryFactory; +import org.elasticsearch.index.analysis.*; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.io.Reader; +import java.util.*; /** * Transport action used to execute analyze requests @@ -63,17 +61,15 @@ import java.util.List; public class TransportAnalyzeAction extends TransportSingleShardAction { private final IndicesService indicesService; - private final IndicesAnalysisService indicesAnalysisService; - - private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + private final Environment environment; @Inject public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, - IndicesService indicesService, IndicesAnalysisService indicesAnalysisService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver) { + IndicesService indicesService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, Environment environment) { super(settings, AnalyzeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, AnalyzeRequest::new, ThreadPool.Names.INDEX); this.indicesService = indicesService; - this.indicesAnalysisService = indicesAnalysisService; + this.environment = environment; } @Override @@ -105,53 +101,69 @@ public class TransportAnalyzeAction extends TransportSingleShardAction tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(request.tokenizer()); if (tokenizerFactoryFactory == null) { throw new IllegalArgumentException("failed to find global tokenizer under [" + request.tokenizer() + "]"); } - tokenizerFactory = tokenizerFactoryFactory.create(request.tokenizer(), DEFAULT_SETTINGS); + tokenizerFactory = tokenizerFactoryFactory.get(environment, request.tokenizer()); } else { - tokenizerFactory = indexService.analysisService().tokenizer(request.tokenizer()); + tokenizerFactory = analysisService.tokenizer(request.tokenizer()); if (tokenizerFactory == null) { throw new IllegalArgumentException("failed to find tokenizer under [" + request.tokenizer() + "]"); } @@ -162,14 +174,14 @@ public class TransportAnalyzeAction extends TransportSingleShardAction tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilterName); if (tokenFilterFactoryFactory == null) { throw new IllegalArgumentException("failed to find global token filter under [" + tokenFilterName + "]"); } - tokenFilterFactories[i] = tokenFilterFactoryFactory.create(tokenFilterName, DEFAULT_SETTINGS); + tokenFilterFactories[i] = tokenFilterFactoryFactory.get(environment, tokenFilterName); } else { - tokenFilterFactories[i] = indexService.analysisService().tokenFilter(tokenFilterName); + tokenFilterFactories[i] = analysisService.tokenFilter(tokenFilterName); if (tokenFilterFactories[i] == null) { throw new IllegalArgumentException("failed to find token filter under [" + tokenFilterName + "]"); } @@ -185,20 +197,20 @@ public class TransportAnalyzeAction extends TransportSingleShardAction charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilterName); if (charFilterFactoryFactory == null) { throw new IllegalArgumentException("failed to find global char filter under [" + charFilterName + "]"); } - charFilterFactories[i] = charFilterFactoryFactory.create(charFilterName, DEFAULT_SETTINGS); + charFilterFactories[i] = charFilterFactoryFactory.get(environment, charFilterName); } else { - charFilterFactories[i] = indexService.analysisService().charFilter(charFilterName); + charFilterFactories[i] = analysisService.charFilter(charFilterName); if (charFilterFactories[i] == null) { - throw new IllegalArgumentException("failed to find token char under [" + charFilterName + "]"); + throw new IllegalArgumentException("failed to find char filter under [" + charFilterName + "]"); } } if (charFilterFactories[i] == null) { - throw new IllegalArgumentException("failed to find token char under [" + charFilterName + "]"); + throw new IllegalArgumentException("failed to find char filter under [" + charFilterName + "]"); } } } @@ -206,16 +218,33 @@ public class TransportAnalyzeAction extends TransportSingleShardAction tokens = null; + DetailAnalyzeResponse detail = null; + + if (request.explain()) { + detail = detailAnalyze(request, analyzer, field); + } else { + tokens = simpleAnalyze(request, analyzer, field); + } + + if (closeAnalyzer) { + analyzer.close(); + } + + return new AnalyzeResponse(tokens, detail); + } + + private static List simpleAnalyze(AnalyzeRequest request, Analyzer analyzer, String field) { List tokens = new ArrayList<>(); int lastPosition = -1; int lastOffset = 0; @@ -232,7 +261,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction 0) { lastPosition = lastPosition + increment; } - tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), type.type())); + tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), type.type(), null)); } stream.end(); @@ -245,11 +274,211 @@ public class TransportAnalyzeAction extends TransportSingleShardAction includeAttributes = new HashSet<>(); + if (request.attributes() != null) { + for (String attribute : request.attributes()) { + includeAttributes.add(attribute.toLowerCase(Locale.ROOT)); + } } - return new AnalyzeResponse(tokens); + CustomAnalyzer customAnalyzer = null; + if (analyzer instanceof CustomAnalyzer) { + customAnalyzer = (CustomAnalyzer) analyzer; + } else if (analyzer instanceof NamedAnalyzer && ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) { + customAnalyzer = (CustomAnalyzer) ((NamedAnalyzer) analyzer).analyzer(); + } + + if (customAnalyzer != null) { + // customAnalyzer = divide charfilter, tokenizer tokenfilters + CharFilterFactory[] charFilterFactories = customAnalyzer.charFilters(); + TokenizerFactory tokenizerFactory = customAnalyzer.tokenizerFactory(); + TokenFilterFactory[] tokenFilterFactories = customAnalyzer.tokenFilters(); + + String[][] charFiltersTexts = new String[charFilterFactories != null ? charFilterFactories.length : 0][request.text().length]; + TokenListCreator[] tokenFiltersTokenListCreator = new TokenListCreator[tokenFilterFactories != null ? tokenFilterFactories.length : 0]; + + TokenListCreator tokenizerTokenListCreator = new TokenListCreator(); + + for (int textIndex = 0; textIndex < request.text().length; textIndex++) { + String charFilteredSource = request.text()[textIndex]; + + Reader reader = new FastStringReader(charFilteredSource); + if (charFilterFactories != null) { + + for (int charFilterIndex = 0; charFilterIndex < charFilterFactories.length; charFilterIndex++) { + reader = charFilterFactories[charFilterIndex].create(reader); + Reader readerForWriteOut = new FastStringReader(charFilteredSource); + readerForWriteOut = charFilterFactories[charFilterIndex].create(readerForWriteOut); + charFilteredSource = writeCharStream(readerForWriteOut); + charFiltersTexts[charFilterIndex][textIndex] = charFilteredSource; + } + } + + // analyzing only tokenizer + Tokenizer tokenizer = tokenizerFactory.create(); + tokenizer.setReader(reader); + tokenizerTokenListCreator.analyze(tokenizer, customAnalyzer, field, includeAttributes); + + // analyzing each tokenfilter + if (tokenFilterFactories != null) { + for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFilterFactories.length; tokenFilterIndex++) { + if (tokenFiltersTokenListCreator[tokenFilterIndex] == null) { + tokenFiltersTokenListCreator[tokenFilterIndex] = new TokenListCreator(); + } + TokenStream stream = createStackedTokenStream(request.text()[textIndex], + charFilterFactories, tokenizerFactory, tokenFilterFactories, tokenFilterIndex + 1); + tokenFiltersTokenListCreator[tokenFilterIndex].analyze(stream, customAnalyzer, field, includeAttributes); + } + } + } + + DetailAnalyzeResponse.CharFilteredText[] charFilteredLists = new DetailAnalyzeResponse.CharFilteredText[charFiltersTexts.length]; + if (charFilterFactories != null) { + for (int charFilterIndex = 0; charFilterIndex < charFiltersTexts.length; charFilterIndex++) { + charFilteredLists[charFilterIndex] = new DetailAnalyzeResponse.CharFilteredText( + charFilterFactories[charFilterIndex].name(), charFiltersTexts[charFilterIndex]); + } + } + DetailAnalyzeResponse.AnalyzeTokenList[] tokenFilterLists = new DetailAnalyzeResponse.AnalyzeTokenList[tokenFiltersTokenListCreator.length]; + if (tokenFilterFactories != null) { + for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFiltersTokenListCreator.length; tokenFilterIndex++) { + tokenFilterLists[tokenFilterIndex] = new DetailAnalyzeResponse.AnalyzeTokenList( + tokenFilterFactories[tokenFilterIndex].name(), tokenFiltersTokenListCreator[tokenFilterIndex].getArrayTokens()); + } + } + detailResponse = new DetailAnalyzeResponse(charFilteredLists, new DetailAnalyzeResponse.AnalyzeTokenList(tokenizerFactory.name(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); + } else { + String name; + if (analyzer instanceof NamedAnalyzer) { + name = ((NamedAnalyzer) analyzer).name(); + } else { + name = analyzer.getClass().getName(); + } + + TokenListCreator tokenListCreator = new TokenListCreator(); + for (String text : request.text()) { + tokenListCreator.analyze(analyzer.tokenStream(field, text), analyzer, field, + includeAttributes); + } + detailResponse = new DetailAnalyzeResponse(new DetailAnalyzeResponse.AnalyzeTokenList(name, tokenListCreator.getArrayTokens())); + } + return detailResponse; + } + + private static TokenStream createStackedTokenStream(String source, CharFilterFactory[] charFilterFactories, TokenizerFactory tokenizerFactory, TokenFilterFactory[] tokenFilterFactories, int current) { + Reader reader = new FastStringReader(source); + for (CharFilterFactory charFilterFactory : charFilterFactories) { + reader = charFilterFactory.create(reader); + } + Tokenizer tokenizer = tokenizerFactory.create(); + tokenizer.setReader(reader); + TokenStream tokenStream = tokenizer; + for (int i = 0; i < current; i++) { + tokenStream = tokenFilterFactories[i].create(tokenStream); + } + return tokenStream; + } + + private static String writeCharStream(Reader input) { + final int BUFFER_SIZE = 1024; + char[] buf = new char[BUFFER_SIZE]; + int len; + StringBuilder sb = new StringBuilder(); + do { + try { + len = input.read(buf, 0, BUFFER_SIZE); + } catch (IOException e) { + throw new ElasticsearchException("failed to analyze (charFiltering)", e); + } + if (len > 0) + sb.append(buf, 0, len); + } while (len == BUFFER_SIZE); + return sb.toString(); + } + + private static class TokenListCreator { + int lastPosition = -1; + int lastOffset = 0; + List tokens; + + TokenListCreator() { + tokens = new ArrayList<>(); + } + + private void analyze(TokenStream stream, Analyzer analyzer, String field, Set includeAttributes) { + try { + stream.reset(); + CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); + PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); + OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); + TypeAttribute type = stream.addAttribute(TypeAttribute.class); + + while (stream.incrementToken()) { + int increment = posIncr.getPositionIncrement(); + if (increment > 0) { + lastPosition = lastPosition + increment; + } + tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), + lastOffset +offset.endOffset(), type.type(), extractExtendedAttributes(stream, includeAttributes))); + + } + stream.end(); + lastOffset += offset.endOffset(); + lastPosition += posIncr.getPositionIncrement(); + + lastPosition += analyzer.getPositionIncrementGap(field); + lastOffset += analyzer.getOffsetGap(field); + + } catch (IOException e) { + throw new ElasticsearchException("failed to analyze", e); + } finally { + IOUtils.closeWhileHandlingException(stream); + } + } + + private AnalyzeResponse.AnalyzeToken[] getArrayTokens() { + return tokens.toArray(new AnalyzeResponse.AnalyzeToken[tokens.size()]); + } + + } + + /** + * other attribute extract object. + * Extracted object group by AttributeClassName + * + * @param stream current TokenStream + * @param includeAttributes filtering attributes + * @return Map<key value> + */ + private static Map extractExtendedAttributes(TokenStream stream, final Set includeAttributes) { + final Map extendedAttributes = new TreeMap<>(); + + stream.reflectWith(new AttributeReflector() { + @Override + public void reflect(Class attClass, String key, Object value) { + if (CharTermAttribute.class.isAssignableFrom(attClass)) + return; + if (PositionIncrementAttribute.class.isAssignableFrom(attClass)) + return; + if (OffsetAttribute.class.isAssignableFrom(attClass)) + return; + if (TypeAttribute.class.isAssignableFrom(attClass)) + return; + if (includeAttributes == null || includeAttributes.isEmpty() || includeAttributes.contains(key.toLowerCase(Locale.ROOT))) { + if (value instanceof BytesRef) { + final BytesRef p = (BytesRef) value; + value = p.toString(); + } + extendedAttributes.put(key, value); + } + } + }); + + return extendedAttributes; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java index c83fc7d177d..2c25ee34f18 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -36,15 +36,15 @@ import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.Arrays; - /** * Close index action */ -public class TransportCloseIndexAction extends TransportMasterNodeAction { +public class TransportCloseIndexAction extends TransportMasterNodeAction implements NodeSettingsService.Listener { private final MetaDataIndexStateService indexStateService; private final DestructiveOperations destructiveOperations; + private volatile boolean closeIndexEnabled; + public static final String SETTING_CLUSTER_INDICES_CLOSE_ENABLE = "cluster.indices.close.enable"; @Inject public TransportCloseIndexAction(Settings settings, TransportService transportService, ClusterService clusterService, @@ -54,6 +54,8 @@ public class TransportCloseIndexAction extends TransportMasterNodeAction listener) { destructiveOperations.failDestructive(request.indices()); + if (closeIndexEnabled == false) { + throw new IllegalStateException("closing indices is disabled - set [" + SETTING_CLUSTER_INDICES_CLOSE_ENABLE + ": true] to enable it. NOTE: closed indices still consume a significant amount of diskspace"); + } super.doExecute(request, listener); } @@ -99,4 +104,13 @@ public class TransportCloseIndexAction extends TransportMasterNodeAction listener) { - String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); if (concreteIndices.length == 0) { listener.onResponse(new DeleteIndexResponse(true)); return; } - // TODO: this API should be improved, currently, if one delete index failed, we send a failure, we should send a response array that includes all the indices that were deleted - final CountDown count = new CountDown(concreteIndices.length); - for (final String index : concreteIndices) { - deleteIndexService.deleteIndex(new MetaDataDeleteIndexService.Request(index).timeout(request.timeout()).masterTimeout(request.masterNodeTimeout()), new MetaDataDeleteIndexService.Listener() { + deleteIndexService.deleteIndices(new MetaDataDeleteIndexService.Request(concreteIndices).timeout(request.timeout()).masterTimeout(request.masterNodeTimeout()), new MetaDataDeleteIndexService.Listener() { - private volatile Throwable lastFailure; - private volatile boolean ack = true; + @Override + public void onResponse(MetaDataDeleteIndexService.Response response) { + listener.onResponse(new DeleteIndexResponse(response.acknowledged())); + } - @Override - public void onResponse(MetaDataDeleteIndexService.Response response) { - if (!response.acknowledged()) { - ack = false; - } - if (count.countDown()) { - if (lastFailure != null) { - listener.onFailure(lastFailure); - } else { - listener.onResponse(new DeleteIndexResponse(ack)); - } - } - } - - @Override - public void onFailure(Throwable t) { - logger.debug("[{}] failed to delete index", t, index); - lastFailure = t; - if (count.countDown()) { - listener.onFailure(t); - } - } - }); - } + @Override + public void onFailure(Throwable t) { + listener.onFailure(t); + } + }); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java index 1fce7d1df61..2fd92451752 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java @@ -74,7 +74,7 @@ public class TransportTypesExistsAction extends TransportMasterNodeReadAction mappings = state.metaData().getIndices().get(concreteIndex).mappings(); + ImmutableOpenMap mappings = state.metaData().getIndices().get(concreteIndex).getMappings(); if (mappings.isEmpty()) { listener.onResponse(new TypesExistsResponse(false)); return; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java index 10db46c1da0..ccf06be8bd0 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.indices.flush; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.shard.ShardId; import java.io.IOException; @@ -29,8 +30,8 @@ public class ShardFlushRequest extends ReplicationRequest { private FlushRequest request = new FlushRequest(); - public ShardFlushRequest(FlushRequest request) { - super(request); + public ShardFlushRequest(FlushRequest request, ShardId shardId) { + super(request, shardId); this.request = request; } @@ -53,5 +54,8 @@ public class ShardFlushRequest extends ReplicationRequest { request.writeTo(out); } - + @Override + public String toString() { + return "flush {" + super.toString() + "}"; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index ac159625420..d2a8f1abcbf 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -19,7 +19,7 @@ package org.elasticsearch.action.admin.indices.flush; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.TransportBroadcastReplicationAction; @@ -36,7 +36,7 @@ import java.util.List; /** * Flush Action. */ -public class TransportFlushAction extends TransportBroadcastReplicationAction { +public class TransportFlushAction extends TransportBroadcastReplicationAction { @Inject public TransportFlushAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, @@ -47,13 +47,13 @@ public class TransportFlushAction extends TransportBroadcastReplicationAction { +public class TransportShardFlushAction extends TransportReplicationAction { public static final String NAME = FlushAction.NAME + "[s]"; @@ -56,20 +53,20 @@ public class TransportShardFlushAction extends TransportReplicationAction shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { - IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).getShard(shardRequest.shardId.id()); - indexShard.flush(shardRequest.request.getRequest()); + protected Tuple shardOperationOnPrimary(MetaData metaData, ShardFlushRequest shardRequest) throws Throwable { + IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id()); + indexShard.flush(shardRequest.getRequest()); logger.trace("{} flush request executed on primary", indexShard.shardId()); - return new Tuple<>(new ActionWriteResponse(), shardRequest.request); + return new Tuple<>(new ReplicationResponse(), shardRequest); } @Override - protected void shardOperationOnReplica(ShardId shardId, ShardFlushRequest request) { + protected void shardOperationOnReplica(ShardFlushRequest request) { IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id()); indexShard.flush(request.getRequest()); logger.trace("{} flush request executed on replica", indexShard.shardId()); @@ -81,18 +78,13 @@ public class TransportShardFlushAction extends TransportReplicationAction { +public class ForceMergeAction extends Action { - public static final OptimizeAction INSTANCE = new OptimizeAction(); - public static final String NAME = "indices:admin/optimize"; + public static final ForceMergeAction INSTANCE = new ForceMergeAction(); + public static final String NAME = "indices:admin/forcemerge"; - private OptimizeAction() { + private ForceMergeAction() { super(NAME); } @Override - public OptimizeResponse newResponse() { - return new OptimizeResponse(); + public ForceMergeResponse newResponse() { + return new ForceMergeResponse(); } @Override - public OptimizeRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new OptimizeRequestBuilder(client, this); + public ForceMergeRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new ForceMergeRequestBuilder(client, this); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java similarity index 59% rename from core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequest.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java index cf3006baa3b..2b8fe5d2b01 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.admin.indices.optimize; +package org.elasticsearch.action.admin.indices.forcemerge; import org.elasticsearch.action.support.broadcast.BroadcastRequest; import org.elasticsearch.common.io.stream.StreamInput; @@ -26,17 +26,18 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; /** - * A request to optimize one or more indices. In order to optimize on all the indices, pass an empty array or - * null for the indices. - *

- * {@link #maxNumSegments(int)} allows to control the number of segments to optimize down to. By default, will - * cause the optimize process to optimize down to half the configured number of segments. + * A request to force merging the segments of one or more indices. In order to + * run a merge on all the indices, pass an empty array or null for the + * indices. + * {@link #maxNumSegments(int)} allows to control the number of segments + * to force merge down to. Defaults to simply checking if a merge needs + * to execute, and if so, executes it * - * @see org.elasticsearch.client.Requests#optimizeRequest(String...) - * @see org.elasticsearch.client.IndicesAdminClient#optimize(OptimizeRequest) - * @see OptimizeResponse + * @see org.elasticsearch.client.Requests#forceMergeRequest(String...) + * @see org.elasticsearch.client.IndicesAdminClient#forceMerge(ForceMergeRequest) + * @see ForceMergeResponse */ -public class OptimizeRequest extends BroadcastRequest { +public class ForceMergeRequest extends BroadcastRequest { public static final class Defaults { public static final int MAX_NUM_SEGMENTS = -1; @@ -49,63 +50,63 @@ public class OptimizeRequest extends BroadcastRequest { private boolean flush = Defaults.FLUSH; /** - * Constructs an optimization request over one or more indices. + * Constructs a merge request over one or more indices. * - * @param indices The indices to optimize, no indices passed means all indices will be optimized. + * @param indices The indices to merge, no indices passed means all indices will be merged. */ - public OptimizeRequest(String... indices) { + public ForceMergeRequest(String... indices) { super(indices); } - public OptimizeRequest() { + public ForceMergeRequest() { } /** - * Will optimize the index down to <= maxNumSegments. By default, will cause the optimize - * process to optimize down to half the configured number of segments. + * Will merge the index down to <= maxNumSegments. By default, will cause the merge + * process to merge down to half the configured number of segments. */ public int maxNumSegments() { return maxNumSegments; } /** - * Will optimize the index down to <= maxNumSegments. By default, will cause the optimize - * process to optimize down to half the configured number of segments. + * Will merge the index down to <= maxNumSegments. By default, will cause the merge + * process to merge down to half the configured number of segments. */ - public OptimizeRequest maxNumSegments(int maxNumSegments) { + public ForceMergeRequest maxNumSegments(int maxNumSegments) { this.maxNumSegments = maxNumSegments; return this; } /** - * Should the optimization only expunge deletes from the index, without full optimization. - * Defaults to full optimization (false). + * Should the merge only expunge deletes from the index, without full merging. + * Defaults to full merging (false). */ public boolean onlyExpungeDeletes() { return onlyExpungeDeletes; } /** - * Should the optimization only expunge deletes from the index, without full optimization. - * Defaults to full optimization (false). + * Should the merge only expunge deletes from the index, without full merge. + * Defaults to full merging (false). */ - public OptimizeRequest onlyExpungeDeletes(boolean onlyExpungeDeletes) { + public ForceMergeRequest onlyExpungeDeletes(boolean onlyExpungeDeletes) { this.onlyExpungeDeletes = onlyExpungeDeletes; return this; } /** - * Should flush be performed after the optimization. Defaults to true. + * Should flush be performed after the merge. Defaults to true. */ public boolean flush() { return flush; } /** - * Should flush be performed after the optimization. Defaults to true. + * Should flush be performed after the merge. Defaults to true. */ - public OptimizeRequest flush(boolean flush) { + public ForceMergeRequest flush(boolean flush) { this.flush = flush; return this; } @@ -128,7 +129,7 @@ public class OptimizeRequest extends BroadcastRequest { @Override public String toString() { - return "OptimizeRequest{" + + return "ForceMergeRequest{" + "maxNumSegments=" + maxNumSegments + ", onlyExpungeDeletes=" + onlyExpungeDeletes + ", flush=" + flush + diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java new file mode 100644 index 00000000000..138db7078ee --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.forcemerge; + +import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +/** + * A request to force merge one or more indices. In order to force merge all + * indices, pass an empty array or null for the indices. + * {@link #setMaxNumSegments(int)} allows to control the number of segments to force + * merge down to. By default, will cause the force merge process to merge down + * to half the configured number of segments. + */ +public class ForceMergeRequestBuilder extends BroadcastOperationRequestBuilder { + + public ForceMergeRequestBuilder(ElasticsearchClient client, ForceMergeAction action) { + super(client, action, new ForceMergeRequest()); + } + + /** + * Will force merge the index down to <= maxNumSegments. By default, will + * cause the merge process to merge down to half the configured number of + * segments. + */ + public ForceMergeRequestBuilder setMaxNumSegments(int maxNumSegments) { + request.maxNumSegments(maxNumSegments); + return this; + } + + /** + * Should the merge only expunge deletes from the index, without full merging. + * Defaults to full merging (false). + */ + public ForceMergeRequestBuilder setOnlyExpungeDeletes(boolean onlyExpungeDeletes) { + request.onlyExpungeDeletes(onlyExpungeDeletes); + return this; + } + + /** + * Should flush be performed after the merge. Defaults to true. + */ + public ForceMergeRequestBuilder setFlush(boolean flush) { + request.flush(flush); + return this; + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeResponse.java similarity index 68% rename from core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeResponse.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeResponse.java index 88341ef2619..5b999c0f5b7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeResponse.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.admin.indices.optimize; +package org.elasticsearch.action.admin.indices.forcemerge; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; @@ -28,27 +28,14 @@ import java.io.IOException; import java.util.List; /** - * A response for optimize action. - * - * + * A response for force merge action. */ -public class OptimizeResponse extends BroadcastResponse { - - OptimizeResponse() { +public class ForceMergeResponse extends BroadcastResponse { + ForceMergeResponse() { } - OptimizeResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { + ForceMergeResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { super(totalShards, successfulShards, failedShards, shardFailures); } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java similarity index 68% rename from core/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java index 764022b28ab..0119b1693aa 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/TransportOptimizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.admin.indices.optimize; +package org.elasticsearch.action.admin.indices.forcemerge; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.ActionFilters; @@ -41,18 +41,18 @@ import java.io.IOException; import java.util.List; /** - * Optimize index/indices action. + * ForceMerge index/indices action. */ -public class TransportOptimizeAction extends TransportBroadcastByNodeAction { +public class TransportForceMergeAction extends TransportBroadcastByNodeAction { private final IndicesService indicesService; @Inject - public TransportOptimizeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, + public TransportForceMergeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, IndicesService indicesService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, OptimizeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, - OptimizeRequest::new, ThreadPool.Names.OPTIMIZE); + super(settings, ForceMergeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, + ForceMergeRequest::new, ThreadPool.Names.FORCE_MERGE); this.indicesService = indicesService; } @@ -62,21 +62,21 @@ public class TransportOptimizeAction extends TransportBroadcastByNodeAction responses, List shardFailures, ClusterState clusterState) { - return new OptimizeResponse(totalShards, successfulShards, failedShards, shardFailures); + protected ForceMergeResponse newResponse(ForceMergeRequest request, int totalShards, int successfulShards, int failedShards, List responses, List shardFailures, ClusterState clusterState) { + return new ForceMergeResponse(totalShards, successfulShards, failedShards, shardFailures); } @Override - protected OptimizeRequest readRequestFrom(StreamInput in) throws IOException { - final OptimizeRequest request = new OptimizeRequest(); + protected ForceMergeRequest readRequestFrom(StreamInput in) throws IOException { + final ForceMergeRequest request = new ForceMergeRequest(); request.readFrom(in); return request; } @Override - protected EmptyResult shardOperation(OptimizeRequest request, ShardRouting shardRouting) throws IOException { + protected EmptyResult shardOperation(ForceMergeRequest request, ShardRouting shardRouting) throws IOException { IndexShard indexShard = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()).getShard(shardRouting.shardId().id()); - indexShard.optimize(request); + indexShard.forceMerge(request); return EmptyResult.INSTANCE; } @@ -84,17 +84,17 @@ public class TransportOptimizeAction extends TransportBroadcastByNodeAction listener) { - final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); - PutMappingClusterStateUpdateRequest updateRequest = new PutMappingClusterStateUpdateRequest() - .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()) - .indices(concreteIndices).type(request.type()) - .updateAllTypes(request.updateAllTypes()) - .source(request.source()); + try { + final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + PutMappingClusterStateUpdateRequest updateRequest = new PutMappingClusterStateUpdateRequest() + .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()) + .indices(concreteIndices).type(request.type()) + .updateAllTypes(request.updateAllTypes()) + .source(request.source()); - metaDataMappingService.putMapping(updateRequest, new ActionListener() { + metaDataMappingService.putMapping(updateRequest, new ActionListener() { - @Override - public void onResponse(ClusterStateUpdateResponse response) { - listener.onResponse(new PutMappingResponse(response.isAcknowledged())); - } + @Override + public void onResponse(ClusterStateUpdateResponse response) { + listener.onResponse(new PutMappingResponse(response.isAcknowledged())); + } - @Override - public void onFailure(Throwable t) { - logger.debug("failed to put mappings on indices [{}], type [{}]", t, concreteIndices, request.type()); - listener.onFailure(t); - } - }); + @Override + public void onFailure(Throwable t) { + logger.debug("failed to put mappings on indices [{}], type [{}]", t, concreteIndices, request.type()); + listener.onFailure(t); + } + }); + } catch (IndexNotFoundException ex) { + logger.debug("failed to put mappings on indices [{}], type [{}]", ex, request.indices(), request.type()); + throw ex; + } } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequestBuilder.java deleted file mode 100644 index d318492e904..00000000000 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/optimize/OptimizeRequestBuilder.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.indices.optimize; - -import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; -import org.elasticsearch.client.ElasticsearchClient; - -/** - * A request to optimize one or more indices. In order to optimize on all the indices, pass an empty array or - * null for the indices. - *

{@link #setMaxNumSegments(int)} allows to control the number of segments to optimize down to. By default, will - * cause the optimize process to optimize down to half the configured number of segments. - */ -public class OptimizeRequestBuilder extends BroadcastOperationRequestBuilder { - - public OptimizeRequestBuilder(ElasticsearchClient client, OptimizeAction action) { - super(client, action, new OptimizeRequest()); - } - - /** - * Will optimize the index down to <= maxNumSegments. By default, will cause the optimize - * process to optimize down to half the configured number of segments. - */ - public OptimizeRequestBuilder setMaxNumSegments(int maxNumSegments) { - request.maxNumSegments(maxNumSegments); - return this; - } - - /** - * Should the optimization only expunge deletes from the index, without full optimization. - * Defaults to full optimization (false). - */ - public OptimizeRequestBuilder setOnlyExpungeDeletes(boolean onlyExpungeDeletes) { - request.onlyExpungeDeletes(onlyExpungeDeletes); - return this; - } - - /** - * Should flush be performed after the optimization. Defaults to true. - */ - public OptimizeRequestBuilder setFlush(boolean flush) { - request.flush(flush); - return this; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java index b003f063a02..9798e189f7b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java @@ -77,7 +77,7 @@ public class TransportRecoveryAction extends TransportBroadcastByNodeAction()); + shardResponses.put(indexName, new ArrayList<>()); } if (request.activeOnly()) { if (recoveryState.getStage() != RecoveryState.Stage.DONE) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index e2d978d306f..a76b714b31d 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -19,7 +19,7 @@ package org.elasticsearch.action.admin.indices.refresh; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.ReplicationRequest; @@ -37,7 +37,7 @@ import java.util.List; /** * Refresh action. */ -public class TransportRefreshAction extends TransportBroadcastReplicationAction { +public class TransportRefreshAction extends TransportBroadcastReplicationAction { @Inject public TransportRefreshAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, @@ -48,13 +48,13 @@ public class TransportRefreshAction extends TransportBroadcastReplicationAction< } @Override - protected ActionWriteResponse newShardResponse() { - return new ActionWriteResponse(); + protected ReplicationResponse newShardResponse() { + return new ReplicationResponse(); } @Override protected ReplicationRequest newShardRequest(RefreshRequest request, ShardId shardId) { - return new ReplicationRequest(request).setShardId(shardId); + return new ReplicationRequest(request, shardId); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index a06483acb1d..c78977fb362 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -19,18 +19,16 @@ package org.elasticsearch.action.admin.indices.refresh; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; -import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -43,7 +41,7 @@ import org.elasticsearch.transport.TransportService; /** * */ -public class TransportShardRefreshAction extends TransportReplicationAction { +public class TransportShardRefreshAction extends TransportReplicationAction { public static final String NAME = RefreshAction.NAME + "[s]"; @@ -57,20 +55,21 @@ public class TransportShardRefreshAction extends TransportReplicationAction shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { - IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).getShard(shardRequest.shardId.id()); + protected Tuple shardOperationOnPrimary(MetaData metaData, ReplicationRequest shardRequest) throws Throwable { + IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id()); indexShard.refresh("api"); logger.trace("{} refresh request executed on primary", indexShard.shardId()); - return new Tuple<>(new ActionWriteResponse(), shardRequest.request); + return new Tuple<>(new ReplicationResponse(), shardRequest); } @Override - protected void shardOperationOnReplica(ShardId shardId, ReplicationRequest request) { + protected void shardOperationOnReplica(ReplicationRequest request) { + final ShardId shardId = request.shardId(); IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); indexShard.refresh("api"); logger.trace("{} refresh request executed on replica", indexShard.shardId()); @@ -82,18 +81,13 @@ public class TransportShardRefreshAction extends TransportReplicationAction shardStatsMap; + private Map shardStatsMap; IndicesStatsResponse() { @@ -56,11 +57,11 @@ public class IndicesStatsResponse extends BroadcastResponse implements ToXConten this.shards = shards; } - public Map asMap() { + public Map asMap() { if (this.shardStatsMap == null) { - Map shardStatsMap = new HashMap<>(); + Map shardStatsMap = new HashMap<>(); for (ShardStats ss : shards) { - shardStatsMap.put(ss.getShardRouting(), ss.getStats()); + shardStatsMap.put(ss.getShardRouting(), ss); } this.shardStatsMap = unmodifiableMap(shardStatsMap); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java index 30aff1f2e6e..5e87e91b255 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java @@ -64,7 +64,7 @@ public class TransportUpgradeAction extends TransportBroadcastByNodeActionnull for the indices. * @see org.elasticsearch.client.Requests#upgradeRequest(String...) * @see org.elasticsearch.client.IndicesAdminClient#upgrade(UpgradeRequest) @@ -43,7 +43,7 @@ public class UpgradeRequest extends BroadcastRequest { /** * Constructs an optimization request over one or more indices. * - * @param indices The indices to optimize, no indices passed means all indices will be optimized. + * @param indices The indices to upgrade, no indices passed means all indices will be optimized. */ public UpgradeRequest(String... indices) { super(indices); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeResponse.java index df611759790..64e958372cd 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeResponse.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.Map; /** - * A response for optimize action. + * A response for the upgrade action. * * */ @@ -80,4 +80,4 @@ public class UpgradeResponse extends BroadcastResponse { public Map> versions() { return versions; } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java index 808d1a5da2f..1242a9087d8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java @@ -22,9 +22,9 @@ package org.elasticsearch.action.admin.indices.validate.query; import org.elasticsearch.action.support.broadcast.BroadcastShardRequest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.shard.ShardId; import java.io.IOException; @@ -34,7 +34,7 @@ import java.io.IOException; */ public class ShardValidateQueryRequest extends BroadcastShardRequest { - private BytesReference source; + private QueryBuilder query; private String[] types = Strings.EMPTY_ARRAY; private boolean explain; private boolean rewrite; @@ -49,7 +49,7 @@ public class ShardValidateQueryRequest extends BroadcastShardRequest { ShardValidateQueryRequest(ShardId shardId, @Nullable String[] filteringAliases, ValidateQueryRequest request) { super(shardId, request); - this.source = request.source(); + this.query = request.query(); this.types = request.types(); this.explain = request.explain(); this.rewrite = request.rewrite(); @@ -57,8 +57,8 @@ public class ShardValidateQueryRequest extends BroadcastShardRequest { this.nowInMillis = request.nowInMillis; } - public BytesReference source() { - return source; + public QueryBuilder query() { + return query; } public String[] types() { @@ -84,7 +84,7 @@ public class ShardValidateQueryRequest extends BroadcastShardRequest { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - source = in.readBytesReference(); + query = in.readQuery(); int typesSize = in.readVInt(); if (typesSize > 0) { @@ -109,7 +109,7 @@ public class ShardValidateQueryRequest extends BroadcastShardRequest { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeBytesReference(source); + out.writeQuery(query); out.writeVInt(types.length); for (String type : types) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index db54fe4278a..326dbc01289 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -36,14 +36,14 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.script.ScriptService; @@ -162,8 +162,8 @@ public class TransportValidateQueryAction extends TransportBroadcastAction 0) { - searchContext.parsedQuery(queryParserService.parseQuery(request.source())); - } + searchContext.parsedQuery(queryShardContext.toQuery(request.query())); searchContext.preProcess(); valid = true; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java index 7c7869b9ce6..1f15cb6ae79 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java @@ -19,34 +19,27 @@ package org.elasticsearch.action.admin.indices.validate.query; -import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import java.io.IOException; import java.util.Arrays; -import java.util.Map; /** * A request to validate a specific query. *

- * The request requires the query source to be set either using {@link #source(QuerySourceBuilder)}, - * or {@link #source(byte[])}. + * The request requires the query to be set using {@link #query(QueryBuilder)} */ public class ValidateQueryRequest extends BroadcastRequest { - private BytesReference source; + private QueryBuilder query = new MatchAllQueryBuilder(); private boolean explain; private boolean rewrite; @@ -71,67 +64,21 @@ public class ValidateQueryRequest extends BroadcastRequest @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = super.validate(); + if (query == null) { + validationException = ValidateActions.addValidationError("query cannot be null", validationException); + } return validationException; } /** - * The source to execute. + * The query to validate. */ - public BytesReference source() { - return source; + public QueryBuilder query() { + return query; } - public ValidateQueryRequest source(QuerySourceBuilder sourceBuilder) { - this.source = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); - return this; - } - - /** - * The source to execute in the form of a map. - */ - public ValidateQueryRequest source(Map source) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE); - builder.map(source); - return source(builder); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); - } - } - - public ValidateQueryRequest source(XContentBuilder builder) { - this.source = builder.bytes(); - return this; - } - - /** - * The query source to validate. It is preferable to use either {@link #source(byte[])} - * or {@link #source(QuerySourceBuilder)}. - */ - public ValidateQueryRequest source(String source) { - this.source = new BytesArray(source); - return this; - } - - /** - * The source to validate. - */ - public ValidateQueryRequest source(byte[] source) { - return source(source, 0, source.length); - } - - /** - * The source to validate. - */ - public ValidateQueryRequest source(byte[] source, int offset, int length) { - return source(new BytesArray(source, offset, length)); - } - - /** - * The source to validate. - */ - public ValidateQueryRequest source(BytesReference source) { - this.source = source; + public ValidateQueryRequest query(QueryBuilder query) { + this.query = query; return this; } @@ -181,9 +128,7 @@ public class ValidateQueryRequest extends BroadcastRequest @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - - source = in.readBytesReference(); - + query = in.readQuery(); int typesSize = in.readVInt(); if (typesSize > 0) { types = new String[typesSize]; @@ -191,7 +136,6 @@ public class ValidateQueryRequest extends BroadcastRequest types[i] = in.readString(); } } - explain = in.readBoolean(); rewrite = in.readBoolean(); } @@ -199,27 +143,18 @@ public class ValidateQueryRequest extends BroadcastRequest @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - - out.writeBytesReference(source); - + out.writeQuery(query); out.writeVInt(types.length); for (String type : types) { out.writeString(type); } - out.writeBoolean(explain); out.writeBoolean(rewrite); } @Override public String toString() { - String sSource = "_na_"; - try { - sSource = XContentHelper.convertToJson(source, false); - } catch (Exception e) { - // ignore - } - return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", source[" + sSource + "], explain:" + explain + + return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", query[" + query + "], explain:" + explain + ", rewrite:" + rewrite; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java index 515ecd1ba1e..bfee7ec6b99 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java @@ -19,10 +19,8 @@ package org.elasticsearch.action.admin.indices.validate.query; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.QueryBuilder; /** @@ -30,8 +28,6 @@ import org.elasticsearch.index.query.QueryBuilder; */ public class ValidateQueryRequestBuilder extends BroadcastOperationRequestBuilder { - private QuerySourceBuilder sourceBuilder; - public ValidateQueryRequestBuilder(ElasticsearchClient client, ValidateQueryAction action) { super(client, action, new ValidateQueryRequest()); } @@ -45,32 +41,12 @@ public class ValidateQueryRequestBuilder extends BroadcastOperationRequestBuilde } /** - * The query source to validate. + * The query to validate. * * @see org.elasticsearch.index.query.QueryBuilders */ public ValidateQueryRequestBuilder setQuery(QueryBuilder queryBuilder) { - sourceBuilder().setQuery(queryBuilder); - return this; - } - - /** - * The source to validate. - * - * @see org.elasticsearch.index.query.QueryBuilders - */ - public ValidateQueryRequestBuilder setSource(BytesReference source) { - request().source(source); - return this; - } - - /** - * The source to validate. - * - * @see org.elasticsearch.index.query.QueryBuilders - */ - public ValidateQueryRequestBuilder setSource(byte[] source) { - request.source(source); + request.query(queryBuilder); return this; } @@ -91,19 +67,4 @@ public class ValidateQueryRequestBuilder extends BroadcastOperationRequestBuilde request.rewrite(rewrite); return this; } - - @Override - protected ValidateQueryRequest beforeExecute(ValidateQueryRequest request) { - if (sourceBuilder != null) { - request.source(sourceBuilder); - } - return request; - } - - private QuerySourceBuilder sourceBuilder() { - if (sourceBuilder == null) { - sourceBuilder = new QuerySourceBuilder(); - } - return sourceBuilder; - } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersRequest.java index 25d82a1bbb6..bebf0d40b6e 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersRequest.java @@ -28,7 +28,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; /** - * An {@ClusterInfoRequest} that fetches {@link org.elasticsearch.search.warmer.IndexWarmersMetaData} for + * A {@link ClusterInfoRequest} that fetches {@link org.elasticsearch.search.warmer.IndexWarmersMetaData} for * a list or all existing index warmers in the cluster-state */ public class GetWarmersRequest extends ClusterInfoRequest { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java index 3ed444c88dd..57e0b746496 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java @@ -20,9 +20,8 @@ package org.elasticsearch.action.admin.indices.warmer.get; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.Version; + import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -69,7 +68,10 @@ public class GetWarmersResponse extends ActionResponse { for (int j = 0; j < valueSize; j++) { String name = in.readString(); String[] types = in.readStringArray(); - BytesReference source = in.readBytesReference(); + IndexWarmersMetaData.SearchSource source = null; + if (in.readBoolean()) { + source = new IndexWarmersMetaData.SearchSource(in); + } Boolean queryCache = null; queryCache = in.readOptionalBoolean(); warmerEntryBuilder.add(new IndexWarmersMetaData.Entry( @@ -94,7 +96,11 @@ public class GetWarmersResponse extends ActionResponse { for (IndexWarmersMetaData.Entry warmerEntry : indexEntry.value) { out.writeString(warmerEntry.name()); out.writeStringArray(warmerEntry.types()); - out.writeBytesReference(warmerEntry.source()); + boolean hasWarmerSource = warmerEntry != null; + out.writeBoolean(hasWarmerSource); + if (hasWarmerSource) { + warmerEntry.source().writeTo(out); + } out.writeOptionalBoolean(warmerEntry.requestCache()); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java index 18246f6df06..d72be81028d 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -114,11 +115,9 @@ public class TransportPutWarmerAction extends TransportMasterNodeAction 0) { - source = request.searchRequest().source(); - } else if (request.searchRequest().extraSource() != null && request.searchRequest().extraSource().length() > 0) { - source = request.searchRequest().extraSource(); + IndexWarmersMetaData.SearchSource source = null; + if (request.searchRequest().source() != null) { + source = new IndexWarmersMetaData.SearchSource(request.searchRequest().source()); } // now replace it on the metadata diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkAction.java index 42d0c22508b..e442f61061a 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkAction.java @@ -47,9 +47,9 @@ public class BulkAction extends Actionnull in * case of failure. */ - public T getResponse() { + public T getResponse() { return (T) response; } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java index 42a9344eabc..2a7c185ad8a 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java @@ -324,9 +324,11 @@ public class BulkProcessor implements Closeable { } } else { boolean success = false; + boolean acquired = false; try { listener.beforeBulk(executionId, bulkRequest); semaphore.acquire(); + acquired = true; client.bulk(bulkRequest, new ActionListener() { @Override public void onResponse(BulkResponse response) { @@ -353,7 +355,7 @@ public class BulkProcessor implements Closeable { } catch (Throwable t) { listener.afterBulk(executionId, bulkRequest, t); } finally { - if (!success) { // if we fail on client.bulk() release the semaphore + if (!success && acquired) { // if we fail on client.bulk() release the semaphore semaphore.release(); } } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index fa6b643eb69..02e0ea40d65 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -46,9 +46,10 @@ import java.util.List; import static org.elasticsearch.action.ValidateActions.addValidationError; /** - * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes - * it in a single batch. + * A bulk request holds an ordered {@link IndexRequest}s, {@link DeleteRequest}s and {@link UpdateRequest}s + * and allows to executes it in a single batch. * + * Note that we only support refresh on the bulk request not per item. * @see org.elasticsearch.client.Client#bulk(BulkRequest) */ public class BulkRequest extends ActionRequest implements CompositeIndicesRequest { @@ -64,6 +65,17 @@ public class BulkRequest extends ActionRequest implements Composite private long sizeInBytes = 0; + public BulkRequest() { + } + + /** + * Creates a bulk request caused by some other request, which is provided as an + * argument so that its headers and context can be copied to the new request + */ + public BulkRequest(ActionRequest request) { + super(request); + } + /** * Adds a list of requests to be executed. Either index or delete requests. */ @@ -78,6 +90,12 @@ public class BulkRequest extends ActionRequest implements Composite return add(request, null); } + /** + * Add a request to the current BulkRequest. + * @param request Request to add + * @param payload Optional payload + * @return the current bulk request + */ public BulkRequest add(ActionRequest request, @Nullable Object payload) { if (request instanceof IndexRequest) { add((IndexRequest) request, payload); @@ -116,7 +134,8 @@ public class BulkRequest extends ActionRequest implements Composite BulkRequest internalAdd(IndexRequest request, @Nullable Object payload) { requests.add(request); addPayload(payload); - sizeInBytes += request.source().length() + REQUEST_OVERHEAD; + // lack of source is validated in validate() method + sizeInBytes += (request.source() != null ? request.source().length() : 0) + REQUEST_OVERHEAD; return this; } @@ -281,7 +300,7 @@ public class BulkRequest extends ActionRequest implements Composite String parent = null; String[] fields = defaultFields; String timestamp = null; - Long ttl = null; + TimeValue ttl = null; String opType = null; long version = Versions.MATCH_ANY; VersionType versionType = VersionType.INTERNAL; @@ -314,9 +333,9 @@ public class BulkRequest extends ActionRequest implements Composite timestamp = parser.text(); } else if ("_ttl".equals(currentFieldName) || "ttl".equals(currentFieldName)) { if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { - ttl = TimeValue.parseTimeValue(parser.text(), null, currentFieldName).millis(); + ttl = TimeValue.parseTimeValue(parser.text(), null, currentFieldName); } else { - ttl = parser.longValue(); + ttl = new TimeValue(parser.longValue()); } } else if ("op_type".equals(currentFieldName) || "opType".equals(currentFieldName)) { opType = parser.text(); @@ -467,8 +486,14 @@ public class BulkRequest extends ActionRequest implements Composite if (requests.isEmpty()) { validationException = addValidationError("no requests added", validationException); } - for (int i = 0; i < requests.size(); i++) { - ActionRequestValidationException ex = requests.get(i).validate(); + for (ActionRequest request : requests) { + // We first check if refresh has been set + if ((request instanceof DeleteRequest && ((DeleteRequest)request).refresh()) || + (request instanceof UpdateRequest && ((UpdateRequest)request).refresh()) || + (request instanceof IndexRequest && ((IndexRequest)request).refresh())) { + validationException = addValidationError("Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", validationException); + } + ActionRequestValidationException ex = request.validate(); if (ex != null) { if (validationException == null) { validationException = new ActionRequestValidationException(); diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java index ec150385678..1edba16220a 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java @@ -40,10 +40,8 @@ public class BulkShardRequest extends ReplicationRequest { public BulkShardRequest() { } - BulkShardRequest(BulkRequest bulkRequest, String index, int shardId, boolean refresh, BulkItemRequest[] items) { - super(bulkRequest); - this.index = index; - this.setShardId(new ShardId(index, shardId)); + BulkShardRequest(BulkRequest bulkRequest, ShardId shardId, boolean refresh, BulkItemRequest[] items) { + super(bulkRequest, shardId); this.items = items; this.refresh = refresh; } @@ -93,4 +91,9 @@ public class BulkShardRequest extends ReplicationRequest { } refresh = in.readBoolean(); } + + @Override + public String toString() { + return "shard bulk {" + super.toString() + "}"; + } } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java index 6b08627f5de..76c80a9b064 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java @@ -19,7 +19,7 @@ package org.elasticsearch.action.bulk; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.shard.ShardId; @@ -29,7 +29,7 @@ import java.io.IOException; /** * */ -public class BulkShardResponse extends ActionWriteResponse { +public class BulkShardResponse extends ReplicationResponse { private ShardId shardId; private BulkItemResponse[] responses; diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 51d32e3ff75..9b18d0328e7 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -275,7 +275,7 @@ public class TransportBulkAction extends HandledTransportAction list = requestsByShard.get(shardId); if (list == null) { list = new ArrayList<>(); @@ -312,7 +312,7 @@ public class TransportBulkAction extends HandledTransportAction> entry : requestsByShard.entrySet()) { final ShardId shardId = entry.getKey(); final List requests = entry.getValue(); - BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId.index().name(), shardId.id(), bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); + BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId, bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel()); bulkShardRequest.timeout(bulkRequest.timeout()); shardBulkAction.execute(bulkShardRequest, new ActionListener() { diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 0f00b87b12a..2597695a1e2 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -25,20 +25,21 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.delete.TransportDeleteAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.action.update.UpdateHelper; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; @@ -49,8 +50,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.VersionConflictEngineException; -import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; @@ -87,11 +86,6 @@ public class TransportShardBulkAction extends TransportReplicationAction shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) { - final BulkShardRequest request = shardRequest.request; + protected Tuple shardOperationOnPrimary(MetaData metaData, BulkShardRequest request) { final IndexService indexService = indicesService.indexServiceSafe(request.index()); - final IndexShard indexShard = indexService.getShard(shardRequest.shardId.id()); + final IndexShard indexShard = indexService.getShard(request.shardId().id()); long[] preVersions = new long[request.items().length]; VersionType[] preVersionTypes = new VersionType[request.items().length]; @@ -128,7 +116,7 @@ public class TransportShardBulkAction extends TransportReplicationAction result = shardIndexOperation(request, indexRequest, clusterState, indexShard, true); + WriteResult result = shardIndexOperation(request, indexRequest, metaData, indexShard, true); location = locationToSync(location, result.location); // add the response IndexResponse indexResponse = result.response(); @@ -143,9 +131,9 @@ public class TransportShardBulkAction extends TransportReplicationAction writeResult = shardDeleteOperation(request, deleteRequest, indexShard); + final WriteResult writeResult = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard); DeleteResponse deleteResponse = writeResult.response(); location = locationToSync(location, writeResult.location); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse)); @@ -178,9 +166,9 @@ public class TransportShardBulkAction extends TransportReplicationAction 0) { Tuple> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true); - updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, shardRequest.request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); + updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); } item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), indexRequest); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); @@ -228,8 +216,8 @@ public class TransportShardBulkAction extends TransportReplicationAction writeResult = updateResult.writeResult; DeleteResponse response = writeResult.response(); DeleteRequest deleteRequest = updateResult.request(); - updateResponse = new UpdateResponse(response.getShardInfo(), response.getIndex(), response.getType(), response.getId(), response.getVersion(), false); - updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, shardRequest.request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null)); + updateResponse = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false); + updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null)); // Replace the update request to the translated delete request to execute on the replica. item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); @@ -264,16 +252,16 @@ public class TransportShardBulkAction extends TransportReplicationAction(new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request); + return new Tuple<>(new BulkShardResponse(request.shardId(), responses), request); } private void setResponse(BulkItemRequest request, BulkItemResponse response) { @@ -320,11 +308,11 @@ public class TransportShardBulkAction extends TransportReplicationAction shardDeleteOperation(BulkShardRequest request, DeleteRequest deleteRequest, IndexShard indexShard) { - Engine.Delete delete = indexShard.prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version(), deleteRequest.versionType(), Engine.Operation.Origin.PRIMARY); - indexShard.delete(delete); - // update the request with the version so it will go to the replicas - deleteRequest.versionType(delete.versionType().versionTypeForReplicationAndRecovery()); - deleteRequest.version(delete.version()); - - assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version()); - - DeleteResponse deleteResponse = new DeleteResponse(request.index(), deleteRequest.type(), deleteRequest.id(), delete.version(), delete.found()); - return new WriteResult(deleteResponse, delete.getTranslogLocation()); + return TransportIndexAction.executeIndexRequestOnPrimary(indexRequest, indexShard, mappingUpdatedAction); } static class UpdateResult { @@ -404,14 +378,14 @@ public class TransportShardBulkAction extends TransportReplicationAction result = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard); return new UpdateResult(translate, deleteRequest, result); } catch (Throwable t) { t = ExceptionsHelper.unwrapCause(t); @@ -445,7 +419,8 @@ public class TransportShardBulkAction extends TransportReplicationAction { - - public static final CountAction INSTANCE = new CountAction(); - public static final String NAME = "indices:data/read/count"; - - private CountAction() { - super(NAME); - } - - @Override - public CountResponse newResponse() { - throw new UnsupportedOperationException("CountAction doesn't have its own transport action, gets executed as a SearchAction internally"); - } - - @Override - public CountRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new CountRequestBuilder(client, this); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/count/CountRequest.java b/core/src/main/java/org/elasticsearch/action/count/CountRequest.java deleted file mode 100644 index 05e193a4266..00000000000 --- a/core/src/main/java/org/elasticsearch/action/count/CountRequest.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.count; - -import org.elasticsearch.ElasticsearchGenerationException; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; - -import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER; - -/** - * A request to count the number of documents matching a specific query. Best created with - * {@link org.elasticsearch.client.Requests#countRequest(String...)}. - *

- * The request requires the query source to be set either using {@link #source(QuerySourceBuilder)}, - * or {@link #source(byte[])}. - * - * @see CountResponse - * @see org.elasticsearch.client.Client#count(CountRequest) - * @see org.elasticsearch.client.Requests#countRequest(String...) - */ -public class CountRequest extends BroadcastRequest { - - public static final float DEFAULT_MIN_SCORE = -1f; - - private float minScore = DEFAULT_MIN_SCORE; - - @Nullable - protected String routing; - - @Nullable - private String preference; - - private BytesReference source; - - private String[] types = Strings.EMPTY_ARRAY; - - private int terminateAfter = DEFAULT_TERMINATE_AFTER; - - /** - * Constructs a new count request against the provided indices. No indices provided means it will - * run against all indices. - */ - public CountRequest(String... indices) { - super(indices); - } - - /** - * The minimum score of the documents to include in the count. - */ - public float minScore() { - return minScore; - } - - /** - * The minimum score of the documents to include in the count. Defaults to -1 which means all - * documents will be included in the count. - */ - public CountRequest minScore(float minScore) { - this.minScore = minScore; - return this; - } - - /** - * The source to execute. - */ - public BytesReference source() { - return source; - } - - /** - * The source to execute. - */ - public CountRequest source(QuerySourceBuilder sourceBuilder) { - this.source = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); - return this; - } - - /** - * The source to execute in the form of a map. - */ - @SuppressWarnings("unchecked") - public CountRequest source(Map querySource) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE); - builder.map(querySource); - return source(builder); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + querySource + "]", e); - } - } - - public CountRequest source(XContentBuilder builder) { - this.source = builder.bytes(); - return this; - } - - /** - * The source to execute. It is preferable to use either {@link #source(byte[])} - * or {@link #source(QuerySourceBuilder)}. - */ - public CountRequest source(String querySource) { - this.source = new BytesArray(querySource); - return this; - } - - /** - * The source to execute. - */ - public CountRequest source(byte[] querySource) { - return source(querySource, 0, querySource.length); - } - - /** - * The source to execute. - */ - public CountRequest source(byte[] querySource, int offset, int length) { - return source(new BytesArray(querySource, offset, length)); - } - - public CountRequest source(BytesReference querySource) { - this.source = querySource; - return this; - } - - /** - * The types of documents the query will run against. Defaults to all types. - */ - public String[] types() { - return this.types; - } - - /** - * The types of documents the query will run against. Defaults to all types. - */ - public CountRequest types(String... types) { - this.types = types; - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public String routing() { - return this.routing; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public CountRequest routing(String routing) { - this.routing = routing; - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public CountRequest routing(String... routings) { - this.routing = Strings.arrayToCommaDelimitedString(routings); - return this; - } - - public CountRequest preference(String preference) { - this.preference = preference; - return this; - } - - public String preference() { - return this.preference; - } - - /** - * Upon reaching terminateAfter counts, the count request will early terminate - */ - public CountRequest terminateAfter(int terminateAfterCount) { - if (terminateAfterCount <= 0) { - throw new IllegalArgumentException("terminateAfter must be > 0"); - } - this.terminateAfter = terminateAfterCount; - return this; - } - - public int terminateAfter() { - return this.terminateAfter; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - throw new UnsupportedOperationException("CountRequest doesn't support being sent over the wire, just a shortcut to the search api"); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException("CountRequest doesn't support being sent over the wire, just a shortcut to the search api"); - } - - @Override - public String toString() { - String sSource = "_na_"; - try { - sSource = XContentHelper.convertToJson(source, false); - } catch (Exception e) { - // ignore - } - return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", source[" + sSource + "]"; - } - - public SearchRequest toSearchRequest() { - SearchRequest searchRequest = new SearchRequest(indices()); - searchRequest.indicesOptions(indicesOptions()); - searchRequest.types(types()); - searchRequest.routing(routing()); - searchRequest.preference(preference()); - searchRequest.source(source()); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.size(0); - if (minScore() != DEFAULT_MIN_SCORE) { - searchSourceBuilder.minScore(minScore()); - } - if (terminateAfter() != DEFAULT_TERMINATE_AFTER) { - searchSourceBuilder.terminateAfter(terminateAfter()); - } - searchRequest.extraSource(searchSourceBuilder); - return searchRequest; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/count/CountRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/count/CountRequestBuilder.java deleted file mode 100644 index 54c60e5736a..00000000000 --- a/core/src/main/java/org/elasticsearch/action/count/CountRequestBuilder.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.count; - -import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.DelegatingActionListener; -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.query.QueryBuilder; - -/** - * A count action request builder. - */ -public class CountRequestBuilder extends BroadcastOperationRequestBuilder { - - private QuerySourceBuilder sourceBuilder; - - public CountRequestBuilder(ElasticsearchClient client, CountAction action) { - super(client, action, new CountRequest()); - } - - /** - * The types of documents the query will run against. Defaults to all types. - */ - public CountRequestBuilder setTypes(String... types) { - request.types(types); - return this; - } - - /** - * The minimum score of the documents to include in the count. Defaults to -1 which means all - * documents will be included in the count. - */ - public CountRequestBuilder setMinScore(float minScore) { - request.minScore(minScore); - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public CountRequestBuilder setRouting(String routing) { - request.routing(routing); - return this; - } - - /** - * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, - * _shards:x,y to operate on shards x & y, or a custom value, which guarantees that the same order - * will be used across different requests. - */ - public CountRequestBuilder setPreference(String preference) { - request.preference(preference); - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public CountRequestBuilder setRouting(String... routing) { - request.routing(routing); - return this; - } - - /** - * The query source to execute. - * - * @see org.elasticsearch.index.query.QueryBuilders - */ - public CountRequestBuilder setQuery(QueryBuilder queryBuilder) { - sourceBuilder().setQuery(queryBuilder); - return this; - } - - /** - * The query binary to execute - */ - public CountRequestBuilder setQuery(BytesReference queryBinary) { - sourceBuilder().setQuery(queryBinary); - return this; - } - - /** - * Constructs a new builder with a raw search query. - */ - public CountRequestBuilder setQuery(XContentBuilder query) { - return setQuery(query.bytes()); - } - - - /** - * The source to execute. - */ - public CountRequestBuilder setSource(BytesReference source) { - request().source(source); - return this; - } - - /** - * The query source to execute. - */ - public CountRequestBuilder setSource(byte[] querySource) { - request.source(querySource); - return this; - } - - public CountRequestBuilder setTerminateAfter(int terminateAfter) { - request().terminateAfter(terminateAfter); - return this; - } - - @Override - protected CountRequest beforeExecute(CountRequest request) { - if (sourceBuilder != null) { - request.source(sourceBuilder); - } - return request; - } - - private QuerySourceBuilder sourceBuilder() { - if (sourceBuilder == null) { - sourceBuilder = new QuerySourceBuilder(); - } - return sourceBuilder; - } - - @Override - public void execute(ActionListener listener) { - CountRequest countRequest = beforeExecute(request); - client.execute(SearchAction.INSTANCE, countRequest.toSearchRequest(), new DelegatingActionListener(listener) { - @Override - protected CountResponse getDelegatedFromInstigator(SearchResponse response) { - return new CountResponse(response); - } - }); - } - - @Override - public String toString() { - if (sourceBuilder != null) { - return sourceBuilder.toString(); - } - if (request.source() != null) { - try { - return XContentHelper.convertToJson(request.source().toBytesArray(), false, true); - } catch (Exception e) { - return "{ \"error\" : \"" + ExceptionsHelper.detailedMessage(e) + "\"}"; - } - } - return new QuerySourceBuilder().toString(); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/count/CountResponse.java b/core/src/main/java/org/elasticsearch/action/count/CountResponse.java deleted file mode 100644 index 916c4ef9373..00000000000 --- a/core/src/main/java/org/elasticsearch/action/count/CountResponse.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.count; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.rest.RestStatus; - -import java.io.IOException; -import java.util.Arrays; - -/** - * The response of the count action. - */ -public class CountResponse extends BroadcastResponse { - - private final boolean terminatedEarly; - private final long count; - - public CountResponse(SearchResponse searchResponse) { - super(searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), searchResponse.getFailedShards(), Arrays.asList(searchResponse.getShardFailures())); - this.count = searchResponse.getHits().totalHits(); - this.terminatedEarly = searchResponse.isTerminatedEarly() != null && searchResponse.isTerminatedEarly(); - } - - /** - * The count of documents matching the query provided. - */ - public long getCount() { - return count; - } - - /** - * True if the request has been terminated early due to enough count - */ - public boolean terminatedEarly() { - return this.terminatedEarly; - } - - public RestStatus status() { - return RestStatus.status(getSuccessfulShards(), getTotalShards(), getShardFailures()); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - throw new UnsupportedOperationException("CountResponse doesn't support being sent over the wire, just a shortcut to the search api"); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException("CountResponse doesn't support being sent over the wire, just a shortcut to the search api"); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java index 26cfa57a13d..57781547266 100644 --- a/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java +++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java @@ -19,9 +19,13 @@ package org.elasticsearch.action.delete; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -31,53 +35,19 @@ import java.io.IOException; * @see org.elasticsearch.action.delete.DeleteRequest * @see org.elasticsearch.client.Client#delete(DeleteRequest) */ -public class DeleteResponse extends ActionWriteResponse { +public class DeleteResponse extends DocWriteResponse { - private String index; - private String id; - private String type; - private long version; private boolean found; public DeleteResponse() { } - public DeleteResponse(String index, String type, String id, long version, boolean found) { - this.index = index; - this.id = id; - this.type = type; - this.version = version; + public DeleteResponse(ShardId shardId, String type, String id, long version, boolean found) { + super(shardId, type, id, version); this.found = found; } - /** - * The index the document was deleted from. - */ - public String getIndex() { - return this.index; - } - - /** - * The type of the document deleted. - */ - public String getType() { - return this.type; - } - - /** - * The id of the document deleted. - */ - public String getId() { - return this.id; - } - - /** - * The version of the delete operation. - */ - public long getVersion() { - return this.version; - } /** * Returns true if a doc was found to delete. @@ -89,20 +59,44 @@ public class DeleteResponse extends ActionWriteResponse { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - index = in.readString(); - type = in.readString(); - id = in.readString(); - version = in.readLong(); found = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(index); - out.writeString(type); - out.writeString(id); - out.writeLong(version); out.writeBoolean(found); } + + @Override + public RestStatus status() { + if (found == false) { + return RestStatus.NOT_FOUND; + } + return super.status(); + } + + static final class Fields { + static final XContentBuilderString FOUND = new XContentBuilderString("found"); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(Fields.FOUND, isFound()); + super.toXContent(builder, params); + return builder; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("DeleteResponse["); + builder.append("index=").append(getIndex()); + builder.append(",type=").append(getType()); + builder.append(",id=").append(getId()); + builder.append(",version=").append(getVersion()); + builder.append(",found=").append(found); + builder.append(",shards=").append(getShardInfo()); + return builder.append("]").toString(); + } } diff --git a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java index c20b203ee6b..ca66b285753 100644 --- a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java +++ b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java @@ -34,7 +34,7 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -94,67 +94,68 @@ public class TransportDeleteAction extends TransportReplicationAction listener) { - request.request().routing(state.metaData().resolveIndexRouting(request.request().routing(), request.request().index())); - if (state.metaData().hasIndex(request.concreteIndex())) { + protected void resolveRequest(final MetaData metaData, String concreteIndex, DeleteRequest request) { + request.routing(metaData.resolveIndexRouting(request.routing(), request.index())); + if (metaData.hasIndex(concreteIndex)) { // check if routing is required, if so, do a broadcast delete - MappingMetaData mappingMd = state.metaData().index(request.concreteIndex()).mappingOrDefault(request.request().type()); + MappingMetaData mappingMd = metaData.index(concreteIndex).mappingOrDefault(request.type()); if (mappingMd != null && mappingMd.routing().required()) { - if (request.request().routing() == null) { - if (request.request().versionType() != VersionType.INTERNAL) { + if (request.routing() == null) { + if (request.versionType() != VersionType.INTERNAL) { // TODO: implement this feature - throw new IllegalArgumentException("routing value is required for deleting documents of type [" + request.request().type() - + "] while using version_type [" + request.request().versionType() + "]"); + throw new IllegalArgumentException("routing value is required for deleting documents of type [" + request.type() + + "] while using version_type [" + request.versionType() + "]"); } - throw new RoutingMissingException(request.concreteIndex(), request.request().type(), request.request().id()); + throw new RoutingMissingException(concreteIndex, request.type(), request.id()); } } } + ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(), concreteIndex, request.id(), request.routing()); + request.setShardId(shardId); } private void innerExecute(final DeleteRequest request, final ActionListener listener) { super.doExecute(request, listener); } - @Override - protected boolean checkWriteConsistency() { - return true; - } - @Override protected DeleteResponse newResponseInstance() { return new DeleteResponse(); } @Override - protected Tuple shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) { - DeleteRequest request = shardRequest.request; - IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).getShard(shardRequest.shardId.id()); - Engine.Delete delete = indexShard.prepareDelete(request.type(), request.id(), request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY); + protected Tuple shardOperationOnPrimary(MetaData metaData, DeleteRequest request) { + IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id()); + final WriteResult result = executeDeleteRequestOnPrimary(request, indexShard); + processAfterWrite(request.refresh(), indexShard, result.location); + return new Tuple<>(result.response, request); + } + + public static WriteResult executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard indexShard) { + Engine.Delete delete = indexShard.prepareDeleteOnPrimary(request.type(), request.id(), request.version(), request.versionType()); indexShard.delete(delete); - // update the request with teh version so it will go to the replicas + // update the request with the version so it will go to the replicas request.versionType(delete.versionType().versionTypeForReplicationAndRecovery()); request.version(delete.version()); assert request.versionType().validateVersionForWrites(request.version()); - processAfter(request.refresh(), indexShard, delete.getTranslogLocation()); - - DeleteResponse response = new DeleteResponse(shardRequest.shardId.getIndex(), request.type(), request.id(), delete.version(), delete.found()); - return new Tuple<>(response, shardRequest.request); + return new WriteResult<>( + new DeleteResponse(indexShard.shardId(), request.type(), request.id(), delete.version(), delete.found()), + delete.getTranslogLocation()); } - @Override - protected void shardOperationOnReplica(ShardId shardId, DeleteRequest request) { - IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); - Engine.Delete delete = indexShard.prepareDelete(request.type(), request.id(), request.version(), request.versionType(), Engine.Operation.Origin.REPLICA); - + public static Engine.Delete executeDeleteRequestOnReplica(DeleteRequest request, IndexShard indexShard) { + Engine.Delete delete = indexShard.prepareDeleteOnReplica(request.type(), request.id(), request.version(), request.versionType()); indexShard.delete(delete); - processAfter(request.refresh(), indexShard, delete.getTranslogLocation()); + return delete; } @Override - protected ShardIterator shards(ClusterState clusterState, InternalRequest request) { - return clusterService.operationRouting() - .deleteShards(clusterService.state(), request.concreteIndex(), request.request().type(), request.request().id(), request.request().routing()); + protected void shardOperationOnReplica(DeleteRequest request) { + final ShardId shardId = request.shardId(); + IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); + Engine.Delete delete = executeDeleteRequestOnReplica(request, indexShard); + processAfterWrite(request.refresh(), indexShard, delete.getTranslogLocation()); } + } diff --git a/core/src/main/java/org/elasticsearch/action/exists/ExistsRequest.java b/core/src/main/java/org/elasticsearch/action/exists/ExistsRequest.java deleted file mode 100644 index 0000676b3ca..00000000000 --- a/core/src/main/java/org/elasticsearch/action/exists/ExistsRequest.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.exists; - -import org.elasticsearch.ElasticsearchGenerationException; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; - -public class ExistsRequest extends BroadcastRequest { - - public static final float DEFAULT_MIN_SCORE = -1f; - private float minScore = DEFAULT_MIN_SCORE; - - @Nullable - protected String routing; - - @Nullable - private String preference; - - private BytesReference source; - - private String[] types = Strings.EMPTY_ARRAY; - - long nowInMillis; - - public ExistsRequest() { - } - - /** - * Constructs a new exists request against the provided indices. No indices provided means it will - * run against all indices. - */ - public ExistsRequest(String... indices) { - super(indices); - } - - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = super.validate(); - return validationException; - } - - /** - * The minimum score of the documents to include in the count. - */ - public float minScore() { - return minScore; - } - - /** - * The minimum score of the documents to include in the count. Defaults to -1 which means all - * documents will be considered. - */ - public ExistsRequest minScore(float minScore) { - this.minScore = minScore; - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public String routing() { - return this.routing; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public ExistsRequest routing(String routing) { - this.routing = routing; - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public ExistsRequest routing(String... routings) { - this.routing = Strings.arrayToCommaDelimitedString(routings); - return this; - } - - /** - * Routing preference for executing the search on shards - */ - public ExistsRequest preference(String preference) { - this.preference = preference; - return this; - } - - public String preference() { - return this.preference; - } - - /** - * The source to execute. - */ - public BytesReference source() { - return source; - } - - /** - * The source to execute. - */ - public ExistsRequest source(QuerySourceBuilder sourceBuilder) { - this.source = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); - return this; - } - - /** - * The source to execute in the form of a map. - */ - public ExistsRequest source(Map querySource) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE); - builder.map(querySource); - return source(builder); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + querySource + "]", e); - } - } - - public ExistsRequest source(XContentBuilder builder) { - this.source = builder.bytes(); - return this; - } - - /** - * The source to execute. It is preferable to use either {@link #source(byte[])} - * or {@link #source(QuerySourceBuilder)}. - */ - public ExistsRequest source(String querySource) { - this.source = new BytesArray(querySource); - return this; - } - - /** - * The source to execute. - */ - public ExistsRequest source(byte[] querySource) { - return source(querySource, 0, querySource.length); - } - - /** - * The source to execute. - */ - public ExistsRequest source(byte[] querySource, int offset, int length) { - return source(new BytesArray(querySource, offset, length)); - } - - public ExistsRequest source(BytesReference querySource) { - this.source = querySource; - return this; - } - - /** - * The types of documents the query will run against. Defaults to all types. - */ - public String[] types() { - return this.types; - } - - /** - * The types of documents the query will run against. Defaults to all types. - */ - public ExistsRequest types(String... types) { - this.types = types; - return this; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - minScore = in.readFloat(); - routing = in.readOptionalString(); - preference = in.readOptionalString(); - source = in.readBytesReference(); - types = in.readStringArray(); - - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeFloat(minScore); - out.writeOptionalString(routing); - out.writeOptionalString(preference); - out.writeBytesReference(source); - out.writeStringArray(types); - - } - - @Override - public String toString() { - String sSource = "_na_"; - try { - sSource = XContentHelper.convertToJson(source, false); - } catch (Exception e) { - // ignore - } - return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", source[" + sSource + "]"; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/exists/ExistsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/exists/ExistsRequestBuilder.java deleted file mode 100644 index c7ef5a10ce2..00000000000 --- a/core/src/main/java/org/elasticsearch/action/exists/ExistsRequestBuilder.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.exists; - -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.index.query.QueryBuilder; - -public class ExistsRequestBuilder extends BroadcastOperationRequestBuilder { - - private QuerySourceBuilder sourceBuilder; - - public ExistsRequestBuilder(ElasticsearchClient client, ExistsAction action) { - super(client, action, new ExistsRequest()); - } - - /** - * The types of documents the query will run against. Defaults to all types. - */ - public ExistsRequestBuilder setTypes(String... types) { - request.types(types); - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public ExistsRequestBuilder setRouting(String routing) { - request.routing(routing); - return this; - } - - /** - * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, - * _shards:x,y to operate on shards x & y, or a custom value, which guarantees that the same order - * will be used across different requests. - */ - public ExistsRequestBuilder setPreference(String preference) { - request.preference(preference); - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public ExistsRequestBuilder setRouting(String... routing) { - request.routing(routing); - return this; - } - - /** - * The query source to execute. - * - * @see org.elasticsearch.index.query.QueryBuilders - */ - public ExistsRequestBuilder setQuery(QueryBuilder queryBuilder) { - sourceBuilder().setQuery(queryBuilder); - return this; - } - - /** - * The query binary to execute - */ - public ExistsRequestBuilder setQuery(BytesReference queryBinary) { - sourceBuilder().setQuery(queryBinary); - return this; - } - - /** - * The source to execute. - */ - public ExistsRequestBuilder setSource(BytesReference source) { - request().source(source); - return this; - } - - /** - * The query source to execute. - */ - public ExistsRequestBuilder setSource(byte[] querySource) { - request.source(querySource); - return this; - } - - @Override - protected ExistsRequest beforeExecute(ExistsRequest request) { - if (sourceBuilder != null) { - request.source(sourceBuilder); - } - return request; - } - - private QuerySourceBuilder sourceBuilder() { - if (sourceBuilder == null) { - sourceBuilder = new QuerySourceBuilder(); - } - return sourceBuilder; - } - -} diff --git a/core/src/main/java/org/elasticsearch/action/exists/ExistsResponse.java b/core/src/main/java/org/elasticsearch/action/exists/ExistsResponse.java deleted file mode 100644 index f271dc65f7f..00000000000 --- a/core/src/main/java/org/elasticsearch/action/exists/ExistsResponse.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.exists; - -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; -import java.util.List; - -public class ExistsResponse extends BroadcastResponse { - - private boolean exists = false; - - ExistsResponse() { - - } - - ExistsResponse(boolean exists, int totalShards, int successfulShards, int failedShards, List shardFailures) { - super(totalShards, successfulShards, failedShards, shardFailures); - this.exists = exists; - } - - /** - * Whether the documents matching the query provided exists - */ - public boolean exists() { - return exists; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - exists = in.readBoolean(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeBoolean(exists); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/exists/ShardExistsRequest.java b/core/src/main/java/org/elasticsearch/action/exists/ShardExistsRequest.java deleted file mode 100644 index d57b1d99010..00000000000 --- a/core/src/main/java/org/elasticsearch/action/exists/ShardExistsRequest.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.exists; - -import org.elasticsearch.action.support.broadcast.BroadcastShardRequest; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.shard.ShardId; - -import java.io.IOException; - -public class ShardExistsRequest extends BroadcastShardRequest { - - private float minScore; - - private BytesReference querySource; - - private String[] types = Strings.EMPTY_ARRAY; - - private long nowInMillis; - - @Nullable - private String[] filteringAliases; - - public ShardExistsRequest() { - } - - ShardExistsRequest(ShardId shardId, @Nullable String[] filteringAliases, ExistsRequest request) { - super(shardId, request); - this.minScore = request.minScore(); - this.querySource = request.source(); - this.types = request.types(); - this.filteringAliases = filteringAliases; - this.nowInMillis = request.nowInMillis; - } - - public float minScore() { - return minScore; - } - - public BytesReference querySource() { - return querySource; - } - - public String[] types() { - return this.types; - } - - public String[] filteringAliases() { - return filteringAliases; - } - - public long nowInMillis() { - return this.nowInMillis; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - minScore = in.readFloat(); - - querySource = in.readBytesReference(); - - int typesSize = in.readVInt(); - if (typesSize > 0) { - types = new String[typesSize]; - for (int i = 0; i < typesSize; i++) { - types[i] = in.readString(); - } - } - int aliasesSize = in.readVInt(); - if (aliasesSize > 0) { - filteringAliases = new String[aliasesSize]; - for (int i = 0; i < aliasesSize; i++) { - filteringAliases[i] = in.readString(); - } - } - nowInMillis = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeFloat(minScore); - - out.writeBytesReference(querySource); - - out.writeVInt(types.length); - for (String type : types) { - out.writeString(type); - } - if (filteringAliases != null) { - out.writeVInt(filteringAliases.length); - for (String alias : filteringAliases) { - out.writeString(alias); - } - } else { - out.writeVInt(0); - } - out.writeVLong(nowInMillis); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/exists/ShardExistsResponse.java b/core/src/main/java/org/elasticsearch/action/exists/ShardExistsResponse.java deleted file mode 100644 index 25f813ee424..00000000000 --- a/core/src/main/java/org/elasticsearch/action/exists/ShardExistsResponse.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.exists; - -import org.elasticsearch.action.support.broadcast.BroadcastShardResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.shard.ShardId; - -import java.io.IOException; - -class ShardExistsResponse extends BroadcastShardResponse { - - private boolean exists; - - ShardExistsResponse() { - } - - ShardExistsResponse(ShardId shardId, boolean exists) { - super(shardId); - this.exists = exists; - } - - public boolean exists() { - return this.exists; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - exists = in.readBoolean(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeBoolean(exists); - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/action/exists/TransportExistsAction.java b/core/src/main/java/org/elasticsearch/action/exists/TransportExistsAction.java deleted file mode 100644 index 46f998fc6a6..00000000000 --- a/core/src/main/java/org/elasticsearch/action/exists/TransportExistsAction.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.exists; - - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.TransportBroadcastAction; -import org.elasticsearch.cache.recycler.PageCacheRecycler; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.GroupShardsIterator; -import org.elasticsearch.cluster.routing.ShardIterator; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.SearchService; -import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.internal.DefaultSearchContext; -import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.internal.ShardSearchLocalRequest; -import org.elasticsearch.search.query.QueryPhaseExecutionException; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReferenceArray; - -import static org.elasticsearch.action.exists.ExistsRequest.DEFAULT_MIN_SCORE; - -public class TransportExistsAction extends TransportBroadcastAction { - - private final IndicesService indicesService; - private final ScriptService scriptService; - private final PageCacheRecycler pageCacheRecycler; - private final BigArrays bigArrays; - - @Inject - public TransportExistsAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, - IndicesService indicesService, ScriptService scriptService, - PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, ExistsAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, - ExistsRequest::new, ShardExistsRequest::new, ThreadPool.Names.SEARCH); - this.indicesService = indicesService; - this.scriptService = scriptService; - this.pageCacheRecycler = pageCacheRecycler; - this.bigArrays = bigArrays; - } - - @Override - protected void doExecute(ExistsRequest request, ActionListener listener) { - request.nowInMillis = System.currentTimeMillis(); - new ExistsAsyncBroadcastAction(request, listener).start(); - } - - @Override - protected ShardExistsRequest newShardRequest(int numShards, ShardRouting shard, ExistsRequest request) { - String[] filteringAliases = indexNameExpressionResolver.filteringAliases(clusterService.state(), shard.index(), request.indices()); - return new ShardExistsRequest(shard.shardId(), filteringAliases, request); - } - - @Override - protected ShardExistsResponse newShardResponse() { - return new ShardExistsResponse(); - } - - @Override - protected GroupShardsIterator shards(ClusterState clusterState, ExistsRequest request, String[] concreteIndices) { - Map> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, request.routing(), request.indices()); - return clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference()); - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, ExistsRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, ExistsRequest countRequest, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); - } - - @Override - protected ExistsResponse newResponse(ExistsRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { - int successfulShards = 0; - int failedShards = 0; - boolean exists = false; - List shardFailures = null; - - // if docs do exist, the last response will have exists = true (since we early terminate the shard requests) - for (int i = shardsResponses.length() - 1; i >= 0 ; i--) { - Object shardResponse = shardsResponses.get(i); - if (shardResponse == null) { - // simply ignore non active shards - } else if (shardResponse instanceof BroadcastShardOperationFailedException) { - failedShards++; - if (shardFailures == null) { - shardFailures = new ArrayList<>(); - } - shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); - } else { - successfulShards++; - if ((exists = ((ShardExistsResponse) shardResponse).exists())) { - successfulShards = shardsResponses.length() - failedShards; - break; - } - } - } - return new ExistsResponse(exists, shardsResponses.length(), successfulShards, failedShards, shardFailures); - } - - @Override - protected ShardExistsResponse shardOperation(ShardExistsRequest request) { - IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); - IndexShard indexShard = indexService.getShard(request.shardId().id()); - - SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(), request.shardId().getIndex(), request.shardId().id()); - SearchContext context = new DefaultSearchContext(0, - new ShardSearchLocalRequest(request.types(), request.nowInMillis(), request.filteringAliases()), - shardTarget, indexShard.acquireSearcher("exists"), indexService, indexShard, - scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher, - SearchService.NO_TIMEOUT - ); - SearchContext.setCurrent(context); - - try { - if (request.minScore() != DEFAULT_MIN_SCORE) { - context.minimumScore(request.minScore()); - } - BytesReference source = request.querySource(); - if (source != null && source.length() > 0) { - try { - QueryShardContext.setTypes(request.types()); - context.parsedQuery(indexService.queryParserService().parseQuery(source)); - } finally { - QueryShardContext.removeTypes(); - } - } - context.preProcess(); - try { - boolean exists; - try { - exists = Lucene.exists(context.searcher(), context.query()); - } finally { - context.clearReleasables(SearchContext.Lifetime.COLLECTION); - } - return new ShardExistsResponse(request.shardId(), exists); - } catch (Exception e) { - throw new QueryPhaseExecutionException(context, "failed to execute exists", e); - } - } finally { - // this will also release the index searcher - context.close(); - SearchContext.removeCurrent(); - } - } - - /** - * An async broadcast action that early terminates shard request - * upon any shard response reporting matched doc existence - */ - final private class ExistsAsyncBroadcastAction extends AsyncBroadcastAction { - - final AtomicBoolean processed = new AtomicBoolean(false); - - ExistsAsyncBroadcastAction(ExistsRequest request, ActionListener listener) { - super(request, listener); - } - - @Override - protected void onOperation(ShardRouting shard, int shardIndex, ShardExistsResponse response) { - super.onOperation(shard, shardIndex, response); - if (response.exists()) { - finishHim(); - } - } - - @Override - protected void performOperation(final ShardIterator shardIt, final ShardRouting shard, final int shardIndex) { - if (processed.get()) { - return; - } - super.performOperation(shardIt, shard, shardIndex); - } - - @Override - protected void finishHim() { - if (processed.compareAndSet(false, true)) { - super.finishHim(); - } - } - } -} diff --git a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java index 2b796b08f9c..08c188ae998 100644 --- a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java +++ b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java @@ -21,13 +21,11 @@ package org.elasticsearch.action.explain; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ValidateActions; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.action.support.single.shard.SingleShardRequest; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; @@ -41,7 +39,7 @@ public class ExplainRequest extends SingleShardRequest { private String id; private String routing; private String preference; - private BytesReference source; + private QueryBuilder query; private String[] fields; private FetchSourceContext fetchSourceContext; @@ -102,17 +100,12 @@ public class ExplainRequest extends SingleShardRequest { return this; } - public BytesReference source() { - return source; + public QueryBuilder query() { + return query; } - public ExplainRequest source(QuerySourceBuilder sourceBuilder) { - this.source = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); - return this; - } - - public ExplainRequest source(BytesReference source) { - this.source = source; + public ExplainRequest query(QueryBuilder query) { + this.query = query; return this; } @@ -159,8 +152,8 @@ public class ExplainRequest extends SingleShardRequest { if (id == null) { validationException = ValidateActions.addValidationError("id is missing", validationException); } - if (source == null) { - validationException = ValidateActions.addValidationError("source is missing", validationException); + if (query == null) { + validationException = ValidateActions.addValidationError("query is missing", validationException); } return validationException; } @@ -172,7 +165,7 @@ public class ExplainRequest extends SingleShardRequest { id = in.readString(); routing = in.readOptionalString(); preference = in.readOptionalString(); - source = in.readBytesReference(); + query = in.readQuery(); filteringAlias = in.readStringArray(); if (in.readBoolean()) { fields = in.readStringArray(); @@ -189,7 +182,7 @@ public class ExplainRequest extends SingleShardRequest { out.writeString(id); out.writeOptionalString(routing); out.writeOptionalString(preference); - out.writeBytesReference(source); + out.writeQuery(query); out.writeStringArray(filteringAlias); if (fields != null) { out.writeBoolean(true); diff --git a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java index f78b0ea2e6d..2910736031f 100644 --- a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java @@ -19,12 +19,10 @@ package org.elasticsearch.action.explain; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.action.support.single.shard.SingleShardOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.fetch.source.FetchSourceContext; @@ -33,8 +31,6 @@ import org.elasticsearch.search.fetch.source.FetchSourceContext; */ public class ExplainRequestBuilder extends SingleShardOperationRequestBuilder { - private QuerySourceBuilder sourceBuilder; - ExplainRequestBuilder(ElasticsearchClient client, ExplainAction action) { super(client, action, new ExplainRequest()); } @@ -87,15 +83,7 @@ public class ExplainRequestBuilder extends SingleShardOperationRequestBuilder> implements Streamable, return sumTotalTermFreq; } + /** + * @return the lowest value in the field. + * + * Note that, documents marked as deleted that haven't yet been merged way aren't taken into account. + */ + public T getMinValue() { + return minValue; + } + + /** + * @return the highest value in the field. + * + * Note that, documents marked as deleted that haven't yet been merged way aren't taken into account. + */ + public T getMaxValue() { + return maxValue; + } + /** * @return the lowest value in the field represented as a string. * * Note that, documents marked as deleted that haven't yet been merged way aren't taken into account. */ - public abstract String getMinValue(); + public abstract String getMinValueAsString(); /** * @return the highest value in the field represented as a string. * * Note that, documents marked as deleted that haven't yet been merged way aren't taken into account. */ - public abstract String getMaxValue(); + public abstract String getMaxValueAsString(); /** * @param value The string to be parsed - * @return The concrete object represented by the string argument + * @param optionalFormat A string describing how to parse the specified value. Whether this parameter is supported + * depends on the implementation. If optionalFormat is specified and the implementation + * doesn't support it an {@link UnsupportedOperationException} is thrown */ - protected abstract T valueOf(String value); + protected abstract T valueOf(String value, String optionalFormat); /** * Merges the provided stats into this stats instance. @@ -153,7 +173,7 @@ public abstract class FieldStats> implements Streamable, */ public boolean match(IndexConstraint constraint) { int cmp; - T value = valueOf(constraint.getValue()); + T value = valueOf(constraint.getValue(), constraint.getOptionalFormat()); if (constraint.getProperty() == IndexConstraint.Property.MIN) { cmp = minValue.compareTo(value); } else if (constraint.getProperty() == IndexConstraint.Property.MAX) { @@ -190,8 +210,10 @@ public abstract class FieldStats> implements Streamable, } protected void toInnerXContent(XContentBuilder builder) throws IOException { - builder.field(Fields.MIN_VALUE, minValue); - builder.field(Fields.MAX_VALUE, maxValue); + builder.field(Fields.MIN_VALUE, getMinValue()); + builder.field(Fields.MIN_VALUE_AS_STRING, getMinValueAsString()); + builder.field(Fields.MAX_VALUE, getMaxValue()); + builder.field(Fields.MAX_VALUE_AS_STRING, getMaxValueAsString()); } @Override @@ -227,12 +249,12 @@ public abstract class FieldStats> implements Streamable, } @Override - public String getMinValue() { + public String getMinValueAsString() { return String.valueOf(minValue.longValue()); } @Override - public String getMaxValue() { + public String getMaxValueAsString() { return String.valueOf(maxValue.longValue()); } @@ -245,7 +267,10 @@ public abstract class FieldStats> implements Streamable, } @Override - protected java.lang.Long valueOf(String value) { + protected java.lang.Long valueOf(String value, String optionalFormat) { + if (optionalFormat != null) { + throw new UnsupportedOperationException("custom format isn't supported"); + } return java.lang.Long.valueOf(value); } @@ -277,12 +302,12 @@ public abstract class FieldStats> implements Streamable, } @Override - public String getMinValue() { + public String getMinValueAsString() { return String.valueOf(minValue.floatValue()); } @Override - public String getMaxValue() { + public String getMaxValueAsString() { return String.valueOf(maxValue.floatValue()); } @@ -295,7 +320,10 @@ public abstract class FieldStats> implements Streamable, } @Override - protected java.lang.Float valueOf(String value) { + protected java.lang.Float valueOf(String value, String optionalFormat) { + if (optionalFormat != null) { + throw new UnsupportedOperationException("custom format isn't supported"); + } return java.lang.Float.valueOf(value); } @@ -327,12 +355,12 @@ public abstract class FieldStats> implements Streamable, } @Override - public String getMinValue() { + public String getMinValueAsString() { return String.valueOf(minValue.doubleValue()); } @Override - public String getMaxValue() { + public String getMaxValueAsString() { return String.valueOf(maxValue.doubleValue()); } @@ -345,7 +373,10 @@ public abstract class FieldStats> implements Streamable, } @Override - protected java.lang.Double valueOf(String value) { + protected java.lang.Double valueOf(String value, String optionalFormat) { + if (optionalFormat != null) { + throw new UnsupportedOperationException("custom format isn't supported"); + } return java.lang.Double.valueOf(value); } @@ -377,12 +408,12 @@ public abstract class FieldStats> implements Streamable, } @Override - public String getMinValue() { + public String getMinValueAsString() { return minValue.utf8ToString(); } @Override - public String getMaxValue() { + public String getMaxValueAsString() { return maxValue.utf8ToString(); } @@ -399,14 +430,17 @@ public abstract class FieldStats> implements Streamable, } @Override - protected BytesRef valueOf(String value) { + protected BytesRef valueOf(String value, String optionalFormat) { + if (optionalFormat != null) { + throw new UnsupportedOperationException("custom format isn't supported"); + } return new BytesRef(value); } @Override protected void toInnerXContent(XContentBuilder builder) throws IOException { - builder.field(Fields.MIN_VALUE, getMinValue()); - builder.field(Fields.MAX_VALUE, getMaxValue()); + builder.field(Fields.MIN_VALUE, getMinValueAsString()); + builder.field(Fields.MAX_VALUE, getMaxValueAsString()); } @Override @@ -438,26 +472,24 @@ public abstract class FieldStats> implements Streamable, } @Override - public String getMinValue() { + public String getMinValueAsString() { return dateFormatter.printer().print(minValue); } @Override - public String getMaxValue() { + public String getMaxValueAsString() { return dateFormatter.printer().print(maxValue); } @Override - protected java.lang.Long valueOf(String value) { + protected java.lang.Long valueOf(String value, String optionalFormat) { + FormatDateTimeFormatter dateFormatter = this.dateFormatter; + if (optionalFormat != null) { + dateFormatter = Joda.forPattern(optionalFormat); + } return dateFormatter.parser().parseMillis(value); } - @Override - protected void toInnerXContent(XContentBuilder builder) throws IOException { - builder.field(Fields.MIN_VALUE, getMinValue()); - builder.field(Fields.MAX_VALUE, getMaxValue()); - } - @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -507,7 +539,9 @@ public abstract class FieldStats> implements Streamable, final static XContentBuilderString SUM_DOC_FREQ = new XContentBuilderString("sum_doc_freq"); final static XContentBuilderString SUM_TOTAL_TERM_FREQ = new XContentBuilderString("sum_total_term_freq"); final static XContentBuilderString MIN_VALUE = new XContentBuilderString("min_value"); + final static XContentBuilderString MIN_VALUE_AS_STRING = new XContentBuilderString("min_value_as_string"); final static XContentBuilderString MAX_VALUE = new XContentBuilderString("max_value"); + final static XContentBuilderString MAX_VALUE_AS_STRING = new XContentBuilderString("max_value_as_string"); } diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java index aa107518110..09411b56e25 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.fieldstats; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.broadcast.BroadcastRequest; @@ -121,22 +122,24 @@ public class FieldStatsRequest extends BroadcastRequest { currentName = parser.currentName(); } else if (fieldToken == Token.START_OBJECT) { IndexConstraint.Property property = IndexConstraint.Property.parse(currentName); - Token propertyToken = parser.nextToken(); - if (propertyToken != Token.FIELD_NAME) { - throw new IllegalArgumentException("unexpected token [" + propertyToken + "]"); - } - IndexConstraint.Comparison comparison = IndexConstraint.Comparison.parse(parser.currentName()); - propertyToken = parser.nextToken(); - if (propertyToken.isValue() == false) { - throw new IllegalArgumentException("unexpected token [" + propertyToken + "]"); - } - String value = parser.text(); - indexConstraints.add(new IndexConstraint(field, property, comparison, value)); - - propertyToken = parser.nextToken(); - if (propertyToken != Token.END_OBJECT) { - throw new IllegalArgumentException("unexpected token [" + propertyToken + "]"); + String value = null; + String optionalFormat = null; + IndexConstraint.Comparison comparison = null; + for (Token propertyToken = parser.nextToken(); propertyToken != Token.END_OBJECT; propertyToken = parser.nextToken()) { + if (propertyToken.isValue()) { + if ("format".equals(parser.currentName())) { + optionalFormat = parser.text(); + } else { + comparison = IndexConstraint.Comparison.parse(parser.currentName()); + value = parser.text(); + } + } else { + if (propertyToken != Token.FIELD_NAME) { + throw new IllegalArgumentException("unexpected token [" + propertyToken + "]"); + } + } } + indexConstraints.add(new IndexConstraint(field, property, comparison, value, optionalFormat)); } else { throw new IllegalArgumentException("unexpected token [" + fieldToken + "]"); } @@ -189,6 +192,9 @@ public class FieldStatsRequest extends BroadcastRequest { out.writeByte(indexConstraint.getProperty().getId()); out.writeByte(indexConstraint.getComparison().getId()); out.writeString(indexConstraint.getValue()); + if (out.getVersion().onOrAfter(Version.V_2_0_1)) { + out.writeOptionalString(indexConstraint.getOptionalFormat()); + } } out.writeString(level); } diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/IndexConstraint.java b/core/src/main/java/org/elasticsearch/action/fieldstats/IndexConstraint.java index 2493e34204d..19e274e785c 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/IndexConstraint.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/IndexConstraint.java @@ -19,10 +19,12 @@ package org.elasticsearch.action.fieldstats; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; import java.util.Locale; +import java.util.Objects; public class IndexConstraint { @@ -30,37 +32,68 @@ public class IndexConstraint { private final Property property; private final Comparison comparison; private final String value; + private final String optionalFormat; IndexConstraint(StreamInput input) throws IOException { this.field = input.readString(); this.property = Property.read(input.readByte()); this.comparison = Comparison.read(input.readByte()); this.value = input.readString(); + if (input.getVersion().onOrAfter(Version.V_2_0_1)) { + this.optionalFormat = input.readOptionalString(); + } else { + this.optionalFormat = null; + } } public IndexConstraint(String field, Property property, Comparison comparison, String value) { - this.field = field; - this.property = property; - this.comparison = comparison; - this.value = value; + this(field, property, comparison, value, null); } + public IndexConstraint(String field, Property property, Comparison comparison, String value, String optionalFormat) { + this.field = Objects.requireNonNull(field); + this.property = Objects.requireNonNull(property); + this.comparison = Objects.requireNonNull(comparison); + this.value = Objects.requireNonNull(value); + this.optionalFormat = optionalFormat; + } + + /** + * @return On what field the constraint is going to be applied on + */ public String getField() { return field; } + /** + * @return How to compare the specified value against the field property (lt, lte, gt and gte) + */ public Comparison getComparison() { return comparison; } + /** + * @return On what property of a field the contraint is going to be applied on (min or max value) + */ public Property getProperty() { return property; } + /** + * @return The value to compare against + */ public String getValue() { return value; } + /** + * @return An optional format, that specifies how the value string is converted in the native value of the field. + * Not all field types support this and right now only date field supports this option. + */ + public String getOptionalFormat() { + return optionalFormat; + } + public enum Property { MIN((byte) 0), diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java index f92571a53df..259003c9b18 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java @@ -119,14 +119,14 @@ public class TransportFieldStatsTransportAction extends TransportBroadcastAction while (iterator.hasNext()) { Map.Entry> entry = iterator.next(); FieldStats indexConstraintFieldStats = entry.getValue().get(indexConstraint.getField()); - if (indexConstraintFieldStats.match(indexConstraint)) { + if (indexConstraintFieldStats != null && indexConstraintFieldStats.match(indexConstraint)) { // If the field stats didn't occur in the list of fields in the original request we need to remove the // field stats, because it was never requested and was only needed to validate the index constraint if (fieldStatFields.contains(indexConstraint.getField()) == false) { entry.getValue().remove(indexConstraint.getField()); } } else { - // The index constraint didn't match, so we remove all the field stats of the index we're checking + // The index constraint didn't match or was empty, so we remove all the field stats of the index we're checking iterator.remove(); } } diff --git a/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java index 89bacf06c3f..f785a3065ed 100644 --- a/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java @@ -108,23 +108,6 @@ public class GetRequestBuilder extends SingleShardOperationRequestBuilder implements I result = 31 * result + id.hashCode(); result = 31 * result + (routing != null ? routing.hashCode() : 0); result = 31 * result + (fields != null ? Arrays.hashCode(fields) : 0); - result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + Long.hashCode(version); result = 31 * result + versionType.hashCode(); result = 31 * result + (fetchSourceContext != null ? fetchSourceContext.hashCode() : 0); return result; diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 0bcadd6c90a..a2cb9873474 100644 --- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -77,7 +77,7 @@ public class TransportGetAction extends TransportSingleShardAction implements Do private String parent; @Nullable private String timestamp; - private long ttl = -1; + @Nullable + private TimeValue ttl; private BytesReference source; @@ -227,6 +231,12 @@ public class IndexRequest extends ReplicationRequest implements Do if (!versionType.validateVersionForWrites(version)) { validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException); } + + if (ttl != null) { + if (ttl.millis() < 0) { + validationException = addValidationError("ttl must not be negative", validationException); + } + } return validationException; } @@ -322,22 +332,33 @@ public class IndexRequest extends ReplicationRequest implements Do } /** - * Sets the relative ttl value. It musts be > 0 as it makes little sense otherwise. Setting it - * to null will reset to have no ttl. + * Sets the ttl value as a time value expression. */ - public IndexRequest ttl(Long ttl) throws ElasticsearchGenerationException { - if (ttl == null) { - this.ttl = -1; - return this; - } - if (ttl <= 0) { - throw new IllegalArgumentException("TTL value must be > 0. Illegal value provided [" + ttl + "]"); - } + public IndexRequest ttl(String ttl) { + this.ttl = TimeValue.parseTimeValue(ttl, null, "ttl"); + return this; + } + + /** + * Sets the ttl as a {@link TimeValue} instance. + */ + public IndexRequest ttl(TimeValue ttl) { this.ttl = ttl; return this; } - public long ttl() { + /** + * Sets the relative ttl value in milliseconds. It musts be greater than 0 as it makes little sense otherwise. + */ + public IndexRequest ttl(long ttl) { + this.ttl = new TimeValue(ttl); + return this; + } + + /** + * Returns the ttl as a {@link TimeValue} + */ + public TimeValue ttl() { return this.ttl; } @@ -561,15 +582,24 @@ public class IndexRequest extends ReplicationRequest implements Do return this.versionType; } + private Version getVersion(MetaData metaData, String concreteIndex) { + // this can go away in 3.0 but is here now for easy backporting - since in 2.x we need the version on the timestamp stuff + final IndexMetaData indexMetaData = metaData.getIndices().get(concreteIndex); + if (indexMetaData == null) { + throw new IndexNotFoundException(concreteIndex); + } + return Version.indexCreated(indexMetaData.getSettings()); + } + public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, boolean allowIdGeneration, String concreteIndex) { // resolve the routing if needed routing(metaData.resolveIndexRouting(routing, index)); + // resolve timestamp if provided externally if (timestamp != null) { - Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings()); timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER, - version); + getVersion(metaData, concreteIndex)); } // extract values if needed if (mappingMd != null) { @@ -592,8 +622,7 @@ public class IndexRequest extends ReplicationRequest implements Do if (parseContext.shouldParseTimestamp()) { timestamp = parseContext.timestamp(); if (timestamp != null) { - Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings()); - timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter(), version); + timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter(), getVersion(metaData, concreteIndex)); } } } catch (MapperParsingException e) { @@ -642,8 +671,7 @@ public class IndexRequest extends ReplicationRequest implements Do if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) { timestamp = Long.toString(System.currentTimeMillis()); } else { - Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings()); - timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), version); + timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), getVersion(metaData, concreteIndex)); } } } @@ -656,7 +684,7 @@ public class IndexRequest extends ReplicationRequest implements Do routing = in.readOptionalString(); parent = in.readOptionalString(); timestamp = in.readOptionalString(); - ttl = in.readLong(); + ttl = in.readBoolean() ? TimeValue.readTimeValue(in) : null; source = in.readBytesReference(); opType = OpType.fromId(in.readByte()); @@ -673,7 +701,12 @@ public class IndexRequest extends ReplicationRequest implements Do out.writeOptionalString(routing); out.writeOptionalString(parent); out.writeOptionalString(timestamp); - out.writeLong(ttl); + if (ttl == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + ttl.writeTo(out); + } out.writeBytesReference(source); out.writeByte(opType.id()); out.writeBoolean(refresh); diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 2df8fec6d22..f7134d84843 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.replication.ReplicationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; @@ -254,9 +255,27 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder 0 as it makes little sense otherwise. + /** + * Sets the ttl value as a time value expression. + */ + public IndexRequestBuilder setTTL(String ttl) { + request.ttl(ttl); + return this; + } + + /** + * Sets the relative ttl value in milliseconds. It musts be greater than 0 as it makes little sense otherwise. + */ public IndexRequestBuilder setTTL(long ttl) { request.ttl(ttl); return this; } + + /** + * Sets the ttl as a {@link TimeValue} instance. + */ + public IndexRequestBuilder setTTL(TimeValue ttl) { + request.ttl(ttl); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java b/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java index 5727b2b673b..665327a749f 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java @@ -19,9 +19,13 @@ package org.elasticsearch.action.index; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -31,54 +35,19 @@ import java.io.IOException; * @see org.elasticsearch.action.index.IndexRequest * @see org.elasticsearch.client.Client#index(IndexRequest) */ -public class IndexResponse extends ActionWriteResponse { +public class IndexResponse extends DocWriteResponse { - private String index; - private String id; - private String type; - private long version; private boolean created; public IndexResponse() { } - public IndexResponse(String index, String type, String id, long version, boolean created) { - this.index = index; - this.id = id; - this.type = type; - this.version = version; + public IndexResponse(ShardId shardId, String type, String id, long version, boolean created) { + super(shardId, type, id, version); this.created = created; } - /** - * The index the document was indexed into. - */ - public String getIndex() { - return this.index; - } - - /** - * The type of the document indexed. - */ - public String getType() { - return this.type; - } - - /** - * The id of the document indexed. - */ - public String getId() { - return this.id; - } - - /** - * Returns the current version of the doc indexed. - */ - public long getVersion() { - return this.version; - } - /** * Returns true if the document was created, false if updated. */ @@ -86,23 +55,23 @@ public class IndexResponse extends ActionWriteResponse { return this.created; } + @Override + public RestStatus status() { + if (created) { + return RestStatus.CREATED; + } + return super.status(); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - index = in.readString(); - type = in.readString(); - id = in.readString(); - version = in.readLong(); created = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(index); - out.writeString(type); - out.writeString(id); - out.writeLong(version); out.writeBoolean(created); } @@ -110,12 +79,23 @@ public class IndexResponse extends ActionWriteResponse { public String toString() { StringBuilder builder = new StringBuilder(); builder.append("IndexResponse["); - builder.append("index=").append(index); - builder.append(",type=").append(type); - builder.append(",id=").append(id); - builder.append(",version=").append(version); + builder.append("index=").append(getIndex()); + builder.append(",type=").append(getType()); + builder.append(",id=").append(getId()); + builder.append(",version=").append(getVersion()); builder.append(",created=").append(created); builder.append(",shards=").append(getShardInfo()); return builder.append("]").toString(); } + + static final class Fields { + static final XContentBuilderString CREATED = new XContentBuilderString("created"); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + super.toXContent(builder, params); + builder.field(Fields.CREATED, isCreated()); + return builder; + } } diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java index 63b82377d8a..620056ded4e 100644 --- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java +++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -120,74 +119,111 @@ public class TransportIndexAction extends TransportReplicationAction indexResponseActionListener) { - MetaData metaData = clusterService.state().metaData(); - + protected void resolveRequest(MetaData metaData, String concreteIndex, IndexRequest request) { MappingMetaData mappingMd = null; - if (metaData.hasIndex(request.concreteIndex())) { - mappingMd = metaData.index(request.concreteIndex()).mappingOrDefault(request.request().type()); + if (metaData.hasIndex(concreteIndex)) { + mappingMd = metaData.index(concreteIndex).mappingOrDefault(request.type()); } - request.request().process(metaData, mappingMd, allowIdGeneration, request.concreteIndex()); + request.process(metaData, mappingMd, allowIdGeneration, concreteIndex); + ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(), concreteIndex, request.id(), request.routing()); + request.setShardId(shardId); } private void innerExecute(final IndexRequest request, final ActionListener listener) { super.doExecute(request, listener); } - @Override - protected boolean checkWriteConsistency() { - return true; - } - @Override protected IndexResponse newResponseInstance() { return new IndexResponse(); } @Override - protected ShardIterator shards(ClusterState clusterState, InternalRequest request) { - return clusterService.operationRouting() - .indexShards(clusterService.state(), request.concreteIndex(), request.request().type(), request.request().id(), request.request().routing()); - } - - @Override - protected Tuple shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { - final IndexRequest request = shardRequest.request; + protected Tuple shardOperationOnPrimary(MetaData metaData, IndexRequest request) throws Throwable { // validate, if routing is required, that we got routing - IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex()); + IndexMetaData indexMetaData = metaData.index(request.shardId().getIndex()); MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type()); if (mappingMd != null && mappingMd.routing().required()) { if (request.routing() == null) { - throw new RoutingMissingException(shardRequest.shardId.getIndex(), request.type(), request.id()); + throw new RoutingMissingException(request.shardId().getIndex(), request.type(), request.id()); } } - IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()); - IndexShard indexShard = indexService.getShard(shardRequest.shardId.id()); + IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); + IndexShard indexShard = indexService.getShard(request.shardId().id()); - final WriteResult result = executeIndexRequestOnPrimary(null, request, indexShard); + final WriteResult result = executeIndexRequestOnPrimary(request, indexShard, mappingUpdatedAction); final IndexResponse response = result.response; final Translog.Location location = result.location; - processAfter(request.refresh(), indexShard, location); - return new Tuple<>(response, shardRequest.request); + processAfterWrite(request.refresh(), indexShard, location); + return new Tuple<>(response, request); } @Override - protected void shardOperationOnReplica(ShardId shardId, IndexRequest request) { + protected void shardOperationOnReplica(IndexRequest request) { + final ShardId shardId = request.shardId(); IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); + final Engine.Index operation = executeIndexRequestOnReplica(request, indexShard); + processAfterWrite(request.refresh(), indexShard, operation.getTranslogLocation()); + } + + /** + * Execute the given {@link IndexRequest} on a replica shard, throwing a + * {@link RetryOnReplicaException} if the operation needs to be re-tried. + */ + public static Engine.Index executeIndexRequestOnReplica(IndexRequest request, IndexShard indexShard) { + final ShardId shardId = indexShard.shardId(); SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, request.source()).index(shardId.getIndex()).type(request.type()).id(request.id()) .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl()); - final Engine.Index operation = indexShard.prepareIndex(sourceToParse, request.version(), request.versionType(), Engine.Operation.Origin.REPLICA); + final Engine.Index operation = indexShard.prepareIndexOnReplica(sourceToParse, request.version(), request.versionType()); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { throw new RetryOnReplicaException(shardId, "Mappings are not available on the replica yet, triggered update: " + update); } indexShard.index(operation); - processAfter(request.refresh(), indexShard, operation.getTranslogLocation()); + return operation; + } + + /** Utility method to prepare an index operation on primary shards */ + public static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard indexShard) { + SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source()).index(request.index()).type(request.type()).id(request.id()) + .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl()); + return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType()); + } + + /** + * Execute the given {@link IndexRequest} on a primary shard, throwing a + * {@link RetryOnPrimaryException} if the operation needs to be re-tried. + */ + public static WriteResult executeIndexRequestOnPrimary(IndexRequest request, IndexShard indexShard, MappingUpdatedAction mappingUpdatedAction) throws Throwable { + Engine.Index operation = prepareIndexOperationOnPrimary(request, indexShard); + Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); + final ShardId shardId = indexShard.shardId(); + if (update != null) { + final String indexName = shardId.getIndex(); + mappingUpdatedAction.updateMappingOnMasterSynchronously(indexName, request.type(), update); + operation = prepareIndexOperationOnPrimary(request, indexShard); + update = operation.parsedDoc().dynamicMappingsUpdate(); + if (update != null) { + throw new RetryOnPrimaryException(shardId, + "Dynamic mappings are not available on the node that holds the primary yet"); + } + } + final boolean created = indexShard.index(operation); + + // update the version on request so it will happen on the replicas + final long version = operation.version(); + request.version(version); + request.versionType(request.versionType().versionTypeForReplicationAndRecovery()); + + assert request.versionType().validateVersionForWrites(request.version()); + + return new WriteResult<>(new IndexResponse(shardId, request.type(), request.id(), request.version(), created), operation.getTranslogLocation()); } } + diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java index 967d5a0a2b7..79f51db59f7 100644 --- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java @@ -238,17 +238,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - XContentType contentType = XContentFactory.xContentType(doc); - if (contentType == builder.contentType()) { - builder.rawField("doc", doc); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(doc)) { - parser.nextToken(); - builder.field("doc"); - builder.copyCurrentStructure(parser); - } - } - return builder; + return builder.rawField("doc", doc); } } diff --git a/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java b/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java index 17343e86912..b390b77504a 100644 --- a/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java @@ -37,6 +37,17 @@ public class ClearScrollRequest extends ActionRequest { private List scrollIds; + public ClearScrollRequest() { + } + + /** + * Creates a clear scroll request caused by some other request, which is provided as an + * argument so that its headers and context can be copied to the new request + */ + public ClearScrollRequest(ActionRequest request) { + super(request); + } + public List getScrollIds() { return scrollIds; } diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index d754d969428..a3236e9653f 100644 --- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -24,22 +24,14 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.*; /** * A multi search API request. @@ -68,107 +60,6 @@ public class MultiSearchRequest extends ActionRequest implem return this; } - public MultiSearchRequest add(byte[] data, int from, int length, - boolean isTemplateRequest, @Nullable String[] indices, @Nullable String[] types, @Nullable String searchType) throws Exception { - return add(new BytesArray(data, from, length), isTemplateRequest, indices, types, searchType, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true); - } - - public MultiSearchRequest add(BytesReference data, boolean isTemplateRequest, @Nullable String[] indices, @Nullable String[] types, @Nullable String searchType, IndicesOptions indicesOptions) throws Exception { - return add(data, isTemplateRequest, indices, types, searchType, null, indicesOptions, true); - } - - public MultiSearchRequest add(BytesReference data, boolean isTemplateRequest, @Nullable String[] indices, @Nullable String[] types, @Nullable String searchType, @Nullable String routing, IndicesOptions indicesOptions, boolean allowExplicitIndex) throws Exception { - XContent xContent = XContentFactory.xContent(data); - int from = 0; - int length = data.length(); - byte marker = xContent.streamSeparator(); - while (true) { - int nextMarker = findNextMarker(marker, from, data, length); - if (nextMarker == -1) { - break; - } - // support first line with \n - if (nextMarker == 0) { - from = nextMarker + 1; - continue; - } - - SearchRequest searchRequest = new SearchRequest(); - if (indices != null) { - searchRequest.indices(indices); - } - if (indicesOptions != null) { - searchRequest.indicesOptions(indicesOptions); - } - if (types != null && types.length > 0) { - searchRequest.types(types); - } - if (routing != null) { - searchRequest.routing(routing); - } - searchRequest.searchType(searchType); - - IndicesOptions defaultOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); - - - // now parse the action - if (nextMarker - from > 0) { - try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) { - Map source = parser.map(); - for (Map.Entry entry : source.entrySet()) { - Object value = entry.getValue(); - if ("index".equals(entry.getKey()) || "indices".equals(entry.getKey())) { - if (!allowExplicitIndex) { - throw new IllegalArgumentException("explicit index in multi percolate is not allowed"); - } - searchRequest.indices(nodeStringArrayValue(value)); - } else if ("type".equals(entry.getKey()) || "types".equals(entry.getKey())) { - searchRequest.types(nodeStringArrayValue(value)); - } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) { - searchRequest.searchType(nodeStringValue(value, null)); - } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) { - searchRequest.requestCache(nodeBooleanValue(value)); - } else if ("preference".equals(entry.getKey())) { - searchRequest.preference(nodeStringValue(value, null)); - } else if ("routing".equals(entry.getKey())) { - searchRequest.routing(nodeStringValue(value, null)); - } - } - defaultOptions = IndicesOptions.fromMap(source, defaultOptions); - } - } - searchRequest.indicesOptions(defaultOptions); - - // move pointers - from = nextMarker + 1; - // now for the body - nextMarker = findNextMarker(marker, from, data, length); - if (nextMarker == -1) { - break; - } - if (isTemplateRequest) { - searchRequest.templateSource(data.slice(from, nextMarker - from)); - } else { - searchRequest.source(data.slice(from, nextMarker - from)); - } - // move pointers - from = nextMarker + 1; - - add(searchRequest); - } - - return this; - } - - private int findNextMarker(byte marker, int from, BytesReference data, int length) { - for (int i = from; i < length; i++) { - if (data.get(i) == marker) { - return i; - } - } - return -1; - } - public List requests() { return this.requests; } diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java index 68bcf6d269c..40e8b0730ff 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java @@ -38,13 +38,11 @@ public class SearchPhaseExecutionException extends ElasticsearchException { private final ShardSearchFailure[] shardFailures; public SearchPhaseExecutionException(String phaseName, String msg, ShardSearchFailure[] shardFailures) { - super(msg); - this.phaseName = phaseName; - this.shardFailures = shardFailures; + this(phaseName, msg, null, shardFailures); } public SearchPhaseExecutionException(String phaseName, String msg, Throwable cause, ShardSearchFailure[] shardFailures) { - super(msg, cause); + super(msg, deduplicateCause(cause, shardFailures)); this.phaseName = phaseName; this.shardFailures = shardFailures; } @@ -63,12 +61,26 @@ public class SearchPhaseExecutionException extends ElasticsearchException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeOptionalString(phaseName); - out.writeVInt(shardFailures == null ? 0 : shardFailures.length); - if (shardFailures != null) { + out.writeVInt(shardFailures.length); + for (ShardSearchFailure failure : shardFailures) { + failure.writeTo(out); + } + } + + private static final Throwable deduplicateCause(Throwable cause, ShardSearchFailure[] shardFailures) { + if (shardFailures == null) { + throw new IllegalArgumentException("shardSearchFailures must not be null"); + } + // if the cause of this exception is also the cause of one of the shard failures we don't add it + // to prevent duplication in stack traces rendered to the REST layer + if (cause != null) { for (ShardSearchFailure failure : shardFailures) { - failure.writeTo(out); + if (failure.getCause() == cause) { + return null; + } } } + return cause; } @Override diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 9348185e513..8014e4acb6c 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -19,31 +19,21 @@ package org.elasticsearch.action.search; -import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.Template; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; -import java.util.Map; import static org.elasticsearch.search.Scroll.readScroll; @@ -53,9 +43,7 @@ import static org.elasticsearch.search.Scroll.readScroll; *

* Note, the search {@link #source(org.elasticsearch.search.builder.SearchSourceBuilder)} * is required. The search source is the different search options, including aggregations and such. - *

- * There is an option to specify an addition search source using the {@link #extraSource(org.elasticsearch.search.builder.SearchSourceBuilder)}. - * + *

* @see org.elasticsearch.client.Requests#searchRequest(String...) * @see org.elasticsearch.client.Client#search(SearchRequest) * @see SearchResponse @@ -71,12 +59,8 @@ public class SearchRequest extends ActionRequest implements Indic @Nullable private String preference; - private BytesReference templateSource; - private Template template; + private SearchSourceBuilder source; - private BytesReference source; - - private BytesReference extraSource; private Boolean requestCache; private Scroll scroll; @@ -87,6 +71,8 @@ public class SearchRequest extends ActionRequest implements Indic private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; + private Template template; + public SearchRequest() { } @@ -100,10 +86,8 @@ public class SearchRequest extends ActionRequest implements Indic this.indices = searchRequest.indices; this.routing = searchRequest.routing; this.preference = searchRequest.preference; - this.templateSource = searchRequest.templateSource; this.template = searchRequest.template; this.source = searchRequest.source; - this.extraSource = searchRequest.extraSource; this.requestCache = searchRequest.requestCache; this.scroll = searchRequest.scroll; this.types = searchRequest.types; @@ -116,6 +100,7 @@ public class SearchRequest extends ActionRequest implements Indic */ public SearchRequest(ActionRequest request) { super(request); + this.source = new SearchSourceBuilder(); } /** @@ -123,15 +108,18 @@ public class SearchRequest extends ActionRequest implements Indic * will run against all indices. */ public SearchRequest(String... indices) { - indices(indices); + this(indices, new SearchSourceBuilder()); } /** * Constructs a new search request against the provided indices with the given search source. */ - public SearchRequest(String[] indices, byte[] source) { + public SearchRequest(String[] indices, SearchSourceBuilder source) { + if (source == null) { + throw new IllegalArgumentException("source must not be null"); + } indices(indices); - this.source = new BytesArray(source); + this.source = source; } @Override @@ -247,60 +235,20 @@ public class SearchRequest extends ActionRequest implements Indic * The source of the search request. */ public SearchRequest source(SearchSourceBuilder sourceBuilder) { - this.source = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); + if (sourceBuilder == null) { + throw new IllegalArgumentException("source must not be null"); + } + this.source = sourceBuilder; return this; } /** * The search source to execute. */ - public SearchRequest source(BytesReference source) { - this.source = source; - return this; - } - - - /** - * The search source to execute. - */ - public BytesReference source() { + public SearchSourceBuilder source() { return source; } - /** - * The search source template to execute. - */ - public BytesReference templateSource() { - return templateSource; - } - - /** - * Allows to provide additional source that will be used as well. - */ - public SearchRequest extraSource(SearchSourceBuilder sourceBuilder) { - if (sourceBuilder == null) { - extraSource = null; - return this; - } - this.extraSource = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); - return this; - } - - /** - * Allows to provide template as source. - */ - public SearchRequest templateSource(BytesReference template) { - this.templateSource = template; - return this; - } - - /** - * The template of the search request. - */ - public SearchRequest templateSource(String template) { - this.templateSource = new BytesArray(template); - return this; - } /** * The stored template @@ -316,88 +264,6 @@ public class SearchRequest extends ActionRequest implements Indic return template; } - /** - * The name of the stored template - * - * @deprecated use {@link #template(Template)} instead. - */ - @Deprecated - public void templateName(String templateName) { - updateOrCreateScript(templateName, null, null, null); - } - - /** - * The type of the stored template - * - * @deprecated use {@link #template(Template)} instead. - */ - @Deprecated - public void templateType(ScriptService.ScriptType templateType) { - updateOrCreateScript(null, templateType, null, null); - } - - /** - * Template parameters used for rendering - * - * @deprecated use {@link #template(Template)} instead. - */ - @Deprecated - public void templateParams(Map params) { - updateOrCreateScript(null, null, null, params); - } - - /** - * The name of the stored template - * - * @deprecated use {@link #template()} instead. - */ - @Deprecated - public String templateName() { - return template == null ? null : template.getScript(); - } - - /** - * The name of the stored template - * - * @deprecated use {@link #template()} instead. - */ - @Deprecated - public ScriptService.ScriptType templateType() { - return template == null ? null : template.getType(); - } - - /** - * Template parameters used for rendering - * - * @deprecated use {@link #template()} instead. - */ - @Deprecated - public Map templateParams() { - return template == null ? null : template.getParams(); - } - - private void updateOrCreateScript(String templateContent, ScriptType type, String lang, Map params) { - Template template = template(); - if (template == null) { - template = new Template(templateContent == null ? "" : templateContent, type == null ? ScriptType.INLINE : type, lang, null, - params); - } else { - String newTemplateContent = templateContent == null ? template.getScript() : templateContent; - ScriptType newTemplateType = type == null ? template.getType() : type; - String newTemplateLang = lang == null ? template.getLang() : lang; - Map newTemplateParams = params == null ? template.getParams() : params; - template = new Template(newTemplateContent, newTemplateType, MustacheScriptEngineService.NAME, null, newTemplateParams); - } - template(template); - } - - /** - * Additional search source to execute. - */ - public BytesReference extraSource() { - return this.extraSource; - } - /** * The tye of search to execute. */ @@ -472,18 +338,15 @@ public class SearchRequest extends ActionRequest implements Indic if (in.readBoolean()) { scroll = readScroll(in); } - - source = in.readBytesReference(); - extraSource = in.readBytesReference(); + if (in.readBoolean()) { + source = SearchSourceBuilder.readSearchSourceFrom(in); + } types = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); - templateSource = in.readBytesReference(); - if (in.readBoolean()) { - template = Template.readTemplate(in); - } requestCache = in.readOptionalBoolean(); + template = in.readOptionalStreamable(Template::new); } @Override @@ -505,18 +368,15 @@ public class SearchRequest extends ActionRequest implements Indic out.writeBoolean(true); scroll.writeTo(out); } - out.writeBytesReference(source); - out.writeBytesReference(extraSource); + if (source == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + source.writeTo(out); + } out.writeStringArray(types); indicesOptions.writeIndicesOptions(out); - - out.writeBytesReference(templateSource); - boolean hasTemplate = template != null; - out.writeBoolean(hasTemplate); - if (hasTemplate) { - template.writeTo(out); - } - out.writeOptionalBoolean(requestCache); + out.writeOptionalStreamable(template); } } diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index a57008056b4..442b0915e3b 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -19,18 +19,13 @@ package org.elasticsearch.action.search; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.Template; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; @@ -42,15 +37,14 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; -import java.util.Map; +import java.util.Arrays; +import java.util.List; /** * A search action request builder. */ public class SearchRequestBuilder extends ActionRequestBuilder { - private SearchSourceBuilder sourceBuilder; - public SearchRequestBuilder(ElasticsearchClient client, SearchAction action) { super(client, action, new SearchRequest()); } @@ -122,14 +116,6 @@ public class SearchRequestBuilder extends ActionRequestBuilder queryBuilder) { sourceBuilder().query(queryBuilder); return this; } - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchRequestBuilder setQuery(String query) { - sourceBuilder().query(query); - return this; - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchRequestBuilder setQuery(BytesReference queryBinary) { - sourceBuilder().query(queryBinary); - return this; - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchRequestBuilder setQuery(byte[] queryBinary) { - sourceBuilder().query(queryBinary); - return this; - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchRequestBuilder setQuery(byte[] queryBinary, int queryBinaryOffset, int queryBinaryLength) { - sourceBuilder().query(queryBinary, queryBinaryOffset, queryBinaryLength); - return this; - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchRequestBuilder setQuery(XContentBuilder query) { - sourceBuilder().query(query); - return this; - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchRequestBuilder setQuery(Map query) { - sourceBuilder().query(query); - return this; - } - /** * Sets a filter that will be executed after the query has been executed and only has affect on the search hits * (not aggregations). This filter is always executed as last filtering mechanism. */ - public SearchRequestBuilder setPostFilter(QueryBuilder postFilter) { - sourceBuilder().postFilter(postFilter); - return this; - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchRequestBuilder setPostFilter(String postFilter) { - sourceBuilder().postFilter(postFilter); - return this; - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchRequestBuilder setPostFilter(BytesReference postFilter) { - sourceBuilder().postFilter(postFilter); - return this; - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchRequestBuilder setPostFilter(byte[] postFilter) { - sourceBuilder().postFilter(postFilter); - return this; - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchRequestBuilder setPostFilter(byte[] postFilter, int postFilterOffset, int postFilterLength) { - sourceBuilder().postFilter(postFilter, postFilterOffset, postFilterLength); - return this; - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchRequestBuilder setPostFilter(XContentBuilder postFilter) { - sourceBuilder().postFilter(postFilter); - return this; - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchRequestBuilder setPostFilter(Map postFilter) { + public SearchRequestBuilder setPostFilter(QueryBuilder postFilter) { sourceBuilder().postFilter(postFilter); return this; } @@ -353,6 +237,14 @@ public class SearchRequestBuilder extends ActionRequestBuilder statsGroups) { sourceBuilder().stats(statsGroups); return this; } @@ -461,11 +353,11 @@ public class SearchRequestBuilder extends ActionRequestBuilderstyled and default. - * - * @param schemaName The tag scheme name - */ - public SearchRequestBuilder setHighlighterTagsSchema(String schemaName) { - highlightBuilder().tagsSchema(schemaName); - return this; - } - - public SearchRequestBuilder setHighlighterFragmentSize(Integer fragmentSize) { - highlightBuilder().fragmentSize(fragmentSize); - return this; - } - - public SearchRequestBuilder setHighlighterNumOfFragments(Integer numOfFragments) { - highlightBuilder().numOfFragments(numOfFragments); - return this; - } - - public SearchRequestBuilder setHighlighterFilter(Boolean highlightFilter) { - highlightBuilder().highlightFilter(highlightFilter); - return this; - } - - /** - * The encoder to set for highlighting - */ - public SearchRequestBuilder setHighlighterEncoder(String encoder) { - highlightBuilder().encoder(encoder); - return this; - } - - /** - * Explicitly set the pre tags that will be used for highlighting. - */ - public SearchRequestBuilder setHighlighterPreTags(String... preTags) { - highlightBuilder().preTags(preTags); - return this; - } - - /** - * Explicitly set the post tags that will be used for highlighting. - */ - public SearchRequestBuilder setHighlighterPostTags(String... postTags) { - highlightBuilder().postTags(postTags); - return this; - } - - /** - * The order of fragments per field. By default, ordered by the order in the - * highlighted text. Can be score, which then it will be ordered - * by score of the fragments. - */ - public SearchRequestBuilder setHighlighterOrder(String order) { - highlightBuilder().order(order); - return this; - } - - public SearchRequestBuilder setHighlighterRequireFieldMatch(boolean requireFieldMatch) { - highlightBuilder().requireFieldMatch(requireFieldMatch); - return this; - } - - public SearchRequestBuilder setHighlighterBoundaryMaxScan(Integer boundaryMaxScan) { - highlightBuilder().boundaryMaxScan(boundaryMaxScan); - return this; - } - - public SearchRequestBuilder setHighlighterBoundaryChars(char[] boundaryChars) { - highlightBuilder().boundaryChars(boundaryChars); - return this; - } - - /** - * The highlighter type to use. - */ - public SearchRequestBuilder setHighlighterType(String type) { - highlightBuilder().highlighterType(type); - return this; - } - - public SearchRequestBuilder setHighlighterFragmenter(String fragmenter) { - highlightBuilder().fragmenter(fragmenter); - return this; - } - - /** - * Sets a query to be used for highlighting all fields instead of the search query. - */ - public SearchRequestBuilder setHighlighterQuery(QueryBuilder highlightQuery) { - highlightBuilder().highlightQuery(highlightQuery); - return this; - } - - /** - * Sets the size of the fragment to return from the beginning of the field if there are no matches to - * highlight and the field doesn't also define noMatchSize. - * - * @param noMatchSize integer to set or null to leave out of request. default is null. - * @return this builder for chaining - */ - public SearchRequestBuilder setHighlighterNoMatchSize(Integer noMatchSize) { - highlightBuilder().noMatchSize(noMatchSize); - return this; - } - - /** - * Sets the maximum number of phrases the fvh will consider if the field doesn't also define phraseLimit. - */ - public SearchRequestBuilder setHighlighterPhraseLimit(Integer phraseLimit) { - highlightBuilder().phraseLimit(phraseLimit); - return this; - } - - public SearchRequestBuilder setHighlighterOptions(Map options) { - highlightBuilder().options(options); - return this; - } - - /** - * Forces to highlight fields based on the source even if fields are stored separately. - */ - public SearchRequestBuilder setHighlighterForceSource(Boolean forceSource) { - highlightBuilder().forceSource(forceSource); - return this; - } - - /** - * Send the fields to be highlighted using a syntax that is specific about the order in which they should be highlighted. - * - * @return this for chaining - */ - public SearchRequestBuilder setHighlighterExplicitFieldOrder(boolean explicitFieldOrder) { - highlightBuilder().useExplicitFieldOrder(explicitFieldOrder); - return this; - } - - public SearchRequestBuilder addParentChildInnerHits(String name, String type, InnerHitsBuilder.InnerHit innerHit) { - innerHitsBuilder().addParentChildInnerHits(name, type, innerHit); - return this; - } - - public SearchRequestBuilder addNestedInnerHits(String name, String path, InnerHitsBuilder.InnerHit innerHit) { - innerHitsBuilder().addNestedInnerHits(name, path, innerHit); - return this; - } - - /** - * Delegates to {@link org.elasticsearch.search.suggest.SuggestBuilder#setText(String)}. - */ - public SearchRequestBuilder setSuggestText(String globalText) { - suggestBuilder().setText(globalText); - return this; - } - - /** - * Delegates to {@link org.elasticsearch.search.suggest.SuggestBuilder#addSuggestion(org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder)}. - */ - public SearchRequestBuilder addSuggestion(SuggestBuilder.SuggestionBuilder suggestion) { - suggestBuilder().addSuggestion(suggestion); + public SearchRequestBuilder innerHits(InnerHitsBuilder innerHitsBuilder) { + sourceBuilder().innerHits(innerHitsBuilder); return this; } @@ -800,46 +448,13 @@ public class SearchRequestBuilder extends ActionRequestBuilder { this.scrollId = scrollId; } + /** + * Creates a scroll request caused by some other request, which is provided as an + * argument so that its headers and context can be copied to the new request + */ + public SearchScrollRequest(ActionRequest request) { + super(request); + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; diff --git a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java index cd4238ccdea..2a953f9b732 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java @@ -114,7 +114,7 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent public void start() { if (scrollId.getContext().length == 0) { - listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", null)); + listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY)); return; } @@ -175,7 +175,7 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { if (successfulOps.get() == 0) { - listener.onFailure(new SearchPhaseExecutionException("query_fetch", "all shards failed", buildShardFailures())); + listener.onFailure(new SearchPhaseExecutionException("query_fetch", "all shards failed", t, buildShardFailures())); } else { finishHim(); } diff --git a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java index 85b06ea7860..8f2df714319 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java @@ -123,7 +123,7 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent public void start() { if (scrollId.getContext().length == 0) { - listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", null)); + listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY)); return; } final AtomicInteger counter = new AtomicInteger(scrollId.getContext().length); @@ -143,7 +143,7 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent try { executeFetchPhase(); } catch (Throwable e) { - listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, null)); + listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, ShardSearchFailure.EMPTY_ARRAY)); return; } } @@ -181,12 +181,12 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { if (successfulOps.get() == 0) { - listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", buildShardFailures())); + listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", t, buildShardFailures())); } else { try { executeFetchPhase(); } catch (Throwable e) { - listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, null)); + listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, ShardSearchFailure.EMPTY_ARRAY)); } } } diff --git a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java index fa5776387dd..31cd3986d2f 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java @@ -220,17 +220,19 @@ public abstract class TransportSearchTypeAction extends TransportAction extends BaseFuture implements ActionFuture, ActionListener { - private Throwable rootFailure; - @Override public T actionGet() { try { @@ -69,7 +67,7 @@ public abstract class AdapterActionFuture extends BaseFuture implements try { return get(timeout, unit); } catch (TimeoutException e) { - throw new ElasticsearchTimeoutException(e.getMessage()); + throw new ElasticsearchTimeoutException(e); } catch (InterruptedException e) { throw new IllegalStateException("Future got interrupted", e); } catch (ExecutionException e) { @@ -105,9 +103,4 @@ public abstract class AdapterActionFuture extends BaseFuture implements } protected abstract T convert(L listenerResponse); - - @Override - public Throwable getRootFailure() { - return rootFailure; - } } diff --git a/core/src/main/java/org/elasticsearch/action/support/QuerySourceBuilder.java b/core/src/main/java/org/elasticsearch/action/support/QuerySourceBuilder.java deleted file mode 100644 index 9266310a48b..00000000000 --- a/core/src/main/java/org/elasticsearch/action/support/QuerySourceBuilder.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.support; - -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilder; - -import java.io.IOException; - -public class QuerySourceBuilder extends ToXContentToBytes { - - private QueryBuilder queryBuilder; - - private BytesReference queryBinary; - - public QuerySourceBuilder setQuery(QueryBuilder query) { - this.queryBuilder = query; - return this; - } - - public QuerySourceBuilder setQuery(BytesReference queryBinary) { - this.queryBinary = queryBinary; - return this; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - innerToXContent(builder, params); - builder.endObject(); - return builder; - } - - public void innerToXContent(XContentBuilder builder, Params params) throws IOException { - if (queryBuilder != null) { - builder.field("query"); - queryBuilder.toXContent(builder, params); - } - - if (queryBinary != null) { - if (XContentFactory.xContentType(queryBinary) == builder.contentType()) { - builder.rawField("query", queryBinary); - } else { - builder.field("query_binary", queryBinary); - } - } - } -} diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 7154c74ce4a..8bcba8ad544 100644 --- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -223,12 +223,20 @@ public abstract class TransportBroadcastByNodeAction(); for (ShardRouting shard : shardIt.asUnordered()) { - if (shard.assignedToNode()) { + // send a request to the shard only if it is assigned to a node that is in the local node's cluster state + // a scenario in which a shard can be assigned but to a node that is not in the local node's cluster state + // is when the shard is assigned to the master node, the local node has detected the master as failed + // and a new master has not yet been elected; in this situation the local node will have removed the + // master node from the local cluster state, but the shards assigned to the master will still be in the + // routing table as such + if (shard.assignedToNode() && nodes.get(shard.currentNodeId()) != null) { String nodeId = shard.currentNodeId(); if (!nodeIds.containsKey(nodeId)) { nodeIds.put(nodeId, new ArrayList<>()); @@ -294,7 +302,9 @@ public abstract class TransportBroadcastByNodeAction shards = request.getShards(); final int totalShards = shards.size(); - logger.trace("[{}] executing operation on [{}] shards", actionName, totalShards); + if (logger.isTraceEnabled()) { + logger.trace("[{}] executing operation on [{}] shards", actionName, totalShards); + } final Object[] shardResultOrExceptions = new Object[totalShards]; int shardIndex = -1; @@ -369,10 +381,14 @@ public abstract class TransportBroadcastByNodeAction extends HandledTransportAction { + private static final ClusterStateObserver.ChangePredicate masterNodeChangedPredicate = new ClusterStateObserver.ChangePredicate() { + @Override + public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, + ClusterState newState, ClusterState.ClusterStateStatus newStatus) { + // The condition !newState.nodes().masterNodeId().equals(previousState.nodes().masterNodeId()) is not sufficient as the same master node might get reelected after a disruption. + return newState.nodes().masterNodeId() != null && newState != previousState; + } + + @Override + public boolean apply(ClusterChangedEvent event) { + return event.nodesDelta().masterNodeChanged(); + } + }; protected final TransportService transportService; protected final ClusterService clusterService; @@ -75,152 +90,125 @@ public abstract class TransportMasterNodeAction listener) { - // TODO do we really need to wrap it in a listener? the handlers should be cheap - if ((listener instanceof ThreadedActionListener) == false) { - listener = new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener); - } - innerExecute(request, listener, new ClusterStateObserver(clusterService, request.masterNodeTimeout(), logger), false); + new AsyncSingleAction(request, listener).start(); } - private void innerExecute(final Request request, final ActionListener listener, final ClusterStateObserver observer, final boolean retrying) { - final ClusterState clusterState = observer.observedState(); - final DiscoveryNodes nodes = clusterState.nodes(); - if (nodes.localNodeMaster() || localExecute(request)) { - // check for block, if blocked, retry, else, execute locally - final ClusterBlockException blockException = checkBlock(request, clusterState); - if (blockException != null) { - if (!blockException.retryable()) { - listener.onFailure(blockException); - return; - } - logger.trace("can't execute due to a cluster block, retrying", blockException); - observer.waitForNextChange( - new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - innerExecute(request, listener, observer, false); - } + class AsyncSingleAction { - @Override - public void onClusterServiceClose() { - listener.onFailure(blockException); - } + private final ActionListener listener; + private final Request request; + private volatile ClusterStateObserver observer; - @Override - public void onTimeout(TimeValue timeout) { - listener.onFailure(blockException); - } - }, new ClusterStateObserver.ValidationPredicate() { - @Override - protected boolean validate(ClusterState newState) { - ClusterBlockException blockException = checkBlock(request, newState); - return (blockException == null || !blockException.retryable()); + private final ClusterStateObserver.ChangePredicate retryableOrNoBlockPredicate = new ClusterStateObserver.ValidationPredicate() { + @Override + protected boolean validate(ClusterState newState) { + ClusterBlockException blockException = checkBlock(request, newState); + return (blockException == null || !blockException.retryable()); + } + }; + + AsyncSingleAction(Request request, ActionListener listener) { + this.request = request; + // TODO do we really need to wrap it in a listener? the handlers should be cheap + if ((listener instanceof ThreadedActionListener) == false) { + listener = new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener); + } + this.listener = listener; + } + + public void start() { + this.observer = new ClusterStateObserver(clusterService, request.masterNodeTimeout(), logger); + doStart(); + } + + protected void doStart() { + final ClusterState clusterState = observer.observedState(); + final DiscoveryNodes nodes = clusterState.nodes(); + if (nodes.localNodeMaster() || localExecute(request)) { + // check for block, if blocked, retry, else, execute locally + final ClusterBlockException blockException = checkBlock(request, clusterState); + if (blockException != null) { + if (!blockException.retryable()) { + listener.onFailure(blockException); + } else { + logger.trace("can't execute due to a cluster block, retrying", blockException); + retry(blockException, retryableOrNoBlockPredicate); + } + } else { + ActionListener delegate = new ActionListener() { + @Override + public void onResponse(Response response) { + listener.onResponse(response); + } + + @Override + public void onFailure(Throwable t) { + if (t instanceof Discovery.FailedToCommitClusterStateException + || (t instanceof NotMasterException)) { + logger.debug("master could not publish cluster state or stepped down before publishing action [{}], scheduling a retry", t, actionName); + retry(t, masterNodeChangedPredicate); + } else { + listener.onFailure(t); } } - ); - + }; + threadPool.executor(executor).execute(new ActionRunnable(delegate) { + @Override + protected void doRun() throws Exception { + masterOperation(request, clusterService.state(), delegate); + } + }); + } } else { - threadPool.executor(executor).execute(new ActionRunnable(listener) { - @Override - protected void doRun() throws Exception { - masterOperation(request, clusterService.state(), listener); - } - }); - } - } else { - if (nodes.masterNode() == null) { - if (retrying) { - listener.onFailure(new MasterNotDiscoveredException()); - } else { + if (nodes.masterNode() == null) { logger.debug("no known master node, scheduling a retry"); - observer.waitForNextChange( - new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - innerExecute(request, listener, observer, true); - } + retry(null, masterNodeChangedPredicate); + } else { + transportService.sendRequest(nodes.masterNode(), actionName, request, new ActionListenerResponseHandler(listener) { + @Override + public Response newInstance() { + return newResponse(); + } - @Override - public void onClusterServiceClose() { - listener.onFailure(new NodeClosedException(clusterService.localNode())); - } - - @Override - public void onTimeout(TimeValue timeout) { - listener.onFailure(new MasterNotDiscoveredException("waited for [" + timeout + "]")); - } - }, new ClusterStateObserver.ChangePredicate() { - @Override - public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, - ClusterState newState, ClusterState.ClusterStateStatus newStatus) { - return newState.nodes().masterNodeId() != null; - } - - @Override - public boolean apply(ClusterChangedEvent event) { - return event.nodesDelta().masterNodeChanged(); - } + @Override + public void handleException(final TransportException exp) { + Throwable cause = exp.unwrapCause(); + if (cause instanceof ConnectTransportException) { + // we want to retry here a bit to see if a new master is elected + logger.debug("connection exception while trying to forward request with action name [{}] to master node [{}], scheduling a retry. Error: [{}]", + actionName, nodes.masterNode(), exp.getDetailedMessage()); + retry(cause, masterNodeChangedPredicate); + } else { + listener.onFailure(exp); } - ); + } + }); } - return; } - processBeforeDelegationToMaster(request, clusterState); - transportService.sendRequest(nodes.masterNode(), actionName, request, new BaseTransportResponseHandler() { - @Override - public Response newInstance() { - return newResponse(); - } + } - @Override - public void handleResponse(Response response) { - listener.onResponse(response); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - - @Override - public void handleException(final TransportException exp) { - if (exp.unwrapCause() instanceof ConnectTransportException) { - // we want to retry here a bit to see if a new master is elected - logger.debug("connection exception while trying to forward request to master node [{}], scheduling a retry. Error: [{}]", - nodes.masterNode(), exp.getDetailedMessage()); - observer.waitForNextChange(new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - innerExecute(request, listener, observer, false); - } - - @Override - public void onClusterServiceClose() { - listener.onFailure(new NodeClosedException(clusterService.localNode())); - } - - @Override - public void onTimeout(TimeValue timeout) { - listener.onFailure(new MasterNotDiscoveredException()); - } - }, new ClusterStateObserver.EventPredicate() { - @Override - public boolean apply(ClusterChangedEvent event) { - return event.nodesDelta().masterNodeChanged(); - } - } - ); - } else { - listener.onFailure(exp); + private void retry(final Throwable failure, final ClusterStateObserver.ChangePredicate changePredicate) { + observer.waitForNextChange( + new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + doStart(); } - } - }); + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + logger.debug("timed out while retrying [{}] after failure (timeout [{}])", failure, actionName, timeout); + listener.onFailure(new MasterNotDiscoveredException(failure)); + } + }, changePredicate + ); } } } diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index 0597c26f636..50b71aecbd2 100644 --- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -126,11 +126,11 @@ public abstract class TransportNodesAction() { + transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(), new BaseTransportResponseHandler() { @Override public NodeResponse newInstance() { return newNodeResponse(); diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java index 77c55ac4a29..adbe199824e 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java @@ -42,7 +42,12 @@ public class ReplicationRequest extends ActionRequ public static final TimeValue DEFAULT_TIMEOUT = new TimeValue(1, TimeUnit.MINUTES); - ShardId internalShardId; + /** + * Target shard the request should execute on. In case of index and delete requests, + * shard id gets resolved by the transport action before performing request operation + * and at request creation time for shard-level bulk, refresh and flush requests. + */ + protected ShardId shardId; protected TimeValue timeout = DEFAULT_TIMEOUT; protected String index; @@ -60,6 +65,15 @@ public class ReplicationRequest extends ActionRequ super(request); } + /** + * Creates a new request with resolved shard id + */ + public ReplicationRequest(ActionRequest request, ShardId shardId) { + super(request); + this.index = shardId.getIndex(); + this.shardId = shardId; + } + /** * Copy constructor that creates a new request that is a copy of the one provided as an argument. */ @@ -124,12 +138,12 @@ public class ReplicationRequest extends ActionRequ /** * @return the shardId of the shard where this operation should be executed on. - * can be null in case the shardId is determined by a single document (index, type, id) for example for index or delete request. + * can be null if the shardID has not yet been resolved */ public @Nullable ShardId shardId() { - return internalShardId; + return shardId; } /** @@ -154,7 +168,9 @@ public class ReplicationRequest extends ActionRequ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); if (in.readBoolean()) { - internalShardId = ShardId.readShardId(in); + shardId = ShardId.readShardId(in); + } else { + shardId = null; } consistencyLevel = WriteConsistencyLevel.fromId(in.readByte()); timeout = TimeValue.readTimeValue(in); @@ -164,15 +180,32 @@ public class ReplicationRequest extends ActionRequ @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeOptionalStreamable(internalShardId); + if (shardId != null) { + out.writeBoolean(true); + shardId.writeTo(out); + } else { + out.writeBoolean(false); + } out.writeByte(consistencyLevel.id()); timeout.writeTo(out); out.writeString(index); } + /** + * Sets the target shard id for the request. The shard id is set when a + * index/delete request is resolved by the transport action + */ public T setShardId(ShardId shardId) { - this.internalShardId = shardId; - this.index = shardId.getIndex(); + this.shardId = shardId; return (T) this; } + + @Override + public String toString() { + if (shardId != null) { + return shardId.toString(); + } else { + return index; + } + } } diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java index ddd4d42f7a6..33a9d349e80 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -22,9 +22,8 @@ package org.elasticsearch.action.support.replication; import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.HandledTransportAction; @@ -53,7 +52,7 @@ import java.util.function.Supplier; * Base class for requests that should be executed on all shards of an index or several indices. * This action sends shard requests to all primary shards of the indices and they are then replicated like write requests */ -public abstract class TransportBroadcastReplicationAction extends HandledTransportAction { +public abstract class TransportBroadcastReplicationAction extends HandledTransportAction { private final TransportReplicationAction replicatedBroadcastShardAction; private final ClusterService clusterService; @@ -91,15 +90,15 @@ public abstract class TransportBroadcastReplicationAction shardFailures = null; for (int i = 0; i < shardsResponses.size(); i++) { - ActionWriteResponse shardResponse = shardsResponses.get(i); + ReplicationResponse shardResponse = shardsResponses.get(i); if (shardResponse == null) { // non active shard, ignore } else { @@ -152,7 +151,7 @@ public abstract class TransportBroadcastReplicationAction(); } - for (ActionWriteResponse.ShardInfo.Failure failure : shardResponse.getShardInfo().getFailures()) { + for (ReplicationResponse.ShardInfo.Failure failure : shardResponse.getShardInfo().getFailures()) { shardFailures.add(new DefaultShardOperationFailedException(new BroadcastShardOperationFailedException(new ShardId(failure.index(), failure.shardId()), failure.getCause()))); } } diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index f3db2b6dd18..26c439c0a3d 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -22,13 +22,9 @@ package org.elasticsearch.action.support.replication; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.WriteConsistencyLevel; -import org.elasticsearch.action.bulk.BulkShardRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexRequest.OpType; -import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.TransportActions; @@ -41,11 +37,10 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.ShardIterator; -import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; @@ -56,10 +51,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.VersionConflictEngineException; -import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; @@ -70,6 +62,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -77,8 +71,16 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; /** + * Base class for requests that should be executed on a primary copy followed by replica copies. + * Subclasses can resolve the target shard and provide implementation for primary and replica operations. + * + * The action samples cluster state on the receiving node to reroute to node with primary copy and on the + * primary node to validate request before primary operation followed by sampling state again for resolving + * nodes with replica copies to perform replication. */ -public abstract class TransportReplicationAction extends TransportAction { +public abstract class TransportReplicationAction extends TransportAction { + + public static final String SHARD_FAILURE_TIMEOUT = "action.support.replication.shard.failure_timeout"; protected final TransportService transportService; protected final ClusterService clusterService; @@ -87,8 +89,10 @@ public abstract class TransportReplicationAction listener) { - new PrimaryPhase(request, listener).run(); + new ReroutePhase(request, listener).run(); } protected abstract Response newResponseInstance(); /** + * Resolves the target shard id of the incoming request. + * Additional processing or validation of the request should be done here. + */ + protected void resolveRequest(MetaData metaData, String concreteIndex, Request request) { + // implementation should be provided if request shardID is not already resolved at request construction + } + + /** + * Primary operation on node with primary copy, the provided metadata should be used for request validation if needed * @return A tuple containing not null values, as first value the result of the primary operation and as second value * the request to be executed on the replica shards. */ - protected abstract Tuple shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable; + protected abstract Tuple shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Throwable; - protected abstract void shardOperationOnReplica(ShardId shardId, ReplicaRequest shardRequest); + /** + * Replica operation on nodes with replica copies + */ + protected abstract void shardOperationOnReplica(ReplicaRequest shardRequest); - protected abstract ShardIterator shards(ClusterState clusterState, InternalRequest request); - - protected abstract boolean checkWriteConsistency(); - - protected ClusterBlockException checkGlobalBlock(ClusterState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); - } - - protected ClusterBlockException checkRequestBlock(ClusterState state, InternalRequest request) { - return state.blocks().indexBlockedException(ClusterBlockLevel.WRITE, request.concreteIndex()); - } - - protected boolean resolveIndex() { + /** + * True if write consistency should be checked for an implementation + */ + protected boolean checkWriteConsistency() { return true; } /** - * Resolves the request, by default doing nothing. Can be subclassed to do - * additional processing or validation depending on the incoming request + * Cluster level block to check before request execution */ - protected void resolveRequest(ClusterState state, InternalRequest request, ActionListener listener) { + protected ClusterBlockLevel globalBlockLevel() { + return ClusterBlockLevel.WRITE; + } + + /** + * Index level block to check before request execution + */ + protected ClusterBlockLevel indexBlockLevel() { + return ClusterBlockLevel.WRITE; + } + + /** + * True if provided index should be resolved when resolving request + */ + protected boolean resolveIndex() { + return true; } protected TransportRequestOptions transportOptions() { @@ -190,7 +214,7 @@ public abstract class TransportReplicationAction { + protected static class WriteResult { public final T response; public final Translog.Location location; @@ -201,10 +225,10 @@ public abstract class TransportReplicationAction T response() { + public T response() { // this sets total, pending and failed to 0 and this is ok, because we will embed this into the replica // request and not use it - response.setShardInfo(new ActionWriteResponse.ShardInfo()); + response.setShardInfo(new ReplicationResponse.ShardInfo()); return (T) response; } @@ -235,6 +259,13 @@ public abstract class TransportReplicationAction { + @Override + public void messageReceived(final Request request, final TransportChannel channel) throws Exception { + new PrimaryPhase(request, channel).run(); + } + } + class ReplicaOperationTransportHandler implements TransportRequestHandler { @Override public void messageReceived(final ReplicaRequest request, final TransportChannel channel) throws Exception { @@ -261,7 +292,6 @@ public abstract class TransportReplicationAction - * Note that as soon as we start sending request to replicas, state responsibility is transferred to {@link ReplicationPhase} + * Responsible for routing and retrying failed operations on the primary. + * The actual primary operation is done in {@link PrimaryPhase} on the + * node with primary copy. + * + * Resolves index and shard id for the request before routing it to target node */ - final class PrimaryPhase extends AbstractRunnable { + final class ReroutePhase extends AbstractRunnable { private final ActionListener listener; - private final InternalRequest internalRequest; + private final Request request; private final ClusterStateObserver observer; - private final AtomicBoolean finished = new AtomicBoolean(false); - private volatile Releasable indexShardReference; + private final AtomicBoolean finished = new AtomicBoolean(); - - PrimaryPhase(Request request, ActionListener listener) { - this.internalRequest = new InternalRequest(request); + ReroutePhase(Request request, ActionListener listener) { + this.request = request; this.listener = listener; - this.observer = new ClusterStateObserver(clusterService, internalRequest.request().timeout(), logger); + this.observer = new ClusterStateObserver(clusterService, request.timeout(), logger); } @Override @@ -364,137 +405,93 @@ public abstract class TransportReplicationAction() { - - @Override - public Response newInstance() { - return newResponseInstance(); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - - @Override - public void handleResponse(Response response) { - finishOnRemoteSuccess(response); - } - - @Override - public void handleException(TransportException exp) { - try { - // if we got disconnected from the node, or the node / shard is not in the right state (being closed) - if (exp.unwrapCause() instanceof ConnectTransportException || exp.unwrapCause() instanceof NodeClosedException || - retryPrimaryException(exp)) { - // we already marked it as started when we executed it (removed the listener) so pass false - // to re-add to the cluster listener - logger.trace("received an error from node the primary was assigned to ({}), scheduling a retry", exp.getMessage()); - retry(exp); - } else { - finishAsFailed(exp); - } - } catch (Throwable t) { - finishWithUnexpectedFailure(t); - } - } - }); + if (logger.isTraceEnabled()) { + logger.trace("send action [{}] on primary [{}] for request [{}] with cluster state version [{}] to [{}]", actionName, request.shardId(), request, state.version(), primary.currentNodeId()); + } + performAction(node, actionName, false); } } + private void handleBlockException(ClusterBlockException blockException) { + if (blockException.retryable()) { + logger.trace("cluster is blocked ({}), scheduling a retry", blockException.getMessage()); + retry(blockException); + } else { + finishAsFailed(blockException); + } + } + + private void performAction(final DiscoveryNode node, final String action, final boolean isPrimaryAction) { + transportService.sendRequest(node, action, request, transportOptions, new BaseTransportResponseHandler() { + + @Override + public Response newInstance() { + return newResponseInstance(); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public void handleResponse(Response response) { + finishOnSuccess(response); + } + + @Override + public void handleException(TransportException exp) { + try { + // if we got disconnected from the node, or the node / shard is not in the right state (being closed) + if (exp.unwrapCause() instanceof ConnectTransportException || exp.unwrapCause() instanceof NodeClosedException || + (isPrimaryAction && retryPrimaryException(exp.unwrapCause()))) { + logger.trace("received an error from node [{}] for request [{}], scheduling a retry", exp, node.id(), request); + retry(exp); + } else { + finishAsFailed(exp); + } + } catch (Throwable t) { + finishWithUnexpectedFailure(t); + } + } + }); + } + void retry(Throwable failure) { assert failure != null; if (observer.isTimedOut()) { @@ -521,22 +518,9 @@ public abstract class TransportReplicationAction + * Note that as soon as we move to replication action, state responsibility is transferred to {@link ReplicationPhase}. + */ + final class PrimaryPhase extends AbstractRunnable { + private final Request request; + private final TransportChannel channel; + private final ClusterState state; + private final AtomicBoolean finished = new AtomicBoolean(); + private Releasable indexShardReference; + + PrimaryPhase(Request request, TransportChannel channel) { + this.state = clusterService.state(); + this.request = request; + this.channel = channel; + } + + @Override + public void onFailure(Throwable e) { + finishAsFailed(e); + } + + @Override + protected void doRun() throws Exception { + // request shardID was set in ReroutePhase + assert request.shardId() != null : "request shardID must be set prior to primary phase"; + final ShardId shardId = request.shardId(); + final String writeConsistencyFailure = checkWriteConsistency(shardId); if (writeConsistencyFailure != null) { - retryBecauseUnavailable(primary.shardId(), writeConsistencyFailure); + finishBecauseUnavailable(shardId, writeConsistencyFailure); return; } final ReplicationPhase replicationPhase; try { - indexShardReference = getIndexShardOperationsCounter(primary.shardId()); - PrimaryOperationRequest por = new PrimaryOperationRequest(primary.id(), internalRequest.concreteIndex(), internalRequest.request()); - Tuple primaryResponse = shardOperationOnPrimary(observer.observedState(), por); - logger.trace("operation completed on primary [{}]", primary); - replicationPhase = new ReplicationPhase(shardsIt, primaryResponse.v2(), primaryResponse.v1(), observer, primary, internalRequest, listener, indexShardReference); - } catch (Throwable e) { - // shard has not been allocated yet, retry it here - if (retryPrimaryException(e)) { - logger.trace("had an error while performing operation on primary ({}), scheduling a retry.", e.getMessage()); - // We have to close here because when we retry we will increment get a new reference on index shard again and we do not want to - // increment twice. - Releasables.close(indexShardReference); - // We have to reset to null here because whe we retry it might be that we never get to the point where we assign a new reference - // (for example, in case the operation was rejected because queue is full). In this case we would release again once one of the finish methods is called. - indexShardReference = null; - retry(e); - return; + indexShardReference = getIndexShardOperationsCounter(shardId); + Tuple primaryResponse = shardOperationOnPrimary(state.metaData(), request); + if (logger.isTraceEnabled()) { + logger.trace("action [{}] completed on shard [{}] for request [{}] with cluster state version [{}]", transportPrimaryAction, shardId, request, state.version()); } + replicationPhase = new ReplicationPhase(primaryResponse.v2(), primaryResponse.v1(), shardId, channel, indexShardReference, shardFailedTimeout); + } catch (Throwable e) { if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { if (logger.isTraceEnabled()) { - logger.trace(primary.shortSummary() + ": Failed to execute [" + internalRequest.request() + "]", e); + logger.trace("failed to execute [{}] on [{}]", e, request, shardId); } } else { if (logger.isDebugEnabled()) { - logger.debug(primary.shortSummary() + ": Failed to execute [" + internalRequest.request() + "]", e); + logger.debug("failed to execute [{}] on [{}]", e, request, shardId); } } finishAsFailed(e); @@ -610,22 +613,22 @@ public abstract class TransportReplicationAction 2) { @@ -646,18 +649,45 @@ public abstract class TransportReplicationAction listener; - private final AtomicBoolean finished = new AtomicBoolean(false); + private final TransportChannel channel; + private final ShardId shardId; + private final List shards; + private final DiscoveryNodes nodes; + private final boolean executeOnReplica; + private final String indexUUID; + private final AtomicBoolean finished = new AtomicBoolean(); private final AtomicInteger success = new AtomicInteger(1); // We already wrote into the primary shard private final ConcurrentMap shardReplicaFailures = ConcurrentCollections.newConcurrentMap(); - private final IndexMetaData indexMetaData; - private final ShardRouting originalPrimaryShard; private final AtomicInteger pending; private final int totalShards; - private final ClusterStateObserver observer; private final Releasable indexShardReference; + private final TimeValue shardFailedTimeout; - /** - * the constructor doesn't take any action, just calculates state. Call {@link #run()} to start - * replicating. - */ - public ReplicationPhase(ShardIterator originalShardIt, ReplicaRequest replicaRequest, Response finalResponse, - ClusterStateObserver observer, ShardRouting originalPrimaryShard, - InternalRequest internalRequest, ActionListener listener, Releasable indexShardReference) { + public ReplicationPhase(ReplicaRequest replicaRequest, Response finalResponse, ShardId shardId, + TransportChannel channel, Releasable indexShardReference, TimeValue shardFailedTimeout) { this.replicaRequest = replicaRequest; - this.listener = listener; + this.channel = channel; this.finalResponse = finalResponse; - this.originalPrimaryShard = originalPrimaryShard; - this.observer = observer; - indexMetaData = observer.observedState().metaData().index(internalRequest.concreteIndex()); this.indexShardReference = indexShardReference; + this.shardFailedTimeout = shardFailedTimeout; + this.shardId = shardId; - ShardRouting shard; - // we double check on the state, if it got changed we need to make sure we take the latest one cause - // maybe a replica shard started its recovery process and we need to apply it there... + // we have to get a new state after successfully indexing into the primary in order to honour recovery semantics. + // we have to make sure that every operation indexed into the primary after recovery start will also be replicated + // to the recovery target. If we use an old cluster state, we may miss a relocation that has started since then. + // If the index gets deleted after primary operation, we skip replication + final ClusterState state = clusterService.state(); + final IndexRoutingTable index = state.getRoutingTable().index(shardId.getIndex()); + final IndexShardRoutingTable shardRoutingTable = (index != null) ? index.shard(shardId.id()) : null; + final IndexMetaData indexMetaData = state.getMetaData().index(shardId.getIndex()); + this.shards = (shardRoutingTable != null) ? shardRoutingTable.shards() : Collections.emptyList(); + this.executeOnReplica = (indexMetaData == null) || shouldExecuteReplication(indexMetaData.getSettings()); + this.indexUUID = (indexMetaData != null) ? indexMetaData.getIndexUUID() : null; + this.nodes = state.getNodes(); - // we also need to make sure if the new state has a new primary shard (that we indexed to before) started - // and assigned to another node (while the indexing happened). In that case, we want to apply it on the - // new primary shard as well... - ClusterState newState = clusterService.state(); + if (shards.isEmpty()) { + logger.debug("replication phase for request [{}] on [{}] is skipped due to index deletion after primary operation", replicaRequest, shardId); + } - int numberOfUnassignedOrIgnoredReplicas = 0; + // we calculate number of target nodes to send replication operations, including nodes with relocating shards + int numberOfIgnoredShardInstances = 0; int numberOfPendingShardInstances = 0; - if (observer.observedState() != newState) { - observer.reset(newState); - shardIt = shards(newState, internalRequest); - while ((shard = shardIt.nextOrNull()) != null) { - if (shard.primary()) { - if (originalPrimaryShard.currentNodeId().equals(shard.currentNodeId()) == false) { - // there is a new primary, we'll have to replicate to it. - numberOfPendingShardInstances++; - } - if (shard.relocating()) { - numberOfPendingShardInstances++; - } - } else if (shouldExecuteReplication(indexMetaData.settings()) == false) { - // If the replicas use shadow replicas, there is no reason to - // perform the action on the replica, so skip it and - // immediately return - - // this delays mapping updates on replicas because they have - // to wait until they get the new mapping through the cluster - // state, which is why we recommend pre-defined mappings for - // indices using shadow replicas - numberOfUnassignedOrIgnoredReplicas++; - } else if (shard.unassigned()) { - numberOfUnassignedOrIgnoredReplicas++; - } else if (shard.relocating()) { - // we need to send to two copies - numberOfPendingShardInstances += 2; - } else { + for (ShardRouting shard : shards) { + if (shard.primary() == false && executeOnReplica == false) { + numberOfIgnoredShardInstances++; + } else if (shard.unassigned()) { + numberOfIgnoredShardInstances++; + } else { + if (shard.currentNodeId().equals(nodes.localNodeId()) == false) { numberOfPendingShardInstances++; } - } - } else { - shardIt = originalShardIt; - shardIt.reset(); - while ((shard = shardIt.nextOrNull()) != null) { - if (shard.unassigned()) { - numberOfUnassignedOrIgnoredReplicas++; - } else if (shard.primary()) { - if (shard.relocating()) { - // we have to replicate to the other copy - numberOfPendingShardInstances += 1; - } - } else if (shouldExecuteReplication(indexMetaData.settings()) == false) { - // If the replicas use shadow replicas, there is no reason to - // perform the action on the replica, so skip it and - // immediately return - - // this delays mapping updates on replicas because they have - // to wait until they get the new mapping through the cluster - // state, which is why we recommend pre-defined mappings for - // indices using shadow replicas - numberOfUnassignedOrIgnoredReplicas++; - } else if (shard.relocating()) { - // we need to send to two copies - numberOfPendingShardInstances += 2; - } else { + if (shard.relocating()) { numberOfPendingShardInstances++; } } } - - // one for the primary already done - this.totalShards = 1 + numberOfPendingShardInstances + numberOfUnassignedOrIgnoredReplicas; + // one for the local primary copy + this.totalShards = 1 + numberOfPendingShardInstances + numberOfIgnoredShardInstances; this.pending = new AtomicInteger(numberOfPendingShardInstances); + if (logger.isTraceEnabled()) { + logger.trace("replication phase started. pending [{}], action [{}], request [{}], cluster state version used [{}]", pending.get(), + transportReplicaAction, replicaRequest, state.version()); + } } /** @@ -817,12 +793,12 @@ public abstract class TransportReplicationAction entry : shardReplicaFailures.entrySet()) { RestStatus restStatus = ExceptionsHelper.status(entry.getValue()); - failuresArray[slot++] = new ActionWriteResponse.ShardInfo.Failure( + failuresArray[slot++] = new ReplicationResponse.ShardInfo.Failure( shardId.getIndex(), shardId.getId(), entry.getKey(), entry.getValue(), restStatus, false ); } } else { - failuresArray = ActionWriteResponse.EMPTY; + failuresArray = ReplicationResponse.EMPTY; } - finalResponse.setShardInfo(new ActionWriteResponse.ShardInfo( + finalResponse.setShardInfo(new ReplicationResponse.ShardInfo( totalShards, success.get(), failuresArray ) ); - listener.onResponse(finalResponse); + try { + channel.sendResponse(finalResponse); + } catch (IOException responseException) { + logger.warn("failed to send error message back to client for action [" + transportReplicaAction + "]", responseException); + } + if (logger.isTraceEnabled()) { + logger.trace("action [{}] completed on all replicas [{}] for request [{}]", transportReplicaAction, shardId, replicaRequest); + } } } + public class ReplicationFailedShardStateListener implements ShardStateAction.Listener { + private final String nodeId; + private Throwable failure; + + public ReplicationFailedShardStateListener(String nodeId, Throwable failure) { + this.nodeId = nodeId; + this.failure = failure; + } + + @Override + public void onSuccess() { + onReplicaFailure(nodeId, failure); + } + + @Override + public void onShardFailedNoMaster() { + onReplicaFailure(nodeId, failure); + } + + @Override + public void onShardFailedFailure(DiscoveryNode master, TransportException e) { + if (e instanceof ReceiveTimeoutTransportException) { + logger.trace("timeout sending shard failure to master [{}]", e, master); + } + onReplicaFailure(nodeId, failure); + } + } } /** @@ -988,34 +976,10 @@ public abstract class TransportReplicationAction executeIndexRequestOnPrimary(BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) throws Throwable { - Engine.Index operation = prepareIndexOperationOnPrimary(shardRequest, request, indexShard); - Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); - final ShardId shardId = indexShard.shardId(); - if (update != null) { - final String indexName = shardId.getIndex(); - mappingUpdatedAction.updateMappingOnMasterSynchronously(indexName, request.type(), update); - operation = prepareIndexOperationOnPrimary(shardRequest, request, indexShard); - update = operation.parsedDoc().dynamicMappingsUpdate(); - if (update != null) { - throw new RetryOnPrimaryException(shardId, - "Dynamics mappings are not available on the node that holds the primary yet"); - } - } - final boolean created = indexShard.index(operation); - - // update the version on request so it will happen on the replicas - final long version = operation.version(); - request.version(version); - request.versionType(request.versionType().versionTypeForReplicationAndRecovery()); - - assert request.versionType().validateVersionForWrites(request.version()); - - return new WriteResult(new IndexResponse(shardId.getIndex(), request.type(), request.id(), request.version(), created), operation.getTranslogLocation()); - } - - protected final void processAfter(boolean refresh, IndexShard indexShard, Translog.Location location) { + protected final void processAfterWrite(boolean refresh, IndexShard indexShard, Translog.Location location) { if (refresh) { try { indexShard.refresh("refresh_flag_index"); diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index ef998922d51..c13e44097bc 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.termvectors; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocumentRequest; @@ -211,7 +210,7 @@ public class TermVectorsRequest extends SingleShardRequest i public String id() { return id; } - + /** * Sets the id of document the term vector is requested for. */ @@ -651,7 +650,7 @@ public class TermVectorsRequest extends SingleShardRequest i if (e.getValue() instanceof String) { mapStrStr.put(e.getKey(), (String) e.getValue()); } else { - throw new ElasticsearchException("expecting the analyzer at [{}] to be a String, but found [{}] instead", e.getKey(), e.getValue().getClass()); + throw new ElasticsearchParseException("expecting the analyzer at [{}] to be a String, but found [{}] instead", e.getKey(), e.getValue().getClass()); } } return mapStrStr; diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java index e654eda85b2..dd78d7a3f65 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.termvectors; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocumentRequest; import org.elasticsearch.action.support.ActionFilters; @@ -79,8 +78,8 @@ public class TransportMultiTermVectorsAction extends HandledTransportAction { @Override public String toString() { String sSource = "_na_"; - try { - sSource = XContentHelper.convertToJson(searchRequest.source(), false); - } catch (IOException e) { - // ignore + if (searchRequest.source() != null) { + sSource = searchRequest.source().toString(); } return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types()) + ", source[" + sSource + "]"; } diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 2a639c83ad1..e5edc1af96b 100644 --- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -175,7 +175,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio indexAction.execute(upsertRequest, new ActionListener() { @Override public void onResponse(IndexResponse response) { - UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getIndex(), response.getType(), response.getId(), response.getVersion(), response.isCreated()); + UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), response.isCreated()); if (request.fields() != null && request.fields().length > 0) { Tuple> sourceAndContent = XContentHelper.convertToMap(upsertSourceBytes, true); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), upsertSourceBytes)); @@ -190,6 +190,8 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio e = ExceptionsHelper.unwrapCause(e); if (e instanceof VersionConflictEngineException) { if (retryCount < request.retryOnConflict()) { + logger.trace("Retry attempt [{}] of [{}] on version conflict on [{}][{}][{}]", + retryCount + 1, request.retryOnConflict(), request.index(), request.shardId(), request.id()); threadPool.executor(executor()).execute(new ActionRunnable(listener) { @Override protected void doRun() { @@ -210,7 +212,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio indexAction.execute(indexRequest, new ActionListener() { @Override public void onResponse(IndexResponse response) { - UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getIndex(), response.getType(), response.getId(), response.getVersion(), response.isCreated()); + UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), response.isCreated()); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), indexSourceBytes)); listener.onResponse(update); } @@ -238,7 +240,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio deleteAction.execute(deleteRequest, new ActionListener() { @Override public void onResponse(DeleteResponse response) { - UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getIndex(), response.getType(), response.getId(), response.getVersion(), false); + UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), null)); listener.onResponse(update); } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index 010142b0b4c..9f8b2a2e7be 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -83,12 +83,13 @@ public class UpdateHelper extends AbstractComponent { @SuppressWarnings("unchecked") protected Result prepare(UpdateRequest request, final GetResult getResult) { long getDateNS = System.nanoTime(); + final ShardId shardId = new ShardId(getResult.getIndex(), request.shardId()); if (!getResult.isExists()) { if (request.upsertRequest() == null && !request.docAsUpsert()) { - throw new DocumentMissingException(new ShardId(request.index(), request.shardId()), request.type(), request.id()); + throw new DocumentMissingException(shardId, request.type(), request.id()); } IndexRequest indexRequest = request.docAsUpsert() ? request.doc() : request.upsertRequest(); - Long ttl = indexRequest.ttl(); + TimeValue ttl = indexRequest.ttl(); if (request.scriptedUpsert() && request.script() != null) { // Run the script to perform the create logic IndexRequest upsert = request.upsertRequest(); @@ -99,7 +100,7 @@ public class UpdateHelper extends AbstractComponent { ctx.put("_source", upsertDoc); ctx = executeScript(request, ctx); //Allow the script to set TTL using ctx._ttl - if (ttl < 0) { + if (ttl == null) { ttl = getTTLFromScriptContext(ctx); } @@ -113,7 +114,7 @@ public class UpdateHelper extends AbstractComponent { logger.warn("Used upsert operation [{}] for script [{}], doing nothing...", scriptOpChoice, request.script.getScript()); } - UpdateResponse update = new UpdateResponse(getResult.getIndex(), getResult.getType(), getResult.getId(), + UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false); update.setGetResult(getResult); return new Result(update, Operation.NONE, upsertDoc, XContentType.JSON); @@ -124,7 +125,7 @@ public class UpdateHelper extends AbstractComponent { indexRequest.index(request.index()).type(request.type()).id(request.id()) // it has to be a "create!" .create(true) - .ttl(ttl == null || ttl < 0 ? null : ttl) + .ttl(ttl) .refresh(request.refresh()) .routing(request.routing()) .parent(request.parent()) @@ -145,13 +146,13 @@ public class UpdateHelper extends AbstractComponent { if (getResult.internalSourceRef() == null) { // no source, we can't do nothing, through a failure... - throw new DocumentSourceMissingException(new ShardId(request.index(), request.shardId()), request.type(), request.id()); + throw new DocumentSourceMissingException(shardId, request.type(), request.id()); } Tuple> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); String operation = null; String timestamp = null; - Long ttl = null; + TimeValue ttl = null; final Map updatedSourceAsMap; final XContentType updateSourceContentType = sourceAndContent.v1(); String routing = getResult.getFields().containsKey(RoutingFieldMapper.NAME) ? getResult.field(RoutingFieldMapper.NAME).getValue().toString() : null; @@ -160,7 +161,7 @@ public class UpdateHelper extends AbstractComponent { if (request.script() == null && request.doc() != null) { IndexRequest indexRequest = request.doc(); updatedSourceAsMap = sourceAndContent.v2(); - if (indexRequest.ttl() > 0) { + if (indexRequest.ttl() != null) { ttl = indexRequest.ttl(); } timestamp = indexRequest.timestamp(); @@ -211,9 +212,9 @@ public class UpdateHelper extends AbstractComponent { // apply script to update the source // No TTL has been given in the update script so we keep previous TTL value if there is one if (ttl == null) { - ttl = getResult.getFields().containsKey(TTLFieldMapper.NAME) ? (Long) getResult.field(TTLFieldMapper.NAME).getValue() : null; - if (ttl != null) { - ttl = ttl - TimeValue.nsecToMSec(System.nanoTime() - getDateNS); // It is an approximation of exact TTL value, could be improved + Long ttlAsLong = getResult.getFields().containsKey(TTLFieldMapper.NAME) ? (Long) getResult.field(TTLFieldMapper.NAME).getValue() : null; + if (ttlAsLong != null) { + ttl = new TimeValue(ttlAsLong - TimeValue.nsecToMSec(System.nanoTime() - getDateNS));// It is an approximation of exact TTL value, could be improved } } @@ -231,12 +232,12 @@ public class UpdateHelper extends AbstractComponent { .consistencyLevel(request.consistencyLevel()); return new Result(deleteRequest, Operation.DELETE, updatedSourceAsMap, updateSourceContentType); } else if ("none".equals(operation)) { - UpdateResponse update = new UpdateResponse(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(), false); + UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false); update.setGetResult(extractGetResult(request, request.index(), getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, getResult.internalSourceRef())); return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType); } else { logger.warn("Used update operation [{}] for script [{}], doing nothing...", operation, request.script.getScript()); - UpdateResponse update = new UpdateResponse(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(), false); + UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false); return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType); } } @@ -256,17 +257,15 @@ public class UpdateHelper extends AbstractComponent { return ctx; } - private Long getTTLFromScriptContext(Map ctx) { - Long ttl = null; + private TimeValue getTTLFromScriptContext(Map ctx) { Object fetchedTTL = ctx.get("_ttl"); if (fetchedTTL != null) { if (fetchedTTL instanceof Number) { - ttl = ((Number) fetchedTTL).longValue(); - } else { - ttl = TimeValue.parseTimeValue((String) fetchedTTL, null, "_ttl").millis(); + return new TimeValue(((Number) fetchedTTL).longValue()); } + return TimeValue.parseTimeValue((String) fetchedTTL, null, "_ttl"); } - return ttl; + return null; } /** @@ -337,13 +336,10 @@ public class UpdateHelper extends AbstractComponent { } } - public static enum Operation { - + public enum Operation { UPSERT, INDEX, DELETE, NONE - } - } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 5b0a5bb9c5b..30b636f4efc 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; @@ -325,7 +326,7 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder SYSTEM_PROPERTIES; + static { + final Dictionary sysprops = System.getProperties(); + SYSTEM_PROPERTIES = new Dictionary() { + + @Override + public int size() { + return sysprops.size(); + } + + @Override + public boolean isEmpty() { + return sysprops.isEmpty(); + } + + @Override + public Enumeration keys() { + return sysprops.keys(); + } + + @Override + public Enumeration elements() { + return sysprops.elements(); + } + + @Override + public Object get(Object key) { + return sysprops.get(key); + } + + @Override + public Object put(Object key, Object value) { + throw new UnsupportedOperationException("treat system properties as immutable"); + } + + @Override + public Object remove(Object key) { + throw new UnsupportedOperationException("treat system properties as immutable"); + } + }; + } + + /** + * Returns a read-only view of all system properties + */ + public static Dictionary getSystemProperties() { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPropertyAccess("*"); + } + return SYSTEM_PROPERTIES; + } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java index 09b4a4d0d94..ee804b1480e 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java @@ -21,14 +21,14 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.common.SuppressForbidden; -import java.net.URI; +import java.net.SocketPermission; import java.net.URL; import java.security.CodeSource; import java.security.Permission; import java.security.PermissionCollection; +import java.security.Permissions; import java.security.Policy; import java.security.ProtectionDomain; -import java.security.URIParameter; import java.util.Map; /** custom policy for union of static and dynamic permissions */ @@ -41,14 +41,18 @@ final class ESPolicy extends Policy { final Policy template; final Policy untrusted; + final Policy system; final PermissionCollection dynamic; - final Map plugins; + final Map plugins; - public ESPolicy(PermissionCollection dynamic, Map plugins) throws Exception { - URI policyUri = getClass().getResource(POLICY_RESOURCE).toURI(); - URI untrustedUri = getClass().getResource(UNTRUSTED_RESOURCE).toURI(); - this.template = Policy.getInstance("JavaPolicy", new URIParameter(policyUri)); - this.untrusted = Policy.getInstance("JavaPolicy", new URIParameter(untrustedUri)); + public ESPolicy(PermissionCollection dynamic, Map plugins, boolean filterBadDefaults) { + this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), JarHell.parseClassPath()); + this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), new URL[0]); + if (filterBadDefaults) { + this.system = new SystemPolicy(Policy.getPolicy()); + } else { + this.system = Policy.getPolicy(); + } this.dynamic = dynamic; this.plugins = plugins; } @@ -69,42 +73,65 @@ final class ESPolicy extends Policy { if (BootstrapInfo.UNTRUSTED_CODEBASE.equals(location.getFile())) { return untrusted.implies(domain, permission); } - // check for an additional plugin permission - PermissionCollection plugin = plugins.get(location.getFile()); - if (plugin != null && plugin.implies(permission)) { + // check for an additional plugin permission: plugin policy is + // only consulted for its codesources. + Policy plugin = plugins.get(location.getFile()); + if (plugin != null && plugin.implies(domain, permission)) { return true; } } - // Special handling for broken AWS code which destroys all SSL security - // REMOVE THIS when https://github.com/aws/aws-sdk-java/pull/432 is fixed - if (permission instanceof RuntimePermission && "accessClassInPackage.sun.security.ssl".equals(permission.getName())) { - for (StackTraceElement element : Thread.currentThread().getStackTrace()) { - if ("com.amazonaws.http.conn.ssl.SdkTLSSocketFactory".equals(element.getClassName()) && - "verifyMasterSecret".equals(element.getMethodName())) { - // we found the horrible method: the hack begins! - // force the aws code to back down, by throwing an exception that it catches. - rethrow(new IllegalAccessException("no amazon, you cannot do this.")); - } + // otherwise defer to template + dynamic file permissions + return template.implies(domain, permission) || dynamic.implies(permission) || system.implies(domain, permission); + } + + @Override + public PermissionCollection getPermissions(CodeSource codesource) { + // code should not rely on this method, or at least use it correctly: + // https://bugs.openjdk.java.net/browse/JDK-8014008 + // return them a new empty permissions object so jvisualvm etc work + for (StackTraceElement element : Thread.currentThread().getStackTrace()) { + if ("sun.rmi.server.LoaderHandler".equals(element.getClassName()) && + "loadClass".equals(element.getMethodName())) { + return new Permissions(); } } - // otherwise defer to template + dynamic file permissions - return template.implies(domain, permission) || dynamic.implies(permission); + // return UNSUPPORTED_EMPTY_COLLECTION since it is safe. + return super.getPermissions(codesource); } + // TODO: remove this hack when insecure defaults are removed from java + + // default policy file states: + // "It is strongly recommended that you either remove this permission + // from this policy file or further restrict it to code sources + // that you specify, because Thread.stop() is potentially unsafe." + // not even sure this method still works... + static final Permission BAD_DEFAULT_NUMBER_ONE = new RuntimePermission("stopThread"); + + // default policy file states: + // "allows anyone to listen on dynamic ports" + // specified exactly because that is what we want, and fastest since it won't imply any + // expensive checks for the implicit "resolve" + static final Permission BAD_DEFAULT_NUMBER_TWO = new SocketPermission("localhost:0", "listen"); + /** - * Classy puzzler to rethrow any checked exception as an unchecked one. + * Wraps the Java system policy, filtering out bad default permissions that + * are granted to all domains. Note, before java 8 these were even worse. */ - private static class Rethrower { - private void rethrow(Throwable t) throws T { - throw (T) t; + static class SystemPolicy extends Policy { + final Policy delegate; + + SystemPolicy(Policy delegate) { + this.delegate = delegate; + } + + @Override + public boolean implies(ProtectionDomain domain, Permission permission) { + if (BAD_DEFAULT_NUMBER_ONE.equals(permission) || BAD_DEFAULT_NUMBER_TWO.equals(permission)) { + return false; + } + return delegate.implies(domain, permission); } } - - /** - * Rethrows t (identical object). - */ - private void rethrow(Throwable t) { - new Rethrower().rethrow(t); - } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java b/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java index 972d1a11cc7..bff22bc19f5 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java @@ -20,6 +20,7 @@ package org.elasticsearch.bootstrap; import com.sun.jna.Native; +import com.sun.jna.NativeLong; import com.sun.jna.Structure; import org.apache.lucene.util.Constants; @@ -55,8 +56,8 @@ final class JNACLibrary { /** corresponds to struct rlimit */ public static final class Rlimit extends Structure implements Structure.ByReference { - public long rlim_cur = 0; - public long rlim_max = 0; + public NativeLong rlim_cur = new NativeLong(0); + public NativeLong rlim_max = new NativeLong(0); @Override protected List getFieldOrder() { @@ -65,6 +66,7 @@ final class JNACLibrary { } static native int getrlimit(int resource, Rlimit rlimit); + static native int setrlimit(int resource, Rlimit rlimit); static native String strerror(int errno); diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java index 8924812e6d6..26e485802f4 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java @@ -217,4 +217,88 @@ final class JNAKernel32Library { * @return true if the function succeeds. */ native boolean CloseHandle(Pointer handle); + + /** + * Creates or opens a new job object + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms682409%28v=vs.85%29.aspx + * + * @param jobAttributes security attributes + * @param name job name + * @return job handle if the function succeeds + */ + native Pointer CreateJobObjectW(Pointer jobAttributes, String name); + + /** + * Associates a process with an existing job + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms681949%28v=vs.85%29.aspx + * + * @param job job handle + * @param process process handle + * @return true if the function succeeds + */ + native boolean AssignProcessToJobObject(Pointer job, Pointer process); + + /** + * Basic limit information for a job object + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms684147%28v=vs.85%29.aspx + */ + public static class JOBOBJECT_BASIC_LIMIT_INFORMATION extends Structure implements Structure.ByReference { + public long PerProcessUserTimeLimit; + public long PerJobUserTimeLimit; + public int LimitFlags; + public SizeT MinimumWorkingSetSize; + public SizeT MaximumWorkingSetSize; + public int ActiveProcessLimit; + public Pointer Affinity; + public int PriorityClass; + public int SchedulingClass; + + @Override + protected List getFieldOrder() { + return Arrays.asList(new String[] { + "PerProcessUserTimeLimit", "PerJobUserTimeLimit", "LimitFlags", "MinimumWorkingSetSize", + "MaximumWorkingSetSize", "ActiveProcessLimit", "Affinity", "PriorityClass", "SchedulingClass" + }); + } + } + + /** + * Constant for JOBOBJECT_BASIC_LIMIT_INFORMATION in Query/Set InformationJobObject + */ + static final int JOBOBJECT_BASIC_LIMIT_INFORMATION_CLASS = 2; + + /** + * Constant for LimitFlags, indicating a process limit has been set + */ + static final int JOB_OBJECT_LIMIT_ACTIVE_PROCESS = 8; + + /** + * Get job limit and state information + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms684925%28v=vs.85%29.aspx + * + * @param job job handle + * @param infoClass information class constant + * @param info pointer to information structure + * @param infoLength size of information structure + * @param returnLength length of data written back to structure (or null if not wanted) + * @return true if the function succeeds + */ + native boolean QueryInformationJobObject(Pointer job, int infoClass, Pointer info, int infoLength, Pointer returnLength); + + /** + * Set job limit and state information + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms686216%28v=vs.85%29.aspx + * + * @param job job handle + * @param infoClass information class constant + * @param info pointer to information structure + * @param infoLength size of information structure + * @return true if the function succeeds + */ + native boolean SetInformationJobObject(Pointer job, int infoClass, Pointer info, int infoLength); } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java index 1b5589a06e1..5356d33bb8e 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java @@ -46,6 +46,9 @@ class JNANatives { static boolean LOCAL_MLOCKALL = false; // Set to true, in case native seccomp call was successful static boolean LOCAL_SECCOMP = false; + // Set to true, in case policy can be applied to all threads of the process (even existing ones) + // otherwise they are only inherited for new threads (ES app threads) + static boolean LOCAL_SECCOMP_ALL = false; static void tryMlockall() { int errno = Integer.MIN_VALUE; @@ -68,8 +71,8 @@ class JNANatives { JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit(); if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_MEMLOCK, rlimit) == 0) { rlimitSuccess = true; - softLimit = rlimit.rlim_cur; - hardLimit = rlimit.rlim_max; + softLimit = rlimit.rlim_cur.longValue(); + hardLimit = rlimit.rlim_max.longValue(); } else { logger.warn("Unable to retrieve resource limits: " + JNACLibrary.strerror(Native.getLastError())); } @@ -177,15 +180,18 @@ class JNANatives { static void trySeccomp(Path tmpFile) { try { - Seccomp.init(tmpFile); + int ret = Seccomp.init(tmpFile); LOCAL_SECCOMP = true; + if (ret == 1) { + LOCAL_SECCOMP_ALL = true; + } } catch (Throwable t) { // this is likely to happen unless the kernel is newish, its a best effort at the moment // so we log stacktrace at debug for now... if (logger.isDebugEnabled()) { logger.debug("unable to install syscall filter", t); } - logger.warn("unable to install syscall filter: " + t.getMessage()); + logger.warn("unable to install syscall filter: ", t); } } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java index 1088225f684..9a4a26c74e3 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java @@ -22,6 +22,7 @@ package org.elasticsearch.bootstrap; import com.sun.jna.Library; import com.sun.jna.Memory; import com.sun.jna.Native; +import com.sun.jna.NativeLong; import com.sun.jna.Pointer; import com.sun.jna.Structure; import com.sun.jna.ptr.PointerByReference; @@ -38,15 +39,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Installs a limited form of secure computing mode, * to filters system calls to block process execution. *

- * This is only supported on the Linux and Mac OS X operating systems. + * This is supported on Linux, Solaris, FreeBSD, OpenBSD, Mac OS X, and Windows. *

- * On Linux it currently supports on the amd64 architecture, on Linux kernels 3.5 or above, and requires + * On Linux it currently supports amd64 and i386 architectures, requires Linux kernel 3.5 or above, and requires * {@code CONFIG_SECCOMP} and {@code CONFIG_SECCOMP_FILTER} compiled into the kernel. *

* On Linux BPF Filters are installed using either {@code seccomp(2)} (3.17+) or {@code prctl(2)} (3.5+). {@code seccomp(2)} @@ -62,6 +65,14 @@ import java.util.List; *

  • {@code execveat}
    • * *

    + * On Solaris 10 or higher, the following privileges are dropped with {@code priv_set(3C)}: + *

      + *
    • {@code PRIV_PROC_FORK}
      • + *
    • {@code PRIV_PROC_EXEC}
      • + *
    + *

    + * On BSD systems, process creation is restricted with {@code setrlimit(RLIMIT_NPROC)}. + *

    * On Mac OS X Leopard or above, a custom {@code sandbox(7)} ("Seatbelt") profile is installed that * denies the following rules: *

      @@ -69,12 +80,16 @@ import java.util.List; *
    • {@code process-exec}
      • *
    *

    + * On Windows, process creation is restricted with {@code SetInformationJobObject/ActiveProcessLimit}. + *

    * This is not intended as a sandbox. It is another level of security, mostly intended to annoy * security researchers and make their lives more difficult in achieving "remote execution" exploits. * @see * http://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt * @see * https://reverse.put.as/wp-content/uploads/2011/06/The-Apple-Sandbox-BHDC2011-Paper.pdf + * @see + * https://docs.oracle.com/cd/E23824_01/html/821-1456/prbac-2.html */ // not an example of how to write code!!! final class Seccomp { @@ -87,16 +102,16 @@ final class Seccomp { /** * maps to prctl(2) */ - int prctl(int option, long arg2, long arg3, long arg4, long arg5); + int prctl(int option, NativeLong arg2, NativeLong arg3, NativeLong arg4, NativeLong arg5); /** * used to call seccomp(2), its too new... * this is the only way, DONT use it on some other architecture unless you know wtf you are doing */ - long syscall(long number, Object... args); + NativeLong syscall(NativeLong number, Object... args); }; // null if unavailable or something goes wrong. - static final LinuxLibrary linux_libc; + private static final LinuxLibrary linux_libc; static { LinuxLibrary lib = null; @@ -111,7 +126,6 @@ final class Seccomp { } /** the preferred method is seccomp(2), since we can apply to all threads of the process */ - static final int SECCOMP_SYSCALL_NR = 317; // since Linux 3.17 static final int SECCOMP_SET_MODE_FILTER = 1; // since Linux 3.17 static final int SECCOMP_FILTER_FLAG_TSYNC = 1; // since Linux 3.17 @@ -120,7 +134,7 @@ final class Seccomp { static final int PR_SET_NO_NEW_PRIVS = 38; // since Linux 3.5 static final int PR_GET_SECCOMP = 21; // since Linux 2.6.23 static final int PR_SET_SECCOMP = 22; // since Linux 2.6.23 - static final int SECCOMP_MODE_FILTER = 2; // since Linux Linux 3.5 + static final long SECCOMP_MODE_FILTER = 2; // since Linux Linux 3.5 /** corresponds to struct sock_filter */ static final class SockFilter { @@ -182,7 +196,6 @@ final class Seccomp { return new SockFilter((short) code, (byte) jt, (byte) jf, k); } - static final int AUDIT_ARCH_X86_64 = 0xC000003E; static final int SECCOMP_RET_ERRNO = 0x00050000; static final int SECCOMP_RET_DATA = 0x0000FFFF; static final int SECCOMP_RET_ALLOW = 0x7FFF0000; @@ -193,28 +206,63 @@ final class Seccomp { static final int EINVAL = 0x16; static final int ENOSYS = 0x26; - // offsets (arch dependent) that our BPF checks + // offsets that our BPF checks + // check with offsetof() when adding a new arch, move to Arch if different. static final int SECCOMP_DATA_NR_OFFSET = 0x00; static final int SECCOMP_DATA_ARCH_OFFSET = 0x04; - - // currently these ranges are blocked (inclusive): - // execve is really the only one needed but why let someone fork a 30G heap? (not really what happens) - // ... - // 57: fork - // 58: vfork - // 59: execve - // ... - // 322: execveat - // ... - static final int NR_SYSCALL_FORK = 57; - static final int NR_SYSCALL_EXECVE = 59; - static final int NR_SYSCALL_EXECVEAT = 322; // since Linux 3.19 + + static class Arch { + /** AUDIT_ARCH_XXX constant from linux/audit.h */ + final int audit; + /** syscall limit (necessary for blacklisting on amd64, to ban 32-bit syscalls) */ + final int limit; + /** __NR_fork */ + final int fork; + /** __NR_vfork */ + final int vfork; + /** __NR_execve */ + final int execve; + /** __NR_execveat */ + final int execveat; + /** __NR_seccomp */ + final int seccomp; + + Arch(int audit, int limit, int fork, int vfork, int execve, int execveat, int seccomp) { + this.audit = audit; + this.limit = limit; + this.fork = fork; + this.vfork = vfork; + this.execve = execve; + this.execveat = execveat; + this.seccomp = seccomp; + } + } + + /** supported architectures map keyed by os.arch */ + private static final Map ARCHITECTURES; + static { + Map m = new HashMap<>(); + m.put("amd64", new Arch(0xC000003E, 0x3FFFFFFF, 57, 58, 59, 322, 317)); + m.put("i386", new Arch(0x40000003, 0xFFFFFFFF, 2, 190, 11, 358, 354)); + ARCHITECTURES = Collections.unmodifiableMap(m); + } + + /** invokes prctl() from linux libc library */ + private static int linux_prctl(int option, long arg2, long arg3, long arg4, long arg5) { + return linux_libc.prctl(option, new NativeLong(arg2), new NativeLong(arg3), new NativeLong(arg4), new NativeLong(arg5)); + } + + /** invokes syscall() from linux libc library */ + private static long linux_syscall(long number, Object... args) { + return linux_libc.syscall(new NativeLong(number), args).longValue(); + } /** try to install our BPF filters via seccomp() or prctl() to block execution */ - private static void linuxImpl() { + private static int linuxImpl() { // first be defensive: we can give nice errors this way, at the very least. // also, some of these security features get backported to old versions, checking kernel version here is a big no-no! - boolean supported = Constants.LINUX && "amd64".equals(Constants.OS_ARCH); + final Arch arch = ARCHITECTURES.get(Constants.OS_ARCH); + boolean supported = Constants.LINUX && arch != null; if (supported == false) { throw new UnsupportedOperationException("seccomp unavailable: '" + Constants.OS_ARCH + "' architecture unsupported"); } @@ -224,24 +272,86 @@ final class Seccomp { throw new UnsupportedOperationException("seccomp unavailable: could not link methods. requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in"); } - // check for kernel version - if (linux_libc.prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) < 0) { + // pure paranoia: + + // check that unimplemented syscalls actually return ENOSYS + // you never know (e.g. https://code.google.com/p/chromium/issues/detail?id=439795) + if (linux_syscall(999) >= 0 || Native.getLastError() != ENOSYS) { + throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade"); + } + + // try to check system calls really are who they claim + // you never know (e.g. https://chromium.googlesource.com/chromium/src.git/+/master/sandbox/linux/seccomp-bpf/sandbox_bpf.cc#57) + final int bogusArg = 0xf7a46a5c; + + // test seccomp(BOGUS) + long ret = linux_syscall(arch.seccomp, bogusArg); + if (ret != -1) { + throw new UnsupportedOperationException("seccomp unavailable: seccomp(BOGUS_OPERATION) returned " + ret); + } else { int errno = Native.getLastError(); switch (errno) { - case ENOSYS: throw new UnsupportedOperationException("seccomp unavailable: requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in"); - default: throw new UnsupportedOperationException("prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(errno)); + case ENOSYS: break; // ok + case EINVAL: break; // ok + default: throw new UnsupportedOperationException("seccomp(BOGUS_OPERATION): " + JNACLibrary.strerror(errno)); } } + + // test seccomp(VALID, BOGUS) + ret = linux_syscall(arch.seccomp, SECCOMP_SET_MODE_FILTER, bogusArg); + if (ret != -1) { + throw new UnsupportedOperationException("seccomp unavailable: seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG) returned " + ret); + } else { + int errno = Native.getLastError(); + switch (errno) { + case ENOSYS: break; // ok + case EINVAL: break; // ok + default: throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG): " + JNACLibrary.strerror(errno)); + } + } + + // test prctl(BOGUS) + ret = linux_prctl(bogusArg, 0, 0, 0, 0); + if (ret != -1) { + throw new UnsupportedOperationException("seccomp unavailable: prctl(BOGUS_OPTION) returned " + ret); + } else { + int errno = Native.getLastError(); + switch (errno) { + case ENOSYS: break; // ok + case EINVAL: break; // ok + default: throw new UnsupportedOperationException("prctl(BOGUS_OPTION): " + JNACLibrary.strerror(errno)); + } + } + + // now just normal defensive checks + + // check for GET_NO_NEW_PRIVS + switch (linux_prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0)) { + case 0: break; // not yet set + case 1: break; // already set by caller + default: + int errno = Native.getLastError(); + if (errno == EINVAL) { + // friendly error, this will be the typical case for an old kernel + throw new UnsupportedOperationException("seccomp unavailable: requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in"); + } else { + throw new UnsupportedOperationException("prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(errno)); + } + } // check for SECCOMP - if (linux_libc.prctl(PR_GET_SECCOMP, 0, 0, 0, 0) < 0) { - int errno = Native.getLastError(); - switch (errno) { - case EINVAL: throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed"); - default: throw new UnsupportedOperationException("prctl(PR_GET_SECCOMP): " + JNACLibrary.strerror(errno)); - } + switch (linux_prctl(PR_GET_SECCOMP, 0, 0, 0, 0)) { + case 0: break; // not yet set + case 2: break; // already in filter mode by caller + default: + int errno = Native.getLastError(); + if (errno == EINVAL) { + throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed"); + } else { + throw new UnsupportedOperationException("prctl(PR_GET_SECCOMP): " + JNACLibrary.strerror(errno)); + } } // check for SECCOMP_MODE_FILTER - if (linux_libc.prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0, 0, 0) < 0) { + if (linux_prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0, 0, 0) != 0) { int errno = Native.getLastError(); switch (errno) { case EFAULT: break; // available @@ -251,35 +361,43 @@ final class Seccomp { } // ok, now set PR_SET_NO_NEW_PRIVS, needed to be able to set a seccomp filter as ordinary user - if (linux_libc.prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) < 0) { + if (linux_prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { throw new UnsupportedOperationException("prctl(PR_SET_NO_NEW_PRIVS): " + JNACLibrary.strerror(Native.getLastError())); } - // BPF installed to check arch, then syscall range. See https://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt for details. - SockFilter insns[] = { - /* 1 */ BPF_STMT(BPF_LD + BPF_W + BPF_ABS, SECCOMP_DATA_ARCH_OFFSET), // - /* 2 */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 0, 4), // if (arch != amd64) goto fail; - /* 3 */ BPF_STMT(BPF_LD + BPF_W + BPF_ABS, SECCOMP_DATA_NR_OFFSET), // - /* 4 */ BPF_JUMP(BPF_JMP + BPF_JGE + BPF_K, NR_SYSCALL_FORK, 0, 3), // if (syscall < SYSCALL_FORK) goto pass; - /* 5 */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, NR_SYSCALL_EXECVEAT, 1, 0), // if (syscall == SYSCALL_EXECVEAT) goto fail; - /* 6 */ BPF_JUMP(BPF_JMP + BPF_JGT + BPF_K, NR_SYSCALL_EXECVE, 1, 0), // if (syscall > SYSCALL_EXECVE) goto pass; - /* 7 */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ERRNO | (EACCES & SECCOMP_RET_DATA)), // fail: return EACCES; - /* 8 */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW) // pass: return OK; - }; + // check it worked + if (linux_prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) != 1) { + throw new UnsupportedOperationException("seccomp filter did not really succeed: prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(Native.getLastError())); + } + // BPF installed to check arch, limit, then syscall. See https://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt for details. + SockFilter insns[] = { + /* 1 */ BPF_STMT(BPF_LD + BPF_W + BPF_ABS, SECCOMP_DATA_ARCH_OFFSET), // + /* 2 */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, arch.audit, 0, 7), // if (arch != audit) goto fail; + /* 3 */ BPF_STMT(BPF_LD + BPF_W + BPF_ABS, SECCOMP_DATA_NR_OFFSET), // + /* 4 */ BPF_JUMP(BPF_JMP + BPF_JGT + BPF_K, arch.limit, 5, 0), // if (syscall > LIMIT) goto fail; + /* 5 */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, arch.fork, 4, 0), // if (syscall == FORK) goto fail; + /* 6 */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, arch.vfork, 3, 0), // if (syscall == VFORK) goto fail; + /* 7 */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, arch.execve, 2, 0), // if (syscall == EXECVE) goto fail; + /* 8 */ BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, arch.execveat, 1, 0), // if (syscall == EXECVEAT) goto fail; + /* 9 */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW), // pass: return OK; + /* 10 */ BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ERRNO | (EACCES & SECCOMP_RET_DATA)), // fail: return EACCES; + }; // seccomp takes a long, so we pass it one explicitly to keep the JNA simple SockFProg prog = new SockFProg(insns); prog.write(); long pointer = Pointer.nativeValue(prog.getPointer()); + int method = 1; // install filter, if this works, after this there is no going back! // first try it with seccomp(SECCOMP_SET_MODE_FILTER), falling back to prctl() - if (linux_libc.syscall(SECCOMP_SYSCALL_NR, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, pointer) != 0) { + if (linux_syscall(arch.seccomp, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, new NativeLong(pointer)) != 0) { + method = 0; int errno1 = Native.getLastError(); if (logger.isDebugEnabled()) { logger.debug("seccomp(SECCOMP_SET_MODE_FILTER): " + JNACLibrary.strerror(errno1) + ", falling back to prctl(PR_SET_SECCOMP)..."); } - if (linux_libc.prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, pointer, 0, 0) < 0) { + if (linux_prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, pointer, 0, 0) != 0) { int errno2 = Native.getLastError(); throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER): " + JNACLibrary.strerror(errno1) + ", prctl(PR_SET_SECCOMP): " + JNACLibrary.strerror(errno2)); @@ -287,11 +405,12 @@ final class Seccomp { } // now check that the filter was really installed, we should be in filter mode. - if (linux_libc.prctl(PR_GET_SECCOMP, 0, 0, 0, 0) != 2) { + if (linux_prctl(PR_GET_SECCOMP, 0, 0, 0, 0) != 2) { throw new UnsupportedOperationException("seccomp filter installation did not really succeed. seccomp(PR_GET_SECCOMP): " + JNACLibrary.strerror(Native.getLastError())); } - logger.debug("Linux seccomp filter installation successful"); + logger.debug("Linux seccomp filter installation successful, threads: [{}]", method == 1 ? "all" : "app" ); + return method; } // OS X implementation via sandbox(7) @@ -310,7 +429,7 @@ final class Seccomp { } // null if unavailable, or something goes wrong. - static final MacLibrary libc_mac; + private static final MacLibrary libc_mac; static { MacLibrary lib = null; @@ -334,7 +453,7 @@ final class Seccomp { // first be defensive: we can give nice errors this way, at the very least. boolean supported = Constants.MAC_OS_X; if (supported == false) { - throw new IllegalStateException("bug: should not be trying to initialize seccomp for an unsupported OS"); + throw new IllegalStateException("bug: should not be trying to initialize seatbelt for an unsupported OS"); } // we couldn't link methods, could be some really ancient OS X (< Leopard) or some bug @@ -367,17 +486,149 @@ final class Seccomp { } } } + + // Solaris implementation via priv_set(3C) + + /** Access to non-standard Solaris libc methods */ + static interface SolarisLibrary extends Library { + /** + * see priv_set(3C), a convenience method for setppriv(2). + */ + int priv_set(int op, String which, String... privs); + } + + // null if unavailable, or something goes wrong. + private static final SolarisLibrary libc_solaris; + + static { + SolarisLibrary lib = null; + if (Constants.SUN_OS) { + try { + lib = (SolarisLibrary) Native.loadLibrary("c", SolarisLibrary.class); + } catch (UnsatisfiedLinkError e) { + logger.warn("unable to link C library. native methods (priv_set) will be disabled.", e); + } + } + libc_solaris = lib; + } + + // constants for priv_set(2) + static final int PRIV_OFF = 1; + static final String PRIV_ALLSETS = null; + // see privileges(5) for complete list of these + static final String PRIV_PROC_FORK = "proc_fork"; + static final String PRIV_PROC_EXEC = "proc_exec"; + + static void solarisImpl() { + // first be defensive: we can give nice errors this way, at the very least. + boolean supported = Constants.SUN_OS; + if (supported == false) { + throw new IllegalStateException("bug: should not be trying to initialize priv_set for an unsupported OS"); + } + + // we couldn't link methods, could be some really ancient Solaris or some bug + if (libc_solaris == null) { + throw new UnsupportedOperationException("priv_set unavailable: could not link methods. requires Solaris 10+"); + } + + // drop a null-terminated list of privileges + if (libc_solaris.priv_set(PRIV_OFF, PRIV_ALLSETS, PRIV_PROC_FORK, PRIV_PROC_EXEC, null) != 0) { + throw new UnsupportedOperationException("priv_set unavailable: priv_set(): " + JNACLibrary.strerror(Native.getLastError())); + } + + logger.debug("Solaris priv_set initialization successful"); + } + + // BSD implementation via setrlimit(2) + + // TODO: add OpenBSD to Lucene Constants + // TODO: JNA doesn't have netbsd support, but this mechanism should work there too. + static final boolean OPENBSD = Constants.OS_NAME.startsWith("OpenBSD"); + + // not a standard limit, means something different on linux, etc! + static final int RLIMIT_NPROC = 7; + + static void bsdImpl() { + boolean supported = Constants.FREE_BSD || OPENBSD || Constants.MAC_OS_X; + if (supported == false) { + throw new IllegalStateException("bug: should not be trying to initialize RLIMIT_NPROC for an unsupported OS"); + } + + JNACLibrary.Rlimit limit = new JNACLibrary.Rlimit(); + limit.rlim_cur.setValue(0); + limit.rlim_max.setValue(0); + if (JNACLibrary.setrlimit(RLIMIT_NPROC, limit) != 0) { + throw new UnsupportedOperationException("RLIMIT_NPROC unavailable: " + JNACLibrary.strerror(Native.getLastError())); + } + + logger.debug("BSD RLIMIT_NPROC initialization successful"); + } + + // windows impl via job ActiveProcessLimit + + static void windowsImpl() { + if (!Constants.WINDOWS) { + throw new IllegalStateException("bug: should not be trying to initialize ActiveProcessLimit for an unsupported OS"); + } + + JNAKernel32Library lib = JNAKernel32Library.getInstance(); + + // create a new Job + Pointer job = lib.CreateJobObjectW(null, null); + if (job == null) { + throw new UnsupportedOperationException("CreateJobObject: " + Native.getLastError()); + } + + try { + // retrieve the current basic limits of the job + int clazz = JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION_CLASS; + JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION limits = new JNAKernel32Library.JOBOBJECT_BASIC_LIMIT_INFORMATION(); + limits.write(); + if (!lib.QueryInformationJobObject(job, clazz, limits.getPointer(), limits.size(), null)) { + throw new UnsupportedOperationException("QueryInformationJobObject: " + Native.getLastError()); + } + limits.read(); + // modify the number of active processes to be 1 (exactly the one process we will add to the job). + limits.ActiveProcessLimit = 1; + limits.LimitFlags = JNAKernel32Library.JOB_OBJECT_LIMIT_ACTIVE_PROCESS; + limits.write(); + if (!lib.SetInformationJobObject(job, clazz, limits.getPointer(), limits.size())) { + throw new UnsupportedOperationException("SetInformationJobObject: " + Native.getLastError()); + } + // assign ourselves to the job + if (!lib.AssignProcessToJobObject(job, lib.GetCurrentProcess())) { + throw new UnsupportedOperationException("AssignProcessToJobObject: " + Native.getLastError()); + } + } finally { + lib.CloseHandle(job); + } + + logger.debug("Windows ActiveProcessLimit initialization successful"); + } /** * Attempt to drop the capability to execute for the process. *

    * This is best effort and OS and architecture dependent. It may throw any Throwable. + * @return 0 if we can do this for application threads, 1 for the entire process */ - static void init(Path tmpFile) throws Throwable { + static int init(Path tmpFile) throws Throwable { if (Constants.LINUX) { - linuxImpl(); + return linuxImpl(); } else if (Constants.MAC_OS_X) { + // try to enable both mechanisms if possible + bsdImpl(); macImpl(tmpFile); + return 1; + } else if (Constants.SUN_OS) { + solarisImpl(); + return 1; + } else if (Constants.FREE_BSD || OPENBSD) { + bsdImpl(); + return 1; + } else if (Constants.WINDOWS) { + windowsImpl(); + return 1; } else { throw new UnsupportedOperationException("syscall filtering not supported for OS: '" + Constants.OS_NAME + "'"); } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java index 26ae76b8b37..2d342eb5743 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -19,11 +19,19 @@ package org.elasticsearch.bootstrap; +import org.elasticsearch.SecureSM; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.http.netty.NettyHttpServerTransport; import org.elasticsearch.plugins.PluginInfo; +import org.elasticsearch.transport.netty.NettyTransport; import java.io.*; +import java.net.SocketPermission; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.AccessMode; import java.nio.file.DirectoryStream; @@ -32,15 +40,14 @@ import java.nio.file.Files; import java.nio.file.NotDirectoryException; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; -import java.security.PermissionCollection; import java.security.Permissions; import java.security.Policy; import java.security.URIParameter; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.IdentityHashMap; +import java.util.List; import java.util.Map; -import java.util.regex.Pattern; /** * Initializes SecurityManager with necessary permissions. @@ -87,6 +94,11 @@ import java.util.regex.Pattern; * 

      * JAVA_OPTS="-Djava.security.debug=access,failure" bin/elasticsearch
  *
    + *

    + * When running tests you have to pass it to the test runner like this: + * 

    + * mvn test -Dtests.jvm.argline="-Djava.security.debug=access,failure" ...
+ *
    * See * Troubleshooting Security for information. */ @@ -97,109 +109,140 @@ final class Security { /** * Initializes SecurityManager for the environment * Can only happen once! + * @param environment configuration for generating dynamic permissions + * @param filterBadDefaults true if we should filter out bad java defaults in the system policy. */ - static void configure(Environment environment) throws Exception { - // set properties for jar locations - setCodebaseProperties(); + static void configure(Environment environment, boolean filterBadDefaults) throws Exception { // enable security policy: union of template and environment-based paths, and possibly plugin permissions - Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment))); + Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults)); // enable security manager - System.setSecurityManager(new SecurityManager() { - // we disable this completely, because its granted otherwise: - // 'Note: The "exitVM.*" permission is automatically granted to - // all code loaded from the application class path, thus enabling - // applications to terminate themselves.' - @Override - public void checkExit(int status) { - throw new SecurityException("exit(" + status + ") not allowed by system policy"); - } - }); + System.setSecurityManager(new SecureSM()); // do some basic tests selfTest(); } - // mapping of jars to codebase properties - // note that this is only read once, when policy is parsed. - private static final Map SPECIAL_JARS; - static { - Map m = new IdentityHashMap<>(); - m.put(Pattern.compile(".*lucene-core-.*\\.jar$"), "es.security.jar.lucene.core"); - m.put(Pattern.compile(".*lucene-test-framework-.*\\.jar$"), "es.security.jar.lucene.testframework"); - m.put(Pattern.compile(".*randomizedtesting-runner-.*\\.jar$"), "es.security.jar.randomizedtesting.runner"); - m.put(Pattern.compile(".*junit4-ant-.*\\.jar$"), "es.security.jar.randomizedtesting.junit4"); - m.put(Pattern.compile(".*securemock-.*\\.jar$"), "es.security.jar.elasticsearch.securemock"); - SPECIAL_JARS = Collections.unmodifiableMap(m); - } - - /** - * Sets properties (codebase URLs) for policy files. - * JAR locations are not fixed so we have to find the locations of - * the ones we want. - */ - @SuppressForbidden(reason = "proper use of URL") - static void setCodebaseProperties() { - for (URL url : JarHell.parseClassPath()) { - for (Map.Entry e : SPECIAL_JARS.entrySet()) { - if (e.getKey().matcher(url.getPath()).matches()) { - String prop = e.getValue(); - if (System.getProperty(prop) != null) { - throw new IllegalStateException("property: " + prop + " is unexpectedly set: " + System.getProperty(prop)); - } - System.setProperty(prop, url.toString()); - } - } - } - for (String prop : SPECIAL_JARS.values()) { - if (System.getProperty(prop) == null) { - System.setProperty(prop, "file:/dev/null"); // no chance to be interpreted as "all" - } - } - } - /** * Sets properties (codebase URLs) for policy files. * we look for matching plugins and set URLs to fit */ @SuppressForbidden(reason = "proper use of URL") - static Map getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException { - Map map = new HashMap<>(); + static Map getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException { + Map map = new HashMap<>(); + // collect up lists of plugins and modules + List pluginsAndModules = new ArrayList<>(); if (Files.exists(environment.pluginsFile())) { try (DirectoryStream stream = Files.newDirectoryStream(environment.pluginsFile())) { for (Path plugin : stream) { - Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY); - if (Files.exists(policyFile)) { - // parse the plugin's policy file into a set of permissions - Policy policy = Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toUri())); - PermissionCollection permissions = policy.getPermissions(Security.class.getProtectionDomain()); - // this method is supported with the specific implementation we use, but just check for safety. - if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) { - throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions"); - } - // grant the permissions to each jar in the plugin - try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) { - for (Path jar : jarStream) { - if (map.put(jar.toUri().toURL().getFile(), permissions) != null) { - // just be paranoid ok? - throw new IllegalStateException("per-plugin permissions already granted for jar file: " + jar); - } - } - } + pluginsAndModules.add(plugin); + } + } + } + if (Files.exists(environment.modulesFile())) { + try (DirectoryStream stream = Files.newDirectoryStream(environment.modulesFile())) { + for (Path plugin : stream) { + pluginsAndModules.add(plugin); + } + } + } + // now process each one + for (Path plugin : pluginsAndModules) { + Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY); + if (Files.exists(policyFile)) { + // first get a list of URLs for the plugins' jars: + // we resolve symlinks so map is keyed on the normalize codebase name + List codebases = new ArrayList<>(); + try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) { + for (Path jar : jarStream) { + codebases.add(jar.toRealPath().toUri().toURL()); + } + } + + // parse the plugin's policy file into a set of permissions + Policy policy = readPolicy(policyFile.toUri().toURL(), codebases.toArray(new URL[codebases.size()])); + + // consult this policy for each of the plugin's jars: + for (URL url : codebases) { + if (map.put(url.getFile(), policy) != null) { + // just be paranoid ok? + throw new IllegalStateException("per-plugin permissions already granted for jar file: " + url); } } } } + return Collections.unmodifiableMap(map); } - /** returns dynamic Permissions to configured paths */ + /** + * Reads and returns the specified {@code policyFile}. + *

    + * Resources (e.g. jar files and directories) listed in {@code codebases} location + * will be provided to the policy file via a system property of the short name: + * e.g. ${codebase.joda-convert-1.2.jar} would map to full URL. + */ + @SuppressForbidden(reason = "accesses fully qualified URLs to configure security") + static Policy readPolicy(URL policyFile, URL codebases[]) { + try { + try { + // set codebase properties + for (URL url : codebases) { + String shortName = PathUtils.get(url.toURI()).getFileName().toString(); + System.setProperty("codebase." + shortName, url.toString()); + } + return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI())); + } finally { + // clear codebase properties + for (URL url : codebases) { + String shortName = PathUtils.get(url.toURI()).getFileName().toString(); + System.clearProperty("codebase." + shortName); + } + } + } catch (NoSuchAlgorithmException | URISyntaxException e) { + throw new IllegalArgumentException("unable to parse policy file `" + policyFile + "`", e); + } + } + + /** returns dynamic Permissions to configured paths and bind ports */ static Permissions createPermissions(Environment environment) throws IOException { Permissions policy = new Permissions(); + addClasspathPermissions(policy); + addFilePermissions(policy, environment); + addBindPermissions(policy, environment.settings()); + return policy; + } + + /** Adds access to classpath jars/classes for jar hell scan, etc */ + @SuppressForbidden(reason = "accesses fully qualified URLs to configure security") + static void addClasspathPermissions(Permissions policy) throws IOException { + // add permissions to everything in classpath + // really it should be covered by lib/, but there could be e.g. agents or similar configured) + for (URL url : JarHell.parseClassPath()) { + Path path; + try { + path = PathUtils.get(url.toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + // resource itself + policy.add(new FilePermission(path.toString(), "read,readlink")); + // classes underneath + if (Files.isDirectory(path)) { + policy.add(new FilePermission(path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink")); + } + } + } + + /** + * Adds access to all configurable paths. + */ + static void addFilePermissions(Permissions policy, Environment environment) { // read-only dirs addPath(policy, "path.home", environment.binFile(), "read,readlink"); addPath(policy, "path.home", environment.libFile(), "read,readlink"); + addPath(policy, "path.home", environment.modulesFile(), "read,readlink"); addPath(policy, "path.plugins", environment.pluginsFile(), "read,readlink"); addPath(policy, "path.conf", environment.configFile(), "read,readlink"); addPath(policy, "path.scripts", environment.scriptsFile(), "read,readlink"); @@ -222,7 +265,40 @@ final class Security { // we just need permission to remove the file if its elsewhere. policy.add(new FilePermission(environment.pidFile().toString(), "delete")); } - return policy; + } + + static void addBindPermissions(Permissions policy, Settings settings) throws IOException { + // http is simple + String httpRange = settings.get("http.netty.port", + settings.get("http.port", + NettyHttpServerTransport.DEFAULT_PORT_RANGE)); + // listen is always called with 'localhost' but use wildcard to be sure, no name service is consulted. + // see SocketPermission implies() code + policy.add(new SocketPermission("*:" + httpRange, "listen,resolve")); + // transport is waaaay overengineered + Map profiles = settings.getGroups("transport.profiles", true); + if (!profiles.containsKey(NettyTransport.DEFAULT_PROFILE)) { + profiles = new HashMap<>(profiles); + profiles.put(NettyTransport.DEFAULT_PROFILE, Settings.EMPTY); + } + + // loop through all profiles and add permissions for each one, if its valid. + // (otherwise NettyTransport is lenient and ignores it) + for (Map.Entry entry : profiles.entrySet()) { + Settings profileSettings = entry.getValue(); + String name = entry.getKey(); + String transportRange = profileSettings.get("port", + settings.get("transport.tcp.port", + NettyTransport.DEFAULT_PORT_RANGE)); + + // a profile is only valid if its the default profile, or if it has an actual name and specifies a port + boolean valid = NettyTransport.DEFAULT_PROFILE.equals(name) || (Strings.hasLength(name) && profileSettings.get("port") != null); + if (valid) { + // listen is always called with 'localhost' but use wildcard to be sure, no name service is consulted. + // see SocketPermission implies() code + policy.add(new SocketPermission("*:" + transportRange, "listen,resolve")); + } + } } /** diff --git a/core/src/main/java/org/elasticsearch/client/Client.java b/core/src/main/java/org/elasticsearch/client/Client.java index 6e0b0b27fd9..a396e183bb7 100644 --- a/core/src/main/java/org/elasticsearch/client/Client.java +++ b/core/src/main/java/org/elasticsearch/client/Client.java @@ -24,15 +24,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountRequestBuilder; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.exists.ExistsRequest; -import org.elasticsearch.action.exists.ExistsRequestBuilder; -import org.elasticsearch.action.exists.ExistsResponse; import org.elasticsearch.action.explain.ExplainRequest; import org.elasticsearch.action.explain.ExplainRequestBuilder; import org.elasticsearch.action.explain.ExplainResponse; @@ -337,52 +331,6 @@ public interface Client extends ElasticsearchClient, Releasable { */ MultiGetRequestBuilder prepareMultiGet(); - /** - * A count of all the documents matching a specific query. - * - * @param request The count request - * @return The result future - * @see Requests#countRequest(String...) - */ - ActionFuture count(CountRequest request); - - /** - * A count of all the documents matching a specific query. - * - * @param request The count request - * @param listener A listener to be notified of the result - * @see Requests#countRequest(String...) - */ - void count(CountRequest request, ActionListener listener); - - /** - * A count of all the documents matching a specific query. - */ - CountRequestBuilder prepareCount(String... indices); - - /** - * Checks existence of any documents matching a specific query. - * - * @param request The exists request - * @return The result future - * @see Requests#existsRequest(String...) - */ - ActionFuture exists(ExistsRequest request); - - /** - * Checks existence of any documents matching a specific query. - * - * @param request The exists request - * @param listener A listener to be notified of the result - * @see Requests#existsRequest(String...) - */ - void exists(ExistsRequest request, ActionListener listener); - - /** - * Checks existence of any documents matching a specific query. - */ - ExistsRequestBuilder prepareExists(String... indices); - /** * Suggestion matching a specific phrase. * diff --git a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index 75cae17ea9d..15def3b273e 100644 --- a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -53,6 +53,9 @@ import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; @@ -63,9 +66,6 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequestBuilder; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; @@ -391,28 +391,27 @@ public interface IndicesAdminClient extends ElasticsearchClient { FlushRequestBuilder prepareFlush(String... indices); /** - * Explicitly optimize one or more indices into a the number of segments. + * Explicitly force merge one or more indices into a the number of segments. * * @param request The optimize request * @return A result future - * @see org.elasticsearch.client.Requests#optimizeRequest(String...) + * @see org.elasticsearch.client.Requests#forceMergeRequest(String...) */ - ActionFuture optimize(OptimizeRequest request); + ActionFuture forceMerge(ForceMergeRequest request); /** - * Explicitly optimize one or more indices into a the number of segments. + * Explicitly force merge one or more indices into a the number of segments. * - * @param request The optimize request + * @param request The force merge request * @param listener A listener to be notified with a result - * @see org.elasticsearch.client.Requests#optimizeRequest(String...) + * @see org.elasticsearch.client.Requests#forceMergeRequest(String...) */ - void optimize(OptimizeRequest request, ActionListener listener); + void forceMerge(ForceMergeRequest request, ActionListener listener); /** - * Explicitly optimize one or more indices into a the number of segments. + * Explicitly force mergee one or more indices into a the number of segments. */ - OptimizeRequestBuilder prepareOptimize(String... indices); - + ForceMergeRequestBuilder prepareForceMerge(String... indices); /** * Explicitly upgrade one or more indices @@ -725,7 +724,6 @@ public interface IndicesAdminClient extends ElasticsearchClient { * * @param request The count request * @return The result future - * @see Requests#countRequest(String...) */ ActionFuture validateQuery(ValidateQueryRequest request); @@ -734,7 +732,6 @@ public interface IndicesAdminClient extends ElasticsearchClient { * * @param request The count request * @param listener A listener to be notified of the result - * @see Requests#countRequest(String...) */ void validateQuery(ValidateQueryRequest request, ActionListener listener); diff --git a/core/src/main/java/org/elasticsearch/client/Requests.java b/core/src/main/java/org/elasticsearch/client/Requests.java index e36c26923d8..7f0decaba52 100644 --- a/core/src/main/java/org/elasticsearch/client/Requests.java +++ b/core/src/main/java/org/elasticsearch/client/Requests.java @@ -45,16 +45,14 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.count.CountRequest; import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.exists.ExistsRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -124,30 +122,6 @@ public class Requests { return new GetRequest(index); } - /** - * Creates a count request which counts the hits matched against a query. Note, the query itself must be set - * either using the JSON source of the query, or using a {@link org.elasticsearch.index.query.QueryBuilder} (using {@link org.elasticsearch.index.query.QueryBuilders}). - * - * @param indices The indices to count matched documents against a query. Use null or _all to execute against all indices - * @return The count request - * @see org.elasticsearch.client.Client#count(org.elasticsearch.action.count.CountRequest) - */ - public static CountRequest countRequest(String... indices) { - return new CountRequest(indices); - } - - /** - * Creates a exists request which checks if any of the hits matched against a query exists. Note, the query itself must be set - * either using the JSON source of the query, or using a {@link org.elasticsearch.index.query.QueryBuilder} (using {@link org.elasticsearch.index.query.QueryBuilders}). - * - * @param indices The indices to count matched documents against a query. Use null or _all to execute against all indices - * @return The exists request - * @see org.elasticsearch.client.Client#exists(org.elasticsearch.action.exists.ExistsRequest) - */ - public static ExistsRequest existsRequest(String... indices) { - return new ExistsRequest(indices); - } - /** * Creates a suggest request for getting suggestions from provided indices. * The suggest query has to be set using the JSON source using {@link org.elasticsearch.action.suggest.SuggestRequest#suggest(org.elasticsearch.common.bytes.BytesReference)}. @@ -292,14 +266,14 @@ public class Requests { } /** - * Creates an optimize request. + * Creates a force merge request. * - * @param indices The indices to optimize. Use null or _all to execute against all indices - * @return The optimize request - * @see org.elasticsearch.client.IndicesAdminClient#optimize(org.elasticsearch.action.admin.indices.optimize.OptimizeRequest) + * @param indices The indices to force merge. Use null or _all to execute against all indices + * @return The force merge request + * @see org.elasticsearch.client.IndicesAdminClient#forceMerge(org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest) */ - public static OptimizeRequest optimizeRequest(String... indices) { - return new OptimizeRequest(indices); + public static ForceMergeRequest forceMergeRequest(String... indices) { + return new ForceMergeRequest(indices); } /** diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 3fa5d789905..1b5e8539ac6 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -96,6 +96,10 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateAction; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequestBuilder; +import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; @@ -139,6 +143,10 @@ import org.elasticsearch.action.admin.indices.flush.FlushAction; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; @@ -152,10 +160,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexAction; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; -import org.elasticsearch.action.admin.indices.optimize.OptimizeAction; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequestBuilder; @@ -177,9 +181,9 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder; -import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; +import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; @@ -208,10 +212,6 @@ import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; -import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateAction; -import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest; -import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequestBuilder; -import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse; import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerAction; import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequest; import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequestBuilder; @@ -228,18 +228,10 @@ import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.count.CountAction; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountRequestBuilder; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.delete.DeleteAction; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.exists.ExistsAction; -import org.elasticsearch.action.exists.ExistsRequest; -import org.elasticsearch.action.exists.ExistsRequestBuilder; -import org.elasticsearch.action.exists.ExistsResponse; import org.elasticsearch.action.explain.ExplainAction; import org.elasticsearch.action.explain.ExplainRequest; import org.elasticsearch.action.explain.ExplainRequestBuilder; @@ -271,8 +263,6 @@ import org.elasticsearch.action.suggest.SuggestAction; import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.action.suggest.SuggestRequestBuilder; import org.elasticsearch.action.suggest.SuggestResponse; -import org.elasticsearch.action.support.AdapterActionFuture; -import org.elasticsearch.action.support.DelegatingActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.termvectors.*; @@ -605,48 +595,6 @@ public abstract class AbstractClient extends AbstractComponent implements Client return new MultiSearchRequestBuilder(this, MultiSearchAction.INSTANCE); } - @Override - public ActionFuture count(final CountRequest request) { - AdapterActionFuture actionFuture = new AdapterActionFuture() { - @Override - protected CountResponse convert(SearchResponse listenerResponse) { - return new CountResponse(listenerResponse); - } - }; - execute(SearchAction.INSTANCE, request.toSearchRequest(), actionFuture); - return actionFuture; - } - - @Override - public void count(final CountRequest request, final ActionListener listener) { - execute(SearchAction.INSTANCE, request.toSearchRequest(), new DelegatingActionListener(listener) { - @Override - protected CountResponse getDelegatedFromInstigator(SearchResponse response) { - return new CountResponse(response); - } - }); - } - - @Override - public CountRequestBuilder prepareCount(String... indices) { - return new CountRequestBuilder(this, CountAction.INSTANCE).setIndices(indices); - } - - @Override - public ActionFuture exists(final ExistsRequest request) { - return execute(ExistsAction.INSTANCE, request); - } - - @Override - public void exists(final ExistsRequest request, final ActionListener listener) { - execute(ExistsAction.INSTANCE, request, listener); - } - - @Override - public ExistsRequestBuilder prepareExists(String... indices) { - return new ExistsRequestBuilder(this, ExistsAction.INSTANCE).setIndices(indices); - } - @Override public ActionFuture suggest(final SuggestRequest request) { return execute(SuggestAction.INSTANCE, request); @@ -1413,18 +1361,18 @@ public abstract class AbstractClient extends AbstractComponent implements Client } @Override - public ActionFuture optimize(final OptimizeRequest request) { - return execute(OptimizeAction.INSTANCE, request); + public ActionFuture forceMerge(final ForceMergeRequest request) { + return execute(ForceMergeAction.INSTANCE, request); } @Override - public void optimize(final OptimizeRequest request, final ActionListener listener) { - execute(OptimizeAction.INSTANCE, request, listener); + public void forceMerge(final ForceMergeRequest request, final ActionListener listener) { + execute(ForceMergeAction.INSTANCE, request, listener); } @Override - public OptimizeRequestBuilder prepareOptimize(String... indices) { - return new OptimizeRequestBuilder(this, OptimizeAction.INSTANCE).setIndices(indices); + public ForceMergeRequestBuilder prepareForceMerge(String... indices) { + return new ForceMergeRequestBuilder(this, ForceMergeAction.INSTANCE).setIndices(indices); } @Override diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 17551012edb..33cf3479419 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -38,7 +38,9 @@ import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.env.Environment; @@ -46,6 +48,7 @@ import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.indices.breaker.CircuitBreakerModule; import org.elasticsearch.monitor.MonitorService; import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsModule; import org.elasticsearch.plugins.PluginsService; @@ -122,13 +125,14 @@ public class TransportClient extends AbstractClient { .put(CLIENT_TYPE_SETTING, CLIENT_TYPE) .build(); - PluginsService pluginsService = new PluginsService(settings, null, pluginClasses); + PluginsService pluginsService = new PluginsService(settings, null, null, pluginClasses); this.settings = pluginsService.updatedSettings(); Version version = Version.CURRENT; final ThreadPool threadPool = new ThreadPool(settings); - + final NetworkService networkService = new NetworkService(settings); + final SettingsFilter settingsFilter = new SettingsFilter(settings); boolean success = false; try { ModulesBuilder modules = new ModulesBuilder(); @@ -138,12 +142,12 @@ public class TransportClient extends AbstractClient { modules.add(pluginModule); } modules.add(new PluginsModule(pluginsService)); - modules.add(new SettingsModule(this.settings)); - modules.add(new NetworkModule()); + modules.add(new SettingsModule(this.settings, settingsFilter )); + modules.add(new NetworkModule(networkService)); modules.add(new ClusterNameModule(this.settings)); modules.add(new ThreadPoolModule(threadPool)); modules.add(new TransportModule(this.settings)); - modules.add(new SearchModule(this.settings) { + modules.add(new SearchModule() { @Override protected void configure() { // noop diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java index 64e2655e463..56befbb9b84 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java @@ -285,7 +285,7 @@ public class TransportClientNodesService extends AbstractComponent { private void ensureNodesAreAvailable(List nodes) { if (nodes.isEmpty()) { - String message = String.format(Locale.ROOT, "None of the configured nodes are available: %s", nodes); + String message = String.format(Locale.ROOT, "None of the configured nodes are available: %s", this.listedNodes); throw new NoNodeAvailableException(message); } } @@ -359,7 +359,7 @@ public class TransportClientNodesService extends AbstractComponent { try { LivenessResponse livenessResponse = transportService.submitRequest(listedNode, TransportLivenessAction.NAME, headers.applyTo(new LivenessRequest()), - TransportRequestOptions.options().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout), + TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout).build(), new FutureTransportResponseHandler() { @Override public LivenessResponse newInstance() { @@ -430,7 +430,7 @@ public class TransportClientNodesService extends AbstractComponent { } transportService.sendRequest(listedNode, ClusterStateAction.NAME, headers.applyTo(Requests.clusterStateRequest().clear().nodes(true).local(true)), - TransportRequestOptions.options().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout), + TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout).build(), new BaseTransportResponseHandler() { @Override diff --git a/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateTaskListener.java b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateTaskListener.java new file mode 100644 index 00000000000..cdd9b2204ff --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateTaskListener.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.unit.TimeValue; + +public interface AckedClusterStateTaskListener extends ClusterStateTaskListener { + + /** + * Called to determine which nodes the acknowledgement is expected from + * + * @param discoveryNode a node + * @return true if the node is expected to send ack back, false otherwise + */ + boolean mustAck(DiscoveryNode discoveryNode); + + /** + * Called once all the nodes have acknowledged the cluster state update request. Must be + * very lightweight execution, since it gets executed on the cluster service thread. + * + * @param t optional error that might have been thrown + */ + void onAllNodesAcked(@Nullable Throwable t); + + /** + * Called once the acknowledgement timeout defined by + * {@link AckedClusterStateUpdateTask#ackTimeout()} has expired + */ + void onAckTimeout(); + + /** + * Acknowledgement timeout, maximum time interval to wait for acknowledgements + */ + TimeValue ackTimeout(); + +} diff --git a/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java index 21c6cd5032a..b833f6e1879 100644 --- a/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java +++ b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java @@ -22,18 +22,24 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.unit.TimeValue; /** * An extension interface to {@link ClusterStateUpdateTask} that allows to be notified when * all the nodes have acknowledged a cluster state update request */ -public abstract class AckedClusterStateUpdateTask extends ClusterStateUpdateTask { +public abstract class AckedClusterStateUpdateTask extends ClusterStateUpdateTask implements AckedClusterStateTaskListener { private final ActionListener listener; private final AckedRequest request; protected AckedClusterStateUpdateTask(AckedRequest request, ActionListener listener) { + this(Priority.NORMAL, request, listener); + } + + protected AckedClusterStateUpdateTask(Priority priority, AckedRequest request, ActionListener listener) { + super(priority); this.listener = listener; this.request = request; } diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java index f55452b5805..0863fbfc4f4 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java @@ -143,7 +143,7 @@ public class ClusterChangedEvent { if (previousMetaData == null) { return true; } - IndexMetaData previousIndexMetaData = previousMetaData.index(current.index()); + IndexMetaData previousIndexMetaData = previousMetaData.index(current.getIndex()); // no need to check on version, since disco modules will make sure to use the // same instance if its a version match if (previousIndexMetaData == current) { diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java index df1529a1957..12be047f17d 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -19,7 +19,9 @@ package org.elasticsearch.cluster; +import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; import org.elasticsearch.action.support.DestructiveOperations; +import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction; import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction; @@ -34,7 +36,6 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService; import org.elasticsearch.cluster.metadata.MetaDataMappingService; -import org.elasticsearch.cluster.metadata.MetaDataService; import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService; import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.cluster.routing.OperationRouting; @@ -80,12 +81,12 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.MergePolicyConfig; import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.index.store.IndexStore; +import org.elasticsearch.index.store.IndexStoreConfig; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.indices.cache.request.IndicesRequestCache; import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.indices.ttl.IndicesTTLService; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.internal.DefaultSearchContext; @@ -158,15 +159,11 @@ public class ClusterModule extends AbstractModule { registerClusterDynamicSetting(FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP + "*", Validator.EMPTY); registerClusterDynamicSetting(FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP + "*", Validator.EMPTY); registerClusterDynamicSetting(FilterAllocationDecider.CLUSTER_ROUTING_REQUIRE_GROUP + "*", Validator.EMPTY); - registerClusterDynamicSetting(IndicesStore.INDICES_STORE_THROTTLE_TYPE, Validator.EMPTY); - registerClusterDynamicSetting(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE); + registerClusterDynamicSetting(IndexStoreConfig.INDICES_STORE_THROTTLE_TYPE, Validator.EMPTY); + registerClusterDynamicSetting(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE); registerClusterDynamicSetting(IndicesTTLService.INDICES_TTL_INTERVAL, Validator.TIME); registerClusterDynamicSetting(MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT, Validator.TIME); registerClusterDynamicSetting(MetaData.SETTING_READ_ONLY, Validator.EMPTY); - registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_FILE_CHUNK_SIZE, Validator.POSITIVE_BYTES_SIZE); - registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_TRANSLOG_OPS, Validator.INTEGER); - registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_TRANSLOG_SIZE, Validator.BYTES_SIZE); - registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_COMPRESS, Validator.EMPTY); registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_CONCURRENT_STREAMS, Validator.POSITIVE_INTEGER); registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_CONCURRENT_SMALL_FILE_STREAMS, Validator.POSITIVE_INTEGER); registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE); @@ -175,8 +172,7 @@ public class ClusterModule extends AbstractModule { registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT, Validator.TIME_NON_NEGATIVE); registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT, Validator.TIME_NON_NEGATIVE); registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT, Validator.TIME_NON_NEGATIVE); - registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_MAX_SIZE_PER_SEC, Validator.BYTES_SIZE); - registerClusterDynamicSetting(ThreadPool.THREADPOOL_GROUP + "*", Validator.EMPTY); + registerClusterDynamicSetting(ThreadPool.THREADPOOL_GROUP + "*", ThreadPool.THREAD_POOL_TYPE_SETTINGS_VALIDATOR); registerClusterDynamicSetting(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES, Validator.INTEGER); registerClusterDynamicSetting(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES, Validator.INTEGER); registerClusterDynamicSetting(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, Validator.EMPTY); @@ -203,6 +199,9 @@ public class ClusterModule extends AbstractModule { registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_INCLUDE + ".*", Validator.EMPTY); registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_EXCLUDE, Validator.EMPTY); registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_EXCLUDE + ".*", Validator.EMPTY); + registerClusterDynamicSetting(TransportCloseIndexAction.SETTING_CLUSTER_INDICES_CLOSE_ENABLE, Validator.BOOLEAN); + registerClusterDynamicSetting(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE, Validator.INTEGER); + registerClusterDynamicSetting(TransportReplicationAction.SHARD_FAILURE_TIMEOUT, Validator.TIME_NON_NEGATIVE); } private void registerBuiltinIndexSettings() { @@ -308,7 +307,6 @@ public class ClusterModule extends AbstractModule { bind(DiscoveryNodeService.class).asEagerSingleton(); bind(ClusterService.class).to(InternalClusterService.class).asEagerSingleton(); bind(OperationRouting.class).asEagerSingleton(); - bind(MetaDataService.class).asEagerSingleton(); bind(MetaDataCreateIndexService.class).asEagerSingleton(); bind(MetaDataDeleteIndexService.class).asEagerSingleton(); bind(MetaDataIndexStateService.class).asEagerSingleton(); diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/ClusterService.java index 2a3d3c12e46..b682b0cc61d 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterService.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterService.java @@ -24,7 +24,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.service.PendingClusterTask; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Priority; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.unit.TimeValue; @@ -101,12 +100,35 @@ public interface ClusterService extends LifecycleComponent { void add(@Nullable TimeValue timeout, TimeoutClusterStateListener listener); /** - * Submits a task that will update the cluster state. + * Submits a cluster state update task; submitted updates will be + * batched across the same instance of executor. The exact batching + * semantics depend on the underlying implementation but a rough + * guideline is that if the update task is submitted while there + * are pending update tasks for the same executor, these update + * tasks will all be executed on the executor in a single batch + * + * @param source the source of the cluster state update task + * @param task the state needed for the cluster state update task + * @param config the cluster state update task configuration + * @param executor the cluster state update task executor; tasks + * that share the same executor will be executed + * batches on this executor + * @param listener callback after the cluster state update task + * completes + * @param the type of the cluster state update task state */ - void submitStateUpdateTask(final String source, Priority priority, final ClusterStateUpdateTask updateTask); + void submitStateUpdateTask(final String source, final T task, + final ClusterStateTaskConfig config, + final ClusterStateTaskExecutor executor, + final ClusterStateTaskListener listener); /** - * Submits a task that will update the cluster state (the task has a default priority of {@link Priority#NORMAL}). + * Submits a cluster state update task; unlike {@link #submitStateUpdateTask(String, Object, ClusterStateTaskConfig, ClusterStateTaskExecutor, ClusterStateTaskListener)}, + * submitted updates will not be batched. + * + * @param source the source of the cluster state update task + * @param updateTask the full context for the cluster state update + * task */ void submitStateUpdateTask(final String source, final ClusterStateUpdateTask updateTask); @@ -123,7 +145,7 @@ public interface ClusterService extends LifecycleComponent { /** * Returns the maximum wait time for tasks in the queue * - * @returns A zero time value if the queue is empty, otherwise the time value oldest task waiting in the queue + * @return A zero time value if the queue is empty, otherwise the time value oldest task waiting in the queue */ TimeValue getMaxTaskWaitTime(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java index b1bdf52e9d4..e20f21b4cec 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -19,10 +19,9 @@ package org.elasticsearch.cluster; +import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - -import org.elasticsearch.cluster.DiffableUtils.KeyedReader; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -31,12 +30,7 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.service.InternalClusterService; import org.elasticsearch.common.Nullable; @@ -57,11 +51,7 @@ import org.elasticsearch.discovery.local.LocalDiscovery; import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction; import java.io.IOException; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * Represents the current state of the cluster. @@ -283,6 +273,7 @@ public class ClusterState implements ToXContent, Diffable { sb.append("state uuid: ").append(stateUUID).append("\n"); sb.append("from_diff: ").append(wasReadFromDiff).append("\n"); sb.append("meta data version: ").append(metaData.version()).append("\n"); + sb.append(blocks().prettyPrint()); sb.append(nodes().prettyPrint()); sb.append(routingTable().prettyPrint()); sb.append(getRoutingNodes().prettyPrint()); @@ -449,17 +440,17 @@ public class ClusterState implements ToXContent, Diffable { builder.startObject("indices"); for (IndexMetaData indexMetaData : metaData()) { - builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); + builder.startObject(indexMetaData.getIndex(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); + builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH)); builder.startObject("settings"); - Settings settings = indexMetaData.settings(); + Settings settings = indexMetaData.getSettings(); settings.toXContent(builder, params); builder.endObject(); builder.startObject("mappings"); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { + for (ObjectObjectCursor cursor : indexMetaData.getMappings()) { byte[] mappingSource = cursor.value.source().uncompressed(); XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); Map mapping = parser.map(); @@ -473,11 +464,21 @@ public class ClusterState implements ToXContent, Diffable { builder.endObject(); builder.startArray("aliases"); - for (ObjectCursor cursor : indexMetaData.aliases().keys()) { + for (ObjectCursor cursor : indexMetaData.getAliases().keys()) { builder.value(cursor.value); } builder.endArray(); + builder.startObject(IndexMetaData.KEY_ACTIVE_ALLOCATIONS); + for (IntObjectCursor> cursor : indexMetaData.getActiveAllocationIds()) { + builder.startArray(String.valueOf(cursor.key)); + for (String allocationId : cursor.value) { + builder.value(allocationId); + } + builder.endArray(); + } + builder.endObject(); + builder.endObject(); } builder.endObject(); @@ -593,6 +594,7 @@ public class ClusterState implements ToXContent, Diffable { public Builder routingResult(RoutingAllocation.Result routingResult) { this.routingTable = routingResult.routingTable(); + this.metaData = routingResult.metaData(); return this; } @@ -768,7 +770,7 @@ public class ClusterState implements ToXContent, Diffable { nodes = after.nodes.diff(before.nodes); metaData = after.metaData.diff(before.metaData); blocks = after.blocks.diff(before.blocks); - customs = DiffableUtils.diff(before.customs, after.customs); + customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer()); } public ClusterStateDiff(StreamInput in, ClusterState proto) throws IOException { @@ -780,14 +782,15 @@ public class ClusterState implements ToXContent, Diffable { nodes = proto.nodes.readDiffFrom(in); metaData = proto.metaData.readDiffFrom(in); blocks = proto.blocks.readDiffFrom(in); - customs = DiffableUtils.readImmutableOpenMapDiff(in, new KeyedReader() { + customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), + new DiffableUtils.DiffableValueSerializer() { @Override - public Custom readFrom(StreamInput in, String key) throws IOException { + public Custom read(StreamInput in, String key) throws IOException { return lookupPrototypeSafe(key).readFrom(in); } @Override - public Diff readDiffFrom(StreamInput in, String key) throws IOException { + public Diff readDiff(StreamInput in, String key) throws IOException { return lookupPrototypeSafe(key).readDiffFrom(in); } }); diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskConfig.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskConfig.java new file mode 100644 index 00000000000..2ef2438991e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskConfig.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.unit.TimeValue; + +/** + * Cluster state update task configuration for timeout and priority + */ +public interface ClusterStateTaskConfig { + /** + * The timeout for this cluster state update task configuration. If + * the cluster state update task isn't processed within this + * timeout, the associated {@link ClusterStateTaskListener#onFailure(String, Throwable)} + * is invoked. + * + * @return the timeout, or null if one is not set + */ + @Nullable + TimeValue timeout(); + + /** + * The {@link Priority} for this cluster state update task configuration. + * + * @return the priority + */ + Priority priority(); + + /** + * Build a cluster state update task configuration with the + * specified {@link Priority} and no timeout. + * + * @param priority the priority for the associated cluster state + * update task + * @return the resulting cluster state update task configuration + */ + static ClusterStateTaskConfig build(Priority priority) { + return new Basic(priority, null); + } + + /** + * Build a cluster state update task configuration with the + * specified {@link Priority} and timeout. + * + * @param priority the priority for the associated cluster state + * update task + * @param timeout the timeout for the associated cluster state + * update task + * @return the result cluster state update task configuration + */ + static ClusterStateTaskConfig build(Priority priority, TimeValue timeout) { + return new Basic(priority, timeout); + } + + class Basic implements ClusterStateTaskConfig { + final TimeValue timeout; + final Priority priority; + + public Basic(Priority priority, TimeValue timeout) { + this.timeout = timeout; + this.priority = priority; + } + + @Override + public TimeValue timeout() { + return timeout; + } + + @Override + public Priority priority() { + return priority; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java new file mode 100644 index 00000000000..ab85d9540f0 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster; + +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +public interface ClusterStateTaskExecutor { + /** + * Update the cluster state based on the current state and the given tasks. Return the *same instance* if no state + * should be changed. + */ + BatchResult execute(ClusterState currentState, List tasks) throws Exception; + + /** + * indicates whether this task should only run if current node is master + */ + default boolean runOnlyOnMaster() { + return true; + } + + /** + * Represents the result of a batched execution of cluster state update tasks + * @param the type of the cluster state update task + */ + class BatchResult { + final public ClusterState resultingState; + final public Map executionResults; + + /** + * Construct an execution result instance with a correspondence between the tasks and their execution result + * @param resultingState the resulting cluster state + * @param executionResults the correspondence between tasks and their outcome + */ + BatchResult(ClusterState resultingState, Map executionResults) { + this.resultingState = resultingState; + this.executionResults = executionResults; + } + + public static Builder builder() { + return new Builder<>(); + } + + public static class Builder { + private final Map executionResults = new IdentityHashMap<>(); + + public Builder success(T task) { + return result(task, TaskResult.success()); + } + + public Builder successes(Iterable tasks) { + for (T task : tasks) { + success(task); + } + return this; + } + + public Builder failure(T task, Throwable t) { + return result(task, TaskResult.failure(t)); + } + + public Builder failures(Iterable tasks, Throwable t) { + for (T task : tasks) { + failure(task, t); + } + return this; + } + + private Builder result(T task, TaskResult executionResult) { + executionResults.put(task, executionResult); + return this; + } + + public BatchResult build(ClusterState resultingState) { + return new BatchResult<>(resultingState, executionResults); + } + } + } + + final class TaskResult { + private final Throwable failure; + + private static final TaskResult SUCCESS = new TaskResult(null); + + public static TaskResult success() { + return SUCCESS; + } + + public static TaskResult failure(Throwable failure) { + return new TaskResult(failure); + } + + private TaskResult(Throwable failure) { + this.failure = failure; + } + + public boolean isSuccess() { + return failure != null; + } + + /** + * Handle the execution result with the provided consumers + * @param onSuccess handler to invoke on success + * @param onFailure handler to invoke on failure; the throwable passed through will not be null + */ + public void handle(Runnable onSuccess, Consumer onFailure) { + if (failure == null) { + onSuccess.run(); + } else { + onFailure.accept(failure); + } + } + } +} diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java new file mode 100644 index 00000000000..3bf7887cd1c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster; + +import java.util.List; + +public interface ClusterStateTaskListener { + + /** + * A callback called when execute fails. + */ + void onFailure(String source, Throwable t); + + /** + * called when the task was rejected because the local node is no longer master + */ + default void onNoLongerMaster(String source) { + onFailure(source, new NotMasterException("no longer master. source: [" + source + "]")); + } + + /** + * Called when the result of the {@link ClusterStateTaskExecutor#execute(ClusterState, List)} have been processed + * properly by all listeners. + */ + default void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + } +} diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java index c0f1438d432..3e2881134f8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java @@ -20,13 +20,31 @@ package org.elasticsearch.cluster; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; + +import java.util.List; /** * A task that can update the cluster state. */ -abstract public class ClusterStateUpdateTask { +abstract public class ClusterStateUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor, ClusterStateTaskListener { + + final private Priority priority; + + public ClusterStateUpdateTask() { + this(Priority.NORMAL); + } + + public ClusterStateUpdateTask(Priority priority) { + this.priority = priority; + } + + @Override + final public BatchResult execute(ClusterState currentState, List tasks) throws Exception { + ClusterState result = execute(currentState); + return BatchResult.builder().successes(tasks).build(result); + } /** * Update the cluster state based on the current state. Return the *same instance* if no state @@ -39,28 +57,6 @@ abstract public class ClusterStateUpdateTask { */ abstract public void onFailure(String source, Throwable t); - - /** - * indicates whether this task should only run if current node is master - */ - public boolean runOnlyOnMaster() { - return true; - } - - /** - * called when the task was rejected because the local node is no longer master - */ - public void onNoLongerMaster(String source) { - onFailure(source, new EsRejectedExecutionException("no longer master. source: [" + source + "]")); - } - - /** - * Called when the result of the {@link #execute(ClusterState)} have been processed - * properly by all listeners. - */ - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - } - /** * If the cluster state update task wasn't processed by the provided timeout, call * {@link #onFailure(String, Throwable)}. May return null to indicate no timeout is needed (default). @@ -70,5 +66,8 @@ abstract public class ClusterStateUpdateTask { return null; } - + @Override + public Priority priority() { + return priority; + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/Diff.java b/core/src/main/java/org/elasticsearch/cluster/Diff.java index 1a9fff246a9..76535a4b763 100644 --- a/core/src/main/java/org/elasticsearch/cluster/Diff.java +++ b/core/src/main/java/org/elasticsearch/cluster/Diff.java @@ -29,7 +29,7 @@ import java.io.IOException; public interface Diff { /** - * Applies difference to the specified part and retunrs the resulted part + * Applies difference to the specified part and returns the resulted part */ T apply(T part); diff --git a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java index 84e0021ee00..1488f059437 100644 --- a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java +++ b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java @@ -19,263 +19,630 @@ package org.elasticsearch.cluster; +import com.carrotsearch.hppc.cursors.IntCursor; +import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public final class DiffableUtils { private DiffableUtils() { } + /** + * Returns a map key serializer for String keys + */ + public static KeySerializer getStringKeySerializer() { + return StringKeySerializer.INSTANCE; + } + + /** + * Returns a map key serializer for Integer keys. Encodes as Int. + */ + public static KeySerializer getIntKeySerializer() { + return IntKeySerializer.INSTANCE; + } + + /** + * Returns a map key serializer for Integer keys. Encodes as VInt. + */ + public static KeySerializer getVIntKeySerializer() { + return VIntKeySerializer.INSTANCE; + } + /** * Calculates diff between two ImmutableOpenMaps of Diffable objects */ - public static > Diff> diff(ImmutableOpenMap before, ImmutableOpenMap after) { + public static > MapDiff> diff(ImmutableOpenMap before, ImmutableOpenMap after, KeySerializer keySerializer) { assert after != null && before != null; - return new ImmutableOpenMapDiff<>(before, after); + return new ImmutableOpenMapDiff<>(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance()); + } + + /** + * Calculates diff between two ImmutableOpenMaps of non-diffable objects + */ + public static MapDiff> diff(ImmutableOpenMap before, ImmutableOpenMap after, KeySerializer keySerializer, NonDiffableValueSerializer valueSerializer) { + assert after != null && before != null; + return new ImmutableOpenMapDiff<>(before, after, keySerializer, valueSerializer); + } + + /** + * Calculates diff between two ImmutableOpenIntMaps of Diffable objects + */ + public static > MapDiff> diff(ImmutableOpenIntMap before, ImmutableOpenIntMap after, KeySerializer keySerializer) { + assert after != null && before != null; + return new ImmutableOpenIntMapDiff<>(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance()); + } + + /** + * Calculates diff between two ImmutableOpenIntMaps of non-diffable objects + */ + public static MapDiff> diff(ImmutableOpenIntMap before, ImmutableOpenIntMap after, KeySerializer keySerializer, NonDiffableValueSerializer valueSerializer) { + assert after != null && before != null; + return new ImmutableOpenIntMapDiff<>(before, after, keySerializer, valueSerializer); } /** * Calculates diff between two Maps of Diffable objects. */ - public static > Diff> diff(Map before, Map after) { + public static > MapDiff> diff(Map before, Map after, KeySerializer keySerializer) { assert after != null && before != null; - return new JdkMapDiff<>(before, after); + return new JdkMapDiff<>(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance()); + } + + /** + * Calculates diff between two Maps of non-diffable objects + */ + public static MapDiff> diff(Map before, Map after, KeySerializer keySerializer, NonDiffableValueSerializer valueSerializer) { + assert after != null && before != null; + return new JdkMapDiff<>(before, after, keySerializer, valueSerializer); } /** * Loads an object that represents difference between two ImmutableOpenMaps */ - public static > Diff> readImmutableOpenMapDiff(StreamInput in, KeyedReader keyedReader) throws IOException { - return new ImmutableOpenMapDiff<>(in, keyedReader); - } - - /** - * Loads an object that represents difference between two Maps. - */ - public static > Diff> readJdkMapDiff(StreamInput in, KeyedReader keyedReader) throws IOException { - return new JdkMapDiff<>(in, keyedReader); + public static MapDiff> readImmutableOpenMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException { + return new ImmutableOpenMapDiff<>(in, keySerializer, valueSerializer); } /** * Loads an object that represents difference between two ImmutableOpenMaps */ - public static > Diff> readImmutableOpenMapDiff(StreamInput in, T proto) throws IOException { - return new ImmutableOpenMapDiff<>(in, new PrototypeReader<>(proto)); + public static MapDiff> readImmutableOpenIntMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException { + return new ImmutableOpenIntMapDiff<>(in, keySerializer, valueSerializer); } /** - * Loads an object that represents difference between two Maps. + * Loads an object that represents difference between two Maps of Diffable objects */ - public static > Diff> readJdkMapDiff(StreamInput in, T proto) throws IOException { - return new JdkMapDiff<>(in, new PrototypeReader<>(proto)); + public static MapDiff> readJdkMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException { + return new JdkMapDiff<>(in, keySerializer, valueSerializer); } /** - * A reader that can deserialize an object. The reader can select the deserialization type based on the key. It's - * used in custom metadata deserialization. + * Loads an object that represents difference between two ImmutableOpenMaps of Diffable objects using Diffable proto object */ - public interface KeyedReader { - - /** - * reads an object of the type T from the stream input - */ - T readFrom(StreamInput in, String key) throws IOException; - - /** - * reads an object that respresents differences between two objects with the type T from the stream input - */ - Diff readDiffFrom(StreamInput in, String key) throws IOException; + public static > MapDiff> readImmutableOpenMapDiff(StreamInput in, KeySerializer keySerializer, T proto) throws IOException { + return new ImmutableOpenMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto)); } /** - * Implementation of the KeyedReader that is using a prototype object for reading operations - * - * Note: this implementation is ignoring the key. + * Loads an object that represents difference between two ImmutableOpenIntMaps of Diffable objects using Diffable proto object */ - public static class PrototypeReader> implements KeyedReader { - private T proto; - - public PrototypeReader(T proto) { - this.proto = proto; - } - - @Override - public T readFrom(StreamInput in, String key) throws IOException { - return proto.readFrom(in); - } - - @Override - public Diff readDiffFrom(StreamInput in, String key) throws IOException { - return proto.readDiffFrom(in); - } + public static > MapDiff> readImmutableOpenIntMapDiff(StreamInput in, KeySerializer keySerializer, T proto) throws IOException { + return new ImmutableOpenIntMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto)); } /** - * Represents differences between two Maps of Diffable objects. + * Loads an object that represents difference between two Maps of Diffable objects using Diffable proto object + */ + public static > MapDiff> readJdkMapDiff(StreamInput in, KeySerializer keySerializer, T proto) throws IOException { + return new JdkMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto)); + } + + /** + * Represents differences between two Maps of (possibly diffable) objects. * * @param the diffable object */ - private static class JdkMapDiff> extends MapDiff> { + private static class JdkMapDiff extends MapDiff> { - protected JdkMapDiff(StreamInput in, KeyedReader reader) throws IOException { - super(in, reader); + protected JdkMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException { + super(in, keySerializer, valueSerializer); } - public JdkMapDiff(Map before, Map after) { + public JdkMapDiff(Map before, Map after, + KeySerializer keySerializer, ValueSerializer valueSerializer) { + super(keySerializer, valueSerializer); assert after != null && before != null; - for (String key : before.keySet()) { + + for (K key : before.keySet()) { if (!after.containsKey(key)) { deletes.add(key); } } - for (Map.Entry partIter : after.entrySet()) { + + for (Map.Entry partIter : after.entrySet()) { T beforePart = before.get(partIter.getKey()); if (beforePart == null) { - adds.put(partIter.getKey(), partIter.getValue()); + upserts.put(partIter.getKey(), partIter.getValue()); } else if (partIter.getValue().equals(beforePart) == false) { - diffs.put(partIter.getKey(), partIter.getValue().diff(beforePart)); + if (valueSerializer.supportsDiffableValues()) { + diffs.put(partIter.getKey(), valueSerializer.diff(partIter.getValue(), beforePart)); + } else { + upserts.put(partIter.getKey(), partIter.getValue()); + } } } } @Override - public Map apply(Map map) { - Map builder = new HashMap<>(); + public Map apply(Map map) { + Map builder = new HashMap<>(); builder.putAll(map); - for (String part : deletes) { + for (K part : deletes) { builder.remove(part); } - for (Map.Entry> diff : diffs.entrySet()) { + for (Map.Entry> diff : diffs.entrySet()) { builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey()))); } - for (Map.Entry additon : adds.entrySet()) { - builder.put(additon.getKey(), additon.getValue()); + for (Map.Entry upsert : upserts.entrySet()) { + builder.put(upsert.getKey(), upsert.getValue()); } return builder; } } /** - * Represents differences between two ImmutableOpenMap of diffable objects + * Represents differences between two ImmutableOpenMap of (possibly diffable) objects * - * @param the diffable object + * @param the object type */ - private static class ImmutableOpenMapDiff> extends MapDiff> { + private static class ImmutableOpenMapDiff extends MapDiff> { - protected ImmutableOpenMapDiff(StreamInput in, KeyedReader reader) throws IOException { - super(in, reader); + protected ImmutableOpenMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException { + super(in, keySerializer, valueSerializer); } - public ImmutableOpenMapDiff(ImmutableOpenMap before, ImmutableOpenMap after) { + public ImmutableOpenMapDiff(ImmutableOpenMap before, ImmutableOpenMap after, + KeySerializer keySerializer, ValueSerializer valueSerializer) { + super(keySerializer, valueSerializer); assert after != null && before != null; - for (ObjectCursor key : before.keys()) { + + for (ObjectCursor key : before.keys()) { if (!after.containsKey(key.value)) { deletes.add(key.value); } } - for (ObjectObjectCursor partIter : after) { + + for (ObjectObjectCursor partIter : after) { T beforePart = before.get(partIter.key); if (beforePart == null) { - adds.put(partIter.key, partIter.value); + upserts.put(partIter.key, partIter.value); } else if (partIter.value.equals(beforePart) == false) { - diffs.put(partIter.key, partIter.value.diff(beforePart)); + if (valueSerializer.supportsDiffableValues()) { + diffs.put(partIter.key, valueSerializer.diff(partIter.value, beforePart)); + } else { + upserts.put(partIter.key, partIter.value); + } } } } @Override - public ImmutableOpenMap apply(ImmutableOpenMap map) { - ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + public ImmutableOpenMap apply(ImmutableOpenMap map) { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); builder.putAll(map); - for (String part : deletes) { + for (K part : deletes) { builder.remove(part); } - for (Map.Entry> diff : diffs.entrySet()) { + for (Map.Entry> diff : diffs.entrySet()) { builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey()))); } - for (Map.Entry additon : adds.entrySet()) { - builder.put(additon.getKey(), additon.getValue()); + for (Map.Entry upsert : upserts.entrySet()) { + builder.put(upsert.getKey(), upsert.getValue()); } return builder.build(); } } /** - * Represents differences between two maps of diffable objects + * Represents differences between two ImmutableOpenIntMap of (possibly diffable) objects * - * This class is used as base class for different map implementations - * - * @param the diffable object + * @param the object type */ - private static abstract class MapDiff, M> implements Diff { + private static class ImmutableOpenIntMapDiff extends MapDiff> { - protected final List deletes; - protected final Map> diffs; - protected final Map adds; - - protected MapDiff() { - deletes = new ArrayList<>(); - diffs = new HashMap<>(); - adds = new HashMap<>(); + protected ImmutableOpenIntMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException { + super(in, keySerializer, valueSerializer); } - protected MapDiff(StreamInput in, KeyedReader reader) throws IOException { + public ImmutableOpenIntMapDiff(ImmutableOpenIntMap before, ImmutableOpenIntMap after, + KeySerializer keySerializer, ValueSerializer valueSerializer) { + super(keySerializer, valueSerializer); + assert after != null && before != null; + + for (IntCursor key : before.keys()) { + if (!after.containsKey(key.value)) { + deletes.add(key.value); + } + } + + for (IntObjectCursor partIter : after) { + T beforePart = before.get(partIter.key); + if (beforePart == null) { + upserts.put(partIter.key, partIter.value); + } else if (partIter.value.equals(beforePart) == false) { + if (valueSerializer.supportsDiffableValues()) { + diffs.put(partIter.key, valueSerializer.diff(partIter.value, beforePart)); + } else { + upserts.put(partIter.key, partIter.value); + } + } + } + } + + @Override + public ImmutableOpenIntMap apply(ImmutableOpenIntMap map) { + ImmutableOpenIntMap.Builder builder = ImmutableOpenIntMap.builder(); + builder.putAll(map); + + for (Integer part : deletes) { + builder.remove(part); + } + + for (Map.Entry> diff : diffs.entrySet()) { + builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey()))); + } + + for (Map.Entry upsert : upserts.entrySet()) { + builder.put(upsert.getKey(), upsert.getValue()); + } + return builder.build(); + } + } + + /** + * Represents differences between two maps of objects and is used as base class for different map implementations. + * + * Implements serialization. How differences are applied is left to subclasses. + * + * @param the type of map keys + * @param the type of map values + * @param the map implementation type + */ + public static abstract class MapDiff implements Diff { + + protected final List deletes; + protected final Map> diffs; // incremental updates + protected final Map upserts; // additions or full updates + protected final KeySerializer keySerializer; + protected final ValueSerializer valueSerializer; + + protected MapDiff(KeySerializer keySerializer, ValueSerializer valueSerializer) { + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; deletes = new ArrayList<>(); diffs = new HashMap<>(); - adds = new HashMap<>(); + upserts = new HashMap<>(); + } + + protected MapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException { + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + deletes = new ArrayList<>(); + diffs = new HashMap<>(); + upserts = new HashMap<>(); int deletesCount = in.readVInt(); for (int i = 0; i < deletesCount; i++) { - deletes.add(in.readString()); + deletes.add(keySerializer.readKey(in)); } - int diffsCount = in.readVInt(); for (int i = 0; i < diffsCount; i++) { - String key = in.readString(); - Diff diff = reader.readDiffFrom(in, key); + K key = keySerializer.readKey(in); + Diff diff = valueSerializer.readDiff(in, key); diffs.put(key, diff); } - - int addsCount = in.readVInt(); - for (int i = 0; i < addsCount; i++) { - String key = in.readString(); - T part = reader.readFrom(in, key); - adds.put(key, part); + int upsertsCount = in.readVInt(); + for (int i = 0; i < upsertsCount; i++) { + K key = keySerializer.readKey(in); + T newValue = valueSerializer.read(in, key); + upserts.put(key, newValue); } } + + /** + * The keys that, when this diff is applied to a map, should be removed from the map. + * + * @return the list of keys that are deleted + */ + public List getDeletes() { + return deletes; + } + + /** + * Map entries that, when this diff is applied to a map, should be + * incrementally updated. The incremental update is represented using + * the {@link Diff} interface. + * + * @return the map entries that are incrementally updated + */ + public Map> getDiffs() { + return diffs; + } + + /** + * Map entries that, when this diff is applied to a map, should be + * added to the map or fully replace the previous value. + * + * @return the map entries that are additions or full updates + */ + public Map getUpserts() { + return upserts; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(deletes.size()); - for (String delete : deletes) { - out.writeString(delete); + for (K delete : deletes) { + keySerializer.writeKey(delete, out); } - out.writeVInt(diffs.size()); - for (Map.Entry> entry : diffs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); + for (Map.Entry> entry : diffs.entrySet()) { + keySerializer.writeKey(entry.getKey(), out); + valueSerializer.writeDiff(entry.getValue(), out); } - - out.writeVInt(adds.size()); - for (Map.Entry entry : adds.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); + out.writeVInt(upserts.size()); + for (Map.Entry entry : upserts.entrySet()) { + keySerializer.writeKey(entry.getKey(), out); + valueSerializer.write(entry.getValue(), out); } } } + + /** + * Provides read and write operations to serialize keys of map + * @param type of key + */ + public interface KeySerializer { + void writeKey(K key, StreamOutput out) throws IOException; + K readKey(StreamInput in) throws IOException; + } + + /** + * Serializes String keys of a map + */ + private static final class StringKeySerializer implements KeySerializer { + private static final StringKeySerializer INSTANCE = new StringKeySerializer(); + + @Override + public void writeKey(String key, StreamOutput out) throws IOException { + out.writeString(key); + } + + @Override + public String readKey(StreamInput in) throws IOException { + return in.readString(); + } + } + + /** + * Serializes Integer keys of a map as an Int + */ + private static final class IntKeySerializer implements KeySerializer { + public static final IntKeySerializer INSTANCE = new IntKeySerializer(); + + @Override + public void writeKey(Integer key, StreamOutput out) throws IOException { + out.writeInt(key); + } + + @Override + public Integer readKey(StreamInput in) throws IOException { + return in.readInt(); + } + } + + /** + * Serializes Integer keys of a map as a VInt. Requires keys to be positive. + */ + private static final class VIntKeySerializer implements KeySerializer { + public static final IntKeySerializer INSTANCE = new IntKeySerializer(); + + @Override + public void writeKey(Integer key, StreamOutput out) throws IOException { + if (key < 0) { + throw new IllegalArgumentException("Map key [" + key + "] must be positive"); + } + out.writeVInt(key); + } + + @Override + public Integer readKey(StreamInput in) throws IOException { + return in.readVInt(); + } + } + + /** + * Provides read and write operations to serialize map values. + * Reading of values can be made dependent on map key. + * + * Also provides operations to distinguish whether map values are diffable. + * + * Should not be directly implemented, instead implement either + * {@link DiffableValueSerializer} or {@link NonDiffableValueSerializer}. + * + * @param key type of map + * @param value type of map + */ + public interface ValueSerializer { + + /** + * Writes value to stream + */ + void write(V value, StreamOutput out) throws IOException; + + /** + * Reads value from stream. Reading operation can be made dependent on map key. + */ + V read(StreamInput in, K key) throws IOException; + + /** + * Whether this serializer supports diffable values + */ + boolean supportsDiffableValues(); + + /** + * Computes diff if this serializer supports diffable values + */ + Diff diff(V value, V beforePart); + + /** + * Writes value as diff to stream if this serializer supports diffable values + */ + void writeDiff(Diff value, StreamOutput out) throws IOException; + + /** + * Reads value as diff from stream if this serializer supports diffable values. + * Reading operation can be made dependent on map key. + */ + Diff readDiff(StreamInput in, K key) throws IOException; + } + + /** + * Serializer for Diffable map values. Needs to implement read and readDiff methods. + * + * @param type of map keys + * @param type of map values + */ + public static abstract class DiffableValueSerializer> implements ValueSerializer { + private static final DiffableValueSerializer WRITE_ONLY_INSTANCE = new DiffableValueSerializer() { + @Override + public Object read(StreamInput in, Object key) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Diff readDiff(StreamInput in, Object key) throws IOException { + throw new UnsupportedOperationException(); + } + }; + + private static > DiffableValueSerializer getWriteOnlyInstance() { + return WRITE_ONLY_INSTANCE; + } + + @Override + public boolean supportsDiffableValues() { + return true; + } + + @Override + public Diff diff(V value, V beforePart) { + return value.diff(beforePart); + } + + @Override + public void write(V value, StreamOutput out) throws IOException { + value.writeTo(out); + } + + public void writeDiff(Diff value, StreamOutput out) throws IOException { + value.writeTo(out); + } + } + + /** + * Serializer for non-diffable map values + * + * @param type of map keys + * @param type of map values + */ + public static abstract class NonDiffableValueSerializer implements ValueSerializer { + @Override + public boolean supportsDiffableValues() { + return false; + } + + @Override + public Diff diff(V value, V beforePart) { + throw new UnsupportedOperationException(); + } + + @Override + public void writeDiff(Diff value, StreamOutput out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Diff readDiff(StreamInput in, K key) throws IOException { + throw new UnsupportedOperationException(); + } + } + + /** + * Implementation of the ValueSerializer that uses a prototype object for reading operations + * + * Note: this implementation is ignoring the key. + */ + public static class DiffablePrototypeValueReader> extends DiffableValueSerializer { + private final V proto; + + public DiffablePrototypeValueReader(V proto) { + this.proto = proto; + } + + @Override + public V read(StreamInput in, K key) throws IOException { + return proto.readFrom(in); + } + + @Override + public Diff readDiff(StreamInput in, K key) throws IOException { + return proto.readDiffFrom(in); + } + } + + /** + * Implementation of ValueSerializer that serializes immutable sets + * + * @param type of map key + */ + public static class StringSetValueSerializer extends NonDiffableValueSerializer> { + private static final StringSetValueSerializer INSTANCE = new StringSetValueSerializer(); + + public static StringSetValueSerializer getInstance() { + return INSTANCE; + } + + @Override + public void write(Set value, StreamOutput out) throws IOException { + out.writeStringArray(value.toArray(new String[value.size()])); + } + + @Override + public Set read(StreamInput in, K key) throws IOException { + return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(in.readStringArray()))); + } + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/DiskUsage.java b/core/src/main/java/org/elasticsearch/cluster/DiskUsage.java index e91adae9e34..fac9a9c479a 100644 --- a/core/src/main/java/org/elasticsearch/cluster/DiskUsage.java +++ b/core/src/main/java/org/elasticsearch/cluster/DiskUsage.java @@ -33,7 +33,7 @@ public class DiskUsage { final long freeBytes; /** - * Create a new DiskUsage, if {@code totalBytes} is 0, {@get getFreeDiskAsPercentage} + * Create a new DiskUsage, if {@code totalBytes} is 0, {@link #getFreeDiskAsPercentage()} * will always return 100.0% free */ public DiskUsage(String nodeId, String nodeName, String path, long totalBytes, long freeBytes) { diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NotMasterException.java b/core/src/main/java/org/elasticsearch/cluster/NotMasterException.java similarity index 87% rename from core/src/main/java/org/elasticsearch/discovery/zen/NotMasterException.java rename to core/src/main/java/org/elasticsearch/cluster/NotMasterException.java index d78d22aa983..892510418e4 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/NotMasterException.java +++ b/core/src/main/java/org/elasticsearch/cluster/NotMasterException.java @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.discovery.zen; +package org.elasticsearch.cluster; /** * Thrown when a node join request or a master ping reaches a node which is not - * currently acting as a master. + * currently acting as a master or when a cluster state update task is to be executed + * on a node that is no longer master. */ public class NotMasterException extends IllegalStateException { diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 821ab3ce1ab..6ee8365d378 100644 --- a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -152,7 +152,7 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus result = 31 * result + shards.hashCode(); result = 31 * result + indices.hashCode(); result = 31 * result + waitingIndices.hashCode(); - result = 31 * result + (int) (startTime ^ (startTime >>> 32)); + result = 31 * result + Long.hashCode(startTime); return result; } diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java index 46f9b7e26ae..4079f14abc7 100644 --- a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java +++ b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; @@ -70,7 +71,7 @@ public class NodeIndexDeletedAction extends AbstractComponent { listeners.remove(listener); } - public void nodeIndexDeleted(final ClusterState clusterState, final String index, final Settings indexSettings, final String nodeId) { + public void nodeIndexDeleted(final ClusterState clusterState, final String index, final IndexSettings indexSettings, final String nodeId) { final DiscoveryNodes nodes = clusterState.nodes(); transportService.sendRequest(clusterState.nodes().masterNode(), INDEX_DELETED_ACTION_NAME, new NodeIndexDeletedMessage(index, nodeId), EmptyTransportResponseHandler.INSTANCE_SAME); @@ -91,7 +92,7 @@ public class NodeIndexDeletedAction extends AbstractComponent { }); } - private void lockIndexAndAck(String index, DiscoveryNodes nodes, String nodeId, ClusterState clusterState, Settings indexSettings) throws IOException { + private void lockIndexAndAck(String index, DiscoveryNodes nodes, String nodeId, ClusterState clusterState, IndexSettings indexSettings) throws IOException { try { // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store to the // master. If we can't acquire the locks here immediately there might be a shard of this index still holding on to the lock @@ -102,6 +103,8 @@ public class NodeIndexDeletedAction extends AbstractComponent { INDEX_STORE_DELETED_ACTION_NAME, new NodeIndexStoreDeletedMessage(index, nodeId), EmptyTransportResponseHandler.INSTANCE_SAME); } catch (LockObtainFailedException exc) { logger.warn("[{}] failed to lock all shards for index - timed out after 30 seconds", index); + } catch (InterruptedException e) { + logger.warn("[{}] failed to lock all shards for index - interrupted", index); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java index d0eb29d6b22..f8507e5b689 100644 --- a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java +++ b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataMappingService; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; @@ -57,7 +56,7 @@ public class NodeMappingRefreshAction extends AbstractComponent { public void nodeMappingRefresh(final ClusterState state, final NodeMappingRefreshRequest request) { final DiscoveryNodes nodes = state.nodes(); if (nodes.masterNode() == null) { - logger.warn("can't send mapping refresh for [{}][{}], no master known.", request.index(), Strings.arrayToCommaDelimitedString(request.types())); + logger.warn("can't send mapping refresh for [{}], no master known.", request.index()); return; } transportService.sendRequest(nodes.masterNode(), ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); @@ -67,7 +66,7 @@ public class NodeMappingRefreshAction extends AbstractComponent { @Override public void messageReceived(NodeMappingRefreshRequest request, TransportChannel channel) throws Exception { - metaDataMappingService.refreshMapping(request.index(), request.indexUUID(), request.types()); + metaDataMappingService.refreshMapping(request.index(), request.indexUUID()); channel.sendResponse(TransportResponse.Empty.INSTANCE); } } @@ -76,16 +75,14 @@ public class NodeMappingRefreshAction extends AbstractComponent { private String index; private String indexUUID = IndexMetaData.INDEX_UUID_NA_VALUE; - private String[] types; private String nodeId; public NodeMappingRefreshRequest() { } - public NodeMappingRefreshRequest(String index, String indexUUID, String[] types, String nodeId) { + public NodeMappingRefreshRequest(String index, String indexUUID, String nodeId) { this.index = index; this.indexUUID = indexUUID; - this.types = types; this.nodeId = nodeId; } @@ -107,11 +104,6 @@ public class NodeMappingRefreshAction extends AbstractComponent { return indexUUID; } - - public String[] types() { - return types; - } - public String nodeId() { return nodeId; } @@ -120,7 +112,6 @@ public class NodeMappingRefreshAction extends AbstractComponent { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(index); - out.writeStringArray(types); out.writeString(nodeId); out.writeString(indexUUID); } @@ -129,7 +120,6 @@ public class NodeMappingRefreshAction extends AbstractComponent { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); index = in.readString(); - types = in.readStringArray(); nodeId = in.readString(); indexUUID = in.readString(); } diff --git a/core/src/main/java/org/elasticsearch/action/count/package-info.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/NoOpShardStateActionListener.java similarity index 85% rename from core/src/main/java/org/elasticsearch/action/count/package-info.java rename to core/src/main/java/org/elasticsearch/cluster/action/shard/NoOpShardStateActionListener.java index 2d4945d3d2c..ed0a7f56b9c 100644 --- a/core/src/main/java/org/elasticsearch/action/count/package-info.java +++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/NoOpShardStateActionListener.java @@ -17,7 +17,7 @@ * under the License. */ -/** - * Count action. - */ -package org.elasticsearch.action.count; \ No newline at end of file +package org.elasticsearch.cluster.action.shard; + +public class NoOpShardStateActionListener implements ShardStateAction.Listener { +} diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index 02867c930be..d09df094a68 100644 --- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -20,9 +20,7 @@ package org.elasticsearch.cluster.action.shard; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingService; @@ -37,14 +35,13 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry; @@ -61,9 +58,6 @@ public class ShardStateAction extends AbstractComponent { private final AllocationService allocationService; private final RoutingService routingService; - private final BlockingQueue startedShardsQueue = ConcurrentCollections.newBlockingQueue(); - private final BlockingQueue failedShardQueue = ConcurrentCollections.newBlockingQueue(); - @Inject public ShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService, AllocationService allocationService, RoutingService routingService) { @@ -77,27 +71,42 @@ public class ShardStateAction extends AbstractComponent { transportService.registerRequestHandler(SHARD_FAILED_ACTION_NAME, ShardRoutingEntry::new, ThreadPool.Names.SAME, new ShardFailedTransportHandler()); } - public void shardFailed(final ShardRouting shardRouting, final String indexUUID, final String message, @Nullable final Throwable failure) { + public void shardFailed(final ShardRouting shardRouting, final String indexUUID, final String message, @Nullable final Throwable failure, Listener listener) { + shardFailed(shardRouting, indexUUID, message, failure, null, listener); + } + + public void shardFailed(final ShardRouting shardRouting, final String indexUUID, final String message, @Nullable final Throwable failure, TimeValue timeout, Listener listener) { DiscoveryNode masterNode = clusterService.state().nodes().masterNode(); if (masterNode == null) { logger.warn("can't send shard failed for {}, no master known.", shardRouting); + listener.onShardFailedNoMaster(); return; } - innerShardFailed(shardRouting, indexUUID, masterNode, message, failure); + innerShardFailed(shardRouting, indexUUID, masterNode, message, failure, timeout, listener); } - public void resendShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, @Nullable final Throwable failure) { + public void resendShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, @Nullable final Throwable failure, Listener listener) { logger.trace("{} re-sending failed shard for {}, indexUUID [{}], reason [{}]", failure, shardRouting.shardId(), shardRouting, indexUUID, message); - innerShardFailed(shardRouting, indexUUID, masterNode, message, failure); + innerShardFailed(shardRouting, indexUUID, masterNode, message, failure, null, listener); } - private void innerShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, final Throwable failure) { + private void innerShardFailed(final ShardRouting shardRouting, final String indexUUID, final DiscoveryNode masterNode, final String message, final Throwable failure, TimeValue timeout, Listener listener) { ShardRoutingEntry shardRoutingEntry = new ShardRoutingEntry(shardRouting, indexUUID, message, failure); + TransportRequestOptions options = TransportRequestOptions.EMPTY; + if (timeout != null) { + options = TransportRequestOptions.builder().withTimeout(timeout).build(); + } transportService.sendRequest(masterNode, - SHARD_FAILED_ACTION_NAME, shardRoutingEntry, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { + SHARD_FAILED_ACTION_NAME, shardRoutingEntry, options, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { + @Override + public void handleResponse(TransportResponse.Empty response) { + listener.onSuccess(); + } + @Override public void handleException(TransportException exp) { logger.warn("failed to send failed shard to {}", exp, masterNode); + listener.onShardFailedFailure(masterNode, exp); } }); } @@ -124,104 +133,94 @@ public class ShardStateAction extends AbstractComponent { }); } + private final ShardFailedClusterStateHandler shardFailedClusterStateHandler = new ShardFailedClusterStateHandler(); + private void handleShardFailureOnMaster(final ShardRoutingEntry shardRoutingEntry) { logger.warn("{} received shard failed for {}", shardRoutingEntry.failure, shardRoutingEntry.shardRouting.shardId(), shardRoutingEntry); - failedShardQueue.add(shardRoutingEntry); - clusterService.submitStateUpdateTask("shard-failed (" + shardRoutingEntry.shardRouting + "), message [" + shardRoutingEntry.message + "]", Priority.HIGH, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask( + "shard-failed (" + shardRoutingEntry.shardRouting + "), message [" + shardRoutingEntry.message + "]", + shardRoutingEntry, + ClusterStateTaskConfig.build(Priority.HIGH), + shardFailedClusterStateHandler, + shardFailedClusterStateHandler); + } - @Override - public ClusterState execute(ClusterState currentState) { - if (shardRoutingEntry.processed) { - return currentState; - } - - List shardRoutingEntries = new ArrayList<>(); - failedShardQueue.drainTo(shardRoutingEntries); - - // nothing to process (a previous event has processed it already) - if (shardRoutingEntries.isEmpty()) { - return currentState; - } - - List shardRoutingsToBeApplied = new ArrayList<>(shardRoutingEntries.size()); - - // mark all entries as processed - for (ShardRoutingEntry entry : shardRoutingEntries) { - entry.processed = true; - shardRoutingsToBeApplied.add(new FailedRerouteAllocation.FailedShard(entry.shardRouting, entry.message, entry.failure)); - } - - RoutingAllocation.Result routingResult = allocationService.applyFailedShards(currentState, shardRoutingsToBeApplied); - if (!routingResult.changed()) { - return currentState; - } - return ClusterState.builder(currentState).routingResult(routingResult).build(); + class ShardFailedClusterStateHandler implements ClusterStateTaskExecutor, ClusterStateTaskListener { + @Override + public BatchResult execute(ClusterState currentState, List tasks) throws Exception { + BatchResult.Builder batchResultBuilder = BatchResult.builder(); + List shardRoutingsToBeApplied = new ArrayList<>(tasks.size()); + for (ShardRoutingEntry task : tasks) { + shardRoutingsToBeApplied.add(new FailedRerouteAllocation.FailedShard(task.shardRouting, task.message, task.failure)); } - - @Override - public void onFailure(String source, Throwable t) { - logger.error("unexpected failure during [{}]", t, source); + ClusterState maybeUpdatedState = currentState; + try { + RoutingAllocation.Result result = allocationService.applyFailedShards(currentState, shardRoutingsToBeApplied); + if (result.changed()) { + maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build(); + } + batchResultBuilder.successes(tasks); + } catch (Throwable t) { + batchResultBuilder.failures(tasks, t); } + return batchResultBuilder.build(maybeUpdatedState); + } - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - if (oldState != newState && newState.getRoutingNodes().hasUnassigned()) { + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + if (oldState != newState && newState.getRoutingNodes().unassigned().size() > 0) { logger.trace("unassigned shards after shard failures. scheduling a reroute."); routingService.reroute("unassigned shards after shard failures, scheduling a reroute"); } - } - }); + } + + @Override + public void onFailure(String source, Throwable t) { + logger.error("unexpected failure during [{}]", t, source); + } } + private final ShardStartedClusterStateHandler shardStartedClusterStateHandler = + new ShardStartedClusterStateHandler(); + private void shardStartedOnMaster(final ShardRoutingEntry shardRoutingEntry) { logger.debug("received shard started for {}", shardRoutingEntry); - // buffer shard started requests, and the state update tasks will simply drain it - // this is to optimize the number of "started" events we generate, and batch them - // possibly, we can do time based batching as well, but usually, we would want to - // process started events as fast as possible, to make shards available - startedShardsQueue.add(shardRoutingEntry); - clusterService.submitStateUpdateTask("shard-started (" + shardRoutingEntry.shardRouting + "), reason [" + shardRoutingEntry.message + "]", Priority.URGENT, - new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { + clusterService.submitStateUpdateTask( + "shard-started (" + shardRoutingEntry.shardRouting + "), reason [" + shardRoutingEntry.message + "]", + shardRoutingEntry, + ClusterStateTaskConfig.build(Priority.URGENT), + shardStartedClusterStateHandler, + shardStartedClusterStateHandler); + } - if (shardRoutingEntry.processed) { - return currentState; - } + class ShardStartedClusterStateHandler implements ClusterStateTaskExecutor, ClusterStateTaskListener { + @Override + public BatchResult execute(ClusterState currentState, List tasks) throws Exception { + BatchResult.Builder builder = BatchResult.builder(); + List shardRoutingsToBeApplied = new ArrayList<>(tasks.size()); + for (ShardRoutingEntry task : tasks) { + shardRoutingsToBeApplied.add(task.shardRouting); + } + ClusterState maybeUpdatedState = currentState; + try { + RoutingAllocation.Result result = + allocationService.applyStartedShards(currentState, shardRoutingsToBeApplied, true); + if (result.changed()) { + maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build(); + } + builder.successes(tasks); + } catch (Throwable t) { + builder.failures(tasks, t); + } - List shardRoutingEntries = new ArrayList<>(); - startedShardsQueue.drainTo(shardRoutingEntries); + return builder.build(maybeUpdatedState); + } - // nothing to process (a previous event has processed it already) - if (shardRoutingEntries.isEmpty()) { - return currentState; - } - - List shardRoutingToBeApplied = new ArrayList<>(shardRoutingEntries.size()); - - // mark all entries as processed - for (ShardRoutingEntry entry : shardRoutingEntries) { - entry.processed = true; - shardRoutingToBeApplied.add(entry.shardRouting); - } - - if (shardRoutingToBeApplied.isEmpty()) { - return currentState; - } - - RoutingAllocation.Result routingResult = allocationService.applyStartedShards(currentState, shardRoutingToBeApplied, true); - if (!routingResult.changed()) { - return currentState; - } - return ClusterState.builder(currentState).routingResult(routingResult).build(); - } - - @Override - public void onFailure(String source, Throwable t) { - logger.error("unexpected failure during [{}]", t, source); - } - }); + @Override + public void onFailure(String source, Throwable t) { + logger.error("unexpected failure during [{}]", t, source); + } } private class ShardFailedTransportHandler implements TransportRequestHandler { @@ -249,8 +248,6 @@ public class ShardStateAction extends AbstractComponent { String message; Throwable failure; - volatile boolean processed; // state field, no need to serialize - public ShardRoutingEntry() { } @@ -284,4 +281,10 @@ public class ShardStateAction extends AbstractComponent { return "" + shardRouting + ", indexUUID [" + indexUUID + "], message [" + message + "], failure [" + ExceptionsHelper.detailedMessage(failure) + "]"; } } + + public interface Listener { + default void onSuccess() {} + default void onShardFailedNoMaster() {} + default void onShardFailedFailure(final DiscoveryNode master, final TransportException e) {} + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java index 39e1068f605..5a7f8f7c0a9 100644 --- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java +++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java @@ -155,8 +155,10 @@ public class ClusterBlock implements Streamable, ToXContent { public String toString() { StringBuilder sb = new StringBuilder(); sb.append(id).append(",").append(description).append(", blocks "); + String delimiter = ""; for (ClusterBlockLevel level : levels) { - sb.append(level.name()).append(","); + sb.append(delimiter).append(level.name()); + delimiter = ","; } return sb.toString(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index 4b1d0569af1..1bab607ee85 100644 --- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -20,7 +20,6 @@ package org.elasticsearch.cluster.block; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; @@ -199,6 +198,28 @@ public class ClusterBlocks extends AbstractDiffable { return new ClusterBlockException(unmodifiableSet(blocks.collect(toSet()))); } + public String prettyPrint() { + if (global.isEmpty() && indices().isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + sb.append("blocks: \n"); + if (global.isEmpty() == false) { + sb.append(" _global_:\n"); + for (ClusterBlock block : global) { + sb.append(" ").append(block); + } + } + for (ObjectObjectCursor> entry : indices()) { + sb.append(" ").append(entry.key).append(":\n"); + for (ClusterBlock block : entry.value) { + sb.append(" ").append(block); + } + } + sb.append("\n"); + return sb.toString(); + } + @Override public void writeTo(StreamOutput out) throws IOException { writeBlockSet(global, out); @@ -282,30 +303,30 @@ public class ClusterBlocks extends AbstractDiffable { } public Builder addBlocks(IndexMetaData indexMetaData) { - if (indexMetaData.state() == IndexMetaData.State.CLOSE) { - addIndexBlock(indexMetaData.index(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK); + if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { + addIndexBlock(indexMetaData.getIndex(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK); } - if (indexMetaData.settings().getAsBoolean(IndexMetaData.SETTING_READ_ONLY, false)) { - addIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_READ_ONLY_BLOCK); + if (indexMetaData.getSettings().getAsBoolean(IndexMetaData.SETTING_READ_ONLY, false)) { + addIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_READ_ONLY_BLOCK); } - if (indexMetaData.settings().getAsBoolean(IndexMetaData.SETTING_BLOCKS_READ, false)) { - addIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_READ_BLOCK); + if (indexMetaData.getSettings().getAsBoolean(IndexMetaData.SETTING_BLOCKS_READ, false)) { + addIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_READ_BLOCK); } - if (indexMetaData.settings().getAsBoolean(IndexMetaData.SETTING_BLOCKS_WRITE, false)) { - addIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_WRITE_BLOCK); + if (indexMetaData.getSettings().getAsBoolean(IndexMetaData.SETTING_BLOCKS_WRITE, false)) { + addIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_WRITE_BLOCK); } - if (indexMetaData.settings().getAsBoolean(IndexMetaData.SETTING_BLOCKS_METADATA, false)) { - addIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_METADATA_BLOCK); + if (indexMetaData.getSettings().getAsBoolean(IndexMetaData.SETTING_BLOCKS_METADATA, false)) { + addIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_METADATA_BLOCK); } return this; } public Builder updateBlocks(IndexMetaData indexMetaData) { - removeIndexBlock(indexMetaData.index(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK); - removeIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_READ_ONLY_BLOCK); - removeIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_READ_BLOCK); - removeIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_WRITE_BLOCK); - removeIndexBlock(indexMetaData.index(), IndexMetaData.INDEX_METADATA_BLOCK); + removeIndexBlock(indexMetaData.getIndex(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK); + removeIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_READ_ONLY_BLOCK); + removeIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_READ_BLOCK); + removeIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_WRITE_BLOCK); + removeIndexBlock(indexMetaData.getIndex(), IndexMetaData.INDEX_METADATA_BLOCK); return addBlocks(indexMetaData); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthStatus.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java similarity index 97% rename from core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthStatus.java rename to core/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java index 969eb0d21f0..6d3e136eb1a 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthStatus.java +++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.admin.cluster.health; +package org.elasticsearch.cluster.health; /** diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterIndexHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java similarity index 96% rename from core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterIndexHealth.java rename to core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java index 345dcb4f254..3fd10fd91da 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterIndexHealth.java +++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.admin.cluster.health; +package org.elasticsearch.cluster.health; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -37,12 +37,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import static org.elasticsearch.action.admin.cluster.health.ClusterShardHealth.readClusterShardHealth; +import static org.elasticsearch.cluster.health.ClusterShardHealth.readClusterShardHealth; -/** - * - */ -public class ClusterIndexHealth implements Iterable, Streamable, ToXContent { +public final class ClusterIndexHealth implements Iterable, Streamable, ToXContent { private String index; @@ -70,7 +67,7 @@ public class ClusterIndexHealth implements Iterable, Streama } public ClusterIndexHealth(IndexMetaData indexMetaData, IndexRoutingTable indexRoutingTable) { - this.index = indexMetaData.index(); + this.index = indexMetaData.getIndex(); this.numberOfShards = indexMetaData.getNumberOfShards(); this.numberOfReplicas = indexMetaData.getNumberOfReplicas(); this.validationFailures = indexRoutingTable.validate(indexMetaData); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterShardHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java similarity index 97% rename from core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterShardHealth.java rename to core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java index 34914d30093..725f89121b7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterShardHealth.java +++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.admin.cluster.health; +package org.elasticsearch.cluster.health; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -27,10 +27,7 @@ import org.elasticsearch.common.io.stream.Streamable; import java.io.IOException; -/** - * - */ -public class ClusterShardHealth implements Streamable { +public final class ClusterShardHealth implements Streamable { private int shardId; diff --git a/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java new file mode 100644 index 00000000000..3b12d874ada --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java @@ -0,0 +1,236 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.health; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.RoutingTableValidation; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.*; + +import static org.elasticsearch.cluster.health.ClusterIndexHealth.readClusterIndexHealth; + +public final class ClusterStateHealth implements Iterable, Streamable { + private int numberOfNodes = 0; + private int numberOfDataNodes = 0; + private int activeShards = 0; + private int relocatingShards = 0; + private int activePrimaryShards = 0; + private int initializingShards = 0; + private int unassignedShards = 0; + private double activeShardsPercent = 100; + private ClusterHealthStatus status = ClusterHealthStatus.RED; + private List validationFailures; + private Map indices = new HashMap<>(); + + public static ClusterStateHealth readClusterHealth(StreamInput in) throws IOException { + ClusterStateHealth clusterStateHealth = new ClusterStateHealth(); + clusterStateHealth.readFrom(in); + return clusterStateHealth; + } + + ClusterStateHealth() { + // only intended for serialization + } + + /** + * Creates a new ClusterStateHealth instance based on cluster meta data and its routing table as a convenience. + * + * @param clusterMetaData Current cluster meta data. Must not be null. + * @param routingTables Current routing table. Must not be null. + */ + public ClusterStateHealth(MetaData clusterMetaData, RoutingTable routingTables) { + this(ClusterState.builder(ClusterName.DEFAULT).metaData(clusterMetaData).routingTable(routingTables).build()); + } + + /** + * Creates a new ClusterStateHealth instance considering the current cluster state and all indices in the cluster. + * + * @param clusterState The current cluster state. Must not be null. + */ + public ClusterStateHealth(ClusterState clusterState) { + this(clusterState, clusterState.metaData().concreteAllIndices()); + } + + /** + * Creates a new ClusterStateHealth instance considering the current cluster state and the provided index names. + * + * @param clusterState The current cluster state. Must not be null. + * @param concreteIndices An array of index names to consider. Must not be null but may be empty. + */ + public ClusterStateHealth(ClusterState clusterState, String[] concreteIndices) { + RoutingTableValidation validation = clusterState.routingTable().validate(clusterState.metaData()); + validationFailures = validation.failures(); + numberOfNodes = clusterState.nodes().size(); + numberOfDataNodes = clusterState.nodes().dataNodes().size(); + + for (String index : concreteIndices) { + IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(index); + IndexMetaData indexMetaData = clusterState.metaData().index(index); + if (indexRoutingTable == null) { + continue; + } + + ClusterIndexHealth indexHealth = new ClusterIndexHealth(indexMetaData, indexRoutingTable); + + indices.put(indexHealth.getIndex(), indexHealth); + } + + status = ClusterHealthStatus.GREEN; + + for (ClusterIndexHealth indexHealth : indices.values()) { + activePrimaryShards += indexHealth.getActivePrimaryShards(); + activeShards += indexHealth.getActiveShards(); + relocatingShards += indexHealth.getRelocatingShards(); + initializingShards += indexHealth.getInitializingShards(); + unassignedShards += indexHealth.getUnassignedShards(); + if (indexHealth.getStatus() == ClusterHealthStatus.RED) { + status = ClusterHealthStatus.RED; + } else if (indexHealth.getStatus() == ClusterHealthStatus.YELLOW && status != ClusterHealthStatus.RED) { + status = ClusterHealthStatus.YELLOW; + } + } + + if (!validationFailures.isEmpty()) { + status = ClusterHealthStatus.RED; + } else if (clusterState.blocks().hasGlobalBlock(RestStatus.SERVICE_UNAVAILABLE)) { + status = ClusterHealthStatus.RED; + } + + // shortcut on green + if (status.equals(ClusterHealthStatus.GREEN)) { + this.activeShardsPercent = 100; + } else { + List shardRoutings = clusterState.getRoutingTable().allShards(); + int activeShardCount = 0; + int totalShardCount = 0; + for (ShardRouting shardRouting : shardRoutings) { + if (shardRouting.active()) activeShardCount++; + totalShardCount++; + } + this.activeShardsPercent = (((double) activeShardCount) / totalShardCount) * 100; + } + } + + public List getValidationFailures() { + return Collections.unmodifiableList(validationFailures); + } + + public int getActiveShards() { + return activeShards; + } + + public int getRelocatingShards() { + return relocatingShards; + } + + public int getActivePrimaryShards() { + return activePrimaryShards; + } + + public int getInitializingShards() { + return initializingShards; + } + + public int getUnassignedShards() { + return unassignedShards; + } + + public int getNumberOfNodes() { + return this.numberOfNodes; + } + + public int getNumberOfDataNodes() { + return this.numberOfDataNodes; + } + + public ClusterHealthStatus getStatus() { + return status; + } + + public Map getIndices() { + return Collections.unmodifiableMap(indices); + } + + public double getActiveShardsPercent() { + return activeShardsPercent; + } + + @Override + public Iterator iterator() { + return indices.values().iterator(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + activePrimaryShards = in.readVInt(); + activeShards = in.readVInt(); + relocatingShards = in.readVInt(); + initializingShards = in.readVInt(); + unassignedShards = in.readVInt(); + numberOfNodes = in.readVInt(); + numberOfDataNodes = in.readVInt(); + status = ClusterHealthStatus.fromValue(in.readByte()); + int size = in.readVInt(); + for (int i = 0; i < size; i++) { + ClusterIndexHealth indexHealth = readClusterIndexHealth(in); + indices.put(indexHealth.getIndex(), indexHealth); + } + size = in.readVInt(); + if (size == 0) { + validationFailures = Collections.emptyList(); + } else { + for (int i = 0; i < size; i++) { + validationFailures.add(in.readString()); + } + } + activeShardsPercent = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(activePrimaryShards); + out.writeVInt(activeShards); + out.writeVInt(relocatingShards); + out.writeVInt(initializingShards); + out.writeVInt(unassignedShards); + out.writeVInt(numberOfNodes); + out.writeVInt(numberOfDataNodes); + out.writeByte(status.value()); + out.writeVInt(indices.size()); + for (ClusterIndexHealth indexHealth : this) { + indexHealth.writeTo(out); + } + out.writeVInt(validationFailures.size()); + for (String failure : validationFailures) { + out.writeString(failure); + } + out.writeDouble(activeShardsPercent); + } +} diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java index 9d1a39d1dac..e5b170b05a6 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.InvalidAliasNameException; @@ -113,14 +112,14 @@ public class AliasValidator extends AbstractComponent { /** * Validates an alias filter by parsing it using the - * provided {@link org.elasticsearch.index.query.IndexQueryParserService} + * provided {@link org.elasticsearch.index.query.QueryShardContext} * @throws IllegalArgumentException if the filter is not valid */ - public void validateAliasFilter(String alias, String filter, IndexQueryParserService indexQueryParserService) { - assert indexQueryParserService != null; + public void validateAliasFilter(String alias, String filter, QueryShardContext queryShardContext) { + assert queryShardContext != null; try { XContentParser parser = XContentFactory.xContent(filter).createParser(filter); - validateAliasFilter(parser, indexQueryParserService); + validateAliasFilter(parser, queryShardContext); } catch (Throwable e) { throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e); } @@ -128,26 +127,25 @@ public class AliasValidator extends AbstractComponent { /** * Validates an alias filter by parsing it using the - * provided {@link org.elasticsearch.index.query.IndexQueryParserService} + * provided {@link org.elasticsearch.index.query.QueryShardContext} * @throws IllegalArgumentException if the filter is not valid */ - public void validateAliasFilter(String alias, byte[] filter, IndexQueryParserService indexQueryParserService) { - assert indexQueryParserService != null; + public void validateAliasFilter(String alias, byte[] filter, QueryShardContext queryShardContext) { + assert queryShardContext != null; try { XContentParser parser = XContentFactory.xContent(filter).createParser(filter); - validateAliasFilter(parser, indexQueryParserService); + validateAliasFilter(parser, queryShardContext); } catch (Throwable e) { throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e); } } - private void validateAliasFilter(XContentParser parser, IndexQueryParserService indexQueryParserService) throws IOException { - QueryShardContext context = indexQueryParserService.getShardContext(); + private void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException { try { - context.reset(parser); - context.parseContext().parseInnerQueryBuilder().toFilter(context); + queryShardContext.reset(parser); + queryShardContext.parseContext().parseInnerQueryBuilder().toFilter(queryShardContext); } finally { - context.reset(null); + queryShardContext.reset(null); parser.close(); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 9d110170f52..669d71477ca 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.Version; @@ -30,6 +31,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; @@ -46,10 +48,13 @@ import org.joda.time.DateTimeZone; import java.io.IOException; import java.text.ParseException; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; import java.util.Map; +import java.util.Set; import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND; import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR; @@ -168,8 +173,10 @@ public class IndexMetaData implements Diffable, FromXContentBuild public static final String SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE = "index.shared_filesystem.recover_on_any_node"; public static final String INDEX_UUID_NA_VALUE = "_na_"; + public static final String KEY_ACTIVE_ALLOCATIONS = "active_allocations"; - + private final int numberOfShards; + private final int numberOfReplicas; private final String index; private final long version; @@ -184,6 +191,8 @@ public class IndexMetaData implements Diffable, FromXContentBuild private final ImmutableOpenMap customs; + private final ImmutableOpenIntMap> activeAllocationIds; + private transient final int totalNumberOfShards; private final DiscoveryNodeFilters requireFilters; @@ -194,68 +203,37 @@ public class IndexMetaData implements Diffable, FromXContentBuild private final Version indexUpgradedVersion; private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion; - private IndexMetaData(String index, long version, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { - if (settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) == null) { - throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]"); - } - if (settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null) == null) { - throw new IllegalArgumentException("must specify numberOfReplicas for index [" + index + "]"); - } + private IndexMetaData(String index, long version, State state, int numberOfShards, int numberOfReplicas, Settings settings, + ImmutableOpenMap mappings, ImmutableOpenMap aliases, + ImmutableOpenMap customs, ImmutableOpenIntMap> activeAllocationIds, + DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters, + Version indexCreatedVersion, Version indexUpgradedVersion, org.apache.lucene.util.Version minimumCompatibleLuceneVersion) { + this.index = index; this.version = version; this.state = state; + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.totalNumberOfShards = numberOfShards * (numberOfReplicas + 1); this.settings = settings; this.mappings = mappings; this.customs = customs; - this.totalNumberOfShards = numberOfShards() * (numberOfReplicas() + 1); this.aliases = aliases; - - Map requireMap = settings.getByPrefix("index.routing.allocation.require.").getAsMap(); - if (requireMap.isEmpty()) { - requireFilters = null; - } else { - requireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, requireMap); - } - Map includeMap = settings.getByPrefix("index.routing.allocation.include.").getAsMap(); - if (includeMap.isEmpty()) { - includeFilters = null; - } else { - includeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, includeMap); - } - Map excludeMap = settings.getByPrefix("index.routing.allocation.exclude.").getAsMap(); - if (excludeMap.isEmpty()) { - excludeFilters = null; - } else { - excludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, excludeMap); - } - indexCreatedVersion = Version.indexCreated(settings); - indexUpgradedVersion = settings.getAsVersion(IndexMetaData.SETTING_VERSION_UPGRADED, indexCreatedVersion); - String stringLuceneVersion = settings.get(SETTING_VERSION_MINIMUM_COMPATIBLE); - if (stringLuceneVersion != null) { - try { - this.minimumCompatibleLuceneVersion = org.apache.lucene.util.Version.parse(stringLuceneVersion); - } catch (ParseException ex) { - throw new IllegalStateException("Cannot parse lucene version [" + stringLuceneVersion + "] in the [" + SETTING_VERSION_MINIMUM_COMPATIBLE +"] setting", ex); - } - } else { - this.minimumCompatibleLuceneVersion = null; - } - } - - public String index() { - return index; + this.activeAllocationIds = activeAllocationIds; + this.requireFilters = requireFilters; + this.includeFilters = includeFilters; + this.excludeFilters = excludeFilters; + this.indexCreatedVersion = indexCreatedVersion; + this.indexUpgradedVersion = indexUpgradedVersion; + this.minimumCompatibleLuceneVersion = minimumCompatibleLuceneVersion; } public String getIndex() { - return index(); - } - - public String indexUUID() { - return settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); + return index; } public String getIndexUUID() { - return indexUUID(); + return settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); } /** @@ -263,17 +241,13 @@ public class IndexMetaData implements Diffable, FromXContentBuild */ public boolean isSameUUID(String otherUUID) { assert otherUUID != null; - assert indexUUID() != null; - if (INDEX_UUID_NA_VALUE.equals(otherUUID) || INDEX_UUID_NA_VALUE.equals(indexUUID())) { + assert getIndexUUID() != null; + if (INDEX_UUID_NA_VALUE.equals(otherUUID) || INDEX_UUID_NA_VALUE.equals(getIndexUUID())) { return true; } return otherUUID.equals(getIndexUUID()); } - public long version() { - return this.version; - } - public long getVersion() { return this.version; } @@ -282,26 +256,18 @@ public class IndexMetaData implements Diffable, FromXContentBuild * Return the {@link Version} on which this index has been created. This * information is typically useful for backward compatibility. */ - public Version creationVersion() { - return indexCreatedVersion; - } - public Version getCreationVersion() { - return creationVersion(); + return indexCreatedVersion; } /** * Return the {@link Version} on which this index has been upgraded. This * information is typically useful for backward compatibility. */ - public Version upgradeVersion() { + public Version getUpgradedVersion() { return indexUpgradedVersion; } - public Version getUpgradeVersion() { - return upgradeVersion(); - } - /** * Return the {@link org.apache.lucene.util.Version} of the oldest lucene segment in the index */ @@ -309,68 +275,36 @@ public class IndexMetaData implements Diffable, FromXContentBuild return minimumCompatibleLuceneVersion; } - public long creationDate() { + public long getCreationDate() { return settings.getAsLong(SETTING_CREATION_DATE, -1l); } - public long getCreationDate() { - return creationDate(); - } - - public State state() { + public State getState() { return this.state; } - public State getState() { - return state(); - } - - public int numberOfShards() { - return settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1); - } - public int getNumberOfShards() { - return numberOfShards(); - } - - public int numberOfReplicas() { - return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1); + return numberOfShards; } public int getNumberOfReplicas() { - return numberOfReplicas(); - } - - public int totalNumberOfShards() { - return totalNumberOfShards; + return numberOfReplicas; } public int getTotalNumberOfShards() { - return totalNumberOfShards(); - } - - public Settings settings() { - return settings; + return totalNumberOfShards; } public Settings getSettings() { - return settings(); - } - - public ImmutableOpenMap aliases() { - return this.aliases; + return settings; } public ImmutableOpenMap getAliases() { - return aliases(); - } - - public ImmutableOpenMap mappings() { - return mappings; + return this.aliases; } public ImmutableOpenMap getMappings() { - return mappings(); + return mappings; } @Nullable @@ -394,10 +328,6 @@ public class IndexMetaData implements Diffable, FromXContentBuild return mappings.get(MapperService.DEFAULT_MAPPING); } - public ImmutableOpenMap customs() { - return this.customs; - } - public ImmutableOpenMap getCustoms() { return this.customs; } @@ -407,6 +337,15 @@ public class IndexMetaData implements Diffable, FromXContentBuild return (T) customs.get(type); } + public ImmutableOpenIntMap> getActiveAllocationIds() { + return activeAllocationIds; + } + + public Set activeAllocationIds(int shardId) { + assert shardId >= 0 && shardId < numberOfShards; + return activeAllocationIds.get(shardId); + } + @Nullable public DiscoveryNodeFilters requireFilters() { return requireFilters; @@ -451,6 +390,9 @@ public class IndexMetaData implements Diffable, FromXContentBuild if (!customs.equals(that.customs)) { return false; } + if (!activeAllocationIds.equals(that.activeAllocationIds)) { + return false; + } return true; } @@ -461,6 +403,7 @@ public class IndexMetaData implements Diffable, FromXContentBuild result = 31 * result + aliases.hashCode(); result = 31 * result + settings.hashCode(); result = 31 * result + mappings.hashCode(); + result = 31 * result + activeAllocationIds.hashCode(); return result; } @@ -493,16 +436,19 @@ public class IndexMetaData implements Diffable, FromXContentBuild private final Settings settings; private final Diff> mappings; private final Diff> aliases; - private Diff> customs; + private final Diff> customs; + private final Diff>> activeAllocationIds; public IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) { index = after.index; version = after.version; state = after.state; settings = after.settings; - mappings = DiffableUtils.diff(before.mappings, after.mappings); - aliases = DiffableUtils.diff(before.aliases, after.aliases); - customs = DiffableUtils.diff(before.customs, after.customs); + mappings = DiffableUtils.diff(before.mappings, after.mappings, DiffableUtils.getStringKeySerializer()); + aliases = DiffableUtils.diff(before.aliases, after.aliases, DiffableUtils.getStringKeySerializer()); + customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer()); + activeAllocationIds = DiffableUtils.diff(before.activeAllocationIds, after.activeAllocationIds, + DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance()); } public IndexMetaDataDiff(StreamInput in) throws IOException { @@ -510,19 +456,22 @@ public class IndexMetaData implements Diffable, FromXContentBuild version = in.readLong(); state = State.fromId(in.readByte()); settings = Settings.readSettingsFromStream(in); - mappings = DiffableUtils.readImmutableOpenMapDiff(in, MappingMetaData.PROTO); - aliases = DiffableUtils.readImmutableOpenMapDiff(in, AliasMetaData.PROTO); - customs = DiffableUtils.readImmutableOpenMapDiff(in, new DiffableUtils.KeyedReader() { + mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MappingMetaData.PROTO); + aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), AliasMetaData.PROTO); + customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), + new DiffableUtils.DiffableValueSerializer() { @Override - public Custom readFrom(StreamInput in, String key) throws IOException { + public Custom read(StreamInput in, String key) throws IOException { return lookupPrototypeSafe(key).readFrom(in); } @Override - public Diff readDiffFrom(StreamInput in, String key) throws IOException { + public Diff readDiff(StreamInput in, String key) throws IOException { return lookupPrototypeSafe(key).readDiffFrom(in); } }); + activeAllocationIds = DiffableUtils.readImmutableOpenIntMapDiff(in, DiffableUtils.getVIntKeySerializer(), + DiffableUtils.StringSetValueSerializer.getInstance()); } @Override @@ -534,6 +483,7 @@ public class IndexMetaData implements Diffable, FromXContentBuild mappings.writeTo(out); aliases.writeTo(out); customs.writeTo(out); + activeAllocationIds.writeTo(out); } @Override @@ -545,6 +495,7 @@ public class IndexMetaData implements Diffable, FromXContentBuild builder.mappings.putAll(mappings.apply(part.mappings)); builder.aliases.putAll(aliases.apply(part.aliases)); builder.customs.putAll(customs.apply(part.customs)); + builder.activeAllocationIds.putAll(activeAllocationIds.apply(part.activeAllocationIds)); return builder.build(); } } @@ -571,6 +522,12 @@ public class IndexMetaData implements Diffable, FromXContentBuild Custom customIndexMetaData = lookupPrototypeSafe(type).readFrom(in); builder.putCustom(type, customIndexMetaData); } + int activeAllocationIdsSize = in.readVInt(); + for (int i = 0; i < activeAllocationIdsSize; i++) { + int key = in.readVInt(); + Set allocationIds = DiffableUtils.StringSetValueSerializer.getInstance().read(in, key); + builder.putActiveAllocationIds(key, allocationIds); + } return builder.build(); } @@ -593,6 +550,11 @@ public class IndexMetaData implements Diffable, FromXContentBuild out.writeString(cursor.key); cursor.value.writeTo(out); } + out.writeVInt(activeAllocationIds.size()); + for (IntObjectCursor> cursor : activeAllocationIds) { + out.writeVInt(cursor.key); + DiffableUtils.StringSetValueSerializer.getInstance().write(cursor.value, out); + } } public static Builder builder(String index) { @@ -612,22 +574,25 @@ public class IndexMetaData implements Diffable, FromXContentBuild private final ImmutableOpenMap.Builder mappings; private final ImmutableOpenMap.Builder aliases; private final ImmutableOpenMap.Builder customs; + private final ImmutableOpenIntMap.Builder> activeAllocationIds; public Builder(String index) { this.index = index; this.mappings = ImmutableOpenMap.builder(); this.aliases = ImmutableOpenMap.builder(); this.customs = ImmutableOpenMap.builder(); + this.activeAllocationIds = ImmutableOpenIntMap.builder(); } public Builder(IndexMetaData indexMetaData) { - this.index = indexMetaData.index(); + this.index = indexMetaData.getIndex(); this.state = indexMetaData.state; this.version = indexMetaData.version; - this.settings = indexMetaData.settings(); + this.settings = indexMetaData.getSettings(); this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings); this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases); this.customs = ImmutableOpenMap.builder(indexMetaData.customs); + this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds); } public String index() { @@ -736,6 +701,15 @@ public class IndexMetaData implements Diffable, FromXContentBuild return this.customs.get(type); } + public Builder putActiveAllocationIds(int shardId, Set allocationIds) { + activeAllocationIds.put(shardId, new HashSet(allocationIds)); + return this; + } + + public Set getActiveAllocationIds(int shardId) { + return activeAllocationIds.get(shardId); + } + public long version() { return this.version; } @@ -757,25 +731,90 @@ public class IndexMetaData implements Diffable, FromXContentBuild } } - return new IndexMetaData(index, version, state, tmpSettings, mappings.build(), tmpAliases.build(), customs.build()); + Integer maybeNumberOfShards = settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null); + if (maybeNumberOfShards == null) { + throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]"); + } + int numberOfShards = maybeNumberOfShards; + if (numberOfShards <= 0) { + throw new IllegalArgumentException("must specify positive number of shards for index [" + index + "]"); + } + + Integer maybeNumberOfReplicas = settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null); + if (maybeNumberOfReplicas == null) { + throw new IllegalArgumentException("must specify numberOfReplicas for index [" + index + "]"); + } + int numberOfReplicas = maybeNumberOfReplicas; + if (numberOfReplicas < 0) { + throw new IllegalArgumentException("must specify non-negative number of shards for index [" + index + "]"); + } + + // fill missing slots in activeAllocationIds with empty set if needed and make all entries immutable + ImmutableOpenIntMap.Builder> filledActiveAllocationIds = ImmutableOpenIntMap.builder(); + for (int i = 0; i < numberOfShards; i++) { + if (activeAllocationIds.containsKey(i)) { + filledActiveAllocationIds.put(i, Collections.unmodifiableSet(new HashSet<>(activeAllocationIds.get(i)))); + } else { + filledActiveAllocationIds.put(i, Collections.emptySet()); + } + } + + Map requireMap = settings.getByPrefix("index.routing.allocation.require.").getAsMap(); + final DiscoveryNodeFilters requireFilters; + if (requireMap.isEmpty()) { + requireFilters = null; + } else { + requireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, requireMap); + } + Map includeMap = settings.getByPrefix("index.routing.allocation.include.").getAsMap(); + final DiscoveryNodeFilters includeFilters; + if (includeMap.isEmpty()) { + includeFilters = null; + } else { + includeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, includeMap); + } + Map excludeMap = settings.getByPrefix("index.routing.allocation.exclude.").getAsMap(); + final DiscoveryNodeFilters excludeFilters; + if (excludeMap.isEmpty()) { + excludeFilters = null; + } else { + excludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, excludeMap); + } + Version indexCreatedVersion = Version.indexCreated(settings); + Version indexUpgradedVersion = settings.getAsVersion(IndexMetaData.SETTING_VERSION_UPGRADED, indexCreatedVersion); + String stringLuceneVersion = settings.get(SETTING_VERSION_MINIMUM_COMPATIBLE); + final org.apache.lucene.util.Version minimumCompatibleLuceneVersion; + if (stringLuceneVersion != null) { + try { + minimumCompatibleLuceneVersion = org.apache.lucene.util.Version.parse(stringLuceneVersion); + } catch (ParseException ex) { + throw new IllegalStateException("Cannot parse lucene version [" + stringLuceneVersion + "] in the [" + SETTING_VERSION_MINIMUM_COMPATIBLE +"] setting", ex); + } + } else { + minimumCompatibleLuceneVersion = null; + } + + return new IndexMetaData(index, version, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(), + tmpAliases.build(), customs.build(), filledActiveAllocationIds.build(), requireFilters, includeFilters, excludeFilters, + indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion); } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); + builder.startObject(indexMetaData.getIndex(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("version", indexMetaData.version()); - builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); + builder.field("version", indexMetaData.getVersion()); + builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH)); boolean binary = params.paramAsBoolean("binary", false); builder.startObject("settings"); - for (Map.Entry entry : indexMetaData.settings().getAsMap().entrySet()) { + for (Map.Entry entry : indexMetaData.getSettings().getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); } builder.endObject(); builder.startArray("mappings"); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { + for (ObjectObjectCursor cursor : indexMetaData.getMappings()) { if (binary) { builder.value(cursor.value.source().compressed()); } else { @@ -788,18 +827,27 @@ public class IndexMetaData implements Diffable, FromXContentBuild } builder.endArray(); - for (ObjectObjectCursor cursor : indexMetaData.customs()) { + for (ObjectObjectCursor cursor : indexMetaData.getCustoms()) { builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); cursor.value.toXContent(builder, params); builder.endObject(); } builder.startObject("aliases"); - for (ObjectCursor cursor : indexMetaData.aliases().values()) { + for (ObjectCursor cursor : indexMetaData.getAliases().values()) { AliasMetaData.Builder.toXContent(cursor.value, builder, params); } builder.endObject(); + builder.startObject(KEY_ACTIVE_ALLOCATIONS); + for (IntObjectCursor> cursor : indexMetaData.activeAllocationIds) { + builder.startArray(String.valueOf(cursor.key)); + for (String allocationId : cursor.value) { + builder.value(allocationId); + } + builder.endArray(); + } + builder.endObject(); builder.endObject(); } @@ -835,6 +883,21 @@ public class IndexMetaData implements Diffable, FromXContentBuild while (parser.nextToken() != XContentParser.Token.END_OBJECT) { builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); } + } else if (KEY_ACTIVE_ALLOCATIONS.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_ARRAY) { + String shardId = currentFieldName; + Set allocationIds = new HashSet<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + allocationIds.add(parser.text()); + } + } + builder.putActiveAllocationIds(Integer.valueOf(shardId), allocationIds); + } + } } else { // check if its a custom index metadata Custom proto = lookupPrototype(currentFieldName); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index adc94a5c74a..4b1514d13e7 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -76,7 +76,7 @@ public class IndexNameExpressionResolver extends AbstractComponent { } /** - * Translates the provided index expression into actual concrete indices. + * Translates the provided index expression into actual concrete indices, properly deduplicated. * * @param state the cluster state containing all the data to resolve to expressions to concrete indices * @param options defines how the aliases or indices need to be resolved to concrete indices @@ -94,7 +94,7 @@ public class IndexNameExpressionResolver extends AbstractComponent { } /** - * Translates the provided index expression into actual concrete indices. + * Translates the provided index expression into actual concrete indices, properly deduplicated. * * @param state the cluster state containing all the data to resolve to expressions to concrete indices * @param options defines how the aliases or indices need to be resolved to concrete indices @@ -141,7 +141,7 @@ public class IndexNameExpressionResolver extends AbstractComponent { } } - List concreteIndices = new ArrayList<>(expressions.size()); + final Set concreteIndices = new HashSet<>(expressions.size()); for (String expression : expressions) { AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression); if (aliasOrIndex == null) { @@ -253,7 +253,7 @@ public class IndexNameExpressionResolver extends AbstractComponent { // Shouldn't happen throw new IndexNotFoundException(index); } - AliasMetaData aliasMetaData = indexMetaData.aliases().get(alias); + AliasMetaData aliasMetaData = indexMetaData.getAliases().get(alias); boolean filteringRequired = aliasMetaData != null && aliasMetaData.filteringRequired(); if (!filteringRequired) { return null; @@ -272,7 +272,7 @@ public class IndexNameExpressionResolver extends AbstractComponent { throw new IndexNotFoundException(index); } - AliasMetaData aliasMetaData = indexMetaData.aliases().get(alias); + AliasMetaData aliasMetaData = indexMetaData.getAliases().get(alias); // Check that this is an alias for the current index // Otherwise - skip it if (aliasMetaData != null) { @@ -806,4 +806,17 @@ public class IndexNameExpressionResolver extends AbstractComponent { } } + /** + * Returns true iff the given expression resolves to the given index name otherwise false + */ + public final boolean matchesIndex(String indexName, String expression, ClusterState state) { + final String[] concreteIndices = concreteIndices(state, IndicesOptions.lenientExpandOpen(), expression); + for (String index : concreteIndices) { + if (Regex.simpleMatch(index, indexName)) { + return true; + } + } + return indexName.equals(expression); + } + } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index d8b98d62564..751f8a09ea5 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -27,7 +27,6 @@ import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; import org.elasticsearch.cluster.DiffableUtils; -import org.elasticsearch.cluster.DiffableUtils.KeyedReader; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -41,6 +40,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.support.LoggerMessageFormat; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; @@ -52,8 +52,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.store.IndexStoreConfig; import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.indices.ttl.IndicesTTLService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.warmer.IndexWarmersMetaData; @@ -184,8 +184,8 @@ public class MetaData implements Iterable, Diffable, Fr int totalNumberOfShards = 0; int numberOfShards = 0; for (ObjectCursor cursor : indices.values()) { - totalNumberOfShards += cursor.value.totalNumberOfShards(); - numberOfShards += cursor.value.numberOfShards(); + totalNumberOfShards += cursor.value.getTotalNumberOfShards(); + numberOfShards += cursor.value.getNumberOfShards(); } this.totalNumberOfShards = totalNumberOfShards; this.numberOfShards = numberOfShards; @@ -353,7 +353,7 @@ public class MetaData implements Iterable, Diffable, Fr } else { filteredMappings = ImmutableOpenMap.builder(); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { + for (ObjectObjectCursor cursor : indexMetaData.getMappings()) { if (Regex.simpleMatch(types, cursor.key)) { filteredMappings.put(cursor.key, cursor.value); } @@ -639,9 +639,9 @@ public class MetaData implements Iterable, Diffable, Fr version = after.version; transientSettings = after.transientSettings; persistentSettings = after.persistentSettings; - indices = DiffableUtils.diff(before.indices, after.indices); - templates = DiffableUtils.diff(before.templates, after.templates); - customs = DiffableUtils.diff(before.customs, after.customs); + indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer()); + templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer()); + customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer()); } public MetaDataDiff(StreamInput in) throws IOException { @@ -649,16 +649,17 @@ public class MetaData implements Iterable, Diffable, Fr version = in.readLong(); transientSettings = Settings.readSettingsFromStream(in); persistentSettings = Settings.readSettingsFromStream(in); - indices = DiffableUtils.readImmutableOpenMapDiff(in, IndexMetaData.PROTO); - templates = DiffableUtils.readImmutableOpenMapDiff(in, IndexTemplateMetaData.PROTO); - customs = DiffableUtils.readImmutableOpenMapDiff(in, new KeyedReader() { + indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexMetaData.PROTO); + templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData.PROTO); + customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), + new DiffableUtils.DiffableValueSerializer() { @Override - public Custom readFrom(StreamInput in, String key) throws IOException { + public Custom read(StreamInput in, String key) throws IOException { return lookupPrototypeSafe(key).readFrom(in); } @Override - public Diff readDiffFrom(StreamInput in, String key) throws IOException { + public Diff readDiff(StreamInput in, String key) throws IOException { return lookupPrototypeSafe(key).readDiffFrom(in); } }); @@ -744,11 +745,8 @@ public class MetaData implements Iterable, Diffable, Fr /** All known byte-sized cluster settings. */ public static final Set CLUSTER_BYTES_SIZE_SETTINGS = unmodifiableSet(newHashSet( - IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, - RecoverySettings.INDICES_RECOVERY_FILE_CHUNK_SIZE, - RecoverySettings.INDICES_RECOVERY_TRANSLOG_SIZE, - RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, - RecoverySettings.INDICES_RECOVERY_MAX_SIZE_PER_SEC)); + IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, + RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC)); /** All known time cluster settings. */ @@ -854,19 +852,19 @@ public class MetaData implements Iterable, Diffable, Fr // we know its a new one, increment the version and store indexMetaDataBuilder.version(indexMetaDataBuilder.version() + 1); IndexMetaData indexMetaData = indexMetaDataBuilder.build(); - indices.put(indexMetaData.index(), indexMetaData); + indices.put(indexMetaData.getIndex(), indexMetaData); return this; } public Builder put(IndexMetaData indexMetaData, boolean incrementVersion) { - if (indices.get(indexMetaData.index()) == indexMetaData) { + if (indices.get(indexMetaData.getIndex()) == indexMetaData) { return this; } // if we put a new index metadata, increment its version if (incrementVersion) { - indexMetaData = IndexMetaData.builder(indexMetaData).version(indexMetaData.version() + 1).build(); + indexMetaData = IndexMetaData.builder(indexMetaData).version(indexMetaData.getVersion() + 1).build(); } - indices.put(indexMetaData.index(), indexMetaData); + indices.put(indexMetaData.getIndex(), indexMetaData); return this; } @@ -937,7 +935,7 @@ public class MetaData implements Iterable, Diffable, Fr throw new IndexNotFoundException(index); } put(IndexMetaData.builder(indexMetaData) - .settings(settingsBuilder().put(indexMetaData.settings()).put(settings))); + .settings(settingsBuilder().put(indexMetaData.getSettings()).put(settings))); } return this; } @@ -1003,7 +1001,7 @@ public class MetaData implements Iterable, Diffable, Fr // do the required operations, the bottleneck isn't resolving expressions into concrete indices. List allIndicesLst = new ArrayList<>(); for (ObjectCursor cursor : indices.values()) { - allIndicesLst.add(cursor.value.index()); + allIndicesLst.add(cursor.value.getIndex()); } String[] allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]); @@ -1011,10 +1009,10 @@ public class MetaData implements Iterable, Diffable, Fr List allClosedIndicesLst = new ArrayList<>(); for (ObjectCursor cursor : indices.values()) { IndexMetaData indexMetaData = cursor.value; - if (indexMetaData.state() == IndexMetaData.State.OPEN) { - allOpenIndicesLst.add(indexMetaData.index()); - } else if (indexMetaData.state() == IndexMetaData.State.CLOSE) { - allClosedIndicesLst.add(indexMetaData.index()); + if (indexMetaData.getState() == IndexMetaData.State.OPEN) { + allOpenIndicesLst.add(indexMetaData.getIndex()); + } else if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { + allClosedIndicesLst.add(indexMetaData.getIndex()); } } String[] allOpenIndices = allOpenIndicesLst.toArray(new String[allOpenIndicesLst.size()]); @@ -1028,12 +1026,18 @@ public class MetaData implements Iterable, Diffable, Fr for (ObjectObjectCursor aliasCursor : indexMetaData.getAliases()) { AliasMetaData aliasMetaData = aliasCursor.value; - AliasOrIndex.Alias aliasOrIndex = (AliasOrIndex.Alias) aliasAndIndexLookup.get(aliasMetaData.getAlias()); + AliasOrIndex aliasOrIndex = aliasAndIndexLookup.get(aliasMetaData.getAlias()); if (aliasOrIndex == null) { aliasOrIndex = new AliasOrIndex.Alias(aliasMetaData, indexMetaData); aliasAndIndexLookup.put(aliasMetaData.getAlias(), aliasOrIndex); + } else if (aliasOrIndex instanceof AliasOrIndex.Alias) { + AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex; + alias.addIndex(indexMetaData); + } else if (aliasOrIndex instanceof AliasOrIndex.Index) { + AliasOrIndex.Index index = (AliasOrIndex.Index) aliasOrIndex; + throw new IllegalStateException("index and alias names need to be unique, but alias [" + aliasMetaData.getAlias() + "] and index [" + index.getIndex().getIndex() + "] have the same name"); } else { - aliasOrIndex.addIndex(indexMetaData); + throw new IllegalStateException("unexpected alias [" + aliasMetaData.getAlias() + "][" + aliasOrIndex + "]"); } } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index f5fd4a0c96d..96d378af042 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -26,7 +26,6 @@ import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; @@ -41,54 +40,38 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.IndexCreationException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.threadpool.ThreadPool; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import java.io.BufferedReader; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; +import java.util.*; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE; @@ -106,33 +89,29 @@ public class MetaDataCreateIndexService extends AbstractComponent { public final static int MAX_INDEX_NAME_BYTES = 255; private static final DefaultIndexTemplateFilter DEFAULT_INDEX_TEMPLATE_FILTER = new DefaultIndexTemplateFilter(); - private final ThreadPool threadPool; private final ClusterService clusterService; private final IndicesService indicesService; private final AllocationService allocationService; - private final MetaDataService metaDataService; private final Version version; private final AliasValidator aliasValidator; private final IndexTemplateFilter indexTemplateFilter; - private final NodeEnvironment nodeEnv; private final Environment env; + private final NodeServicesProvider nodeServicesProvider; + @Inject - public MetaDataCreateIndexService(Settings settings, ThreadPool threadPool, ClusterService clusterService, - IndicesService indicesService, AllocationService allocationService, MetaDataService metaDataService, + public MetaDataCreateIndexService(Settings settings, ClusterService clusterService, + IndicesService indicesService, AllocationService allocationService, Version version, AliasValidator aliasValidator, - Set indexTemplateFilters, Environment env, - NodeEnvironment nodeEnv) { + Set indexTemplateFilters, Environment env, NodeServicesProvider nodeServicesProvider) { super(settings); - this.threadPool = threadPool; this.clusterService = clusterService; this.indicesService = indicesService; this.allocationService = allocationService; - this.metaDataService = metaDataService; this.version = version; this.aliasValidator = aliasValidator; - this.nodeEnv = nodeEnv; this.env = env; + this.nodeServicesProvider = nodeServicesProvider; if (indexTemplateFilters.isEmpty()) { this.indexTemplateFilter = DEFAULT_INDEX_TEMPLATE_FILTER; @@ -147,29 +126,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - public void createIndex(final CreateIndexClusterStateUpdateRequest request, final ActionListener listener) { - - // we lock here, and not within the cluster service callback since we don't want to - // block the whole cluster state handling - final Semaphore mdLock = metaDataService.indexMetaDataLock(request.index()); - - // quick check to see if we can acquire a lock, otherwise spawn to a thread pool - if (mdLock.tryAcquire()) { - createIndex(request, listener, mdLock); - return; - } - threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new ActionRunnable(listener) { - @Override - public void doRun() throws InterruptedException { - if (!mdLock.tryAcquire(request.masterNodeTimeout().nanos(), TimeUnit.NANOSECONDS)) { - listener.onFailure(new ProcessClusterEventTimeoutException(request.masterNodeTimeout(), "acquire index lock")); - return; - } - createIndex(request, listener, mdLock); - } - }); - } - public void validateIndexName(String index, ClusterState state) { if (state.routingTable().hasIndex(index)) { throw new IndexAlreadyExistsException(new Index(index)); @@ -209,36 +165,17 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - private void createIndex(final CreateIndexClusterStateUpdateRequest request, final ActionListener listener, final Semaphore mdLock) { - + public void createIndex(final CreateIndexClusterStateUpdateRequest request, final ActionListener listener) { Settings.Builder updatedSettingsBuilder = Settings.settingsBuilder(); updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX); request.settings(updatedSettingsBuilder.build()); - clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]", Priority.URGENT, new AckedClusterStateUpdateTask(request, listener) { - - @Override - protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { - return new ClusterStateUpdateResponse(acknowledged); - } - - @Override - public void onAllNodesAcked(@Nullable Throwable t) { - mdLock.release(); - super.onAllNodesAcked(t); - } - - @Override - public void onAckTimeout() { - mdLock.release(); - super.onAckTimeout(); - } - - @Override - public void onFailure(String source, Throwable t) { - mdLock.release(); - super.onFailure(source, t); - } + clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]", + new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { + @Override + protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { + return new ClusterStateUpdateResponse(acknowledged); + } @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -362,7 +299,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { // Set up everything, now locally create the index to see that things are ok, and apply final IndexMetaData tmpImd = IndexMetaData.builder(request.index()).settings(actualIndexSettings).build(); // create the index here (on the master) to validate it can be created, as well as adding the mapping - indicesService.createIndex(tmpImd); + indicesService.createIndex(nodeServicesProvider, tmpImd, Collections.emptyList()); indexCreated = true; // now add the mappings IndexService indexService = indicesService.indexServiceSafe(request.index()); @@ -373,7 +310,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), false, request.updateAllTypes()); } catch (Exception e) { removalReason = "failed on parsing default mapping on index creation"; - throw new MapperParsingException("mapping [" + MapperService.DEFAULT_MAPPING + "]", e); + throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, MapperService.DEFAULT_MAPPING, e.getMessage()); } } for (Map.Entry> entry : mappings.entrySet()) { @@ -385,19 +322,19 @@ public class MetaDataCreateIndexService extends AbstractComponent { mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), true, request.updateAllTypes()); } catch (Exception e) { removalReason = "failed on parsing mappings on index creation"; - throw new MapperParsingException("mapping [" + entry.getKey() + "]", e); + throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage()); } } - IndexQueryParserService indexQueryParserService = indexService.queryParserService(); + QueryShardContext queryShardContext = indexService.getQueryShardContext(); for (Alias alias : request.aliases()) { if (Strings.hasLength(alias.filter())) { - aliasValidator.validateAliasFilter(alias.name(), alias.filter(), indexQueryParserService); + aliasValidator.validateAliasFilter(alias.name(), alias.filter(), queryShardContext); } } for (AliasMetaData aliasMetaData : templatesAliases.values()) { if (aliasMetaData.filter() != null) { - aliasValidator.validateAliasFilter(aliasMetaData.alias(), aliasMetaData.filter().uncompressed(), indexQueryParserService); + aliasValidator.validateAliasFilter(aliasMetaData.alias(), aliasMetaData.filter().uncompressed(), queryShardContext); } } @@ -436,17 +373,17 @@ public class MetaDataCreateIndexService extends AbstractComponent { throw e; } - indexService.indicesLifecycle().beforeIndexAddedToCluster(new Index(request.index()), - indexMetaData.settings()); + indexService.getIndexEventListener().beforeIndexAddedToCluster(new Index(request.index()), + indexMetaData.getSettings()); MetaData newMetaData = MetaData.builder(currentState.metaData()) .put(indexMetaData, false) .build(); - String maybeShadowIndicator = IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.settings()) ? "s" : ""; + String maybeShadowIndicator = IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings()) ? "s" : ""; logger.info("[{}] creating index, cause [{}], templates {}, shards [{}]/[{}{}], mappings {}", - request.index(), request.cause(), templateNames, indexMetaData.numberOfShards(), - indexMetaData.numberOfReplicas(), maybeShadowIndicator, mappings.keySet()); + request.index(), request.cause(), templateNames, indexMetaData.getNumberOfShards(), + indexMetaData.getNumberOfReplicas(), maybeShadowIndicator, mappings.keySet()); ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); if (!request.blocks().isEmpty()) { @@ -461,7 +398,9 @@ public class MetaDataCreateIndexService extends AbstractComponent { if (request.state() == State.OPEN) { RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable()) .addAsNew(updatedState.metaData().index(request.index())); - RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build()); + RoutingAllocation.Result routingResult = allocationService.reroute( + ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(), + "index [" + request.index() + "] created"); updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build(); } removalReason = "cleaning up after validating index on master"; @@ -482,29 +421,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - private void addMappings(Map> mappings, Path mappingsDir) throws IOException { - try (DirectoryStream stream = Files.newDirectoryStream(mappingsDir)) { - for (Path mappingFile : stream) { - final String fileName = mappingFile.getFileName().toString(); - if (FileSystemUtils.isHidden(mappingFile)) { - continue; - } - int lastDotIndex = fileName.lastIndexOf('.'); - String mappingType = lastDotIndex != -1 ? mappingFile.getFileName().toString().substring(0, lastDotIndex) : mappingFile.getFileName().toString(); - try (BufferedReader reader = Files.newBufferedReader(mappingFile, StandardCharsets.UTF_8)) { - String mappingSource = Streams.copyToString(reader); - if (mappings.containsKey(mappingType)) { - XContentHelper.mergeDefaults(mappings.get(mappingType), parseMapping(mappingSource)); - } else { - mappings.put(mappingType, parseMapping(mappingSource)); - } - } catch (Exception e) { - logger.warn("failed to read / parse mapping [" + mappingType + "] from location [" + mappingFile + "], ignoring...", e); - } - } - } - } - private List findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException { List templates = new ArrayList<>(); for (ObjectCursor cursor : state.metaData().templates().values()) { diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java index 88e1aad5614..54c014fb4ed 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java @@ -37,9 +37,9 @@ import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Arrays; +import java.util.Collection; import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -56,51 +56,21 @@ public class MetaDataDeleteIndexService extends AbstractComponent { private final NodeIndexDeletedAction nodeIndexDeletedAction; - private final MetaDataService metaDataService; - @Inject public MetaDataDeleteIndexService(Settings settings, ThreadPool threadPool, ClusterService clusterService, AllocationService allocationService, - NodeIndexDeletedAction nodeIndexDeletedAction, MetaDataService metaDataService) { + NodeIndexDeletedAction nodeIndexDeletedAction) { super(settings); this.threadPool = threadPool; this.clusterService = clusterService; this.allocationService = allocationService; this.nodeIndexDeletedAction = nodeIndexDeletedAction; - this.metaDataService = metaDataService; } - public void deleteIndex(final Request request, final Listener userListener) { - // we lock here, and not within the cluster service callback since we don't want to - // block the whole cluster state handling - final Semaphore mdLock = metaDataService.indexMetaDataLock(request.index); + public void deleteIndices(final Request request, final Listener userListener) { + Collection indices = Arrays.asList(request.indices); + final DeleteIndexListener listener = new DeleteIndexListener(userListener); - // quick check to see if we can acquire a lock, otherwise spawn to a thread pool - if (mdLock.tryAcquire()) { - deleteIndex(request, userListener, mdLock); - return; - } - - threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new Runnable() { - @Override - public void run() { - try { - if (!mdLock.tryAcquire(request.masterTimeout.nanos(), TimeUnit.NANOSECONDS)) { - userListener.onFailure(new ProcessClusterEventTimeoutException(request.masterTimeout, "acquire index lock")); - return; - } - } catch (InterruptedException e) { - userListener.onFailure(e); - return; - } - - deleteIndex(request, userListener, mdLock); - } - }); - } - - private void deleteIndex(final Request request, final Listener userListener, Semaphore mdLock) { - final DeleteIndexListener listener = new DeleteIndexListener(mdLock, userListener); - clusterService.submitStateUpdateTask("delete-index [" + request.index + "]", Priority.URGENT, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("delete-index " + indices, new ClusterStateUpdateTask(Priority.URGENT) { @Override public TimeValue timeout() { @@ -114,34 +84,32 @@ public class MetaDataDeleteIndexService extends AbstractComponent { @Override public ClusterState execute(final ClusterState currentState) { - if (!currentState.metaData().hasConcreteIndex(request.index)) { - throw new IndexNotFoundException(request.index); - } - - logger.info("[{}] deleting index", request.index); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable()); - routingTableBuilder.remove(request.index); + MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData()); + ClusterBlocks.Builder clusterBlocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks()); - MetaData newMetaData = MetaData.builder(currentState.metaData()) - .remove(request.index) - .build(); + for (final String index: indices) { + if (!currentState.metaData().hasConcreteIndex(index)) { + throw new IndexNotFoundException(index); + } - RoutingAllocation.Result routingResult = allocationService.reroute( - ClusterState.builder(currentState).routingTable(routingTableBuilder.build()).metaData(newMetaData).build()); - - ClusterBlocks blocks = ClusterBlocks.builder().blocks(currentState.blocks()).removeIndexBlocks(request.index).build(); + logger.debug("[{}] deleting index", index); + routingTableBuilder.remove(index); + clusterBlocksBuilder.removeIndexBlocks(index); + metaDataBuilder.remove(index); + } // wait for events from all nodes that it has been removed from their respective metadata... int count = currentState.nodes().size(); // add the notifications that the store was deleted from *data* nodes count += currentState.nodes().dataNodes().size(); - final AtomicInteger counter = new AtomicInteger(count); + final AtomicInteger counter = new AtomicInteger(count * indices.size()); + // this listener will be notified once we get back a notification based on the cluster state change below. final NodeIndexDeletedAction.Listener nodeIndexDeleteListener = new NodeIndexDeletedAction.Listener() { @Override - public void onNodeIndexDeleted(String index, String nodeId) { - if (index.equals(request.index)) { + public void onNodeIndexDeleted(String deleted, String nodeId) { + if (indices.contains(deleted)) { if (counter.decrementAndGet() == 0) { listener.onResponse(new Response(true)); nodeIndexDeletedAction.remove(this); @@ -150,8 +118,8 @@ public class MetaDataDeleteIndexService extends AbstractComponent { } @Override - public void onNodeIndexStoreDeleted(String index, String nodeId) { - if (index.equals(request.index)) { + public void onNodeIndexStoreDeleted(String deleted, String nodeId) { + if (indices.contains(deleted)) { if (counter.decrementAndGet() == 0) { listener.onResponse(new Response(true)); nodeIndexDeletedAction.remove(this); @@ -160,15 +128,16 @@ public class MetaDataDeleteIndexService extends AbstractComponent { } }; nodeIndexDeletedAction.add(nodeIndexDeleteListener); - - listener.future = threadPool.schedule(request.timeout, ThreadPool.Names.SAME, new Runnable() { - @Override - public void run() { - listener.onResponse(new Response(false)); - nodeIndexDeletedAction.remove(nodeIndexDeleteListener); - } + listener.future = threadPool.schedule(request.timeout, ThreadPool.Names.SAME, () -> { + listener.onResponse(new Response(false)); + nodeIndexDeletedAction.remove(nodeIndexDeleteListener); }); + MetaData newMetaData = metaDataBuilder.build(); + ClusterBlocks blocks = clusterBlocksBuilder.build(); + RoutingAllocation.Result routingResult = allocationService.reroute( + ClusterState.builder(currentState).routingTable(routingTableBuilder.build()).metaData(newMetaData).build(), + "deleted indices [" + indices + "]"); return ClusterState.builder(currentState).routingResult(routingResult).metaData(newMetaData).blocks(blocks).build(); } @@ -181,19 +150,16 @@ public class MetaDataDeleteIndexService extends AbstractComponent { class DeleteIndexListener implements Listener { private final AtomicBoolean notified = new AtomicBoolean(); - private final Semaphore mdLock; private final Listener listener; volatile ScheduledFuture future; - private DeleteIndexListener(Semaphore mdLock, Listener listener) { - this.mdLock = mdLock; + private DeleteIndexListener(Listener listener) { this.listener = listener; } @Override public void onResponse(final Response response) { if (notified.compareAndSet(false, true)) { - mdLock.release(); FutureUtils.cancel(future); listener.onResponse(response); } @@ -202,15 +168,13 @@ public class MetaDataDeleteIndexService extends AbstractComponent { @Override public void onFailure(Throwable t) { if (notified.compareAndSet(false, true)) { - mdLock.release(); FutureUtils.cancel(future); listener.onFailure(t); } } } - - public static interface Listener { + public interface Listener { void onResponse(Response response); @@ -219,13 +183,13 @@ public class MetaDataDeleteIndexService extends AbstractComponent { public static class Request { - final String index; + final String[] indices; TimeValue timeout = TimeValue.timeValueSeconds(10); TimeValue masterTimeout = MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT; - public Request(String index) { - this.index = index; + public Request(String[] indices) { + this.indices = indices; } public Request timeout(TimeValue timeout) { diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java index 82d7f5511eb..71ef9c22c33 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java @@ -33,13 +33,11 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesService; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Service responsible for submitting add and remove aliases requests @@ -52,16 +50,19 @@ public class MetaDataIndexAliasesService extends AbstractComponent { private final AliasValidator aliasValidator; + private final NodeServicesProvider nodeServicesProvider; + @Inject - public MetaDataIndexAliasesService(Settings settings, ClusterService clusterService, IndicesService indicesService, AliasValidator aliasValidator) { + public MetaDataIndexAliasesService(Settings settings, ClusterService clusterService, IndicesService indicesService, AliasValidator aliasValidator, NodeServicesProvider nodeServicesProvider) { super(settings); this.clusterService = clusterService; this.indicesService = indicesService; this.aliasValidator = aliasValidator; + this.nodeServicesProvider = nodeServicesProvider; } public void indicesAliases(final IndicesAliasesClusterStateUpdateRequest request, final ActionListener listener) { - clusterService.submitStateUpdateTask("index-aliases", Priority.URGENT, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("index-aliases", new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { @Override protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { return new ClusterStateUpdateResponse(acknowledged); @@ -92,30 +93,30 @@ public class MetaDataIndexAliasesService extends AbstractComponent { String filter = aliasAction.filter(); if (Strings.hasLength(filter)) { // parse the filter, in order to validate it - IndexService indexService = indices.get(indexMetaData.index()); + IndexService indexService = indices.get(indexMetaData.getIndex()); if (indexService == null) { - indexService = indicesService.indexService(indexMetaData.index()); + indexService = indicesService.indexService(indexMetaData.getIndex()); if (indexService == null) { // temporarily create the index and add mappings so we can parse the filter try { - indexService = indicesService.createIndex(indexMetaData); - if (indexMetaData.mappings().containsKey(MapperService.DEFAULT_MAPPING)) { - indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source(), false, false); + indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, Collections.emptyList()); + if (indexMetaData.getMappings().containsKey(MapperService.DEFAULT_MAPPING)) { + indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.getMappings().get(MapperService.DEFAULT_MAPPING).source(), false, false); } - for (ObjectCursor cursor : indexMetaData.mappings().values()) { + for (ObjectCursor cursor : indexMetaData.getMappings().values()) { MappingMetaData mappingMetaData = cursor.value; indexService.mapperService().merge(mappingMetaData.type(), mappingMetaData.source(), false, false); } } catch (Exception e) { - logger.warn("[{}] failed to temporary create in order to apply alias action", e, indexMetaData.index()); + logger.warn("[{}] failed to temporary create in order to apply alias action", e, indexMetaData.getIndex()); continue; } - indicesToClose.add(indexMetaData.index()); + indicesToClose.add(indexMetaData.getIndex()); } - indices.put(indexMetaData.index(), indexService); + indices.put(indexMetaData.getIndex(), indexService); } - aliasValidator.validateAliasFilter(aliasAction.alias(), filter, indexService.queryParserService()); + aliasValidator.validateAliasFilter(aliasAction.alias(), filter, indexService.getQueryShardContext()); } AliasMetaData newAliasMd = AliasMetaData.newAliasMetaDataBuilder( aliasAction.alias()) @@ -124,14 +125,14 @@ public class MetaDataIndexAliasesService extends AbstractComponent { .searchRouting(aliasAction.searchRouting()) .build(); // Check if this alias already exists - AliasMetaData aliasMd = indexMetaData.aliases().get(aliasAction.alias()); + AliasMetaData aliasMd = indexMetaData.getAliases().get(aliasAction.alias()); if (aliasMd != null && aliasMd.equals(newAliasMd)) { // It's the same alias - ignore it continue; } indexMetaDataBuilder.putAlias(newAliasMd); } else if (aliasAction.actionType() == AliasAction.Type.REMOVE) { - if (!indexMetaData.aliases().containsKey(aliasAction.alias())) { + if (!indexMetaData.getAliases().containsKey(aliasAction.alias())) { // This alias doesn't exist - ignore continue; } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java index e4452e4ee35..1fa1b702f66 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java @@ -47,6 +47,7 @@ import org.elasticsearch.rest.RestStatus; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; /** * Service responsible for submitting open/close index requests @@ -75,7 +76,7 @@ public class MetaDataIndexStateService extends AbstractComponent { } final String indicesAsString = Arrays.toString(request.indices()); - clusterService.submitStateUpdateTask("close-indices " + indicesAsString, Priority.URGENT, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("close-indices " + indicesAsString, new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { @Override protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { return new ClusterStateUpdateResponse(acknowledged); @@ -90,7 +91,7 @@ public class MetaDataIndexStateService extends AbstractComponent { throw new IndexNotFoundException(index); } - if (indexMetaData.state() != IndexMetaData.State.CLOSE) { + if (indexMetaData.getState() != IndexMetaData.State.CLOSE) { IndexRoutingTable indexRoutingTable = currentState.routingTable().index(index); for (IndexShardRoutingTable shard : indexRoutingTable) { for (ShardRouting shardRouting : shard) { @@ -124,7 +125,9 @@ public class MetaDataIndexStateService extends AbstractComponent { rtBuilder.remove(index); } - RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build()); + RoutingAllocation.Result routingResult = allocationService.reroute( + ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build(), + "indices closed [" + indicesAsString + "]"); //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask return ClusterState.builder(updatedState).routingResult(routingResult).build(); } @@ -137,7 +140,7 @@ public class MetaDataIndexStateService extends AbstractComponent { } final String indicesAsString = Arrays.toString(request.indices()); - clusterService.submitStateUpdateTask("open-indices " + indicesAsString, Priority.URGENT, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("open-indices " + indicesAsString, new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { @Override protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { return new ClusterStateUpdateResponse(acknowledged); @@ -151,7 +154,7 @@ public class MetaDataIndexStateService extends AbstractComponent { if (indexMetaData == null) { throw new IndexNotFoundException(index); } - if (indexMetaData.state() != IndexMetaData.State.OPEN) { + if (indexMetaData.getState() != IndexMetaData.State.OPEN) { indicesToOpen.add(index); } } @@ -181,7 +184,9 @@ public class MetaDataIndexStateService extends AbstractComponent { rtBuilder.addAsFromCloseToOpen(updatedState.metaData().index(index)); } - RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build()); + RoutingAllocation.Result routingResult = allocationService.reroute( + ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build(), + "indices opened [" + indicesAsString + "]"); //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask return ClusterState.builder(updatedState).routingResult(routingResult).build(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java index 13823e8ebdd..790cb99c64b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java @@ -56,7 +56,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent { } public void removeTemplates(final RemoveRequest request, final RemoveListener listener) { - clusterService.submitStateUpdateTask("remove-index-template [" + request.name + "]", Priority.URGENT, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("remove-index-template [" + request.name + "]", new ClusterStateUpdateTask(Priority.URGENT) { @Override public TimeValue timeout() { @@ -143,7 +143,8 @@ public class MetaDataIndexTemplateService extends AbstractComponent { } final IndexTemplateMetaData template = templateBuilder.build(); - clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]", Priority.URGENT, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]", + new ClusterStateUpdateTask(Priority.URGENT) { @Override public TimeValue timeout() { @@ -216,6 +217,9 @@ public class MetaDataIndexTemplateService extends AbstractComponent { for (Alias alias : request.aliases) { //we validate the alias only partially, as we don't know yet to which index it'll get applied to aliasValidator.validateAliasStandalone(alias); + if (request.template.equals(alias.name())) { + throw new IllegalArgumentException("Alias [" + alias.name() + "] cannot be the same as the template pattern [" + request.template + "]"); + } } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index cdde49170d4..00904af8915 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -26,14 +26,14 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.script.ScriptService; +import org.elasticsearch.indices.mapper.MapperRegistry; -import java.util.Locale; +import java.util.Collections; import java.util.Set; import static java.util.Collections.unmodifiableSet; @@ -49,12 +49,12 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet; */ public class MetaDataIndexUpgradeService extends AbstractComponent { - private final ScriptService scriptService; + private final MapperRegistry mapperRegistry; @Inject - public MetaDataIndexUpgradeService(Settings settings, ScriptService scriptService) { + public MetaDataIndexUpgradeService(Settings settings, MapperRegistry mapperRegistry) { super(settings); - this.scriptService = scriptService; + this.mapperRegistry = mapperRegistry; } /** @@ -82,7 +82,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { * Checks if the index was already opened by this version of Elasticsearch and doesn't require any additional checks. */ private boolean isUpgraded(IndexMetaData indexMetaData) { - return indexMetaData.upgradeVersion().onOrAfter(Version.V_3_0_0); + return indexMetaData.getUpgradedVersion().onOrAfter(Version.V_3_0_0); } /** @@ -102,7 +102,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { * Returns true if this index can be supported by the current version of elasticsearch */ private static boolean isSupportedVersion(IndexMetaData indexMetaData) { - if (indexMetaData.creationVersion().onOrAfter(Version.V_2_0_0_beta1)) { + if (indexMetaData.getCreationVersion().onOrAfter(Version.V_2_0_0_beta1)) { // The index was created with elasticsearch that was using Lucene 5.2.1 return true; } @@ -150,6 +150,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { "index.translog.flush_threshold_period", "index.translog.interval", "index.translog.sync_interval", + "index.shard.inactive_time", UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING)); /** @@ -160,7 +161,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { if (indexMetaData.getCreationVersion().before(Version.V_2_0_0_beta1)) { // TODO: can we somehow only do this *once* for a pre-2.0 index? Maybe we could stuff a "fake marker setting" here? Seems hackish... // Created lazily if we find any settings that are missing units: - Settings settings = indexMetaData.settings(); + Settings settings = indexMetaData.getSettings(); Settings.Builder newSettings = null; for(String byteSizeSetting : INDEX_BYTES_SIZE_SETTINGS) { String value = settings.get(byteSizeSetting); @@ -199,7 +200,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { if (newSettings != null) { // At least one setting was changed: return IndexMetaData.builder(indexMetaData) - .version(indexMetaData.version()) + .version(indexMetaData.getVersion()) .settings(newSettings.build()) .build(); } @@ -214,14 +215,14 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { * Checks the mappings for compatibility with the current version */ private void checkMappingsCompatibility(IndexMetaData indexMetaData) { - Index index = new Index(indexMetaData.getIndex()); - Settings settings = indexMetaData.settings(); try { - SimilarityService similarityService = new SimilarityService(index, settings); // We cannot instantiate real analysis server at this point because the node might not have // been started yet. However, we don't really need real analyzers at this stage - so we can fake it - try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) { - try (MapperService mapperService = new MapperService(index, settings, analysisService, similarityService, scriptService)) { + IndexSettings indexSettings = new IndexSettings(indexMetaData, this.settings, Collections.emptyList()); + SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); + + try (AnalysisService analysisService = new FakeAnalysisService(indexSettings)) { + try (MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry)) { for (ObjectCursor cursor : indexMetaData.getMappings().values()) { MappingMetaData mappingMetaData = cursor.value; mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), false, false); @@ -230,7 +231,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { } } catch (Exception ex) { // Wrap the inner exception so we have the index name in the exception message - throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetaData.getIndex() + "], reason: [" + ex.getMessage() + "]", ex); + throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetaData.getIndex() + "]", ex); } } @@ -238,7 +239,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { * Marks index as upgraded so we don't have to test it again */ private IndexMetaData markAsUpgraded(IndexMetaData indexMetaData) { - Settings settings = Settings.builder().put(indexMetaData.settings()).put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.CURRENT).build(); + Settings settings = Settings.builder().put(indexMetaData.getSettings()).put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.CURRENT).build(); return IndexMetaData.builder(indexMetaData).settings(settings).build(); } @@ -254,8 +255,8 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { } }; - public FakeAnalysisService(Index index, Settings indexSettings) { - super(index, indexSettings); + public FakeAnalysisService(IndexSettings indexSettings) { + super(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); } @Override diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 36d727a1134..957125703b6 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -22,27 +22,27 @@ package org.elasticsearch.cluster.metadata; import com.carrotsearch.hppc.cursors.ObjectCursor; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidTypeNameException; import org.elasticsearch.percolator.PercolatorService; +import java.io.IOException; import java.util.*; /** * Service responsible for submitting mapping changes @@ -52,102 +52,57 @@ public class MetaDataMappingService extends AbstractComponent { private final ClusterService clusterService; private final IndicesService indicesService; - // the mutex protect all the refreshOrUpdate variables! - private final Object refreshOrUpdateMutex = new Object(); - private final List refreshOrUpdateQueue = new ArrayList<>(); - private long refreshOrUpdateInsertOrder; - private long refreshOrUpdateProcessedInsertOrder; + final ClusterStateTaskExecutor refreshExecutor = new RefreshTaskExecutor(); + final ClusterStateTaskExecutor putMappingExecutor = new PutMappingExecutor(); + private final NodeServicesProvider nodeServicesProvider; + @Inject - public MetaDataMappingService(Settings settings, ClusterService clusterService, IndicesService indicesService) { + public MetaDataMappingService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) { super(settings); this.clusterService = clusterService; this.indicesService = indicesService; + this.nodeServicesProvider = nodeServicesProvider; } - static class MappingTask { + static class RefreshTask { final String index; final String indexUUID; - MappingTask(String index, final String indexUUID) { + RefreshTask(String index, final String indexUUID) { this.index = index; this.indexUUID = indexUUID; } } - static class RefreshTask extends MappingTask { - final String[] types; - - RefreshTask(String index, final String indexUUID, String[] types) { - super(index, indexUUID); - this.types = types; - } - } - - static class UpdateTask extends MappingTask { - final String type; - final CompressedXContent mappingSource; - final String nodeId; // null fr unknown - final ActionListener listener; - - UpdateTask(String index, String indexUUID, String type, CompressedXContent mappingSource, String nodeId, ActionListener listener) { - super(index, indexUUID); - this.type = type; - this.mappingSource = mappingSource; - this.nodeId = nodeId; - this.listener = listener; + class RefreshTaskExecutor implements ClusterStateTaskExecutor { + @Override + public BatchResult execute(ClusterState currentState, List tasks) throws Exception { + ClusterState newClusterState = executeRefresh(currentState, tasks); + return BatchResult.builder().successes(tasks).build(newClusterState); } } /** - * Batch method to apply all the queued refresh or update operations. The idea is to try and batch as much + * Batch method to apply all the queued refresh operations. The idea is to try and batch as much * as possible so we won't create the same index all the time for example for the updates on the same mapping * and generate a single cluster change event out of all of those. */ - Tuple> executeRefreshOrUpdate(final ClusterState currentState, final long insertionOrder) throws Exception { - final List allTasks = new ArrayList<>(); - - synchronized (refreshOrUpdateMutex) { - if (refreshOrUpdateQueue.isEmpty()) { - return Tuple.tuple(currentState, allTasks); - } - - // we already processed this task in a bulk manner in a previous cluster event, simply ignore - // it so we will let other tasks get in and processed ones, we will handle the queued ones - // later on in a subsequent cluster state event - if (insertionOrder < refreshOrUpdateProcessedInsertOrder) { - return Tuple.tuple(currentState, allTasks); - } - - allTasks.addAll(refreshOrUpdateQueue); - refreshOrUpdateQueue.clear(); - - refreshOrUpdateProcessedInsertOrder = refreshOrUpdateInsertOrder; - } - - if (allTasks.isEmpty()) { - return Tuple.tuple(currentState, allTasks); - } - + ClusterState executeRefresh(final ClusterState currentState, final List allTasks) throws Exception { // break down to tasks per index, so we can optimize the on demand index service creation // to only happen for the duration of a single index processing of its respective events - Map> tasksPerIndex = new HashMap<>(); - for (MappingTask task : allTasks) { + Map> tasksPerIndex = new HashMap<>(); + for (RefreshTask task : allTasks) { if (task.index == null) { logger.debug("ignoring a mapping task of type [{}] with a null index.", task); } - List indexTasks = tasksPerIndex.get(task.index); - if (indexTasks == null) { - indexTasks = new ArrayList<>(); - tasksPerIndex.put(task.index, indexTasks); - } - indexTasks.add(task); + tasksPerIndex.computeIfAbsent(task.index, k -> new ArrayList<>()).add(task); } boolean dirty = false; MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); - for (Map.Entry> entry : tasksPerIndex.entrySet()) { + for (Map.Entry> entry : tasksPerIndex.entrySet()) { String index = entry.getKey(); IndexMetaData indexMetaData = mdBuilder.get(index); if (indexMetaData == null) { @@ -157,14 +112,17 @@ public class MetaDataMappingService extends AbstractComponent { } // the tasks lists to iterate over, filled with the list of mapping tasks, trying to keep // the latest (based on order) update mapping one per node - List allIndexTasks = entry.getValue(); - List tasks = new ArrayList<>(); - for (MappingTask task : allIndexTasks) { - if (!indexMetaData.isSameUUID(task.indexUUID)) { + List allIndexTasks = entry.getValue(); + boolean hasTaskWithRightUUID = false; + for (RefreshTask task : allIndexTasks) { + if (indexMetaData.isSameUUID(task.indexUUID)) { + hasTaskWithRightUUID = true; + } else { logger.debug("[{}] ignoring task [{}] - index meta data doesn't match task uuid", index, task); - continue; } - tasks.add(task); + } + if (hasTaskWithRightUUID == false) { + continue; } // construct the actual index if needed, and make sure the relevant mappings are there @@ -172,28 +130,17 @@ public class MetaDataMappingService extends AbstractComponent { IndexService indexService = indicesService.indexService(index); if (indexService == null) { // we need to create the index here, and add the current mapping to it, so we can merge - indexService = indicesService.createIndex(indexMetaData); + indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, Collections.emptyList()); removeIndex = true; - Set typesToIntroduce = new HashSet<>(); - for (MappingTask task : tasks) { - if (task instanceof UpdateTask) { - typesToIntroduce.add(((UpdateTask) task).type); - } else if (task instanceof RefreshTask) { - Collections.addAll(typesToIntroduce, ((RefreshTask) task).types); - } - } - for (String type : typesToIntroduce) { - // only add the current relevant mapping (if exists) - if (indexMetaData.mappings().containsKey(type)) { - // don't apply the default mapping, it has been applied when the mapping was created - indexService.mapperService().merge(type, indexMetaData.mappings().get(type).source(), false, true); - } + for (ObjectCursor metaData : indexMetaData.getMappings().values()) { + // don't apply the default mapping, it has been applied when the mapping was created + indexService.mapperService().merge(metaData.value.type(), metaData.value.source(), false, true); } } IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData); try { - boolean indexDirty = processIndexMappingTasks(tasks, indexService, builder); + boolean indexDirty = refreshIndexMapping(indexService, builder); if (indexDirty) { mdBuilder.put(builder); dirty = true; @@ -206,81 +153,33 @@ public class MetaDataMappingService extends AbstractComponent { } if (!dirty) { - return Tuple.tuple(currentState, allTasks); + return currentState; } - return Tuple.tuple(ClusterState.builder(currentState).metaData(mdBuilder).build(), allTasks); + return ClusterState.builder(currentState).metaData(mdBuilder).build(); } - private boolean processIndexMappingTasks(List tasks, IndexService indexService, IndexMetaData.Builder builder) { + private boolean refreshIndexMapping(IndexService indexService, IndexMetaData.Builder builder) { boolean dirty = false; String index = indexService.index().name(); - // keep track of what we already refreshed, no need to refresh it again... - Set processedRefreshes = new HashSet<>(); - for (MappingTask task : tasks) { - if (task instanceof RefreshTask) { - RefreshTask refreshTask = (RefreshTask) task; - try { - List updatedTypes = new ArrayList<>(); - for (String type : refreshTask.types) { - if (processedRefreshes.contains(type)) { - continue; - } - DocumentMapper mapper = indexService.mapperService().documentMapper(type); - if (mapper == null) { - continue; - } - if (!mapper.mappingSource().equals(builder.mapping(type).source())) { - updatedTypes.add(type); - builder.putMapping(new MappingMetaData(mapper)); - } - processedRefreshes.add(type); - } - - if (updatedTypes.isEmpty()) { - continue; - } - - logger.warn("[{}] re-syncing mappings with cluster state for types [{}]", index, updatedTypes); - dirty = true; - } catch (Throwable t) { - logger.warn("[{}] failed to refresh-mapping in cluster state, types [{}]", index, refreshTask.types); + try { + List updatedTypes = new ArrayList<>(); + for (DocumentMapper mapper : indexService.mapperService().docMappers(true)) { + final String type = mapper.type(); + if (!mapper.mappingSource().equals(builder.mapping(type).source())) { + updatedTypes.add(type); } - } else if (task instanceof UpdateTask) { - UpdateTask updateTask = (UpdateTask) task; - try { - String type = updateTask.type; - CompressedXContent mappingSource = updateTask.mappingSource; - - MappingMetaData mappingMetaData = builder.mapping(type); - if (mappingMetaData != null && mappingMetaData.source().equals(mappingSource)) { - logger.debug("[{}] update_mapping [{}] ignoring mapping update task as its source is equal to ours", index, updateTask.type); - continue; - } - - DocumentMapper updatedMapper = indexService.mapperService().merge(type, mappingSource, false, true); - processedRefreshes.add(type); - - // if we end up with the same mapping as the original once, ignore - if (mappingMetaData != null && mappingMetaData.source().equals(updatedMapper.mappingSource())) { - logger.debug("[{}] update_mapping [{}] ignoring mapping update task as it results in the same source as what we have", index, updateTask.type); - continue; - } - - // build the updated mapping source - if (logger.isDebugEnabled()) { - logger.debug("[{}] update_mapping [{}] (dynamic) with source [{}]", index, type, updatedMapper.mappingSource()); - } else if (logger.isInfoEnabled()) { - logger.info("[{}] update_mapping [{}] (dynamic)", index, type); - } - - builder.putMapping(new MappingMetaData(updatedMapper)); - dirty = true; - } catch (Throwable t) { - logger.warn("[{}] failed to update-mapping in cluster state, type [{}]", index, updateTask.type); - } - } else { - logger.warn("illegal state, got wrong mapping task type [{}]", task); } + + // if a single type is not up-to-date, re-send everything + if (updatedTypes.isEmpty() == false) { + logger.warn("[{}] re-syncing mappings with cluster state because of types [{}]", index, updatedTypes); + dirty = true; + for (DocumentMapper mapper : indexService.mapperService().docMappers(true)) { + builder.putMapping(new MappingMetaData(mapper)); + } + } + } catch (Throwable t) { + logger.warn("[{}] failed to refresh-mapping in cluster state", t, index); } return dirty; } @@ -288,198 +187,198 @@ public class MetaDataMappingService extends AbstractComponent { /** * Refreshes mappings if they are not the same between original and parsed version */ - public void refreshMapping(final String index, final String indexUUID, final String... types) { - final long insertOrder; - synchronized (refreshOrUpdateMutex) { - insertOrder = ++refreshOrUpdateInsertOrder; - refreshOrUpdateQueue.add(new RefreshTask(index, indexUUID, types)); - } - clusterService.submitStateUpdateTask("refresh-mapping [" + index + "][" + Arrays.toString(types) + "]", Priority.HIGH, new ClusterStateUpdateTask() { - private volatile List allTasks; + public void refreshMapping(final String index, final String indexUUID) { + final RefreshTask refreshTask = new RefreshTask(index, indexUUID); + clusterService.submitStateUpdateTask("refresh-mapping [" + index + "]", + refreshTask, + ClusterStateTaskConfig.build(Priority.HIGH), + refreshExecutor, + (source, t) -> logger.warn("failure during [{}]", t, source) + ); + } - @Override - public void onFailure(String source, Throwable t) { - logger.warn("failure during [{}]", t, source); - } - - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - Tuple> tuple = executeRefreshOrUpdate(currentState, insertOrder); - this.allTasks = tuple.v2(); - return tuple.v1(); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - if (allTasks == null) { - return; + class PutMappingExecutor implements ClusterStateTaskExecutor { + @Override + public BatchResult execute(ClusterState currentState, List tasks) throws Exception { + Set indicesToClose = new HashSet<>(); + BatchResult.Builder builder = BatchResult.builder(); + try { + // precreate incoming indices; + for (PutMappingClusterStateUpdateRequest request : tasks) { + // failures here mean something is broken with our cluster state - fail all tasks by letting exceptions bubble up + for (String index : request.indices()) { + final IndexMetaData indexMetaData = currentState.metaData().index(index); + if (indexMetaData != null && indicesService.hasIndex(index) == false) { + // if we don't have the index, we will throw exceptions later; + indicesToClose.add(index); + IndexService indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, Collections.emptyList()); + // add mappings for all types, we need them for cross-type validation + for (ObjectCursor mapping : indexMetaData.getMappings().values()) { + indexService.mapperService().merge(mapping.value.type(), mapping.value.source(), false, request.updateAllTypes()); + } + } + } } - for (Object task : allTasks) { - if (task instanceof UpdateTask) { - UpdateTask uTask = (UpdateTask) task; - ClusterStateUpdateResponse response = new ClusterStateUpdateResponse(true); - uTask.listener.onResponse(response); + for (PutMappingClusterStateUpdateRequest request : tasks) { + try { + currentState = applyRequest(currentState, request); + builder.success(request); + } catch (Throwable t) { + builder.failure(request, t); + } + } + + return builder.build(currentState); + } finally { + for (String index : indicesToClose) { + indicesService.removeIndex(index, "created for mapping processing"); + } + } + } + + private ClusterState applyRequest(ClusterState currentState, PutMappingClusterStateUpdateRequest request) throws IOException { + Map newMappers = new HashMap<>(); + Map existingMappers = new HashMap<>(); + for (String index : request.indices()) { + IndexService indexService = indicesService.indexServiceSafe(index); + // try and parse it (no need to add it here) so we can bail early in case of parsing exception + DocumentMapper newMapper; + DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.type()); + if (MapperService.DEFAULT_MAPPING.equals(request.type())) { + // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default + newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), false); + } else { + newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), existingMapper == null); + if (existingMapper != null) { + // first, simulate + MergeResult mergeResult = existingMapper.merge(newMapper.mapping(), true, request.updateAllTypes()); + // if we have conflicts, throw an exception + if (mergeResult.hasConflicts()) { + throw new IllegalArgumentException("Merge failed with failures {" + Arrays.toString(mergeResult.buildConflicts()) + "}"); + } + } else { + // TODO: can we find a better place for this validation? + // The reason this validation is here is that the mapper service doesn't learn about + // new types all at once , which can create a false error. + + // For example in MapperService we can't distinguish between a create index api call + // and a put mapping api call, so we don't which type did exist before. + // Also the order of the mappings may be backwards. + if (newMapper.parentFieldMapper().active()) { + IndexMetaData indexMetaData = currentState.metaData().index(index); + for (ObjectCursor mapping : indexMetaData.getMappings().values()) { + if (newMapper.parentFieldMapper().type().equals(mapping.value.type())) { + throw new IllegalArgumentException("can't add a _parent field that points to an already existing type"); + } + } + } + } + } + newMappers.put(index, newMapper); + if (existingMapper != null) { + existingMappers.put(index, existingMapper); + } + } + + String mappingType = request.type(); + if (mappingType == null) { + mappingType = newMappers.values().iterator().next().type(); + } else if (!mappingType.equals(newMappers.values().iterator().next().type())) { + throw new InvalidTypeNameException("Type name provided does not match type name within mapping definition"); + } + if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && !PercolatorService.TYPE_NAME.equals(mappingType) && mappingType.charAt(0) == '_') { + throw new InvalidTypeNameException("Document mapping type name can't start with '_'"); + } + final Map mappings = new HashMap<>(); + for (Map.Entry entry : newMappers.entrySet()) { + String index = entry.getKey(); + // do the actual merge here on the master, and update the mapping source + DocumentMapper newMapper = entry.getValue(); + IndexService indexService = indicesService.indexService(index); + if (indexService == null) { + continue; + } + + CompressedXContent existingSource = null; + if (existingMappers.containsKey(entry.getKey())) { + existingSource = existingMappers.get(entry.getKey()).mappingSource(); + } + DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource(), false, request.updateAllTypes()); + CompressedXContent updatedSource = mergedMapper.mappingSource(); + + if (existingSource != null) { + if (existingSource.equals(updatedSource)) { + // same source, no changes, ignore it + } else { + // use the merged mapping source + mappings.put(index, new MappingMetaData(mergedMapper)); + if (logger.isDebugEnabled()) { + logger.debug("[{}] update_mapping [{}] with source [{}]", index, mergedMapper.type(), updatedSource); + } else if (logger.isInfoEnabled()) { + logger.info("[{}] update_mapping [{}]", index, mergedMapper.type()); + } + + } + } else { + mappings.put(index, new MappingMetaData(mergedMapper)); + if (logger.isDebugEnabled()) { + logger.debug("[{}] create_mapping [{}] with source [{}]", index, newMapper.type(), updatedSource); + } else if (logger.isInfoEnabled()) { + logger.info("[{}] create_mapping [{}]", index, newMapper.type()); } } } - }); + if (mappings.isEmpty()) { + // no changes, return + return currentState; + } + MetaData.Builder builder = MetaData.builder(currentState.metaData()); + for (String indexName : request.indices()) { + IndexMetaData indexMetaData = currentState.metaData().index(indexName); + if (indexMetaData == null) { + throw new IndexNotFoundException(indexName); + } + MappingMetaData mappingMd = mappings.get(indexName); + if (mappingMd != null) { + builder.put(IndexMetaData.builder(indexMetaData).putMapping(mappingMd)); + } + } + + return ClusterState.builder(currentState).metaData(builder).build(); + } } public void putMapping(final PutMappingClusterStateUpdateRequest request, final ActionListener listener) { + clusterService.submitStateUpdateTask("put-mapping [" + request.type() + "]", + request, + ClusterStateTaskConfig.build(Priority.HIGH, request.masterNodeTimeout()), + putMappingExecutor, + new AckedClusterStateTaskListener() { - clusterService.submitStateUpdateTask("put-mapping [" + request.type() + "]", Priority.HIGH, new AckedClusterStateUpdateTask(request, listener) { - - @Override - protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { - return new ClusterStateUpdateResponse(acknowledged); - } - - @Override - public ClusterState execute(final ClusterState currentState) throws Exception { - List indicesToClose = new ArrayList<>(); - try { - for (String index : request.indices()) { - if (!currentState.metaData().hasIndex(index)) { - throw new IndexNotFoundException(index); - } + @Override + public void onFailure(String source, Throwable t) { + listener.onFailure(t); } - // pre create indices here and add mappings to them so we can merge the mappings here if needed - for (String index : request.indices()) { - if (indicesService.hasIndex(index)) { - continue; - } - final IndexMetaData indexMetaData = currentState.metaData().index(index); - IndexService indexService = indicesService.createIndex(indexMetaData); - indicesToClose.add(indexMetaData.index()); - // make sure to add custom default mapping if exists - if (indexMetaData.mappings().containsKey(MapperService.DEFAULT_MAPPING)) { - indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, indexMetaData.mappings().get(MapperService.DEFAULT_MAPPING).source(), false, request.updateAllTypes()); - } - // only add the current relevant mapping (if exists) - if (indexMetaData.mappings().containsKey(request.type())) { - indexService.mapperService().merge(request.type(), indexMetaData.mappings().get(request.type()).source(), false, request.updateAllTypes()); - } + @Override + public boolean mustAck(DiscoveryNode discoveryNode) { + return true; } - Map newMappers = new HashMap<>(); - Map existingMappers = new HashMap<>(); - for (String index : request.indices()) { - IndexService indexService = indicesService.indexServiceSafe(index); - // try and parse it (no need to add it here) so we can bail early in case of parsing exception - DocumentMapper newMapper; - DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.type()); - if (MapperService.DEFAULT_MAPPING.equals(request.type())) { - // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default - newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), false); - } else { - newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), existingMapper == null); - if (existingMapper != null) { - // first, simulate - MergeResult mergeResult = existingMapper.merge(newMapper.mapping(), true, request.updateAllTypes()); - // if we have conflicts, throw an exception - if (mergeResult.hasConflicts()) { - throw new MergeMappingException(mergeResult.buildConflicts()); - } - } else { - // TODO: can we find a better place for this validation? - // The reason this validation is here is that the mapper service doesn't learn about - // new types all at once , which can create a false error. - - // For example in MapperService we can't distinguish between a create index api call - // and a put mapping api call, so we don't which type did exist before. - // Also the order of the mappings may be backwards. - if (newMapper.parentFieldMapper().active()) { - IndexMetaData indexMetaData = currentState.metaData().index(index); - for (ObjectCursor mapping : indexMetaData.mappings().values()) { - if (newMapper.parentFieldMapper().type().equals(mapping.value.type())) { - throw new IllegalArgumentException("can't add a _parent field that points to an already existing type"); - } - } - } - } - } - - - newMappers.put(index, newMapper); - if (existingMapper != null) { - existingMappers.put(index, existingMapper); - } + @Override + public void onAllNodesAcked(@Nullable Throwable t) { + listener.onResponse(new ClusterStateUpdateResponse(true)); } - String mappingType = request.type(); - if (mappingType == null) { - mappingType = newMappers.values().iterator().next().type(); - } else if (!mappingType.equals(newMappers.values().iterator().next().type())) { - throw new InvalidTypeNameException("Type name provided does not match type name within mapping definition"); - } - if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && !PercolatorService.TYPE_NAME.equals(mappingType) && mappingType.charAt(0) == '_') { - throw new InvalidTypeNameException("Document mapping type name can't start with '_'"); + @Override + public void onAckTimeout() { + listener.onResponse(new ClusterStateUpdateResponse(false)); } - final Map mappings = new HashMap<>(); - for (Map.Entry entry : newMappers.entrySet()) { - String index = entry.getKey(); - // do the actual merge here on the master, and update the mapping source - DocumentMapper newMapper = entry.getValue(); - IndexService indexService = indicesService.indexService(index); - if (indexService == null) { - continue; - } - - CompressedXContent existingSource = null; - if (existingMappers.containsKey(entry.getKey())) { - existingSource = existingMappers.get(entry.getKey()).mappingSource(); - } - DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource(), false, request.updateAllTypes()); - CompressedXContent updatedSource = mergedMapper.mappingSource(); - - if (existingSource != null) { - if (existingSource.equals(updatedSource)) { - // same source, no changes, ignore it - } else { - // use the merged mapping source - mappings.put(index, new MappingMetaData(mergedMapper)); - if (logger.isDebugEnabled()) { - logger.debug("[{}] update_mapping [{}] with source [{}]", index, mergedMapper.type(), updatedSource); - } else if (logger.isInfoEnabled()) { - logger.info("[{}] update_mapping [{}]", index, mergedMapper.type()); - } - } - } else { - mappings.put(index, new MappingMetaData(mergedMapper)); - if (logger.isDebugEnabled()) { - logger.debug("[{}] create_mapping [{}] with source [{}]", index, newMapper.type(), updatedSource); - } else if (logger.isInfoEnabled()) { - logger.info("[{}] create_mapping [{}]", index, newMapper.type()); - } - } + @Override + public TimeValue ackTimeout() { + return request.ackTimeout(); } - - if (mappings.isEmpty()) { - // no changes, return - return currentState; - } - - MetaData.Builder builder = MetaData.builder(currentState.metaData()); - for (String indexName : request.indices()) { - IndexMetaData indexMetaData = currentState.metaData().index(indexName); - if (indexMetaData == null) { - throw new IndexNotFoundException(indexName); - } - MappingMetaData mappingMd = mappings.get(indexName); - if (mappingMd != null) { - builder.put(IndexMetaData.builder(indexMetaData).putMapping(mappingMd)); - } - } - - return ClusterState.builder(currentState).metaData(builder).build(); - } finally { - for (String index : indicesToClose) { - indicesService.removeIndex(index, "created for mapping processing"); - } - } - } - }); + }); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java index 58dffd83614..eaa1eefd25e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java @@ -24,11 +24,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeSettingsClusterStateUpdateRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.routing.RoutingTable; @@ -44,13 +40,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.settings.IndexDynamicSettings; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -93,7 +83,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements // we need to do this each time in case it was changed by update settings for (final IndexMetaData indexMetaData : event.state().metaData()) { - String autoExpandReplicas = indexMetaData.settings().get(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS); + String autoExpandReplicas = indexMetaData.getSettings().get(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS); if (autoExpandReplicas != null && Booleans.parseBoolean(autoExpandReplicas, true)) { // Booleans only work for false values, just as we want it here try { final int min; @@ -102,7 +92,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements final int dash = autoExpandReplicas.indexOf('-'); if (-1 == dash) { logger.warn("failed to set [{}] for index [{}], it should be dash delimited [{}]", - IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, indexMetaData.index(), autoExpandReplicas); + IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, indexMetaData.getIndex(), autoExpandReplicas); continue; } final String sMin = autoExpandReplicas.substring(0, dash); @@ -110,7 +100,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements min = Integer.parseInt(sMin); } catch (NumberFormatException e) { logger.warn("failed to set [{}] for index [{}], minimum value is not a number [{}]", - e, IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, indexMetaData.index(), sMin); + e, IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, indexMetaData.getIndex(), sMin); continue; } String sMax = autoExpandReplicas.substring(dash + 1); @@ -121,7 +111,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements max = Integer.parseInt(sMax); } catch (NumberFormatException e) { logger.warn("failed to set [{}] for index [{}], maximum value is neither [{}] nor a number [{}]", - e, IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, indexMetaData.index(), ALL_NODES_VALUE, sMax); + e, IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, indexMetaData.getIndex(), ALL_NODES_VALUE, sMax); continue; } } @@ -134,7 +124,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements } // same value, nothing to do there - if (numberOfReplicas == indexMetaData.numberOfReplicas()) { + if (numberOfReplicas == indexMetaData.getNumberOfReplicas()) { continue; } @@ -144,10 +134,10 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements nrReplicasChanged.put(numberOfReplicas, new ArrayList()); } - nrReplicasChanged.get(numberOfReplicas).add(indexMetaData.index()); + nrReplicasChanged.get(numberOfReplicas).add(indexMetaData.getIndex()); } } catch (Exception e) { - logger.warn("[{}] failed to parse auto expand replicas", e, indexMetaData.index()); + logger.warn("[{}] failed to parse auto expand replicas", e, indexMetaData.getIndex()); } } } @@ -219,7 +209,8 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements } final Settings openSettings = updatedSettingsBuilder.build(); - clusterService.submitStateUpdateTask("update-settings", Priority.URGENT, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("update-settings", + new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { @Override protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { @@ -237,7 +228,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements Set openIndices = new HashSet<>(); Set closeIndices = new HashSet<>(); for (String index : actualIndices) { - if (currentState.metaData().index(index).state() == IndexMetaData.State.OPEN) { + if (currentState.metaData().index(index).getState() == IndexMetaData.State.OPEN) { openIndices.add(index); } else { closeIndices.add(index); @@ -323,7 +314,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements ClusterState updatedState = ClusterState.builder(currentState).metaData(metaDataBuilder).routingTable(routingTableBuilder.build()).blocks(blocks).build(); // now, reroute in case things change that require it (like number of replicas) - RoutingAllocation.Result routingResult = allocationService.reroute(updatedState); + RoutingAllocation.Result routingResult = allocationService.reroute(updatedState, "settings update"); updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build(); return updatedState; @@ -334,7 +325,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements public void upgradeIndexSettings(final UpgradeSettingsClusterStateUpdateRequest request, final ActionListener listener) { - clusterService.submitStateUpdateTask("update-index-compatibility-versions", Priority.URGENT, new AckedClusterStateUpdateTask(request, listener) { + clusterService.submitStateUpdateTask("update-index-compatibility-versions", new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { @Override protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { @@ -348,10 +339,10 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements String index = entry.getKey(); IndexMetaData indexMetaData = metaDataBuilder.get(index); if (indexMetaData != null) { - if (Version.CURRENT.equals(indexMetaData.creationVersion()) == false) { + if (Version.CURRENT.equals(indexMetaData.getCreationVersion()) == false) { // No reason to pollute the settings, we didn't really upgrade anything metaDataBuilder.put(IndexMetaData.builder(indexMetaData) - .settings(settingsBuilder().put(indexMetaData.settings()) + .settings(settingsBuilder().put(indexMetaData.getSettings()) .put(IndexMetaData.SETTING_VERSION_MINIMUM_COMPATIBLE, entry.getValue().v2()) .put(IndexMetaData.SETTING_VERSION_UPGRADED, entry.getValue().v1()) ) diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java index efc987a59af..528ed8b1c3f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java @@ -19,13 +19,18 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.Objects; /** * Uniquely identifies an allocation. An allocation is a shard moving from unassigned to initializing, @@ -37,8 +42,35 @@ import java.io.IOException; * behavior to how ShardRouting#currentNodeId is used. */ public class AllocationId implements ToXContent { + private static final String ID_KEY = "id"; + private static final String RELOCATION_ID_KEY = "relocation_id"; + + private static final ObjectParser ALLOCATION_ID_PARSER = new ObjectParser<>("allocationId"); + + static { + ALLOCATION_ID_PARSER.declareString(AllocationId.Builder::setId, new ParseField(ID_KEY)); + ALLOCATION_ID_PARSER.declareString(AllocationId.Builder::setRelocationId, new ParseField(RELOCATION_ID_KEY)); + } + + private static class Builder { + private String id; + private String relocationId; + + public void setId(String id) { + this.id = id; + } + + public void setRelocationId(String relocationId) { + this.relocationId = relocationId; + } + + public AllocationId build() { + return new AllocationId(id, relocationId); + } + } private final String id; + @Nullable private final String relocationId; AllocationId(StreamInput in) throws IOException { @@ -52,6 +84,7 @@ public class AllocationId implements ToXContent { } private AllocationId(String id, String relocationId) { + Objects.requireNonNull(id, "Argument [id] must be non-null"); this.id = id; this.relocationId = relocationId; } @@ -148,12 +181,16 @@ public class AllocationId implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("allocation_id"); - builder.field("id", id); + builder.startObject(); + builder.field(ID_KEY, id); if (relocationId != null) { - builder.field("relocation_id", relocationId); + builder.field(RELOCATION_ID_KEY, relocationId); } builder.endObject(); return builder; } + + public static AllocationId fromXContent(XContentParser parser) throws IOException { + return ALLOCATION_ID_PARSER.parse(parser, new AllocationId.Builder()).build(); + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 42b12930f71..ca071c811e3 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -130,9 +130,9 @@ public class IndexRoutingTable extends AbstractDiffable imple ArrayList failures = new ArrayList<>(); // check the number of shards - if (indexMetaData.numberOfShards() != shards().size()) { + if (indexMetaData.getNumberOfShards() != shards().size()) { Set expected = new HashSet<>(); - for (int i = 0; i < indexMetaData.numberOfShards(); i++) { + for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) { expected.add(i); } for (IndexShardRoutingTable indexShardRoutingTable : this) { @@ -143,9 +143,9 @@ public class IndexRoutingTable extends AbstractDiffable imple // check the replicas for (IndexShardRoutingTable indexShardRoutingTable : this) { int routingNumberOfReplicas = indexShardRoutingTable.size() - 1; - if (routingNumberOfReplicas != indexMetaData.numberOfReplicas()) { + if (routingNumberOfReplicas != indexMetaData.getNumberOfReplicas()) { failures.add("Shard [" + indexShardRoutingTable.shardId().id() - + "] routing table has wrong number of replicas, expected [" + indexMetaData.numberOfReplicas() + "], got [" + routingNumberOfReplicas + "]"); + + "] routing table has wrong number of replicas, expected [" + indexMetaData.getNumberOfReplicas() + "], got [" + routingNumberOfReplicas + "]"); } for (ShardRouting shardRouting : indexShardRoutingTable) { if (!shardRouting.index().equals(index())) { @@ -419,9 +419,9 @@ public class IndexRoutingTable extends AbstractDiffable imple if (!shards.isEmpty()) { throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } - for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) { - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId)); - for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) { + for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) { + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId)); + for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { if (asNew && ignoreShards.contains(shardId)) { // This shards wasn't completely snapshotted - restore it as new shard indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo)); @@ -441,9 +441,9 @@ public class IndexRoutingTable extends AbstractDiffable imple if (!shards.isEmpty()) { throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } - for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) { - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.index(), shardId)); - for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) { + for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) { + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId)); + for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo)); } shards.put(shardId, indexShardRoutingBuilder.build()); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index c142b754aa2..267dae80d55 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.math.MathUtils; @@ -55,19 +54,16 @@ public class OperationRouting extends AbstractComponent { } public ShardIterator indexShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing) { - return shards(clusterState, index, type, id, routing).shardsIt(); - } - - public ShardIterator deleteShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing) { - return shards(clusterState, index, type, id, routing).shardsIt(); + return shards(clusterState, index, id, routing).shardsIt(); } public ShardIterator getShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing, @Nullable String preference) { - return preferenceActiveShardIterator(shards(clusterState, index, type, id, routing), clusterState.nodes().localNodeId(), clusterState.nodes(), preference); + return preferenceActiveShardIterator(shards(clusterState, index, id, routing), clusterState.nodes().localNodeId(), clusterState.nodes(), preference); } public ShardIterator getShards(ClusterState clusterState, String index, int shardId, @Nullable String preference) { - return preferenceActiveShardIterator(shards(clusterState, index, shardId), clusterState.nodes().localNodeId(), clusterState.nodes(), preference); + final IndexShardRoutingTable indexShard = clusterState.getRoutingTable().shardRoutingTable(index, shardId); + return preferenceActiveShardIterator(indexShard, clusterState.nodes().localNodeId(), clusterState.nodes(), preference); } public GroupShardsIterator broadcastDeleteShards(ClusterState clusterState, String index) { @@ -102,7 +98,7 @@ public class OperationRouting extends AbstractComponent { final Set effectiveRouting = routing.get(index); if (effectiveRouting != null) { for (String r : effectiveRouting) { - int shardId = shardId(clusterState, index, null, null, r); + int shardId = generateShardId(clusterState, index, null, r); IndexShardRoutingTable indexShard = indexRouting.shard(shardId); if (indexShard == null) { throw new ShardNotFoundException(new ShardId(index, shardId)); @@ -200,14 +196,6 @@ public class OperationRouting extends AbstractComponent { } } - public IndexMetaData indexMetaData(ClusterState clusterState, String index) { - IndexMetaData indexMetaData = clusterState.metaData().index(index); - if (indexMetaData == null) { - throw new IndexNotFoundException(index); - } - return indexMetaData; - } - protected IndexRoutingTable indexRoutingTable(ClusterState clusterState, String index) { IndexRoutingTable indexRouting = clusterState.routingTable().index(index); if (indexRouting == null) { @@ -216,32 +204,27 @@ public class OperationRouting extends AbstractComponent { return indexRouting; } - - // either routing is set, or type/id are set - - protected IndexShardRoutingTable shards(ClusterState clusterState, String index, String type, String id, String routing) { - int shardId = shardId(clusterState, index, type, id, routing); - return shards(clusterState, index, shardId); + protected IndexShardRoutingTable shards(ClusterState clusterState, String index, String id, String routing) { + int shardId = generateShardId(clusterState, index, id, routing); + return clusterState.getRoutingTable().shardRoutingTable(index, shardId); } - protected IndexShardRoutingTable shards(ClusterState clusterState, String index, int shardId) { - IndexShardRoutingTable indexShard = indexRoutingTable(clusterState, index).shard(shardId); - if (indexShard == null) { - throw new ShardNotFoundException(new ShardId(index, shardId)); + public ShardId shardId(ClusterState clusterState, String index, String id, @Nullable String routing) { + return new ShardId(index, generateShardId(clusterState, index, id, routing)); + } + + private int generateShardId(ClusterState clusterState, String index, String id, @Nullable String routing) { + IndexMetaData indexMetaData = clusterState.metaData().index(index); + if (indexMetaData == null) { + throw new IndexNotFoundException(index); } - return indexShard; - } - - @SuppressForbidden(reason = "Math#abs is trappy") - private int shardId(ClusterState clusterState, String index, String type, String id, @Nullable String routing) { - final IndexMetaData indexMetaData = indexMetaData(clusterState, index); final int hash; if (routing == null) { hash = Murmur3HashFunction.hash(id); } else { hash = Murmur3HashFunction.hash(routing); } - return MathUtils.mod(hash, indexMetaData.numberOfShards()); + return MathUtils.mod(hash, indexMetaData.getNumberOfShards()); } private void ensureNodeIdExists(DiscoveryNodes nodes, String nodeId) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java b/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java index 01bbfc33558..c091f71798b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java @@ -68,7 +68,7 @@ public class RestoreSource implements Streamable, ToXContent { } public static RestoreSource readOptionalRestoreSource(StreamInput in) throws IOException { - return in.readOptionalStreamable(new RestoreSource()); + return in.readOptionalStreamable(RestoreSource::new); } @Override diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index d5ed922b120..5d17a59339a 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -27,18 +27,11 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.shard.ShardId; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.function.Predicate; /** @@ -183,13 +176,7 @@ public class RoutingNodes implements Iterable { return this.customs; } - public T custom(String type) { - return (T) customs.get(type); - } - - public boolean hasUnassigned() { - return !unassignedShards.isEmpty(); - } + public T custom(String type) { return (T) customs.get(type); } public UnassignedShards unassigned() { return this.unassignedShards; @@ -217,12 +204,22 @@ public class RoutingNodes implements Iterable { return nodesPerAttributesCounts; } + /** + * Returns true iff this {@link RoutingNodes} instance has any unassigned primaries even if the + * primaries are marked as temporarily ignored. + */ public boolean hasUnassignedPrimaries() { - return unassignedShards.numPrimaries() > 0; + return unassignedShards.getNumPrimaries() + unassignedShards.getNumIgnoredPrimaries() > 0; } + /** + * Returns true iff this {@link RoutingNodes} instance has any unassigned shards even if the + * shards are marked as temporarily ignored. + * @see UnassignedShards#isEmpty() + * @see UnassignedShards#isIgnoredEmpty() + */ public boolean hasUnassignedShards() { - return !unassignedShards.isEmpty(); + return unassignedShards.isEmpty() == false || unassignedShards.isIgnoredEmpty() == false; } public boolean hasInactivePrimaries() { @@ -524,25 +521,12 @@ public class RoutingNodes implements Iterable { private final List ignored; private int primaries = 0; - private long transactionId = 0; - private final UnassignedShards source; - private final long sourceTransactionId; - - public UnassignedShards(UnassignedShards other) { - this.nodes = other.nodes; - source = other; - sourceTransactionId = other.transactionId; - unassigned = new ArrayList<>(other.unassigned); - ignored = new ArrayList<>(other.ignored); - primaries = other.primaries; - } + private int ignoredPrimaries = 0; public UnassignedShards(RoutingNodes nodes) { this.nodes = nodes; unassigned = new ArrayList<>(); ignored = new ArrayList<>(); - source = null; - sourceTransactionId = -1; } public void add(ShardRouting shardRouting) { @@ -550,21 +534,34 @@ public class RoutingNodes implements Iterable { primaries++; } unassigned.add(shardRouting); - transactionId++; } public void sort(Comparator comparator) { CollectionUtil.timSort(unassigned, comparator); } - public int size() { - return unassigned.size(); - } + /** + * Returns the size of the non-ignored unassigned shards + */ + public int size() { return unassigned.size(); } - public int numPrimaries() { + /** + * Returns the size of the temporarily marked as ignored unassigned shards + */ + public int ignoredSize() { return ignored.size(); } + + /** + * Returns the number of non-ignored unassigned primaries + */ + public int getNumPrimaries() { return primaries; } + /** + * Returns the number of temporarily marked as ignored unassigned primaries + */ + public int getNumIgnoredPrimaries() { return ignoredPrimaries; } + @Override public UnassignedIterator iterator() { return new UnassignedIterator(); @@ -580,12 +577,18 @@ public class RoutingNodes implements Iterable { } /** - * Adds a shard to the ignore unassigned list. Should be used with caution, typically, + * Marks a shard as temporarily ignored and adds it to the ignore unassigned list. + * Should be used with caution, typically, * the correct usage is to removeAndIgnore from the iterator. + * @see #ignored() + * @see UnassignedIterator#removeAndIgnore() + * @see #isIgnoredEmpty() */ public void ignoreShard(ShardRouting shard) { + if (shard.primary()) { + ignoredPrimaries++; + } ignored.add(shard); - transactionId++; } public class UnassignedIterator implements Iterator { @@ -618,6 +621,8 @@ public class RoutingNodes implements Iterable { /** * Removes and ignores the unassigned shard (will be ignored for this run, but * will be added back to unassigned once the metadata is constructed again). + * Typically this is used when an allocation decision prevents a shard from being allocated such + * that subsequent consumers of this API won't try to allocate this shard again. */ public void removeAndIgnore() { innerRemove(); @@ -639,45 +644,37 @@ public class RoutingNodes implements Iterable { if (current.primary()) { primaries--; } - transactionId++; } } + /** + * Returns true iff this collection contains one or more non-ignored unassigned shards. + */ public boolean isEmpty() { return unassigned.isEmpty(); } + /** + * Returns true iff any unassigned shards are marked as temporarily ignored. + * @see UnassignedShards#ignoreShard(ShardRouting) + * @see UnassignedIterator#removeAndIgnore() + */ + public boolean isIgnoredEmpty() { + return ignored.isEmpty(); + } + public void shuffle() { - Collections.shuffle(unassigned); - } - - public void clear() { - transactionId++; - unassigned.clear(); - ignored.clear(); - primaries = 0; - } - - public void transactionEnd(UnassignedShards shards) { - assert shards.source == this && shards.sourceTransactionId == transactionId : - "Expected ID: " + shards.sourceTransactionId + " actual: " + transactionId + " Expected Source: " + shards.source + " actual: " + this; - transactionId++; - this.unassigned.clear(); - this.unassigned.addAll(shards.unassigned); - this.ignored.clear(); - this.ignored.addAll(shards.ignored); - this.primaries = shards.primaries; - } - - public UnassignedShards transactionBegin() { - return new UnassignedShards(this); + Randomness.shuffle(unassigned); } + /** + * Drains all unassigned shards and returns it. + * This method will not drain ignored shards. + */ public ShardRouting[] drain() { ShardRouting[] mutableShardRoutings = unassigned.toArray(new ShardRouting[unassigned.size()]); unassigned.clear(); primaries = 0; - transactionId++; return mutableShardRoutings; } } @@ -698,10 +695,10 @@ public class RoutingNodes implements Iterable { return true; } int unassignedPrimaryCount = 0; + int unassignedIgnoredPrimaryCount = 0; int inactivePrimaryCount = 0; int inactiveShardCount = 0; int relocating = 0; - final Set seenShards = new HashSet<>(); Map indicesAndShards = new HashMap<>(); for (RoutingNode node : routingNodes) { for (ShardRouting shard : node) { @@ -716,7 +713,6 @@ public class RoutingNodes implements Iterable { if (shard.relocating()) { relocating++; } - seenShards.add(shard.shardId()); Integer i = indicesAndShards.get(shard.index()); if (i == null) { i = shard.id(); @@ -751,11 +747,18 @@ public class RoutingNodes implements Iterable { if (shard.primary()) { unassignedPrimaryCount++; } - seenShards.add(shard.shardId()); } - assert unassignedPrimaryCount == routingNodes.unassignedShards.numPrimaries() : - "Unassigned primaries is [" + unassignedPrimaryCount + "] but RoutingNodes returned unassigned primaries [" + routingNodes.unassigned().numPrimaries() + "]"; + for (ShardRouting shard : routingNodes.unassigned().ignored()) { + if (shard.primary()) { + unassignedIgnoredPrimaryCount++; + } + } + + assert unassignedPrimaryCount == routingNodes.unassignedShards.getNumPrimaries() : + "Unassigned primaries is [" + unassignedPrimaryCount + "] but RoutingNodes returned unassigned primaries [" + routingNodes.unassigned().getNumPrimaries() + "]"; + assert unassignedIgnoredPrimaryCount == routingNodes.unassignedShards.getNumIgnoredPrimaries() : + "Unassigned ignored primaries is [" + unassignedIgnoredPrimaryCount + "] but RoutingNodes returned unassigned ignored primaries [" + routingNodes.unassigned().getNumIgnoredPrimaries() + "]"; assert inactivePrimaryCount == routingNodes.inactivePrimaryCount : "Inactive Primary count [" + inactivePrimaryCount + "] but RoutingNodes returned inactive primaries [" + routingNodes.inactivePrimaryCount + "]"; assert inactiveShardCount == routingNodes.inactiveShardCount : diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java index a3aa9b2ed07..5cd4366bea4 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java @@ -55,9 +55,8 @@ public class RoutingService extends AbstractLifecycleComponent i private final AllocationService allocationService; private AtomicBoolean rerouting = new AtomicBoolean(); - private volatile long registeredNextDelaySetting = Long.MAX_VALUE; + private volatile long minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; private volatile ScheduledFuture registeredNextDelayFuture; - private volatile long unassignedShardsAllocatedTimestamp = 0; @Inject public RoutingService(Settings settings, ThreadPool threadPool, ClusterService clusterService, AllocationService allocationService) { @@ -88,19 +87,6 @@ public class RoutingService extends AbstractLifecycleComponent i return this.allocationService; } - /** - * Update the last time the allocator tried to assign unassigned shards - * - * This is used so that both the GatewayAllocator and RoutingService use a - * consistent timestamp for comparing which shards have been delayed to - * avoid a race condition where GatewayAllocator thinks the shard should - * be delayed and the RoutingService thinks it has already passed the delay - * and that the GatewayAllocator has/will handle it. - */ - public void setUnassignedShardsAllocatedTimestamp(long timeInMillis) { - this.unassignedShardsAllocatedTimestamp = timeInMillis; - } - /** * Initiates a reroute. */ @@ -110,50 +96,44 @@ public class RoutingService extends AbstractLifecycleComponent i @Override public void clusterChanged(ClusterChangedEvent event) { - if (event.source().startsWith(CLUSTER_UPDATE_TASK_SOURCE)) { - // that's us, ignore this event - return; - } if (event.state().nodes().localNodeMaster()) { - // figure out when the next unassigned allocation need to happen from now. If this is larger or equal - // then the last time we checked and scheduled, we are guaranteed to have a reroute until then, so no need - // to schedule again - long nextDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSetting(settings, event.state()); - if (nextDelaySetting > 0 && nextDelaySetting < registeredNextDelaySetting) { + // Figure out if an existing scheduled reroute is good enough or whether we need to cancel and reschedule. + // If the minimum of the currently relevant delay settings is larger than something we scheduled in the past, + // we are guaranteed that the planned schedule will happen before any of the current shard delays are expired. + long minDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSettingNanos(settings, event.state()); + if (minDelaySetting <= 0) { + logger.trace("no need to schedule reroute - no delayed unassigned shards, minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastSchedulingNanos); + minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; FutureUtils.cancel(registeredNextDelayFuture); - registeredNextDelaySetting = nextDelaySetting; - // We use System.currentTimeMillis here because we want the - // next delay from the "now" perspective, rather than the - // delay from the last time the GatewayAllocator tried to - // assign/delay the shard - TimeValue nextDelay = TimeValue.timeValueMillis(UnassignedInfo.findNextDelayedAllocationIn(System.currentTimeMillis(), settings, event.state())); - int unassignedDelayedShards = UnassignedInfo.getNumberOfDelayedUnassigned(unassignedShardsAllocatedTimestamp, settings, event.state()); - if (unassignedDelayedShards > 0) { - logger.info("delaying allocation for [{}] unassigned shards, next check in [{}]", - unassignedDelayedShards, nextDelay); - registeredNextDelayFuture = threadPool.schedule(nextDelay, ThreadPool.Names.SAME, new AbstractRunnable() { - @Override - protected void doRun() throws Exception { - registeredNextDelaySetting = Long.MAX_VALUE; - reroute("assign delayed unassigned shards"); - } + } else if (minDelaySetting < minDelaySettingAtLastSchedulingNanos) { + FutureUtils.cancel(registeredNextDelayFuture); + minDelaySettingAtLastSchedulingNanos = minDelaySetting; + TimeValue nextDelay = TimeValue.timeValueNanos(UnassignedInfo.findNextDelayedAllocationIn(event.state())); + assert nextDelay.nanos() > 0 : "next delay must be non 0 as minDelaySetting is [" + minDelaySetting + "]"; + logger.info("delaying allocation for [{}] unassigned shards, next check in [{}]", + UnassignedInfo.getNumberOfDelayedUnassigned(event.state()), nextDelay); + registeredNextDelayFuture = threadPool.schedule(nextDelay, ThreadPool.Names.SAME, new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; + reroute("assign delayed unassigned shards"); + } - @Override - public void onFailure(Throwable t) { - logger.warn("failed to schedule/execute reroute post unassigned shard", t); - registeredNextDelaySetting = Long.MAX_VALUE; - } - }); - } + @Override + public void onFailure(Throwable t) { + logger.warn("failed to schedule/execute reroute post unassigned shard", t); + minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; + } + }); } else { - logger.trace("no need to schedule reroute due to delayed unassigned, next_delay_setting [{}], registered [{}]", nextDelaySetting, registeredNextDelaySetting); + logger.trace("no need to schedule reroute - current schedule reroute is enough. minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastSchedulingNanos); } } } // visible for testing - long getRegisteredNextDelaySetting() { - return this.registeredNextDelaySetting; + long getMinDelaySettingAtLastSchedulingNanos() { + return this.minDelaySettingAtLastSchedulingNanos; } // visible for testing @@ -167,11 +147,11 @@ public class RoutingService extends AbstractLifecycleComponent i return; } logger.trace("rerouting {}", reason); - clusterService.submitStateUpdateTask(CLUSTER_UPDATE_TASK_SOURCE + "(" + reason + ")", Priority.HIGH, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask(CLUSTER_UPDATE_TASK_SOURCE + "(" + reason + ")", new ClusterStateUpdateTask(Priority.HIGH) { @Override public ClusterState execute(ClusterState currentState) { rerouting.set(false); - RoutingAllocation.Result routingResult = allocationService.reroute(currentState); + RoutingAllocation.Result routingResult = allocationService.reroute(currentState, reason); if (!routingResult.changed()) { // no state changed return currentState; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 10d7ff9deaf..fbabacd79fd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -33,6 +33,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -95,6 +97,24 @@ public class RoutingTable implements Iterable, Diffable, Diffable ACTIVE_PREDICATE = shardRouting -> shardRouting.active(); private static Predicate ASSIGNED_PREDICATE = shardRouting -> shardRouting.assignedToNode(); - // TODO: replace with JDK 8 native java.util.function.Predicate private GroupShardsIterator allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate predicate) { // use list here since we need to maintain identity across shards ArrayList set = new ArrayList<>(); @@ -222,7 +241,6 @@ public class RoutingTable implements Iterable, Diffable true, true); } - // TODO: replace with JDK 8 native java.util.function.Predicate private ShardsIterator allShardsSatisfyingPredicate(String[] indices, Predicate predicate, boolean includeRelocationTargets) { // use list here since we need to maintain identity across shards List shards = new ArrayList<>(); @@ -316,12 +334,12 @@ public class RoutingTable implements Iterable, Diffable, Diffable, Diffable, Diffable, Diffable, Diffabletrue if this shard is a relocation target for another shard (i.e., was created with {@link #buildTargetRelocatingShard()} + */ + public boolean isRelocationTarget() { + return state == ShardRoutingState.INITIALIZING && relocatingNodeId != null; + } + /** returns true if the routing is the relocation target of the given routing */ public boolean isRelocationTargetOf(ShardRouting other) { boolean b = this.allocationId != null && other.allocationId != null && this.state == ShardRoutingState.INITIALIZING && - this.allocationId.getId().equals(other.allocationId.getRelocationId()); + this.allocationId.getId().equals(other.allocationId.getRelocationId()); assert b == false || other.state == ShardRoutingState.RELOCATING : - "ShardRouting is a relocation target but the source shard state isn't relocating. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation target but the source shard state isn't relocating. This [" + this + "], other [" + other + "]"; assert b == false || other.allocationId.getId().equals(this.allocationId.getRelocationId()) : - "ShardRouting is a relocation target but the source id isn't equal to source's allocationId.getRelocationId. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation target but the source id isn't equal to source's allocationId.getRelocationId. This [" + this + "], other [" + other + "]"; assert b == false || other.currentNodeId().equals(this.relocatingNodeId) : - "ShardRouting is a relocation target but source current node id isn't equal to target relocating node. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation target but source current node id isn't equal to target relocating node. This [" + this + "], other [" + other + "]"; assert b == false || this.currentNodeId().equals(other.relocatingNodeId) : - "ShardRouting is a relocation target but current node id isn't equal to source relocating node. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation target but current node id isn't equal to source relocating node. This [" + this + "], other [" + other + "]"; assert b == false || isSameShard(other) : - "ShardRouting is a relocation target but both routings are not of the same shard. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation target but both routings are not of the same shard. This [" + this + "], other [" + other + "]"; assert b == false || this.primary == other.primary : - "ShardRouting is a relocation target but primary flag is different. This [" + this + "], target [" + other + "]"; + "ShardRouting is a relocation target but primary flag is different. This [" + this + "], target [" + other + "]"; return b; } @@ -568,26 +575,26 @@ public final class ShardRouting implements Streamable, ToXContent { /** returns true if the routing is the relocation source for the given routing */ public boolean isRelocationSourceOf(ShardRouting other) { boolean b = this.allocationId != null && other.allocationId != null && other.state == ShardRoutingState.INITIALIZING && - other.allocationId.getId().equals(this.allocationId.getRelocationId()); + other.allocationId.getId().equals(this.allocationId.getRelocationId()); assert b == false || this.state == ShardRoutingState.RELOCATING : - "ShardRouting is a relocation source but shard state isn't relocating. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation source but shard state isn't relocating. This [" + this + "], other [" + other + "]"; assert b == false || this.allocationId.getId().equals(other.allocationId.getRelocationId()) : - "ShardRouting is a relocation source but the allocation id isn't equal to other.allocationId.getRelocationId. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation source but the allocation id isn't equal to other.allocationId.getRelocationId. This [" + this + "], other [" + other + "]"; assert b == false || this.currentNodeId().equals(other.relocatingNodeId) : - "ShardRouting is a relocation source but current node isn't equal to other's relocating node. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation source but current node isn't equal to other's relocating node. This [" + this + "], other [" + other + "]"; assert b == false || other.currentNodeId().equals(this.relocatingNodeId) : - "ShardRouting is a relocation source but relocating node isn't equal to other's current node. This [" + this + "], other [" + other + "]"; + "ShardRouting is a relocation source but relocating node isn't equal to other's current node. This [" + this + "], other [" + other + "]"; assert b == false || isSameShard(other) : - "ShardRouting is a relocation source but both routings are not of the same shard. This [" + this + "], target [" + other + "]"; + "ShardRouting is a relocation source but both routings are not of the same shard. This [" + this + "], target [" + other + "]"; assert b == false || this.primary == other.primary : - "ShardRouting is a relocation source but primary flag is different. This [" + this + "], target [" + other + "]"; + "ShardRouting is a relocation source but primary flag is different. This [" + this + "], target [" + other + "]"; return b; } @@ -654,7 +661,7 @@ public final class ShardRouting implements Streamable, ToXContent { result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0); result = 31 * result + (primary ? 1 : 0); result = 31 * result + (state != null ? state.hashCode() : 0); - result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + Long.hashCode(version); result = 31 * result + (restoreSource != null ? restoreSource.hashCode() : 0); result = 31 * result + (allocationId != null ? allocationId.hashCode() : 0); result = 31 * result + (unassignedInfo != null ? unassignedInfo.hashCode() : 0); @@ -701,14 +708,14 @@ public final class ShardRouting implements Streamable, ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject() - .field("state", state()) - .field("primary", primary()) - .field("node", currentNodeId()) - .field("relocating_node", relocatingNodeId()) - .field("shard", shardId().id()) - .field("index", shardId().index().name()) - .field("version", version); - if (expectedShardSize != UNAVAILABLE_EXPECTED_SHARD_SIZE){ + .field("state", state()) + .field("primary", primary()) + .field("node", currentNodeId()) + .field("relocating_node", relocatingNodeId()) + .field("shard", shardId().id()) + .field("index", shardId().index().name()) + .field("version", version); + if (expectedShardSize != UNAVAILABLE_EXPECTED_SHARD_SIZE) { builder.field("expected_shard_size_in_bytes", expectedShardSize); } if (restoreSource() != null) { @@ -716,6 +723,7 @@ public final class ShardRouting implements Streamable, ToXContent { restoreSource().toXContent(builder, params); } if (allocationId != null) { + builder.field("allocation_id"); allocationId.toXContent(builder, params); } if (unassignedInfo != null) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java index e3295415389..23733b96f0c 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java @@ -103,21 +103,33 @@ public class UnassignedInfo implements ToXContent, Writeable { } private final Reason reason; - private final long timestamp; + private final long unassignedTimeMillis; // used for display and log messages, in milliseconds + private final long unassignedTimeNanos; // in nanoseconds, used to calculate delay for delayed shard allocation + private volatile long lastComputedLeftDelayNanos = 0l; // how long to delay shard allocation, not serialized (always positive, 0 means no delay) private final String message; private final Throwable failure; + /** + * creates an UnassingedInfo object based **current** time + * + * @param reason the cause for making this shard unassigned. See {@link Reason} for more information. + * @param message more information about cause. + **/ public UnassignedInfo(Reason reason, String message) { - this(reason, System.currentTimeMillis(), message, null); + this(reason, message, null, System.nanoTime(), System.currentTimeMillis()); } - public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure) { - this(reason, System.currentTimeMillis(), message, failure); - } - - private UnassignedInfo(Reason reason, long timestamp, String message, Throwable failure) { + /** + * @param reason the cause for making this shard unassigned. See {@link Reason} for more information. + * @param message more information about cause. + * @param failure the shard level failure that caused this shard to be unassigned, if exists. + * @param unassignedTimeNanos the time to use as the base for any delayed re-assignment calculation + * @param unassignedTimeMillis the time of unassignment used to display to in our reporting. + */ + public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure, long unassignedTimeNanos, long unassignedTimeMillis) { this.reason = reason; - this.timestamp = timestamp; + this.unassignedTimeMillis = unassignedTimeMillis; + this.unassignedTimeNanos = unassignedTimeNanos; this.message = message; this.failure = failure; assert !(message == null && failure != null) : "provide a message if a failure exception is provided"; @@ -125,14 +137,18 @@ public class UnassignedInfo implements ToXContent, Writeable { UnassignedInfo(StreamInput in) throws IOException { this.reason = Reason.values()[(int) in.readByte()]; - this.timestamp = in.readLong(); + this.unassignedTimeMillis = in.readLong(); + // As System.nanoTime() cannot be compared across different JVMs, reset it to now. + // This means that in master failover situations, elapsed delay time is forgotten. + this.unassignedTimeNanos = System.nanoTime(); this.message = in.readOptionalString(); this.failure = in.readThrowable(); } public void writeTo(StreamOutput out) throws IOException { out.writeByte((byte) reason.ordinal()); - out.writeLong(timestamp); + out.writeLong(unassignedTimeMillis); + // Do not serialize unassignedTimeNanos as System.nanoTime() cannot be compared across different JVMs out.writeOptionalString(message); out.writeThrowable(failure); } @@ -149,13 +165,20 @@ public class UnassignedInfo implements ToXContent, Writeable { } /** - * The timestamp in milliseconds since epoch. Note, we use timestamp here since - * we want to make sure its preserved across node serializations. Extra care need - * to be made if its used to calculate diff (handle negative values) in case of - * time drift. + * The timestamp in milliseconds when the shard became unassigned, based on System.currentTimeMillis(). + * Note, we use timestamp here since we want to make sure its preserved across node serializations. */ - public long getTimestampInMillis() { - return this.timestamp; + public long getUnassignedTimeInMillis() { + return this.unassignedTimeMillis; + } + + /** + * The timestamp in nanoseconds when the shard became unassigned, based on System.nanoTime(). + * Used to calculate the delay for delayed shard allocation. + * ONLY EXPOSED FOR TESTS! + */ + public long getUnassignedTimeInNanos() { + return this.unassignedTimeNanos; } /** @@ -186,42 +209,50 @@ public class UnassignedInfo implements ToXContent, Writeable { } /** - * The allocation delay value associated with the index (defaulting to node settings if not set). + * The allocation delay value in nano seconds associated with the index (defaulting to node settings if not set). */ - public long getAllocationDelayTimeoutSetting(Settings settings, Settings indexSettings) { + public long getAllocationDelayTimeoutSettingNanos(Settings settings, Settings indexSettings) { if (reason != Reason.NODE_LEFT) { return 0; } TimeValue delayTimeout = indexSettings.getAsTime(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, settings.getAsTime(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, DEFAULT_DELAYED_NODE_LEFT_TIMEOUT)); - return Math.max(0l, delayTimeout.millis()); + return Math.max(0l, delayTimeout.nanos()); } /** - * The time in millisecond until this unassigned shard can be reassigned. + * The delay in nanoseconds until this unassigned shard can be reassigned. This value is cached and might be slightly out-of-date. + * See also the {@link #updateDelay(long, Settings, Settings)} method. */ - public long getDelayAllocationExpirationIn(long unassignedShardsAllocatedTimestamp, Settings settings, Settings indexSettings) { - long delayTimeout = getAllocationDelayTimeoutSetting(settings, indexSettings); - if (delayTimeout == 0) { - return 0; - } - long delta = unassignedShardsAllocatedTimestamp - timestamp; - // account for time drift, treat it as no timeout - if (delta < 0) { - return 0; - } - return delayTimeout - delta; + public long getLastComputedLeftDelayNanos() { + return lastComputedLeftDelayNanos; } + /** + * Updates delay left based on current time (in nanoseconds) and index/node settings. + * + * @return updated delay in nanoseconds + */ + public long updateDelay(long nanoTimeNow, Settings settings, Settings indexSettings) { + long delayTimeoutNanos = getAllocationDelayTimeoutSettingNanos(settings, indexSettings); + final long newComputedLeftDelayNanos; + if (delayTimeoutNanos == 0l) { + newComputedLeftDelayNanos = 0l; + } else { + assert nanoTimeNow >= unassignedTimeNanos; + newComputedLeftDelayNanos = Math.max(0L, delayTimeoutNanos - (nanoTimeNow - unassignedTimeNanos)); + } + lastComputedLeftDelayNanos = newComputedLeftDelayNanos; + return newComputedLeftDelayNanos; + } /** * Returns the number of shards that are unassigned and currently being delayed. */ - public static int getNumberOfDelayedUnassigned(long unassignedShardsAllocatedTimestamp, Settings settings, ClusterState state) { + public static int getNumberOfDelayedUnassigned(ClusterState state) { int count = 0; for (ShardRouting shard : state.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED)) { if (shard.primary() == false) { - IndexMetaData indexMetaData = state.metaData().index(shard.getIndex()); - long delay = shard.unassignedInfo().getDelayAllocationExpirationIn(unassignedShardsAllocatedTimestamp, settings, indexMetaData.getSettings()); + long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos(); if (delay > 0) { count++; } @@ -231,32 +262,32 @@ public class UnassignedInfo implements ToXContent, Writeable { } /** - * Finds the smallest delay expiration setting of an unassigned shard. Returns 0 if there are none. + * Finds the smallest delay expiration setting in nanos of all unassigned shards that are still delayed. Returns 0 if there are none. */ - public static long findSmallestDelayedAllocationSetting(Settings settings, ClusterState state) { - long nextDelaySetting = Long.MAX_VALUE; + public static long findSmallestDelayedAllocationSettingNanos(Settings settings, ClusterState state) { + long minDelaySetting = Long.MAX_VALUE; for (ShardRouting shard : state.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED)) { if (shard.primary() == false) { IndexMetaData indexMetaData = state.metaData().index(shard.getIndex()); - long delayTimeoutSetting = shard.unassignedInfo().getAllocationDelayTimeoutSetting(settings, indexMetaData.getSettings()); - if (delayTimeoutSetting > 0 && delayTimeoutSetting < nextDelaySetting) { - nextDelaySetting = delayTimeoutSetting; + boolean delayed = shard.unassignedInfo().getLastComputedLeftDelayNanos() > 0; + long delayTimeoutSetting = shard.unassignedInfo().getAllocationDelayTimeoutSettingNanos(settings, indexMetaData.getSettings()); + if (delayed && delayTimeoutSetting > 0 && delayTimeoutSetting < minDelaySetting) { + minDelaySetting = delayTimeoutSetting; } } } - return nextDelaySetting == Long.MAX_VALUE ? 0l : nextDelaySetting; + return minDelaySetting == Long.MAX_VALUE ? 0l : minDelaySetting; } /** - * Finds the next (closest) delay expiration of an unassigned shard. Returns 0 if there are none. + * Finds the next (closest) delay expiration of an unassigned shard in nanoseconds. Returns 0 if there are none. */ - public static long findNextDelayedAllocationIn(long unassignedShardsAllocatedTimestamp, Settings settings, ClusterState state) { + public static long findNextDelayedAllocationIn(ClusterState state) { long nextDelay = Long.MAX_VALUE; for (ShardRouting shard : state.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED)) { if (shard.primary() == false) { - IndexMetaData indexMetaData = state.metaData().index(shard.getIndex()); - long nextShardDelay = shard.unassignedInfo().getDelayAllocationExpirationIn(unassignedShardsAllocatedTimestamp, settings, indexMetaData.getSettings()); + long nextShardDelay = shard.unassignedInfo().getLastComputedLeftDelayNanos(); if (nextShardDelay > 0 && nextShardDelay < nextDelay) { nextDelay = nextShardDelay; } @@ -268,7 +299,7 @@ public class UnassignedInfo implements ToXContent, Writeable { public String shortSummary() { StringBuilder sb = new StringBuilder(); sb.append("[reason=").append(reason).append("]"); - sb.append(", at[").append(DATE_TIME_FORMATTER.printer().print(timestamp)).append("]"); + sb.append(", at[").append(DATE_TIME_FORMATTER.printer().print(unassignedTimeMillis)).append("]"); String details = getDetails(); if (details != null) { sb.append(", details[").append(details).append("]"); @@ -285,7 +316,7 @@ public class UnassignedInfo implements ToXContent, Writeable { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("unassigned_info"); builder.field("reason", reason); - builder.field("at", DATE_TIME_FORMATTER.printer().print(timestamp)); + builder.field("at", DATE_TIME_FORMATTER.printer().print(unassignedTimeMillis)); String details = getDetails(); if (details != null) { builder.field("details", details); @@ -296,14 +327,24 @@ public class UnassignedInfo implements ToXContent, Writeable { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } UnassignedInfo that = (UnassignedInfo) o; - if (timestamp != that.timestamp) return false; - if (reason != that.reason) return false; - if (message != null ? !message.equals(that.message) : that.message != null) return false; + if (unassignedTimeMillis != that.unassignedTimeMillis) { + return false; + } + if (reason != that.reason) { + return false; + } + if (message != null ? !message.equals(that.message) : that.message != null) { + return false; + } return !(failure != null ? !failure.equals(that.failure) : that.failure != null); } @@ -311,7 +352,7 @@ public class UnassignedInfo implements ToXContent, Writeable { @Override public int hashCode() { int result = reason != null ? reason.hashCode() : 0; - result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); + result = 31 * result + Long.hashCode(unassignedTimeMillis); result = 31 * result + (message != null ? message.hashCode() : 0); result = 31 * result + (failure != null ? failure.hashCode() : 0); return result; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index a3050f60e5f..2268bf1d995 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -22,9 +22,14 @@ package org.elasticsearch.cluster.routing.allocation; import com.carrotsearch.hppc.cursors.ObjectCursor; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; @@ -34,13 +39,14 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators; import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; /** @@ -79,13 +85,81 @@ public class AllocationService extends AbstractComponent { StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState.nodes(), startedShards, clusterInfoService.getClusterInfo()); boolean changed = applyStartedShards(routingNodes, startedShards); if (!changed) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } shardsAllocators.applyStartedShards(allocation); if (withReroute) { reroute(allocation); } - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData())); + final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes); + + String startedShardsAsString = firstListElementsToCommaDelimitedString(startedShards, s -> s.shardId().toString()); + logClusterHealthStateChange( + new ClusterStateHealth(clusterState), + new ClusterStateHealth(clusterState.metaData(), result.routingTable()), + "shards started [" + startedShardsAsString + "] ..." + ); + return result; + } + + + protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes) { + return buildChangedResult(metaData, routingNodes, new RoutingExplanations()); + + } + protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes, RoutingExplanations explanations) { + final RoutingTable routingTable = new RoutingTable.Builder().updateNodes(routingNodes).build(); + MetaData newMetaData = updateMetaDataWithRoutingTable(metaData,routingTable); + return new RoutingAllocation.Result(true, routingTable.validateRaiseException(newMetaData), newMetaData, explanations); + } + + /** + * Updates the current {@link MetaData} based on the newly created {@link RoutingTable}. + * + * @param currentMetaData {@link MetaData} object from before the routing table was changed. + * @param newRoutingTable new {@link RoutingTable} created by the allocation change + * @return adapted {@link MetaData}, potentially the original one if no change was needed. + */ + static MetaData updateMetaDataWithRoutingTable(MetaData currentMetaData, RoutingTable newRoutingTable) { + // make sure index meta data and routing tables are in sync w.r.t active allocation ids + MetaData.Builder metaDataBuilder = null; + for (IndexRoutingTable indexRoutingTable : newRoutingTable) { + final IndexMetaData indexMetaData = currentMetaData.index(indexRoutingTable.getIndex()); + if (indexMetaData == null) { + throw new IllegalStateException("no metadata found for index [" + indexRoutingTable.index() + "]"); + } + IndexMetaData.Builder indexMetaDataBuilder = null; + for (IndexShardRoutingTable shardRoutings : indexRoutingTable) { + Set activeAllocationIds = shardRoutings.activeShards().stream() + .map(ShardRouting::allocationId) + .filter(Objects::nonNull) + .map(AllocationId::getId) + .collect(Collectors.toSet()); + // only update active allocation ids if there is an active shard + if (activeAllocationIds.isEmpty() == false) { + // get currently stored allocation ids + Set storedAllocationIds = indexMetaData.activeAllocationIds(shardRoutings.shardId().id()); + if (activeAllocationIds.equals(storedAllocationIds) == false) { + if (indexMetaDataBuilder == null) { + indexMetaDataBuilder = IndexMetaData.builder(indexMetaData); + } + + indexMetaDataBuilder.putActiveAllocationIds(shardRoutings.shardId().id(), activeAllocationIds); + } + } + } + if (indexMetaDataBuilder != null) { + if (metaDataBuilder == null) { + metaDataBuilder = MetaData.builder(currentMetaData); + } + metaDataBuilder.put(indexMetaDataBuilder); + } + } + if (metaDataBuilder != null) { + return metaDataBuilder.build(); + } else { + return currentMetaData; + } } public RoutingAllocation.Result applyFailedShard(ClusterState clusterState, ShardRouting failedShard) { @@ -104,14 +178,39 @@ public class AllocationService extends AbstractComponent { FailedRerouteAllocation allocation = new FailedRerouteAllocation(allocationDeciders, routingNodes, clusterState.nodes(), failedShards, clusterInfoService.getClusterInfo()); boolean changed = false; for (FailedRerouteAllocation.FailedShard failedShard : failedShards) { - changed |= applyFailedShard(allocation, failedShard.shard, true, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShard.message, failedShard.failure)); + changed |= applyFailedShard(allocation, failedShard.shard, true, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShard.message, failedShard.failure, + System.nanoTime(), System.currentTimeMillis())); } if (!changed) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } shardsAllocators.applyFailedShards(allocation); reroute(allocation); - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData())); + final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes); + String failedShardsAsString = firstListElementsToCommaDelimitedString(failedShards, s -> s.shard.shardId().toString()); + logClusterHealthStateChange( + new ClusterStateHealth(clusterState), + new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()), + "shards failed [" + failedShardsAsString + "] ..." + ); + return result; + } + + /** + * Internal helper to cap the number of elements in a potentially long list for logging. + * + * @param elements The elements to log. May be any non-null list. Must not be null. + * @param formatter A function that can convert list elements to a String. Must not be null. + * @param The list element type. + * @return A comma-separated string of the first few elements. + */ + private String firstListElementsToCommaDelimitedString(List elements, Function formatter) { + final int maxNumberOfElements = 10; + return elements + .stream() + .limit(maxNumberOfElements) + .map(formatter) + .collect(Collectors.joining(", ")); } public RoutingAllocation.Result reroute(ClusterState clusterState, AllocationCommands commands) { @@ -123,7 +222,7 @@ public class AllocationService extends AbstractComponent { // we don't shuffle the unassigned shards here, to try and get as close as possible to // a consistent result of the effect the commands have on the routing // this allows systems to dry run the commands, see the resulting cluster state, and act on it - RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo()); + RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo(), currentNanoTime()); // don't short circuit deciders, we want a full explanation allocation.debugDecision(true); // we ignore disable allocation, because commands are explicit @@ -134,7 +233,13 @@ public class AllocationService extends AbstractComponent { // the assumption is that commands will move / act on shards (or fail through exceptions) // so, there will always be shard "movements", so no need to check on reroute reroute(allocation); - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData()), explanations); + RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes, explanations); + logClusterHealthStateChange( + new ClusterStateHealth(clusterState), + new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()), + "reroute commands" + ); + return result; } /** @@ -142,8 +247,8 @@ public class AllocationService extends AbstractComponent { *

    * If the same instance of the routing table is returned, then no change has been made. */ - public RoutingAllocation.Result reroute(ClusterState clusterState) { - return reroute(clusterState, false); + public RoutingAllocation.Result reroute(ClusterState clusterState, String reason) { + return reroute(clusterState, reason, false); } /** @@ -151,16 +256,30 @@ public class AllocationService extends AbstractComponent { *

    * If the same instance of the routing table is returned, then no change has been made. */ - public RoutingAllocation.Result reroute(ClusterState clusterState, boolean debug) { + protected RoutingAllocation.Result reroute(ClusterState clusterState, String reason, boolean debug) { RoutingNodes routingNodes = getMutableRoutingNodes(clusterState); // shuffle the unassigned nodes, just so we won't have things like poison failed shards routingNodes.unassigned().shuffle(); - RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo()); + RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo(), currentNanoTime()); allocation.debugDecision(debug); if (!reroute(allocation)) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); + } + RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes); + logClusterHealthStateChange( + new ClusterStateHealth(clusterState), + new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()), + reason + ); + return result; + } + + private void logClusterHealthStateChange(ClusterStateHealth previousStateHealth, ClusterStateHealth newStateHealth, String reason) { + ClusterHealthStatus previousHealth = previousStateHealth.getStatus(); + ClusterHealthStatus currentHealth = newStateHealth.getStatus(); + if (!previousHealth.equals(currentHealth)) { + logger.info("Cluster health status changed from [{}] to [{}] (reason: [{}]).", previousHealth, currentHealth, reason); } - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData())); } private boolean reroute(RoutingAllocation allocation) { @@ -176,7 +295,9 @@ public class AllocationService extends AbstractComponent { changed |= electPrimariesAndUnassignedDanglingReplicas(allocation); // now allocate all the unassigned to available nodes - if (allocation.routingNodes().hasUnassigned()) { + if (allocation.routingNodes().unassigned().size() > 0) { + updateLeftDelayOfUnassignedShards(allocation, settings); + changed |= shardsAllocators.allocateUnassigned(allocation); } @@ -189,6 +310,15 @@ public class AllocationService extends AbstractComponent { return changed; } + // public for testing + public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) { + for (ShardRouting shardRouting : allocation.routingNodes().unassigned()) { + final MetaData metaData = allocation.metaData(); + final IndexMetaData indexMetaData = metaData.index(shardRouting.index()); + shardRouting.unassignedInfo().updateDelay(allocation.getCurrentNanoTime(), settings, indexMetaData.getSettings()); + } + } + private boolean moveShards(RoutingAllocation allocation) { boolean changed = false; @@ -232,7 +362,7 @@ public class AllocationService extends AbstractComponent { private boolean electPrimariesAndUnassignedDanglingReplicas(RoutingAllocation allocation) { boolean changed = false; RoutingNodes routingNodes = allocation.routingNodes(); - if (!routingNodes.hasUnassignedPrimaries()) { + if (routingNodes.unassigned().getNumPrimaries() == 0) { // move out if we don't have unassigned primaries return changed; } @@ -250,7 +380,9 @@ public class AllocationService extends AbstractComponent { } } for (ShardRouting shardToFail : shardsToFail) { - changed |= applyFailedShard(allocation, shardToFail, false, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "primary failed while replica initializing")); + changed |= applyFailedShard(allocation, shardToFail, false, + new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "primary failed while replica initializing", + null, allocation.getCurrentNanoTime(), System.currentTimeMillis())); } // now, go over and elect a new primary if possible, not, from this code block on, if one is elected, @@ -275,7 +407,7 @@ public class AllocationService extends AbstractComponent { } } } - if (IndexMetaData.isIndexUsingShadowReplicas(index.settings())) { + if (IndexMetaData.isIndexUsingShadowReplicas(index.getSettings())) { routingNodes.reinitShadowPrimary(candidate); changed = true; } @@ -310,8 +442,9 @@ public class AllocationService extends AbstractComponent { } changed = true; // now, go over all the shards routing on the node, and fail them - UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]"); for (ShardRouting shardRouting : node.copyShards()) { + UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]", null, + allocation.getCurrentNanoTime(), System.currentTimeMillis()); applyFailedShard(allocation, shardRouting, false, unassignedInfo); } // its a dead node, remove it, note, its important to remove it *after* we apply failed shard @@ -469,4 +602,9 @@ public class AllocationService extends AbstractComponent { RoutingNodes routingNodes = new RoutingNodes(clusterState, false); // this is a costly operation - only call this once! return routingNodes; } + + /** ovrride this to control time based decisions during allocation */ + protected long currentNanoTime() { + return System.nanoTime(); + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java index 24e38279f4d..835556a265b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java @@ -58,7 +58,7 @@ public class FailedRerouteAllocation extends RoutingAllocation { private final List failedShards; public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, List failedShards, ClusterInfo clusterInfo) { - super(deciders, routingNodes, nodes, clusterInfo); + super(deciders, routingNodes, nodes, clusterInfo, System.nanoTime()); this.failedShards = failedShards; } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index 1874a7b020b..4e6ba0fb5ad 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -52,29 +52,33 @@ public class RoutingAllocation { private final RoutingTable routingTable; + private final MetaData metaData; + private RoutingExplanations explanations = new RoutingExplanations(); /** * Creates a new {@link RoutingAllocation.Result} - * * @param changed a flag to determine whether the actual {@link RoutingTable} has been changed * @param routingTable the {@link RoutingTable} this Result references + * @param metaData the {@link MetaData} this Result references */ - public Result(boolean changed, RoutingTable routingTable) { + public Result(boolean changed, RoutingTable routingTable, MetaData metaData) { this.changed = changed; this.routingTable = routingTable; + this.metaData = metaData; } /** * Creates a new {@link RoutingAllocation.Result} - * * @param changed a flag to determine whether the actual {@link RoutingTable} has been changed * @param routingTable the {@link RoutingTable} this Result references + * @param metaData the {@link MetaData} this Result references * @param explanations Explanation for the reroute actions */ - public Result(boolean changed, RoutingTable routingTable, RoutingExplanations explanations) { + public Result(boolean changed, RoutingTable routingTable, MetaData metaData, RoutingExplanations explanations) { this.changed = changed; this.routingTable = routingTable; + this.metaData = metaData; this.explanations = explanations; } @@ -85,6 +89,14 @@ public class RoutingAllocation { return this.changed; } + /** + * Get the {@link MetaData} referenced by this result + * @return referenced {@link MetaData} + */ + public MetaData metaData() { + return metaData; + } + /** * Get the {@link RoutingTable} referenced by this result * @return referenced {@link RoutingTable} @@ -118,18 +130,29 @@ public class RoutingAllocation { private boolean debugDecision = false; + private boolean hasPendingAsyncFetch = false; + + private final long currentNanoTime; + + /** * Creates a new {@link RoutingAllocation} - * - * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations - * @param routingNodes Routing nodes in the current cluster + * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations + * @param routingNodes Routing nodes in the current cluster * @param nodes TODO: Documentation + * @param currentNanoTime the nano time to use for all delay allocation calculation (typically {@link System#nanoTime()}) */ - public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, ClusterInfo clusterInfo) { + public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, ClusterInfo clusterInfo, long currentNanoTime) { this.deciders = deciders; this.routingNodes = routingNodes; this.nodes = nodes; this.clusterInfo = clusterInfo; + this.currentNanoTime = currentNanoTime; + } + + /** returns the nano time captured at the beginning of the allocation. used to make sure all time based decisions are aligned */ + public long getCurrentNanoTime() { + return currentNanoTime; } /** @@ -246,4 +269,20 @@ public class RoutingAllocation { return decision; } } + + /** + * Returns true iff the current allocation run has not processed all of the in-flight or available + * shard or store fetches. Otherwise true + */ + public boolean hasPendingAsyncFetch() { + return hasPendingAsyncFetch; + } + + /** + * Sets a flag that signals that current allocation run has not processed all of the in-flight or available shard or store fetches. + * This state is anti-viral and can be reset in on allocation run. + */ + public void setHasPendingAsyncFetch() { + this.hasPendingAsyncFetch = true; + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java index da69419d948..00f3944ae03 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java @@ -36,7 +36,7 @@ public class StartedRerouteAllocation extends RoutingAllocation { private final List startedShards; public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, List startedShards, ClusterInfo clusterInfo) { - super(deciders, routingNodes, nodes, clusterInfo); + super(deciders, routingNodes, nodes, clusterInfo, System.nanoTime()); this.startedShards = startedShards; } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index efb5c96bf50..b9ce532a611 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -118,7 +118,8 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards @Override public boolean allocateUnassigned(RoutingAllocation allocation) { - return rebalance(allocation); + final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold); + return balancer.allocateUnassigned(); } @Override @@ -262,7 +263,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards * Returns the average of shards per node for the given index */ public float avgShardsPerNode(String index) { - return ((float) metaData.index(index).totalNumberOfShards()) / nodes.size(); + return ((float) metaData.index(index).getTotalNumberOfShards()) / nodes.size(); } /** @@ -313,6 +314,15 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards return delta <= (threshold + 0.001f); } + /** + * Allocates all possible unassigned shards + * @return true if the current configuration has been + * changed, otherwise false + */ + final boolean allocateUnassigned() { + return balance(true); + } + /** * Balances the nodes on the cluster model according to the weight * function. The configured threshold is the minimum delta between the @@ -328,16 +338,24 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards * changed, otherwise false */ public boolean balance() { + return balance(false); + } + + private boolean balance(boolean onlyAssign) { if (this.nodes.isEmpty()) { /* with no nodes this is pointless */ return false; } if (logger.isTraceEnabled()) { - logger.trace("Start balancing cluster"); + if (onlyAssign) { + logger.trace("Start balancing cluster"); + } else { + logger.trace("Start assigning unassigned shards"); + } } - final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned().transactionBegin(); + final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned(); boolean changed = initialize(routingNodes, unassigned); - if (!changed && allocation.deciders().canRebalance(allocation).type() == Type.YES) { + if (onlyAssign == false && changed == false && allocation.deciders().canRebalance(allocation).type() == Type.YES) { NodeSorter sorter = newNodeSorter(); if (nodes.size() > 1) { /* skip if we only have one node */ for (String index : buildWeightOrderedIndidces(Operation.BALANCE, sorter)) { @@ -415,7 +433,6 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards } } } - routingNodes.unassigned().transactionEnd(unassigned); return changed; } @@ -490,7 +507,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards if (logger.isTraceEnabled()) { logger.trace("Try moving shard [{}] from [{}]", shard, node); } - final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned().transactionBegin(); + final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned(); boolean changed = initialize(routingNodes, unassigned); if (!changed) { final ModelNode sourceNode = nodes.get(node.nodeId()); @@ -511,7 +528,9 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards continue; } RoutingNode target = routingNodes.node(currentNode.getNodeId()); - Decision decision = allocation.deciders().canAllocate(shard, target, allocation); + Decision allocationDecision = allocation.deciders().canAllocate(shard, target, allocation); + Decision rebalanceDecision = allocation.deciders().canRebalance(shard, allocation); + Decision decision = new Decision.Multi().add(allocationDecision).add(rebalanceDecision); if (decision.type() == Type.YES) { // TODO maybe we can respect throttling here too? sourceNode.removeShard(shard); ShardRouting targetRelocatingShard = routingNodes.relocate(shard, target.nodeId(), allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE)); @@ -524,7 +543,6 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards } } } - routingNodes.unassigned().transactionEnd(unassigned); return changed; } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java index 8fb65bbfe9b..a9ce43c5f76 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java @@ -19,8 +19,8 @@ package org.elasticsearch.cluster.routing.allocation.allocator; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation; @@ -74,9 +74,25 @@ public class ShardsAllocators extends AbstractComponent implements ShardsAllocat return changed; } + protected long nanoTime() { + return System.nanoTime(); + } + @Override public boolean rebalance(RoutingAllocation allocation) { - return allocator.rebalance(allocation); + if (allocation.hasPendingAsyncFetch() == false) { + /* + * see https://github.com/elastic/elasticsearch/issues/14387 + * if we allow rebalance operations while we are still fetching shard store data + * we might end up with unnecessary rebalance operations which can be super confusion/frustrating + * since once the fetches come back we might just move all the shards back again. + * Therefore we only do a rebalance if we have fetched all information. + */ + return allocator.rebalance(allocation); + } else { + logger.debug("skipping rebalance due to in-flight shard/store fetches"); + return false; + } } @Override diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java index b210557b687..5646d308dda 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java @@ -229,7 +229,8 @@ public class AllocateAllocationCommand implements AllocationCommand { // it was index creation if (unassigned.primary() && unassigned.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED) { unassigned.updateUnassignedInfo(new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, - "force allocation from previous reason " + unassigned.unassignedInfo().getReason() + ", " + unassigned.unassignedInfo().getMessage(), unassigned.unassignedInfo().getFailure())); + "force allocation from previous reason " + unassigned.unassignedInfo().getReason() + ", " + unassigned.unassignedInfo().getMessage(), + unassigned.unassignedInfo().getFailure(), System.nanoTime(), System.currentTimeMillis())); } it.initialize(routingNode.nodeId(), unassigned.version(), allocation.clusterInfo().getShardSize(unassigned, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE)); break; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java index d9ad8c4f871..7554fa4c46f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java @@ -21,7 +21,10 @@ package org.elasticsearch.cluster.routing.allocation.command; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.RerouteExplanation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.Decision; @@ -34,7 +37,6 @@ import org.elasticsearch.index.shard.ShardId; import java.io.IOException; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; /** diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java index 54b6d403d2e..6f7bbac8aea 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -166,7 +166,7 @@ public class AwarenessAllocationDecider extends AllocationDecider { } IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.index()); - int shardCount = indexMetaData.numberOfReplicas() + 1; // 1 for primary + int shardCount = indexMetaData.getNumberOfReplicas() + 1; // 1 for primary for (String awarenessAttribute : awarenessAttributes) { // the node the shard exists on must be associated with an awareness attribute if (!node.node().attributes().containsKey(awarenessAttribute)) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java index 00f6575287f..7638c7aeee8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java @@ -51,15 +51,12 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider { public static final String NAME = "cluster_rebalance"; public static final String CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE = "cluster.routing.allocation.allow_rebalance"; - public static final Validator ALLOCATION_ALLOW_REBALANCE_VALIDATOR = new Validator() { - @Override - public String validate(String setting, String value, ClusterState clusterState) { - try { - ClusterRebalanceType.parseString(value); - return null; - } catch (IllegalArgumentException e) { - return "the value of " + setting + " must be one of: [always, indices_primaries_active, indices_all_active]"; - } + public static final Validator ALLOCATION_ALLOW_REBALANCE_VALIDATOR = (setting, value, clusterState) -> { + try { + ClusterRebalanceType.parseString(value); + return null; + } catch (IllegalArgumentException e) { + return "the value of " + setting + " must be one of: [always, indices_primaries_active, indices_all_active]"; } }; @@ -153,7 +150,7 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider { } if (type == ClusterRebalanceType.INDICES_ALL_ACTIVE) { // check if there are unassigned shards. - if ( allocation.routingNodes().hasUnassignedShards() ) { + if (allocation.routingNodes().hasUnassignedShards() ) { return allocation.decision(Decision.NO, NAME, "cluster has unassigned shards"); } // in case all indices are assigned, are there initializing shards which diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java index f83aa56964f..6bd1b437acf 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java @@ -70,7 +70,7 @@ public class ConcurrentRebalanceAllocationDecider extends AllocationDecider { return allocation.decision(Decision.YES, NAME, "all concurrent rebalances are allowed"); } if (allocation.routingNodes().getRelocatingShardCount() >= clusterConcurrentRebalance) { - return allocation.decision(Decision.NO, NAME, "too man concurrent rebalances [%d], limit: [%d]", + return allocation.decision(Decision.NO, NAME, "too many concurrent rebalances [%d], limit: [%d]", allocation.routingNodes().getRelocatingShardCount(), clusterConcurrentRebalance); } return allocation.decision(Decision.YES, NAME, "below threshold [%d] for concurrent rebalances", clusterConcurrentRebalance); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index f21ced8cc0a..a02c72c5745 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -466,8 +466,8 @@ public class DiskThresholdDecider extends AllocationDecider { // If this node is already above the high threshold, the shard cannot remain (get it off!) final double freeDiskPercentage = usage.getFreeDiskAsPercentage(); final long freeBytes = usage.getFreeBytes(); - if (logger.isDebugEnabled()) { - logger.debug("node [{}] has {}% free disk ({} bytes)", node.nodeId(), freeDiskPercentage, freeBytes); + if (logger.isTraceEnabled()) { + logger.trace("node [{}] has {}% free disk ({} bytes)", node.nodeId(), freeDiskPercentage, freeBytes); } if (dataPath == null || usage.getPath().equals(dataPath) == false) { return allocation.decision(Decision.YES, NAME, "shard is not allocated on the most utilized disk"); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java index 8fc6b4f109d..0bbd4935044 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java @@ -82,7 +82,7 @@ public class EnableAllocationDecider extends AllocationDecider implements NodeSe return allocation.decision(Decision.YES, NAME, "allocation disabling is ignored"); } - Settings indexSettings = allocation.routingNodes().metaData().index(shardRouting.index()).settings(); + Settings indexSettings = allocation.routingNodes().metaData().index(shardRouting.index()).getSettings(); String enableIndexValue = indexSettings.get(INDEX_ROUTING_ALLOCATION_ENABLE); final Allocation enable; if (enableIndexValue != null) { @@ -118,7 +118,7 @@ public class EnableAllocationDecider extends AllocationDecider implements NodeSe return allocation.decision(Decision.YES, NAME, "rebalance disabling is ignored"); } - Settings indexSettings = allocation.routingNodes().metaData().index(shardRouting.index()).settings(); + Settings indexSettings = allocation.routingNodes().metaData().index(shardRouting.index()).getSettings(); String enableIndexValue = indexSettings.get(INDEX_ROUTING_REBALANCE_ENABLE); final Rebalance enable; if (enableIndexValue != null) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java index bf03896257b..3d68ed50d27 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java @@ -24,13 +24,16 @@ import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.settings.NodeSettingsService; /** * This {@link AllocationDecider} limits the number of shards per node on a per - * index basis. The allocator prevents a single node to hold more than - * {@value #INDEX_TOTAL_SHARDS_PER_NODE} per index during the allocation + * index or node-wide basis. The allocator prevents a single node to hold more + * than {@value #INDEX_TOTAL_SHARDS_PER_NODE} per index and + * {@value #CLUSTER_TOTAL_SHARDS_PER_NODE} globally during the allocation * process. The limits of this decider can be changed in real-time via a the * index settings API. *

    @@ -50,66 +53,140 @@ public class ShardsLimitAllocationDecider extends AllocationDecider { public static final String NAME = "shards_limit"; + private volatile int clusterShardLimit; + /** * Controls the maximum number of shards per index on a single Elasticsearch * node. Negative values are interpreted as unlimited. */ public static final String INDEX_TOTAL_SHARDS_PER_NODE = "index.routing.allocation.total_shards_per_node"; + /** + * Controls the maximum number of shards per node on a global level. + * Negative values are interpreted as unlimited. + */ + public static final String CLUSTER_TOTAL_SHARDS_PER_NODE = "cluster.routing.allocation.total_shards_per_node"; + + class ApplySettings implements NodeSettingsService.Listener { + @Override + public void onRefreshSettings(Settings settings) { + Integer newClusterLimit = settings.getAsInt(CLUSTER_TOTAL_SHARDS_PER_NODE, null); + + if (newClusterLimit != null) { + logger.info("updating [{}] from [{}] to [{}]", CLUSTER_TOTAL_SHARDS_PER_NODE, + ShardsLimitAllocationDecider.this.clusterShardLimit, newClusterLimit); + ShardsLimitAllocationDecider.this.clusterShardLimit = newClusterLimit; + } + } + } @Inject - public ShardsLimitAllocationDecider(Settings settings) { + public ShardsLimitAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) { super(settings); + this.clusterShardLimit = settings.getAsInt(CLUSTER_TOTAL_SHARDS_PER_NODE, -1); + nodeSettingsService.addListener(new ApplySettings()); } @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index()); - int totalShardsPerNode = indexMd.settings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); - if (totalShardsPerNode <= 0) { - return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [%d] <= 0", totalShardsPerNode); + int indexShardLimit = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); + // Capture the limit here in case it changes during this method's + // execution + final int clusterShardLimit = this.clusterShardLimit; + + if (indexShardLimit <= 0 && clusterShardLimit <= 0) { + return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [index: %d, cluster: %d] <= 0", + indexShardLimit, clusterShardLimit); } - int nodeCount = 0; + int indexShardCount = 0; + int nodeShardCount = 0; for (ShardRouting nodeShard : node) { - if (!nodeShard.index().equals(shardRouting.index())) { - continue; - } // don't count relocating shards... if (nodeShard.relocating()) { continue; } - nodeCount++; + nodeShardCount++; + if (nodeShard.index().equals(shardRouting.index())) { + indexShardCount++; + } } - if (nodeCount >= totalShardsPerNode) { - return allocation.decision(Decision.NO, NAME, "too many shards for this index on node [%d], limit: [%d]", - nodeCount, totalShardsPerNode); + if (clusterShardLimit > 0 && nodeShardCount >= clusterShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]", + nodeShardCount, clusterShardLimit); } - return allocation.decision(Decision.YES, NAME, "shard count under limit [%d] of total shards per node", totalShardsPerNode); + if (indexShardLimit > 0 && indexShardCount >= indexShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this index [%s] on node [%d], limit: [%d]", + shardRouting.index(), indexShardCount, indexShardLimit); + } + return allocation.decision(Decision.YES, NAME, "shard count under index limit [%d] and node limit [%d] of total shards per node", + indexShardLimit, clusterShardLimit); } @Override public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index()); - int totalShardsPerNode = indexMd.settings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); - if (totalShardsPerNode <= 0) { - return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [%d] <= 0", totalShardsPerNode); + int indexShardLimit = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1); + // Capture the limit here in case it changes during this method's + // execution + final int clusterShardLimit = this.clusterShardLimit; + + if (indexShardLimit <= 0 && clusterShardLimit <= 0) { + return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [index: %d, cluster: %d] <= 0", + indexShardLimit, clusterShardLimit); } - int nodeCount = 0; + int indexShardCount = 0; + int nodeShardCount = 0; for (ShardRouting nodeShard : node) { - if (!nodeShard.index().equals(shardRouting.index())) { - continue; - } // don't count relocating shards... if (nodeShard.relocating()) { continue; } - nodeCount++; + nodeShardCount++; + if (nodeShard.index().equals(shardRouting.index())) { + indexShardCount++; + } } - if (nodeCount > totalShardsPerNode) { - return allocation.decision(Decision.NO, NAME, "too many shards for this index on node [%d], limit: [%d]", - nodeCount, totalShardsPerNode); + // Subtle difference between the `canAllocate` and `canRemain` is that + // this checks > while canAllocate checks >= + if (clusterShardLimit > 0 && nodeShardCount > clusterShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]", + nodeShardCount, clusterShardLimit); } - return allocation.decision(Decision.YES, NAME, "shard count under limit [%d] of total shards per node", totalShardsPerNode); + if (indexShardLimit > 0 && indexShardCount > indexShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this index [%s] on node [%d], limit: [%d]", + shardRouting.index(), indexShardCount, indexShardLimit); + } + return allocation.decision(Decision.YES, NAME, "shard count under index limit [%d] and node limit [%d] of total shards per node", + indexShardLimit, clusterShardLimit); + } + + @Override + public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) { + // Only checks the node-level limit, not the index-level + // Capture the limit here in case it changes during this method's + // execution + final int clusterShardLimit = this.clusterShardLimit; + + if (clusterShardLimit <= 0) { + return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [cluster: %d] <= 0", + clusterShardLimit); + } + + int nodeShardCount = 0; + for (ShardRouting nodeShard : node) { + // don't count relocating shards... + if (nodeShard.relocating()) { + continue; + } + nodeShardCount++; + } + if (clusterShardLimit >= 0 && nodeShardCount >= clusterShardLimit) { + return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]", + nodeShardCount, clusterShardLimit); + } + return allocation.decision(Decision.YES, NAME, "shard count under node limit [%d] of total shards per node", + clusterShardLimit); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java index c2300739a7d..d4b15861846 100644 --- a/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java +++ b/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java @@ -20,16 +20,8 @@ package org.elasticsearch.cluster.service; import org.elasticsearch.Version; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.ClusterState.Builder; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.LocalNodeMasterListener; -import org.elasticsearch.cluster.TimeoutClusterStateListener; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.MetaData; @@ -41,6 +33,7 @@ import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; @@ -49,13 +42,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.StringText; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.util.concurrent.CountDown; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.common.util.concurrent.FutureUtils; -import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor; -import org.elasticsearch.common.util.concurrent.PrioritizedRunnable; +import org.elasticsearch.common.util.concurrent.*; import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoveryService; @@ -63,18 +50,10 @@ import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; @@ -111,6 +90,7 @@ public class InternalClusterService extends AbstractLifecycleComponent priorityClusterStateListeners = new CopyOnWriteArrayList<>(); private final Collection clusterStateListeners = new CopyOnWriteArrayList<>(); private final Collection lastClusterStateListeners = new CopyOnWriteArrayList<>(); + private final Map> updateTasksPerExecutor = new HashMap<>(); // TODO this is rather frequently changing I guess a Synced Set would be better here and a dedicated remove API private final Collection postAppliedListeners = new CopyOnWriteArrayList<>(); private final Iterable preAppliedListeners = Iterables.concat(priorityClusterStateListeners, clusterStateListeners, lastClusterStateListeners); @@ -289,30 +269,33 @@ public class InternalClusterService extends AbstractLifecycleComponent void submitStateUpdateTask(final String source, final T task, + final ClusterStateTaskConfig config, + final ClusterStateTaskExecutor executor, + final ClusterStateTaskListener listener + ) { if (!lifecycle.started()) { return; } try { - final UpdateTask task = new UpdateTask(source, priority, updateTask); - if (updateTask.timeout() != null) { - updateTasksExecutor.execute(task, threadPool.scheduler(), updateTask.timeout(), new Runnable() { - @Override - public void run() { - threadPool.generic().execute(new Runnable() { - @Override - public void run() { - updateTask.onFailure(task.source(), new ProcessClusterEventTimeoutException(updateTask.timeout(), task.source())); - } - }); - } - }); + final UpdateTask updateTask = new UpdateTask<>(source, task, config, executor, listener); + + synchronized (updateTasksPerExecutor) { + updateTasksPerExecutor.computeIfAbsent(executor, k -> new ArrayList<>()).add(updateTask); + } + + if (config.timeout() != null) { + updateTasksExecutor.execute(updateTask, threadPool.scheduler(), config.timeout(), () -> threadPool.generic().execute(() -> { + if (updateTask.processed.getAndSet(true) == false) { + listener.onFailure(source, new ProcessClusterEventTimeoutException(config.timeout(), source)); + }})); } else { - updateTasksExecutor.execute(task); + updateTasksExecutor.execute(updateTask); } } catch (EsRejectedExecutionException e) { // ignore cases where we are shutting down..., there is really nothing interesting @@ -379,188 +362,238 @@ public class InternalClusterService extends AbstractLifecycleComponent void runTasksForExecutor(ClusterStateTaskExecutor executor) { + final ArrayList> toExecute = new ArrayList<>(); + final ArrayList sources = new ArrayList<>(); + synchronized (updateTasksPerExecutor) { + List pending = updateTasksPerExecutor.remove(executor); + if (pending != null) { + for (UpdateTask task : pending) { + if (task.processed.getAndSet(true) == false) { + logger.trace("will process [{}]", task.source); + toExecute.add(task); + sources.add(task.source); + } else { + logger.trace("skipping [{}], already processed", task.source); + } + } + } + } + if (toExecute.isEmpty()) { + return; + } + final String source = Strings.collectionToCommaDelimitedString(sources); + if (!lifecycle.started()) { + logger.debug("processing [{}]: ignoring, cluster_service not started", source); + return; + } + logger.debug("processing [{}]: execute", source); + ClusterState previousClusterState = clusterState; + if (!previousClusterState.nodes().localNodeMaster() && executor.runOnlyOnMaster()) { + logger.debug("failing [{}]: local node is no longer master", source); + toExecute.stream().forEach(task -> task.listener.onNoLongerMaster(task.source)); + return; + } + ClusterStateTaskExecutor.BatchResult batchResult; + long startTimeNS = System.nanoTime(); + try { + List inputs = toExecute.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList()); + batchResult = executor.execute(previousClusterState, inputs); + } catch (Throwable e) { + TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); + if (logger.isTraceEnabled()) { + StringBuilder sb = new StringBuilder("failed to execute cluster state update in ").append(executionTime).append(", state:\nversion [").append(previousClusterState.version()).append("], source [").append(source).append("]\n"); + sb.append(previousClusterState.nodes().prettyPrint()); + sb.append(previousClusterState.routingTable().prettyPrint()); + sb.append(previousClusterState.getRoutingNodes().prettyPrint()); + logger.trace(sb.toString(), e); + } + warnAboutSlowTaskIfNeeded(executionTime, source); + batchResult = ClusterStateTaskExecutor.BatchResult.builder().failures(toExecute.stream().map(updateTask -> updateTask.task)::iterator, e).build(previousClusterState); } - @Override - public void run() { - if (!lifecycle.started()) { - logger.debug("processing [{}]: ignoring, cluster_service not started", source); - return; - } - logger.debug("processing [{}]: execute", source); - ClusterState previousClusterState = clusterState; - if (!previousClusterState.nodes().localNodeMaster() && updateTask.runOnlyOnMaster()) { - logger.debug("failing [{}]: local node is no longer master", source); - updateTask.onNoLongerMaster(source); - return; - } - ClusterState newClusterState; - long startTimeNS = System.nanoTime(); - try { - newClusterState = updateTask.execute(previousClusterState); - } catch (Throwable e) { - TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); - if (logger.isTraceEnabled()) { - StringBuilder sb = new StringBuilder("failed to execute cluster state update in ").append(executionTime).append(", state:\nversion [").append(previousClusterState.version()).append("], source [").append(source).append("]\n"); - sb.append(previousClusterState.nodes().prettyPrint()); - sb.append(previousClusterState.routingTable().prettyPrint()); - sb.append(previousClusterState.getRoutingNodes().prettyPrint()); - logger.trace(sb.toString(), e); - } - warnAboutSlowTaskIfNeeded(executionTime, source); - updateTask.onFailure(source, e); - return; - } + assert batchResult.executionResults != null; - if (previousClusterState == newClusterState) { - if (updateTask instanceof AckedClusterStateUpdateTask) { + ClusterState newClusterState = batchResult.resultingState; + final ArrayList> proccessedListeners = new ArrayList<>(); + // fail all tasks that have failed and extract those that are waiting for results + for (UpdateTask updateTask : toExecute) { + assert batchResult.executionResults.containsKey(updateTask.task) : "missing " + updateTask.task.toString(); + final ClusterStateTaskExecutor.TaskResult executionResult = + batchResult.executionResults.get(updateTask.task); + executionResult.handle(() -> proccessedListeners.add(updateTask), ex -> updateTask.listener.onFailure(updateTask.source, ex)); + } + + if (previousClusterState == newClusterState) { + for (UpdateTask task : proccessedListeners) { + if (task.listener instanceof AckedClusterStateTaskListener) { //no need to wait for ack if nothing changed, the update can be counted as acknowledged - ((AckedClusterStateUpdateTask) updateTask).onAllNodesAcked(null); + ((AckedClusterStateTaskListener) task.listener).onAllNodesAcked(null); } - updateTask.clusterStateProcessed(source, previousClusterState, newClusterState); - TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); - logger.debug("processing [{}]: took {} no change in cluster_state", source, executionTime); - warnAboutSlowTaskIfNeeded(executionTime, source); - return; + task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState); } + TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); + logger.debug("processing [{}]: took {} no change in cluster_state", source, executionTime); + warnAboutSlowTaskIfNeeded(executionTime, source); + return; + } - try { - Discovery.AckListener ackListener = new NoOpAckListener(); - if (newClusterState.nodes().localNodeMaster()) { - // only the master controls the version numbers - Builder builder = ClusterState.builder(newClusterState).incrementVersion(); - if (previousClusterState.routingTable() != newClusterState.routingTable()) { - builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1).build()); - } - if (previousClusterState.metaData() != newClusterState.metaData()) { - builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1)); - } - newClusterState = builder.build(); - - if (updateTask instanceof AckedClusterStateUpdateTask) { - final AckedClusterStateUpdateTask ackedUpdateTask = (AckedClusterStateUpdateTask) updateTask; - if (ackedUpdateTask.ackTimeout() == null || ackedUpdateTask.ackTimeout().millis() == 0) { - ackedUpdateTask.onAckTimeout(); + try { + ArrayList ackListeners = new ArrayList<>(); + if (newClusterState.nodes().localNodeMaster()) { + // only the master controls the version numbers + Builder builder = ClusterState.builder(newClusterState).incrementVersion(); + if (previousClusterState.routingTable() != newClusterState.routingTable()) { + builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1).build()); + } + if (previousClusterState.metaData() != newClusterState.metaData()) { + builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1)); + } + newClusterState = builder.build(); + for (UpdateTask task : proccessedListeners) { + if (task.listener instanceof AckedClusterStateTaskListener) { + final AckedClusterStateTaskListener ackedListener = (AckedClusterStateTaskListener) task.listener; + if (ackedListener.ackTimeout() == null || ackedListener.ackTimeout().millis() == 0) { + ackedListener.onAckTimeout(); } else { try { - ackListener = new AckCountDownListener(ackedUpdateTask, newClusterState.version(), newClusterState.nodes(), threadPool); + ackListeners.add(new AckCountDownListener(ackedListener, newClusterState.version(), newClusterState.nodes(), threadPool)); } catch (EsRejectedExecutionException ex) { if (logger.isDebugEnabled()) { logger.debug("Couldn't schedule timeout thread - node might be shutting down", ex); } //timeout straightaway, otherwise we could wait forever as the timeout thread has not started - ackedUpdateTask.onAckTimeout(); + ackedListener.onAckTimeout(); } } } } - - newClusterState.status(ClusterState.ClusterStateStatus.BEING_APPLIED); - - if (logger.isTraceEnabled()) { - StringBuilder sb = new StringBuilder("cluster state updated, source [").append(source).append("]\n"); - sb.append(newClusterState.prettyPrint()); - logger.trace(sb.toString()); - } else if (logger.isDebugEnabled()) { - logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), source); - } - - ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(source, newClusterState, previousClusterState); - // new cluster state, notify all listeners - final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta(); - if (nodesDelta.hasChanges() && logger.isInfoEnabled()) { - String summary = nodesDelta.shortSummary(); - if (summary.length() > 0) { - logger.info("{}, reason: {}", summary, source); - } - } - - // TODO, do this in parallel (and wait) - for (DiscoveryNode node : nodesDelta.addedNodes()) { - if (!nodeRequiresConnection(node)) { - continue; - } - try { - transportService.connectToNode(node); - } catch (Throwable e) { - // the fault detection will detect it as failed as well - logger.warn("failed to connect to node [" + node + "]", e); - } - } - - // if we are the master, publish the new state to all nodes - // we publish here before we send a notification to all the listeners, since if it fails - // we don't want to notify - if (newClusterState.nodes().localNodeMaster()) { - logger.debug("publishing cluster state version [{}]", newClusterState.version()); - try { - discoveryService.publish(clusterChangedEvent, ackListener); - } catch (Discovery.FailedToCommitClusterStateException t) { - logger.warn("failing [{}]: failed to commit cluster state version [{}]", t, source, newClusterState.version()); - updateTask.onFailure(source, t); - return; - } - } - - // update the current cluster state - clusterState = newClusterState; - logger.debug("set local cluster state to version {}", newClusterState.version()); - for (ClusterStateListener listener : preAppliedListeners) { - try { - listener.clusterChanged(clusterChangedEvent); - } catch (Exception ex) { - logger.warn("failed to notify ClusterStateListener", ex); - } - } - - for (DiscoveryNode node : nodesDelta.removedNodes()) { - try { - transportService.disconnectFromNode(node); - } catch (Throwable e) { - logger.warn("failed to disconnect to node [" + node + "]", e); - } - } - - newClusterState.status(ClusterState.ClusterStateStatus.APPLIED); - - for (ClusterStateListener listener : postAppliedListeners) { - try { - listener.clusterChanged(clusterChangedEvent); - } catch (Exception ex) { - logger.warn("failed to notify ClusterStateListener", ex); - } - } - - //manual ack only from the master at the end of the publish - if (newClusterState.nodes().localNodeMaster()) { - try { - ackListener.onNodeAck(newClusterState.nodes().localNode(), null); - } catch (Throwable t) { - logger.debug("error while processing ack for master node [{}]", t, newClusterState.nodes().localNode()); - } - } - - updateTask.clusterStateProcessed(source, previousClusterState, newClusterState); - - TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); - logger.debug("processing [{}]: took {} done applying updated cluster_state (version: {}, uuid: {})", source, executionTime, newClusterState.version(), newClusterState.stateUUID()); - warnAboutSlowTaskIfNeeded(executionTime, source); - } catch (Throwable t) { - TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); - StringBuilder sb = new StringBuilder("failed to apply updated cluster state in ").append(executionTime).append(":\nversion [").append(newClusterState.version()).append("], uuid [").append(newClusterState.stateUUID()).append("], source [").append(source).append("]\n"); - sb.append(newClusterState.nodes().prettyPrint()); - sb.append(newClusterState.routingTable().prettyPrint()); - sb.append(newClusterState.getRoutingNodes().prettyPrint()); - logger.warn(sb.toString(), t); - // TODO: do we want to call updateTask.onFailure here? } + final Discovery.AckListener ackListener = new DelegetingAckListener(ackListeners); + + newClusterState.status(ClusterState.ClusterStateStatus.BEING_APPLIED); + + if (logger.isTraceEnabled()) { + StringBuilder sb = new StringBuilder("cluster state updated, source [").append(source).append("]\n"); + sb.append(newClusterState.prettyPrint()); + logger.trace(sb.toString()); + } else if (logger.isDebugEnabled()) { + logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), source); + } + + ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(source, newClusterState, previousClusterState); + // new cluster state, notify all listeners + final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta(); + if (nodesDelta.hasChanges() && logger.isInfoEnabled()) { + String summary = nodesDelta.shortSummary(); + if (summary.length() > 0) { + logger.info("{}, reason: {}", summary, source); + } + } + + // TODO, do this in parallel (and wait) + for (DiscoveryNode node : nodesDelta.addedNodes()) { + if (!nodeRequiresConnection(node)) { + continue; + } + try { + transportService.connectToNode(node); + } catch (Throwable e) { + // the fault detection will detect it as failed as well + logger.warn("failed to connect to node [" + node + "]", e); + } + } + + // if we are the master, publish the new state to all nodes + // we publish here before we send a notification to all the listeners, since if it fails + // we don't want to notify + if (newClusterState.nodes().localNodeMaster()) { + logger.debug("publishing cluster state version [{}]", newClusterState.version()); + try { + discoveryService.publish(clusterChangedEvent, ackListener); + } catch (Discovery.FailedToCommitClusterStateException t) { + logger.warn("failing [{}]: failed to commit cluster state version [{}]", t, source, newClusterState.version()); + proccessedListeners.forEach(task -> task.listener.onFailure(task.source, t)); + return; + } + } + + // update the current cluster state + clusterState = newClusterState; + logger.debug("set local cluster state to version {}", newClusterState.version()); + for (ClusterStateListener listener : preAppliedListeners) { + try { + listener.clusterChanged(clusterChangedEvent); + } catch (Exception ex) { + logger.warn("failed to notify ClusterStateListener", ex); + } + } + + for (DiscoveryNode node : nodesDelta.removedNodes()) { + try { + transportService.disconnectFromNode(node); + } catch (Throwable e) { + logger.warn("failed to disconnect to node [" + node + "]", e); + } + } + + newClusterState.status(ClusterState.ClusterStateStatus.APPLIED); + + for (ClusterStateListener listener : postAppliedListeners) { + try { + listener.clusterChanged(clusterChangedEvent); + } catch (Exception ex) { + logger.warn("failed to notify ClusterStateListener", ex); + } + } + + //manual ack only from the master at the end of the publish + if (newClusterState.nodes().localNodeMaster()) { + try { + ackListener.onNodeAck(newClusterState.nodes().localNode(), null); + } catch (Throwable t) { + logger.debug("error while processing ack for master node [{}]", t, newClusterState.nodes().localNode()); + } + } + + for (UpdateTask task : proccessedListeners) { + task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState); + } + + TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); + logger.debug("processing [{}]: took {} done applying updated cluster_state (version: {}, uuid: {})", source, executionTime, newClusterState.version(), newClusterState.stateUUID()); + warnAboutSlowTaskIfNeeded(executionTime, source); + } catch (Throwable t) { + TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS))); + StringBuilder sb = new StringBuilder("failed to apply updated cluster state in ").append(executionTime).append(":\nversion [").append(newClusterState.version()).append("], uuid [").append(newClusterState.stateUUID()).append("], source [").append(source).append("]\n"); + sb.append(newClusterState.nodes().prettyPrint()); + sb.append(newClusterState.routingTable().prettyPrint()); + sb.append(newClusterState.getRoutingNodes().prettyPrint()); + logger.warn(sb.toString(), t); + // TODO: do we want to call updateTask.onFailure here? + } + + } + + class UpdateTask extends SourcePrioritizedRunnable { + + public final T task; + public final ClusterStateTaskConfig config; + public final ClusterStateTaskExecutor executor; + public final ClusterStateTaskListener listener; + public final AtomicBoolean processed = new AtomicBoolean(); + + UpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor executor, ClusterStateTaskListener listener) { + super(config.priority(), source); + this.task = task; + this.config = config; + this.executor = executor; + this.listener = listener; + } + + @Override + public void run() { + runTasksForExecutor(executor); } } @@ -729,13 +762,24 @@ public class InternalClusterService extends AbstractLifecycleComponent listeners; + + private DelegetingAckListener(List listeners) { + this.listeners = listeners; + } + @Override public void onNodeAck(DiscoveryNode node, @Nullable Throwable t) { + for (Discovery.AckListener listener : listeners) { + listener.onNodeAck(node, t); + } } @Override public void onTimeout() { + throw new UnsupportedOperationException("no timeout delegation"); } } @@ -743,20 +787,20 @@ public class InternalClusterService extends AbstractLifecycleComponent ackTimeoutCallback; private Throwable lastFailure; - AckCountDownListener(AckedClusterStateUpdateTask ackedUpdateTask, long clusterStateVersion, DiscoveryNodes nodes, ThreadPool threadPool) { - this.ackedUpdateTask = ackedUpdateTask; + AckCountDownListener(AckedClusterStateTaskListener ackedTaskListener, long clusterStateVersion, DiscoveryNodes nodes, ThreadPool threadPool) { + this.ackedTaskListener = ackedTaskListener; this.clusterStateVersion = clusterStateVersion; this.nodes = nodes; int countDown = 0; for (DiscoveryNode node : nodes) { - if (ackedUpdateTask.mustAck(node)) { + if (ackedTaskListener.mustAck(node)) { countDown++; } } @@ -764,7 +808,7 @@ public class InternalClusterService extends AbstractLifecycleComponent parseFlags; public ParseFieldMatcher(Settings settings) { - if (settings.getAsBoolean(IndexQueryParserService.PARSE_STRICT, false)) { + if (settings.getAsBoolean(PARSE_STRICT, false)) { this.parseFlags = EnumSet.of(ParseField.Flag.STRICT); } else { this.parseFlags = ParseField.EMPTY_FLAGS; @@ -49,7 +48,7 @@ public class ParseFieldMatcher { /** * Matches a {@link ParseField} against a field name, and throws deprecation exception depending on the current - * value of the {@link IndexQueryParserService#PARSE_STRICT} setting. + * value of the {@link #PARSE_STRICT} setting. * @param fieldName the field name found in the request while parsing * @param parseField the parse field that we are looking for * @throws IllegalArgumentException whenever we are in strict mode and the request contained a deprecated field diff --git a/core/src/main/java/org/elasticsearch/common/Randomness.java b/core/src/main/java/org/elasticsearch/common/Randomness.java new file mode 100644 index 00000000000..dbfa8034b99 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/Randomness.java @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common; + +import org.elasticsearch.common.settings.Settings; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Provides factory methods for producing reproducible sources of + * randomness. Reproducible sources of randomness contribute to + * reproducible tests. When running the Elasticsearch test suite, the + * test runner will establish a global random seed accessible via the + * system property "tests.seed". By seeding a random number generator + * with this global seed, we ensure that instances of Random produced + * with this class produce reproducible sources of randomness under + * when running under the Elasticsearch test suite. Alternatively, + * a reproducible source of randomness can be produced by providing a + * setting a reproducible seed. When running the Elasticsearch server + * process, non-reproducible sources of randomness are provided (unless + * a setting is provided for a module that exposes a seed setting (e.g., + * DiscoveryService#SETTING_DISCOVERY_SEED)). + */ +public final class Randomness { + private static final Method currentMethod; + private static final Method getRandomMethod; + + static { + Method maybeCurrentMethod; + Method maybeGetRandomMethod; + try { + Class clazz = Class.forName("com.carrotsearch.randomizedtesting.RandomizedContext"); + maybeCurrentMethod = clazz.getMethod("current"); + maybeGetRandomMethod = clazz.getMethod("getRandom"); + } catch (Throwable t) { + maybeCurrentMethod = null; + maybeGetRandomMethod = null; + } + currentMethod = maybeCurrentMethod; + getRandomMethod = maybeGetRandomMethod; + } + + private Randomness() {} + + /** + * Provides a reproducible source of randomness seeded by a long + * seed in the settings with the key setting. + * + * @param settings the settings containing the seed + * @param setting the key to access the seed + * @return a reproducible source of randomness + */ + public static Random get(Settings settings, String setting) { + Long maybeSeed = settings.getAsLong(setting, null); + if (maybeSeed != null) { + return new Random(maybeSeed); + } else { + return get(); + } + } + + /** + * Provides a source of randomness that is reproducible when + * running under the Elasticsearch test suite, and otherwise + * produces a non-reproducible source of randomness. Reproducible + * sources of randomness are created when the system property + * "tests.seed" is set and the security policy allows reading this + * system property. Otherwise, non-reproducible sources of + * randomness are created. + * + * @return a source of randomness + * @throws IllegalStateException if running tests but was not able + * to acquire an instance of Random from + * RandomizedContext or tests are + * running but tests.seed is not set + */ + public static Random get() { + if (currentMethod != null && getRandomMethod != null) { + try { + Object randomizedContext = currentMethod.invoke(null); + return (Random) getRandomMethod.invoke(randomizedContext); + } catch (ReflectiveOperationException e) { + // unexpected, bail + throw new IllegalStateException("running tests but failed to invoke RandomizedContext#getRandom", e); + } + } else { + return getWithoutSeed(); + } + } + + private static Random getWithoutSeed() { + assert currentMethod == null && getRandomMethod == null : "running under tests but tried to create non-reproducible random"; + return ThreadLocalRandom.current(); + } + + public static void shuffle(List list) { + Collections.shuffle(list, get()); + } +} diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractLegacyBlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractLegacyBlobContainer.java deleted file mode 100644 index b95a7f28c58..00000000000 --- a/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractLegacyBlobContainer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.blobstore.support; - -import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -/** - * Temporary compatibility interface. - * - * This class should be removed after S3 and Azure containers migrate to the new model - */ -@Deprecated -public abstract class AbstractLegacyBlobContainer extends AbstractBlobContainer { - - protected AbstractLegacyBlobContainer(BlobPath path) { - super(path); - } - - /** - * Creates a new {@link InputStream} for the given blob name - *

    - * This method is deprecated and is used only for compatibility with older blob containers - * The new blob containers should use readBlob/writeBlob methods instead - */ - @Deprecated - protected abstract InputStream openInput(String blobName) throws IOException; - - /** - * Creates a new OutputStream for the given blob name - *

    - * This method is deprecated and is used only for compatibility with older blob containers - * The new blob containers should override readBlob/writeBlob methods instead - */ - @Deprecated - protected abstract OutputStream createOutput(String blobName) throws IOException; - - @Override - public InputStream readBlob(String blobName) throws IOException { - return openInput(blobName); - } - - @Override - public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { - try (OutputStream stream = createOutput(blobName)) { - Streams.copy(inputStream, stream); - } - } - - @Override - public void writeBlob(String blobName, BytesReference data) throws IOException { - try (OutputStream stream = createOutput(blobName)) { - data.writeTo(stream); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java index add383b75fa..14771791a61 100644 --- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java +++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java @@ -311,6 +311,10 @@ public class PagedBytesReference implements BytesReference { return true; } + if (obj == null) { + return false; + } + if (!(obj instanceof PagedBytesReference)) { return BytesReference.Helper.bytesEqual(this, (BytesReference) obj); } diff --git a/core/src/main/java/org/elasticsearch/common/cache/Cache.java b/core/src/main/java/org/elasticsearch/common/cache/Cache.java index a686ecc9645..a6c3bc81afd 100644 --- a/core/src/main/java/org/elasticsearch/common/cache/Cache.java +++ b/core/src/main/java/org/elasticsearch/common/cache/Cache.java @@ -25,12 +25,11 @@ import org.elasticsearch.common.util.concurrent.ReleasableLock; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.BiFunction; import java.util.function.ToLongBiFunction; /** @@ -175,7 +174,7 @@ public class Cache { ReleasableLock readLock = new ReleasableLock(segmentLock.readLock()); ReleasableLock writeLock = new ReleasableLock(segmentLock.writeLock()); - Map>> map = new HashMap<>(); + Map>> map = new HashMap<>(); SegmentStats segmentStats = new SegmentStats(); @@ -187,20 +186,28 @@ public class Cache { * @return the entry if there was one, otherwise null */ Entry get(K key, long now) { - Future> future; + CompletableFuture> future; Entry entry = null; try (ReleasableLock ignored = readLock.acquire()) { future = map.get(key); } if (future != null) { - segmentStats.hit(); - try { - entry = future.get(); - entry.accessTime = now; - } catch (ExecutionException | InterruptedException e) { - throw new IllegalStateException("future should be a completedFuture for which get should not throw", e); - } - } else { + try { + entry = future.handle((ok, ex) -> { + if (ok != null) { + segmentStats.hit(); + ok.accessTime = now; + return ok; + } else { + segmentStats.miss(); + return null; + } + }).get(); + } catch (ExecutionException | InterruptedException e) { + throw new IllegalStateException(e); + } + } + else { segmentStats.miss(); } return entry; @@ -216,13 +223,21 @@ public class Cache { */ Tuple, Entry> put(K key, V value, long now) { Entry entry = new Entry<>(key, value, now); - Entry existing; + Entry existing = null; try (ReleasableLock ignored = writeLock.acquire()) { try { - Future> future = map.put(key, CompletableFuture.completedFuture(entry)); - existing = future != null ? future.get() : null; + CompletableFuture> future = map.put(key, CompletableFuture.completedFuture(entry)); + if (future != null) { + existing = future.handle((ok, ex) -> { + if (ok != null) { + return ok; + } else { + return null; + } + }).get(); + } } catch (ExecutionException | InterruptedException e) { - throw new IllegalStateException("future should be a completedFuture for which get should not throw", e); + throw new IllegalStateException(e); } } return Tuple.tuple(entry, existing); @@ -235,17 +250,23 @@ public class Cache { * @return the removed entry if there was one, otherwise null */ Entry remove(K key) { - Future> future; + CompletableFuture> future; Entry entry = null; try (ReleasableLock ignored = writeLock.acquire()) { future = map.remove(key); } if (future != null) { - segmentStats.eviction(); try { - entry = future.get(); + entry = future.handle((ok, ex) -> { + if (ok != null) { + segmentStats.eviction(); + return ok; + } else { + return null; + } + }).get(); } catch (ExecutionException | InterruptedException e) { - throw new IllegalStateException("future should be a completedFuture for which get should not throw", e); + throw new IllegalStateException(e); } } return entry; @@ -327,39 +348,57 @@ public class Cache { // the segment lock; to do this, we atomically put a future in the map that can load the value, and then // get the value from this future on the thread that won the race to place the future into the segment map CacheSegment segment = getCacheSegment(key); - Future> future; - FutureTask> task = new FutureTask<>(() -> new Entry<>(key, loader.load(key), now)); + CompletableFuture> future; + CompletableFuture> completableFuture = new CompletableFuture<>(); + try (ReleasableLock ignored = segment.writeLock.acquire()) { - future = segment.map.putIfAbsent(key, task); - } - if (future == null) { - future = task; - task.run(); + future = segment.map.putIfAbsent(key, completableFuture); } - Entry entry; - try { - entry = future.get(); - } catch (ExecutionException | InterruptedException e) { - // if the future ended exceptionally, we do not want to pollute the cache - // however, we have to take care to ensure that the polluted entry has not already been replaced - try (ReleasableLock ignored = segment.writeLock.acquire()) { - Future> sanity = segment.map.get(key); - try { - sanity.get(); - } catch (ExecutionException | InterruptedException gotcha) { - segment.map.remove(key); + BiFunction, Throwable, ? extends V> handler = (ok, ex) -> { + if (ok != null) { + try (ReleasableLock ignored = lruLock.acquire()) { + promote(ok, now); } + return ok.value; + } else { + try (ReleasableLock ignored = segment.writeLock.acquire()) { + CompletableFuture> sanity = segment.map.get(key); + if (sanity != null && sanity.isCompletedExceptionally()) { + segment.map.remove(key); + } + } + return null; } - throw (e instanceof ExecutionException) ? (ExecutionException)e : new ExecutionException(e); + }; + + CompletableFuture completableValue; + if (future == null) { + future = completableFuture; + completableValue = future.handle(handler); + V loaded; + try { + loaded = loader.load(key); + } catch (Exception e) { + future.completeExceptionally(e); + throw new ExecutionException(e); + } + if (loaded == null) { + NullPointerException npe = new NullPointerException("loader returned a null value"); + future.completeExceptionally(npe); + throw new ExecutionException(npe); + } else { + future.complete(new Entry<>(key, loaded, now)); + } + } else { + completableValue = future.handle(handler); } - if (entry.value == null) { - throw new ExecutionException(new NullPointerException("loader returned a null value")); + + try { + value = completableValue.get(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); } - try (ReleasableLock ignored = lruLock.acquire()) { - promote(entry, now); - } - value = entry.value; } return value; } diff --git a/core/src/main/java/org/elasticsearch/common/cli/Terminal.java b/core/src/main/java/org/elasticsearch/common/cli/Terminal.java index 583eb9cc3a9..9523115b024 100644 --- a/core/src/main/java/org/elasticsearch/common/cli/Terminal.java +++ b/core/src/main/java/org/elasticsearch/common/cli/Terminal.java @@ -116,7 +116,7 @@ public abstract class Terminal { } public void printError(Throwable t) { - printError("%s", t.getMessage()); + printError("%s", t.toString()); if (isDebugEnabled) { printStackTrace(t); } diff --git a/core/src/main/java/org/elasticsearch/common/collect/Iterators.java b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java index 34546120b0a..d44bf7341c4 100644 --- a/core/src/main/java/org/elasticsearch/common/collect/Iterators.java +++ b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java @@ -28,7 +28,8 @@ public class Iterators { throw new NullPointerException("iterators"); } - return new ConcatenatedIterator<>(iterators); + // explicit generic type argument needed for type inference + return new ConcatenatedIterator(iterators); } static class ConcatenatedIterator implements Iterator { diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java index 4af2e962d85..e6c43a524ca 100644 --- a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java +++ b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java @@ -23,7 +23,6 @@ import org.apache.lucene.store.IndexInput; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.deflate.DeflateCompressor; -import org.elasticsearch.common.compress.lzf.LZFCompressor; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -42,7 +41,6 @@ public class CompressorFactory { static { compressors = new Compressor[] { - new LZFCompressor(), new DeflateCompressor() }; defaultCompressor = new DeflateCompressor(); @@ -82,12 +80,23 @@ public class CompressorFactory { XContentType contentType = XContentFactory.xContentType(bytes); if (contentType == null) { + if (isAncient(bytes)) { + throw new IllegalStateException("unsupported compression: index was created before v2.0.0.beta1 and wasn't upgraded?"); + } throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } return null; } + /** true if the bytes were compressed with LZF: only used before elasticsearch 2.0 */ + private static boolean isAncient(BytesReference bytes) { + return bytes.length() >= 3 && + bytes.get(0) == 'Z' && + bytes.get(1) == 'V' && + (bytes.get(2) == 0 || bytes.get(2) == 1); + } + public static Compressor compressor(ChannelBuffer buffer) { for (Compressor compressor : compressors) { if (compressor.isCompressed(buffer)) { diff --git a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java b/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java deleted file mode 100644 index 93bd583662b..00000000000 --- a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.LZFChunk; -import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.store.IndexInput; -import org.elasticsearch.common.compress.CompressedIndexInput; -import org.elasticsearch.common.lucene.store.InputStreamIndexInput; - -import java.io.IOException; -import java.util.Arrays; - -/** - */ -@Deprecated -public class LZFCompressedIndexInput extends CompressedIndexInput { - - private final ChunkDecoder decoder; - // scratch area buffer - private byte[] inputBuffer; - - public LZFCompressedIndexInput(IndexInput in, ChunkDecoder decoder) throws IOException { - super(in); - - this.decoder = decoder; - this.uncompressed = new byte[LZFChunk.MAX_CHUNK_LEN]; - this.uncompressedLength = LZFChunk.MAX_CHUNK_LEN; - this.inputBuffer = new byte[LZFChunk.MAX_CHUNK_LEN]; - } - - @Override - protected void readHeader(IndexInput in) throws IOException { - byte[] header = new byte[LZFCompressor.LUCENE_HEADER.length]; - in.readBytes(header, 0, header.length, false); - if (!Arrays.equals(header, LZFCompressor.LUCENE_HEADER)) { - throw new IOException("wrong lzf compressed header [" + Arrays.toString(header) + "]"); - } - } - - @Override - protected int uncompress(IndexInput in, byte[] out) throws IOException { - return decoder.decodeChunk(new InputStreamIndexInput(in, Long.MAX_VALUE), inputBuffer, out); - } - - @Override - protected void doClose() throws IOException { - // nothing to do here... - } - - @Override - public IndexInput clone() { - LZFCompressedIndexInput cloned = (LZFCompressedIndexInput) super.clone(); - cloned.inputBuffer = new byte[LZFChunk.MAX_CHUNK_LEN]; - return cloned; - } - - @Override - public IndexInput slice(String description, long offset, long length) throws IOException { - return BufferedIndexInput.wrap(description, this, offset, length); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java b/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java deleted file mode 100644 index baefcaa8928..00000000000 --- a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.BufferRecycler; -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.LZFChunk; -import org.elasticsearch.common.compress.CompressedStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; - -import java.io.IOException; - -/** - */ -public class LZFCompressedStreamInput extends CompressedStreamInput { - - private final BufferRecycler recycler; - - private final ChunkDecoder decoder; - - // scratch area buffer - private byte[] inputBuffer; - - public LZFCompressedStreamInput(StreamInput in, ChunkDecoder decoder) throws IOException { - super(in); - this.recycler = BufferRecycler.instance(); - this.decoder = decoder; - - this.uncompressed = recycler.allocDecodeBuffer(LZFChunk.MAX_CHUNK_LEN); - this.inputBuffer = recycler.allocInputBuffer(LZFChunk.MAX_CHUNK_LEN); - } - - @Override - public void readHeader(StreamInput in) throws IOException { - // nothing to do here, each chunk has a header - } - - @Override - public int uncompress(StreamInput in, byte[] out) throws IOException { - return decoder.decodeChunk(in, inputBuffer, out); - } - - @Override - protected void doClose() throws IOException { - byte[] buf = inputBuffer; - if (buf != null) { - inputBuffer = null; - recycler.releaseInputBuffer(buf); - } - buf = uncompressed; - if (buf != null) { - uncompressed = null; - recycler.releaseDecodeBuffer(uncompressed); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java b/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java deleted file mode 100644 index bb7a642c987..00000000000 --- a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.LZFChunk; -import com.ning.compress.lzf.util.ChunkDecoderFactory; -import org.apache.lucene.store.IndexInput; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedIndexInput; -import org.elasticsearch.common.compress.Compressor; -import org.elasticsearch.common.compress.deflate.DeflateCompressor; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.Loggers; -import org.jboss.netty.buffer.ChannelBuffer; - -import java.io.IOException; - -/** - * @deprecated Use {@link DeflateCompressor} instead - */ -@Deprecated -public class LZFCompressor implements Compressor { - - static final byte[] LUCENE_HEADER = {'L', 'Z', 'F', 0}; - - private ChunkDecoder decoder; - - public LZFCompressor() { - this.decoder = ChunkDecoderFactory.safeInstance(); - Loggers.getLogger(LZFCompressor.class).debug("using decoder[{}] ", this.decoder.getClass().getSimpleName()); - } - - @Override - public boolean isCompressed(BytesReference bytes) { - return bytes.length() >= 3 && - bytes.get(0) == LZFChunk.BYTE_Z && - bytes.get(1) == LZFChunk.BYTE_V && - (bytes.get(2) == LZFChunk.BLOCK_TYPE_COMPRESSED || bytes.get(2) == LZFChunk.BLOCK_TYPE_NON_COMPRESSED); - } - - @Override - public boolean isCompressed(ChannelBuffer buffer) { - int offset = buffer.readerIndex(); - return buffer.readableBytes() >= 3 && - buffer.getByte(offset) == LZFChunk.BYTE_Z && - buffer.getByte(offset + 1) == LZFChunk.BYTE_V && - (buffer.getByte(offset + 2) == LZFChunk.BLOCK_TYPE_COMPRESSED || buffer.getByte(offset + 2) == LZFChunk.BLOCK_TYPE_NON_COMPRESSED); - } - - @Override - public boolean isCompressed(IndexInput in) throws IOException { - long currentPointer = in.getFilePointer(); - // since we have some metdata before the first compressed header, we check on our specific header - if (in.length() - currentPointer < (LUCENE_HEADER.length)) { - return false; - } - for (int i = 0; i < LUCENE_HEADER.length; i++) { - if (in.readByte() != LUCENE_HEADER[i]) { - in.seek(currentPointer); - return false; - } - } - in.seek(currentPointer); - return true; - } - - @Override - public StreamInput streamInput(StreamInput in) throws IOException { - return new LZFCompressedStreamInput(in, decoder); - } - - @Override - public StreamOutput streamOutput(StreamOutput out) throws IOException { - throw new UnsupportedOperationException("LZF is only here for back compat, no write support"); - } - - @Override - public CompressedIndexInput indexInput(IndexInput in) throws IOException { - return new LZFCompressedIndexInput(in, decoder); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java index 0bc9455bea3..7130537fceb 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java @@ -20,8 +20,8 @@ package org.elasticsearch.common.geo; import org.apache.lucene.util.BitUtil; -import org.apache.lucene.util.XGeoHashUtils; -import org.apache.lucene.util.XGeoUtils; +import org.apache.lucene.util.GeoHashUtils; +import org.apache.lucene.util.GeoUtils; /** * @@ -30,8 +30,7 @@ public final class GeoPoint { private double lat; private double lon; - private final static double TOLERANCE = XGeoUtils.TOLERANCE; - + public GeoPoint() { } @@ -82,14 +81,14 @@ public final class GeoPoint { } public GeoPoint resetFromIndexHash(long hash) { - lon = XGeoUtils.mortonUnhashLon(hash); - lat = XGeoUtils.mortonUnhashLat(hash); + lon = GeoUtils.mortonUnhashLon(hash); + lat = GeoUtils.mortonUnhashLat(hash); return this; } public GeoPoint resetFromGeoHash(String geohash) { - final long hash = XGeoHashUtils.mortonEncode(geohash); - return this.reset(XGeoUtils.mortonUnhashLat(hash), XGeoUtils.mortonUnhashLon(hash)); + final long hash = GeoHashUtils.mortonEncode(geohash); + return this.reset(GeoUtils.mortonUnhashLat(hash), GeoUtils.mortonUnhashLon(hash)); } public GeoPoint resetFromGeoHash(long geohashLong) { @@ -114,11 +113,11 @@ public final class GeoPoint { } public final String geohash() { - return XGeoHashUtils.stringEncode(lon, lat); + return GeoHashUtils.stringEncode(lon, lat); } public final String getGeohash() { - return XGeoHashUtils.stringEncode(lon, lat); + return GeoHashUtils.stringEncode(lon, lat); } @Override @@ -126,14 +125,10 @@ public final class GeoPoint { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - final GeoPoint geoPoint = (GeoPoint) o; - final double lonCompare = geoPoint.lon - lon; - final double latCompare = geoPoint.lat - lat; + GeoPoint geoPoint = (GeoPoint) o; - if ((lonCompare < -TOLERANCE || lonCompare > TOLERANCE) - || (latCompare < -TOLERANCE || latCompare > TOLERANCE)) { - return false; - } + if (Double.compare(geoPoint.lat, lat) != 0) return false; + if (Double.compare(geoPoint.lon, lon) != 0) return false; return true; } @@ -143,9 +138,9 @@ public final class GeoPoint { int result; long temp; temp = lat != +0.0d ? Double.doubleToLongBits(lat) : 0L; - result = (int) (temp ^ (temp >>> 32)); + result = Long.hashCode(temp); temp = lon != +0.0d ? Double.doubleToLongBits(lon) : 0L; - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + Long.hashCode(temp); return result; } diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index 62fe81a5f15..83a8adab1e0 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -66,6 +66,22 @@ public class GeoUtils { /** Earth ellipsoid polar distance in meters */ public static final double EARTH_POLAR_DISTANCE = Math.PI * EARTH_SEMI_MINOR_AXIS; + /** Returns the maximum distance/radius from the point 'center' before overlapping */ + public static double maxRadialDistance(GeoPoint center) { + if (Math.abs(center.lat()) == 90.0) { + return SloppyMath.haversin(center.lat(), center.lon(), 0, center.lon())*1000.0; + } + return SloppyMath.haversin(center.lat(), center.lon(), center.lat(), (180.0 + center.lon()) % 360)*1000.0; + } + + /** Returns the minimum between the provided distance 'initialRadius' and the + * maximum distance/radius from the point 'center' before overlapping + **/ + public static double maxRadialDistance(GeoPoint center, double initialRadius) { + final double maxRadius = maxRadialDistance(center); + return Math.min(initialRadius, maxRadius); + } + /** Returns true if latitude is actually a valid latitude value.*/ public static boolean isValidLatitude(double latitude) { if (Double.isNaN(latitude) || Double.isInfinite(latitude) || latitude < GeoUtils.MIN_LAT || latitude > GeoUtils.MAX_LAT) { @@ -285,38 +301,46 @@ public class GeoUtils { * @param normLon Whether to normalize longitude. */ public static void normalizePoint(GeoPoint point, boolean normLat, boolean normLon) { - double lat = point.lat(); - double lon = point.lon(); - - normLat = normLat && (lat>90 || lat <= -90); - normLon = normLon && (lon>180 || lon <= -180); - + double[] pt = {point.lon(), point.lat()}; + normalizePoint(pt, normLon, normLat); + point.reset(pt[1], pt[0]); + } + + public static void normalizePoint(double[] lonLat) { + normalizePoint(lonLat, true, true); + } + + public static void normalizePoint(double[] lonLat, boolean normLon, boolean normLat) { + assert lonLat != null && lonLat.length == 2; + + normLat = normLat && (lonLat[1] > 90 || lonLat[1] < -90); + normLon = normLon && (lonLat[0] > 180 || lonLat[0] < -180); + if (normLat) { - lat = centeredModulus(lat, 360); + lonLat[1] = centeredModulus(lonLat[1], 360); boolean shift = true; - if (lat < -90) { - lat = -180 - lat; - } else if (lat > 90) { - lat = 180 - lat; + if (lonLat[1] < -90) { + lonLat[1] = -180 - lonLat[1]; + } else if (lonLat[1] > 90) { + lonLat[1] = 180 - lonLat[1]; } else { // No need to shift the longitude, and the latitude is normalized shift = false; } if (shift) { if (normLon) { - lon += 180; + lonLat[0] += 180; } else { // Longitude won't be normalized, // keep it in the form x+k*360 (with x in ]-180;180]) // by only changing x, assuming k is meaningful for the user application. - lon += normalizeLon(lon) > 0 ? -180 : 180; + lonLat[0] += normalizeLon(lonLat[0]) > 0 ? -180 : 180; } } } if (normLon) { - lon = centeredModulus(lon, 360); + lonLat[0] = centeredModulus(lonLat[0], 360); } - point.reset(lat, lon); } private static double centeredModulus(double dividend, double divisor) { diff --git a/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java b/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java index 67287b6cb30..ada4f60e179 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java +++ b/core/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java @@ -34,7 +34,8 @@ public enum ShapeRelation implements Writeable{ INTERSECTS("intersects"), DISJOINT("disjoint"), - WITHIN("within"); + WITHIN("within"), + CONTAINS("contains"); private final String relationName; diff --git a/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java b/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java index 695db015eda..64c657c8b6f 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java +++ b/core/src/main/java/org/elasticsearch/common/geo/XShapeCollection.java @@ -28,11 +28,7 @@ import java.util.Collection; import java.util.List; /** - * Overrides bounding box logic in ShapeCollection base class to comply with - * OGC OpenGIS Abstract Specification: An Object Model for Interoperable Geoprocessing. - * - * NOTE: This algorithm is O(N) and can possibly be improved O(log n) using an internal R*-Tree - * data structure for a collection of bounding boxes + * Extends spatial4j ShapeCollection for points_only shape indexing support */ public class XShapeCollection extends ShapeCollection { @@ -49,42 +45,4 @@ public class XShapeCollection extends ShapeCollection { public void setPointsOnly(boolean pointsOnly) { this.pointsOnly = pointsOnly; } - - @Override - protected Rectangle computeBoundingBox(Collection shapes, SpatialContext ctx) { - Rectangle retBox = shapes.iterator().next().getBoundingBox(); - for (Shape geom : shapes) { - retBox = expandBBox(retBox, geom.getBoundingBox()); - } - return retBox; - } - - /** - * Spatial4J shapes have no knowledge of directed edges. For this reason, a bounding box - * that wraps the dateline can have a min longitude that is mathematically > than the - * Rectangles' minX value. This is an issue for geometric collections (e.g., MultiPolygon - * and ShapeCollection) Until geometry logic can be cleaned up in Spatial4J, ES provides - * the following expansion algorithm for GeometryCollections - */ - private Rectangle expandBBox(Rectangle bbox, Rectangle expand) { - if (bbox.equals(expand) || bbox.equals(SpatialContext.GEO.getWorldBounds())) { - return bbox; - } - - double minX = bbox.getMinX(); - double eMinX = expand.getMinX(); - double maxX = bbox.getMaxX(); - double eMaxX = expand.getMaxX(); - double minY = bbox.getMinY(); - double eMinY = expand.getMinY(); - double maxY = bbox.getMaxY(); - double eMaxY = expand.getMaxY(); - - bbox.reset(Math.min(Math.min(minX, maxX), Math.min(eMinX, eMaxX)), - Math.max(Math.max(minX, maxX), Math.max(eMinX, eMaxX)), - Math.min(Math.min(minY, maxY), Math.min(eMinY, eMaxY)), - Math.max(Math.max(minY, maxY), Math.max(eMinY, eMaxY))); - - return bbox; - } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/BaseLineStringBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/BaseLineStringBuilder.java deleted file mode 100644 index bf5beb9beb9..00000000000 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/BaseLineStringBuilder.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.geo.builders; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; - -import com.spatial4j.core.shape.ShapeCollection; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import com.spatial4j.core.shape.Shape; -import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.Geometry; -import com.vividsolutions.jts.geom.GeometryFactory; -import com.vividsolutions.jts.geom.LineString; - -public abstract class BaseLineStringBuilder> extends PointCollection { - - protected BaseLineStringBuilder() { - this(new ArrayList()); - } - - protected BaseLineStringBuilder(ArrayList points) { - super(points); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return coordinatesToXcontent(builder, false); - } - - @Override - public Shape build() { - Coordinate[] coordinates = points.toArray(new Coordinate[points.size()]); - Geometry geometry; - if(wrapdateline) { - ArrayList strings = decompose(FACTORY, coordinates, new ArrayList()); - - if(strings.size() == 1) { - geometry = strings.get(0); - } else { - LineString[] linestrings = strings.toArray(new LineString[strings.size()]); - geometry = FACTORY.createMultiLineString(linestrings); - } - - } else { - geometry = FACTORY.createLineString(coordinates); - } - return jtsGeometry(geometry); - } - - protected static ArrayList decompose(GeometryFactory factory, Coordinate[] coordinates, ArrayList strings) { - for(Coordinate[] part : decompose(+DATELINE, coordinates)) { - for(Coordinate[] line : decompose(-DATELINE, part)) { - strings.add(factory.createLineString(line)); - } - } - return strings; - } - - /** - * Decompose a linestring given as array of coordinates at a vertical line. - * - * @param dateline x-axis intercept of the vertical line - * @param coordinates coordinates forming the linestring - * @return array of linestrings given as coordinate arrays - */ - protected static Coordinate[][] decompose(double dateline, Coordinate[] coordinates) { - int offset = 0; - ArrayList parts = new ArrayList<>(); - - double shift = coordinates[0].x > DATELINE ? DATELINE : (coordinates[0].x < -DATELINE ? -DATELINE : 0); - - for (int i = 1; i < coordinates.length; i++) { - double t = intersection(coordinates[i-1], coordinates[i], dateline); - if(!Double.isNaN(t)) { - Coordinate[] part; - if(t<1) { - part = Arrays.copyOfRange(coordinates, offset, i+1); - part[part.length-1] = Edge.position(coordinates[i-1], coordinates[i], t); - coordinates[offset+i-1] = Edge.position(coordinates[i-1], coordinates[i], t); - shift(shift, part); - offset = i-1; - shift = coordinates[i].x > DATELINE ? DATELINE : (coordinates[i].x < -DATELINE ? -DATELINE : 0); - } else { - part = shift(shift, Arrays.copyOfRange(coordinates, offset, i+1)); - offset = i; - } - parts.add(part); - } - } - - if(offset == 0) { - parts.add(shift(shift, coordinates)); - } else if(offset < coordinates.length-1) { - Coordinate[] part = Arrays.copyOfRange(coordinates, offset, coordinates.length); - parts.add(shift(shift, part)); - } - return parts.toArray(new Coordinate[parts.size()][]); - } - - private static Coordinate[] shift(double shift, Coordinate...coordinates) { - if(shift != 0) { - for (int j = 0; j < coordinates.length; j++) { - coordinates[j] = new Coordinate(coordinates[j].x - 2 * shift, coordinates[j].y); - } - } - return coordinates; - } -} diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/BasePolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/BasePolygonBuilder.java deleted file mode 100644 index aabba08936d..00000000000 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/BasePolygonBuilder.java +++ /dev/null @@ -1,558 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.geo.builders; - -import com.spatial4j.core.exception.InvalidShapeException; -import com.spatial4j.core.shape.Shape; -import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.Geometry; -import com.vividsolutions.jts.geom.GeometryFactory; -import com.vividsolutions.jts.geom.LinearRing; -import com.vividsolutions.jts.geom.MultiPolygon; -import com.vividsolutions.jts.geom.Polygon; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; - -/** - * The {@link BasePolygonBuilder} implements the groundwork to create polygons. This contains - * Methods to wrap polygons at the dateline and building shapes from the data held by the - * builder. - * Since this Builder can be embedded to other builders (i.e. {@link MultiPolygonBuilder}) - * the class of the embedding builder is given by the generic argument E - - * @param type of the embedding class - */ -public abstract class BasePolygonBuilder> extends ShapeBuilder { - - public static final GeoShapeType TYPE = GeoShapeType.POLYGON; - - // Linear ring defining the shell of the polygon - protected Ring shell; - - // List of linear rings defining the holes of the polygon - protected final ArrayList> holes = new ArrayList<>(); - - public BasePolygonBuilder(Orientation orientation) { - super(orientation); - } - - @SuppressWarnings("unchecked") - private E thisRef() { - return (E)this; - } - - public E point(double longitude, double latitude) { - shell.point(longitude, latitude); - return thisRef(); - } - - /** - * Add a point to the shell of the polygon - * @param coordinate coordinate of the new point - * @return this - */ - public E point(Coordinate coordinate) { - shell.point(coordinate); - return thisRef(); - } - - /** - * Add a array of points to the shell of the polygon - * @param coordinates coordinates of the new points to add - * @return this - */ - public E points(Coordinate...coordinates) { - shell.points(coordinates); - return thisRef(); - } - - /** - * Add a new hole to the polygon - * @param hole linear ring defining the hole - * @return this - */ - public E hole(BaseLineStringBuilder hole) { - holes.add(hole); - return thisRef(); - } - - /** - * build new hole to the polygon - * @return this - */ - public Ring hole() { - Ring hole = new Ring<>(thisRef()); - this.holes.add(hole); - return hole; - } - - /** - * Close the shell of the polygon - * @return parent - */ - public ShapeBuilder close() { - return shell.close(); - } - - /** - * Validates only 1 vertex is tangential (shared) between the interior and exterior of a polygon - */ - protected void validateHole(BaseLineStringBuilder shell, BaseLineStringBuilder hole) { - HashSet exterior = Sets.newHashSet(shell.points); - HashSet interior = Sets.newHashSet(hole.points); - exterior.retainAll(interior); - if (exterior.size() >= 2) { - throw new InvalidShapeException("Invalid polygon, interior cannot share more than one point with the exterior"); - } - } - - /** - * The coordinates setup by the builder will be assembled to a polygon. The result will consist of - * a set of polygons. Each of these components holds a list of linestrings defining the polygon: the - * first set of coordinates will be used as the shell of the polygon. The others are defined to holes - * within the polygon. - * This Method also wraps the polygons at the dateline. In order to this fact the result may - * contains more polygons and less holes than defined in the builder it self. - * - * @return coordinates of the polygon - */ - public Coordinate[][][] coordinates() { - int numEdges = shell.points.size()-1; // Last point is repeated - for (int i = 0; i < holes.size(); i++) { - numEdges += holes.get(i).points.size()-1; - validateHole(shell, this.holes.get(i)); - } - - Edge[] edges = new Edge[numEdges]; - Edge[] holeComponents = new Edge[holes.size()]; - int offset = createEdges(0, orientation, shell, null, edges, 0); - for (int i = 0; i < holes.size(); i++) { - int length = createEdges(i+1, orientation, shell, this.holes.get(i), edges, offset); - holeComponents[i] = edges[offset]; - offset += length; - } - - int numHoles = holeComponents.length; - - numHoles = merge(edges, 0, intersections(+DATELINE, edges), holeComponents, numHoles); - numHoles = merge(edges, 0, intersections(-DATELINE, edges), holeComponents, numHoles); - - return compose(edges, holeComponents, numHoles); - } - - @Override - public Shape build() { - return jtsGeometry(buildGeometry(FACTORY, wrapdateline)); - } - - protected XContentBuilder coordinatesArray(XContentBuilder builder, Params params) throws IOException { - shell.coordinatesToXcontent(builder, true); - for(BaseLineStringBuilder hole : holes) { - hole.coordinatesToXcontent(builder, true); - } - return builder; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); - builder.startArray(FIELD_COORDINATES); - coordinatesArray(builder, params); - builder.endArray(); - builder.endObject(); - return builder; - } - - public Geometry buildGeometry(GeometryFactory factory, boolean fixDateline) { - if(fixDateline) { - Coordinate[][][] polygons = coordinates(); - return polygons.length == 1 - ? polygon(factory, polygons[0]) - : multipolygon(factory, polygons); - } else { - return toPolygon(factory); - } - } - - public Polygon toPolygon() { - return toPolygon(FACTORY); - } - - protected Polygon toPolygon(GeometryFactory factory) { - final LinearRing shell = linearRing(factory, this.shell.points); - final LinearRing[] holes = new LinearRing[this.holes.size()]; - Iterator> iterator = this.holes.iterator(); - for (int i = 0; iterator.hasNext(); i++) { - holes[i] = linearRing(factory, iterator.next().points); - } - return factory.createPolygon(shell, holes); - } - - protected static LinearRing linearRing(GeometryFactory factory, ArrayList coordinates) { - return factory.createLinearRing(coordinates.toArray(new Coordinate[coordinates.size()])); - } - - @Override - public GeoShapeType type() { - return TYPE; - } - - protected static Polygon polygon(GeometryFactory factory, Coordinate[][] polygon) { - LinearRing shell = factory.createLinearRing(polygon[0]); - LinearRing[] holes; - - if(polygon.length > 1) { - holes = new LinearRing[polygon.length-1]; - for (int i = 0; i < holes.length; i++) { - holes[i] = factory.createLinearRing(polygon[i+1]); - } - } else { - holes = null; - } - return factory.createPolygon(shell, holes); - } - - /** - * Create a Multipolygon from a set of coordinates. Each primary array contains a polygon which - * in turn contains an array of linestrings. These line Strings are represented as an array of - * coordinates. The first linestring will be the shell of the polygon the others define holes - * within the polygon. - * - * @param factory {@link GeometryFactory} to use - * @param polygons definition of polygons - * @return a new Multipolygon - */ - protected static MultiPolygon multipolygon(GeometryFactory factory, Coordinate[][][] polygons) { - Polygon[] polygonSet = new Polygon[polygons.length]; - for (int i = 0; i < polygonSet.length; i++) { - polygonSet[i] = polygon(factory, polygons[i]); - } - return factory.createMultiPolygon(polygonSet); - } - - /** - * This method sets the component id of all edges in a ring to a given id and shifts the - * coordinates of this component according to the dateline - * - * @param edge An arbitrary edge of the component - * @param id id to apply to the component - * @param edges a list of edges to which all edges of the component will be added (could be null) - * @return number of edges that belong to this component - */ - private static int component(final Edge edge, final int id, final ArrayList edges) { - // find a coordinate that is not part of the dateline - Edge any = edge; - while(any.coordinate.x == +DATELINE || any.coordinate.x == -DATELINE) { - if((any = any.next) == edge) { - break; - } - } - - double shiftOffset = any.coordinate.x > DATELINE ? DATELINE : (any.coordinate.x < -DATELINE ? -DATELINE : 0); - if (debugEnabled()) { - LOGGER.debug("shift: {[]}", shiftOffset); - } - - // run along the border of the component, collect the - // edges, shift them according to the dateline and - // update the component id - int length = 0, connectedComponents = 0; - // if there are two connected components, splitIndex keeps track of where to split the edge array - // start at 1 since the source coordinate is shared - int splitIndex = 1; - Edge current = edge; - Edge prev = edge; - // bookkeep the source and sink of each visited coordinate - HashMap> visitedEdge = new HashMap<>(); - do { - current.coordinate = shift(current.coordinate, shiftOffset); - current.component = id; - - if (edges != null) { - // found a closed loop - we have two connected components so we need to slice into two distinct components - if (visitedEdge.containsKey(current.coordinate)) { - if (connectedComponents > 0 && current.next != edge) { - throw new InvalidShapeException("Shape contains more than one shared point"); - } - - // a negative id flags the edge as visited for the edges(...) method. - // since we're splitting connected components, we want the edges method to visit - // the newly separated component - final int visitID = -id; - Edge firstAppearance = visitedEdge.get(current.coordinate).v2(); - // correct the graph pointers by correcting the 'next' pointer for both the - // first appearance and this appearance of the edge - Edge temp = firstAppearance.next; - firstAppearance.next = current.next; - current.next = temp; - current.component = visitID; - // backtrack until we get back to this coordinate, setting the visit id to - // a non-visited value (anything positive) - do { - prev.component = visitID; - prev = visitedEdge.get(prev.coordinate).v1(); - ++splitIndex; - } while (!current.coordinate.equals(prev.coordinate)); - ++connectedComponents; - } else { - visitedEdge.put(current.coordinate, new Tuple(prev, current)); - } - edges.add(current); - prev = current; - } - length++; - } while(connectedComponents == 0 && (current = current.next) != edge); - - return (splitIndex != 1) ? length-splitIndex: length; - } - - /** - * Compute all coordinates of a component - * @param component an arbitrary edge of the component - * @param coordinates Array of coordinates to write the result to - * @return the coordinates parameter - */ - private static Coordinate[] coordinates(Edge component, Coordinate[] coordinates) { - for (int i = 0; i < coordinates.length; i++) { - coordinates[i] = (component = component.next).coordinate; - } - return coordinates; - } - - private static Coordinate[][][] buildCoordinates(ArrayList> components) { - Coordinate[][][] result = new Coordinate[components.size()][][]; - for (int i = 0; i < result.length; i++) { - ArrayList component = components.get(i); - result[i] = component.toArray(new Coordinate[component.size()][]); - } - - if(debugEnabled()) { - for (int i = 0; i < result.length; i++) { - LOGGER.debug("Component {[]}:", i); - for (int j = 0; j < result[i].length; j++) { - LOGGER.debug("\t" + Arrays.toString(result[i][j])); - } - } - } - - return result; - } - - private static final Coordinate[][] EMPTY = new Coordinate[0][]; - - private static Coordinate[][] holes(Edge[] holes, int numHoles) { - if (numHoles == 0) { - return EMPTY; - } - final Coordinate[][] points = new Coordinate[numHoles][]; - - for (int i = 0; i < numHoles; i++) { - int length = component(holes[i], -(i+1), null); // mark as visited by inverting the sign - points[i] = coordinates(holes[i], new Coordinate[length+1]); - } - - return points; - } - - private static Edge[] edges(Edge[] edges, int numHoles, ArrayList> components) { - ArrayList mainEdges = new ArrayList<>(edges.length); - - for (int i = 0; i < edges.length; i++) { - if (edges[i].component >= 0) { - int length = component(edges[i], -(components.size()+numHoles+1), mainEdges); - ArrayList component = new ArrayList<>(); - component.add(coordinates(edges[i], new Coordinate[length+1])); - components.add(component); - } - } - - return mainEdges.toArray(new Edge[mainEdges.size()]); - } - - private static Coordinate[][][] compose(Edge[] edges, Edge[] holes, int numHoles) { - final ArrayList> components = new ArrayList<>(); - assign(holes, holes(holes, numHoles), numHoles, edges(edges, numHoles, components), components); - return buildCoordinates(components); - } - - private static void assign(Edge[] holes, Coordinate[][] points, int numHoles, Edge[] edges, ArrayList> components) { - // Assign Hole to related components - // To find the new component the hole belongs to all intersections of the - // polygon edges with a vertical line are calculated. This vertical line - // is an arbitrary point of the hole. The polygon edge next to this point - // is part of the polygon the hole belongs to. - if (debugEnabled()) { - LOGGER.debug("Holes: " + Arrays.toString(holes)); - } - for (int i = 0; i < numHoles; i++) { - final Edge current = new Edge(holes[i].coordinate, holes[i].next); - // the edge intersects with itself at its own coordinate. We need intersect to be set this way so the binary search - // will get the correct position in the edge list and therefore the correct component to add the hole - current.intersect = current.coordinate; - final int intersections = intersections(current.coordinate.x, edges); - // if no intersection is found then the hole is not within the polygon, so - // don't waste time calling a binary search - final int pos; - boolean sharedVertex = false; - if (intersections == 0 || ((pos = Arrays.binarySearch(edges, 0, intersections, current, INTERSECTION_ORDER)) >= 0) - && !(sharedVertex = (edges[pos].intersect.compareTo(current.coordinate) == 0)) ) { - throw new InvalidShapeException("Invalid shape: Hole is not within polygon"); - } - final int index = -((sharedVertex) ? 0 : pos+2); - final int component = -edges[index].component - numHoles - 1; - - if(debugEnabled()) { - LOGGER.debug("\tposition ("+index+") of edge "+current+": " + edges[index]); - LOGGER.debug("\tComponent: " + component); - LOGGER.debug("\tHole intersections ("+current.coordinate.x+"): " + Arrays.toString(edges)); - } - - components.get(component).add(points[i]); - } - } - - private static int merge(Edge[] intersections, int offset, int length, Edge[] holes, int numHoles) { - // Intersections appear pairwise. On the first edge the inner of - // of the polygon is entered. On the second edge the outer face - // is entered. Other kinds of intersections are discard by the - // intersection function - - for (int i = 0; i < length; i += 2) { - Edge e1 = intersections[offset + i + 0]; - Edge e2 = intersections[offset + i + 1]; - - // If two segments are connected maybe a hole must be deleted - // Since Edges of components appear pairwise we need to check - // the second edge only (the first edge is either polygon or - // already handled) - if (e2.component > 0) { - //TODO: Check if we could save the set null step - numHoles--; - holes[e2.component-1] = holes[numHoles]; - holes[numHoles] = null; - } - // only connect edges if intersections are pairwise - // 1. per the comment above, the edge array is sorted by y-value of the intersection - // with the dateline. Two edges have the same y intercept when they cross the - // dateline thus they appear sequentially (pairwise) in the edge array. Two edges - // do not have the same y intercept when we're forming a multi-poly from a poly - // that wraps the dateline (but there are 2 ordered intercepts). - // The connect method creates a new edge for these paired edges in the linked list. - // For boundary conditions (e.g., intersect but not crossing) there is no sibling edge - // to connect. Thus the first logic check enforces the pairwise rule - // 2. the second logic check ensures the two candidate edges aren't already connected by an - // existing edge along the dateline - this is necessary due to a logic change in - // ShapeBuilder.intersection that computes dateline edges as valid intersect points - // in support of OGC standards - if (e1.intersect != Edge.MAX_COORDINATE && e2.intersect != Edge.MAX_COORDINATE - && !(e1.next.next.coordinate.equals3D(e2.coordinate) && Math.abs(e1.next.coordinate.x) == DATELINE - && Math.abs(e2.coordinate.x) == DATELINE) ) { - connect(e1, e2); - } - } - return numHoles; - } - - private static void connect(Edge in, Edge out) { - assert in != null && out != null; - assert in != out; - // Connecting two Edges by inserting the point at - // dateline intersection and connect these by adding - // two edges between this points. One per direction - if(in.intersect != in.next.coordinate) { - // NOTE: the order of the object creation is crucial here! Don't change it! - // first edge has no point on dateline - Edge e1 = new Edge(in.intersect, in.next); - - if(out.intersect != out.next.coordinate) { - // second edge has no point on dateline - Edge e2 = new Edge(out.intersect, out.next); - in.next = new Edge(in.intersect, e2, in.intersect); - } else { - // second edge intersects with dateline - in.next = new Edge(in.intersect, out.next, in.intersect); - } - out.next = new Edge(out.intersect, e1, out.intersect); - } else if (in.next != out && in.coordinate != out.intersect) { - // first edge intersects with dateline - Edge e2 = new Edge(out.intersect, in.next, out.intersect); - - if(out.intersect != out.next.coordinate) { - // second edge has no point on dateline - Edge e1 = new Edge(out.intersect, out.next); - in.next = new Edge(in.intersect, e1, in.intersect); - - } else { - // second edge intersects with dateline - in.next = new Edge(in.intersect, out.next, in.intersect); - } - out.next = e2; - } - } - - private static int createEdges(int component, Orientation orientation, BaseLineStringBuilder shell, - BaseLineStringBuilder hole, - Edge[] edges, int offset) { - // inner rings (holes) have an opposite direction than the outer rings - // XOR will invert the orientation for outer ring cases (Truth Table:, T/T = F, T/F = T, F/T = T, F/F = F) - boolean direction = (component == 0 ^ orientation == Orientation.RIGHT); - // set the points array accordingly (shell or hole) - Coordinate[] points = (hole != null) ? hole.coordinates(false) : shell.coordinates(false); - Edge.ring(component, direction, orientation == Orientation.LEFT, shell, points, 0, edges, offset, points.length-1); - return points.length-1; - } - - public static class Ring

    extends BaseLineStringBuilder> { - - private final P parent; - - protected Ring(P parent) { - this(parent, new ArrayList()); - } - - protected Ring(P parent, ArrayList points) { - super(points); - this.parent = parent; - } - - public P close() { - Coordinate start = points.get(0); - Coordinate end = points.get(points.size()-1); - if(start.x != end.x || start.y != end.y) { - points.add(start); - } - return parent; - } - - @Override - public GeoShapeType type() { - return null; - } - } -} diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java index f1054e18663..5f11d12a4bf 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java @@ -21,24 +21,30 @@ package org.elasticsearch.common.geo.builders; import com.spatial4j.core.shape.Circle; import com.vividsolutions.jts.geom.Coordinate; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.unit.DistanceUnit.Distance; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Objects; public class CircleBuilder extends ShapeBuilder { public static final String FIELD_RADIUS = "radius"; public static final GeoShapeType TYPE = GeoShapeType.CIRCLE; + public static final CircleBuilder PROTOTYPE = new CircleBuilder(); + private DistanceUnit unit; private double radius; private Coordinate center; - + /** * Set the center of the circle - * + * * @param center coordinate of the circles center * @return this */ @@ -57,6 +63,13 @@ public class CircleBuilder extends ShapeBuilder { return center(new Coordinate(lon, lat)); } + /** + * Get the center of the circle + */ + public Coordinate center() { + return center; + } + /** * Set the radius of the circle. The String value will be parsed by {@link DistanceUnit} * @param radius Value and unit of the circle combined in a string @@ -97,10 +110,24 @@ public class CircleBuilder extends ShapeBuilder { return this; } + /** + * Get the radius of the circle without unit + */ + public double radius() { + return this.radius; + } + + /** + * Get the radius unit of the circle + */ + public DistanceUnit unit() { + return this.unit; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); builder.field(FIELD_RADIUS, unit.toString(radius)); builder.field(FIELD_COORDINATES); toXContent(builder, center); @@ -116,4 +143,37 @@ public class CircleBuilder extends ShapeBuilder { public GeoShapeType type() { return TYPE; } + + @Override + public int hashCode() { + return Objects.hash(center, radius, unit.ordinal()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + CircleBuilder other = (CircleBuilder) obj; + return Objects.equals(center, other.center) && + Objects.equals(radius, other.radius) && + Objects.equals(unit.ordinal(), other.unit.ordinal()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeCoordinateTo(center, out); + out.writeDouble(radius); + DistanceUnit.writeDistanceUnit(out, unit); + } + + @Override + public CircleBuilder readFrom(StreamInput in) throws IOException { + return new CircleBuilder() + .center(readCoordinateFrom(in)) + .radius(in.readDouble(), DistanceUnit.readDistanceUnit(in)); + } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java index a296b3406ef..62f29d2bad7 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java @@ -21,13 +21,19 @@ package org.elasticsearch.common.geo.builders; import com.spatial4j.core.shape.Rectangle; import com.vividsolutions.jts.geom.Coordinate; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Locale; +import java.util.Objects; public class EnvelopeBuilder extends ShapeBuilder { - public static final GeoShapeType TYPE = GeoShapeType.ENVELOPE; + public static final GeoShapeType TYPE = GeoShapeType.ENVELOPE; + public static final EnvelopeBuilder PROTOTYPE = new EnvelopeBuilder(); protected Coordinate topLeft; protected Coordinate bottomRight; @@ -61,7 +67,8 @@ public class EnvelopeBuilder extends ShapeBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); + builder.field(FIELD_ORIENTATION, orientation.name().toLowerCase(Locale.ROOT)); builder.startArray(FIELD_COORDINATES); toXContent(builder, topLeft); toXContent(builder, bottomRight); @@ -78,4 +85,38 @@ public class EnvelopeBuilder extends ShapeBuilder { public GeoShapeType type() { return TYPE; } + + @Override + public int hashCode() { + return Objects.hash(orientation, topLeft, bottomRight); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + EnvelopeBuilder other = (EnvelopeBuilder) obj; + return Objects.equals(orientation, other.orientation) && + Objects.equals(topLeft, other.topLeft) && + Objects.equals(bottomRight, other.bottomRight); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(orientation == Orientation.RIGHT); + writeCoordinateTo(topLeft, out); + writeCoordinateTo(bottomRight, out); + } + + @Override + public EnvelopeBuilder readFrom(StreamInput in) throws IOException { + Orientation orientation = in.readBoolean() ? Orientation.RIGHT : Orientation.LEFT; + return new EnvelopeBuilder(orientation) + .topLeft(readCoordinateFrom(in)) + .bottomRight(readCoordinateFrom(in)); + } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java index ede6bcf62eb..45397ed962f 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java @@ -29,7 +29,7 @@ import java.util.ArrayList; import java.util.List; public class GeometryCollectionBuilder extends ShapeBuilder { - + public static final GeoShapeType TYPE = GeoShapeType.GEOMETRYCOLLECTION; protected final ArrayList shapes = new ArrayList<>(); @@ -46,42 +46,42 @@ public class GeometryCollectionBuilder extends ShapeBuilder { this.shapes.add(shape); return this; } - + public GeometryCollectionBuilder point(PointBuilder point) { this.shapes.add(point); return this; } - + public GeometryCollectionBuilder multiPoint(MultiPointBuilder multiPoint) { this.shapes.add(multiPoint); return this; } - - public GeometryCollectionBuilder line(BaseLineStringBuilder line) { + + public GeometryCollectionBuilder line(LineStringBuilder line) { this.shapes.add(line); return this; } - + public GeometryCollectionBuilder multiLine(MultiLineStringBuilder multiLine) { this.shapes.add(multiLine); return this; } - - public GeometryCollectionBuilder polygon(BasePolygonBuilder polygon) { + + public GeometryCollectionBuilder polygon(PolygonBuilder polygon) { this.shapes.add(polygon); return this; } - + public GeometryCollectionBuilder multiPolygon(MultiPolygonBuilder multiPolygon) { this.shapes.add(multiPolygon); return this; } - + public GeometryCollectionBuilder envelope(EnvelopeBuilder envelope) { this.shapes.add(envelope); return this; } - + public GeometryCollectionBuilder circle(CircleBuilder circle) { this.shapes.add(circle); return this; @@ -102,7 +102,7 @@ public class GeometryCollectionBuilder extends ShapeBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); builder.startArray(FIELD_GEOMETRIES); for (ShapeBuilder shape : shapes) { shape.toXContent(builder, params); @@ -120,11 +120,11 @@ public class GeometryCollectionBuilder extends ShapeBuilder { public Shape build() { List shapes = new ArrayList<>(this.shapes.size()); - + for (ShapeBuilder shape : this.shapes) { shapes.add(shape.build()); } - + if (shapes.size() == 1) return shapes.get(0); else diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java index c581475b21a..c7ba9b72f55 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java @@ -19,27 +19,124 @@ package org.elasticsearch.common.geo.builders; -import org.elasticsearch.common.xcontent.XContentBuilder; - import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; -public class LineStringBuilder extends BaseLineStringBuilder { +import org.elasticsearch.common.xcontent.XContentBuilder; +import com.spatial4j.core.shape.Shape; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.Geometry; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.LineString; + +public class LineStringBuilder extends PointCollection { public static final GeoShapeType TYPE = GeoShapeType.LINESTRING; @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); builder.field(FIELD_COORDINATES); coordinatesToXcontent(builder, false); builder.endObject(); return builder; } + /** + * Closes the current lineString by adding the starting point as the end point + */ + public LineStringBuilder close() { + Coordinate start = points.get(0); + Coordinate end = points.get(points.size()-1); + if(start.x != end.x || start.y != end.y) { + points.add(start); + } + return this; + } + @Override public GeoShapeType type() { return TYPE; } + @Override + public Shape build() { + Coordinate[] coordinates = points.toArray(new Coordinate[points.size()]); + Geometry geometry; + if(wrapdateline) { + ArrayList strings = decompose(FACTORY, coordinates, new ArrayList()); + + if(strings.size() == 1) { + geometry = strings.get(0); + } else { + LineString[] linestrings = strings.toArray(new LineString[strings.size()]); + geometry = FACTORY.createMultiLineString(linestrings); + } + + } else { + geometry = FACTORY.createLineString(coordinates); + } + return jtsGeometry(geometry); + } + + static ArrayList decompose(GeometryFactory factory, Coordinate[] coordinates, ArrayList strings) { + for(Coordinate[] part : decompose(+DATELINE, coordinates)) { + for(Coordinate[] line : decompose(-DATELINE, part)) { + strings.add(factory.createLineString(line)); + } + } + return strings; + } + + /** + * Decompose a linestring given as array of coordinates at a vertical line. + * + * @param dateline x-axis intercept of the vertical line + * @param coordinates coordinates forming the linestring + * @return array of linestrings given as coordinate arrays + */ + private static Coordinate[][] decompose(double dateline, Coordinate[] coordinates) { + int offset = 0; + ArrayList parts = new ArrayList<>(); + + double shift = coordinates[0].x > DATELINE ? DATELINE : (coordinates[0].x < -DATELINE ? -DATELINE : 0); + + for (int i = 1; i < coordinates.length; i++) { + double t = intersection(coordinates[i-1], coordinates[i], dateline); + if(!Double.isNaN(t)) { + Coordinate[] part; + if(t<1) { + part = Arrays.copyOfRange(coordinates, offset, i+1); + part[part.length-1] = Edge.position(coordinates[i-1], coordinates[i], t); + coordinates[offset+i-1] = Edge.position(coordinates[i-1], coordinates[i], t); + shift(shift, part); + offset = i-1; + shift = coordinates[i].x > DATELINE ? DATELINE : (coordinates[i].x < -DATELINE ? -DATELINE : 0); + } else { + part = shift(shift, Arrays.copyOfRange(coordinates, offset, i+1)); + offset = i; + } + parts.add(part); + } + } + + if(offset == 0) { + parts.add(shift(shift, coordinates)); + } else if(offset < coordinates.length-1) { + Coordinate[] part = Arrays.copyOfRange(coordinates, offset, coordinates.length); + parts.add(shift(shift, part)); + } + return parts.toArray(new Coordinate[parts.size()][]); + } + + private static Coordinate[] shift(double shift, Coordinate...coordinates) { + if(shift != 0) { + for (int j = 0; j < coordinates.length; j++) { + coordinates[j] = new Coordinate(coordinates[j].x - 2 * shift, coordinates[j].y); + } + } + return coordinates; + } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java index dcef02d7a68..a004b90a2dc 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java @@ -22,7 +22,6 @@ package org.elasticsearch.common.geo.builders; import org.elasticsearch.common.xcontent.XContentBuilder; import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.jts.JtsGeometry; import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Geometry; import com.vividsolutions.jts.geom.LineString; @@ -35,15 +34,9 @@ public class MultiLineStringBuilder extends ShapeBuilder { public static final GeoShapeType TYPE = GeoShapeType.MULTILINESTRING; - private final ArrayList> lines = new ArrayList<>(); + private final ArrayList lines = new ArrayList<>(); - public InternalLineStringBuilder linestring() { - InternalLineStringBuilder line = new InternalLineStringBuilder(this); - this.lines.add(line); - return line; - } - - public MultiLineStringBuilder linestring(BaseLineStringBuilder line) { + public MultiLineStringBuilder linestring(LineStringBuilder line) { this.lines.add(line); return this; } @@ -64,10 +57,10 @@ public class MultiLineStringBuilder extends ShapeBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); builder.field(FIELD_COORDINATES); builder.startArray(); - for(BaseLineStringBuilder line : lines) { + for(LineStringBuilder line : lines) { line.coordinatesToXcontent(builder, false); } builder.endArray(); @@ -80,8 +73,8 @@ public class MultiLineStringBuilder extends ShapeBuilder { final Geometry geometry; if(wrapdateline) { ArrayList parts = new ArrayList<>(); - for (BaseLineStringBuilder line : lines) { - BaseLineStringBuilder.decompose(FACTORY, line.coordinates(false), parts); + for (LineStringBuilder line : lines) { + LineStringBuilder.decompose(FACTORY, line.coordinates(false), parts); } if(parts.size() == 1) { geometry = parts.get(0); @@ -91,7 +84,7 @@ public class MultiLineStringBuilder extends ShapeBuilder { } } else { LineString[] lineStrings = new LineString[lines.size()]; - Iterator> iterator = lines.iterator(); + Iterator iterator = lines.iterator(); for (int i = 0; iterator.hasNext(); i++) { lineStrings[i] = FACTORY.createLineString(iterator.next().coordinates(false)); } @@ -99,27 +92,4 @@ public class MultiLineStringBuilder extends ShapeBuilder { } return jtsGeometry(geometry); } - - public static class InternalLineStringBuilder extends BaseLineStringBuilder { - - private final MultiLineStringBuilder collection; - - public InternalLineStringBuilder(MultiLineStringBuilder collection) { - super(); - this.collection = collection; - } - - public MultiLineStringBuilder end() { - return collection; - } - - public Coordinate[] coordinates() { - return super.coordinates(false); - } - - @Override - public GeoShapeType type() { - return null; - } - } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java index 5a9aaa90927..8d5cfabdabb 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java @@ -31,12 +31,13 @@ import java.util.List; public class MultiPointBuilder extends PointCollection { + public static final GeoShapeType TYPE = GeoShapeType.MULTIPOINT; @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); builder.field(FIELD_COORDINATES); super.coordinatesToXcontent(builder, false); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java index ee06fa64376..e7762e51b61 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java @@ -33,7 +33,7 @@ public class MultiPolygonBuilder extends ShapeBuilder { public static final GeoShapeType TYPE = GeoShapeType.MULTIPOLYGON; - protected final ArrayList> polygons = new ArrayList<>(); + protected final ArrayList polygons = new ArrayList<>(); public MultiPolygonBuilder() { this(Orientation.RIGHT); @@ -43,27 +43,17 @@ public class MultiPolygonBuilder extends ShapeBuilder { super(orientation); } - public MultiPolygonBuilder polygon(BasePolygonBuilder polygon) { + public MultiPolygonBuilder polygon(PolygonBuilder polygon) { this.polygons.add(polygon); return this; } - public InternalPolygonBuilder polygon() { - return polygon(Orientation.RIGHT); - } - - public InternalPolygonBuilder polygon(Orientation orientation) { - InternalPolygonBuilder polygon = new InternalPolygonBuilder(this, orientation); - this.polygon(polygon); - return polygon; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); builder.startArray(FIELD_COORDINATES); - for(BasePolygonBuilder polygon : polygons) { + for(PolygonBuilder polygon : polygons) { builder.startArray(); polygon.coordinatesArray(builder, params); builder.endArray(); @@ -81,15 +71,15 @@ public class MultiPolygonBuilder extends ShapeBuilder { public Shape build() { List shapes = new ArrayList<>(this.polygons.size()); - + if(wrapdateline) { - for (BasePolygonBuilder polygon : this.polygons) { + for (PolygonBuilder polygon : this.polygons) { for(Coordinate[][] part : polygon.coordinates()) { shapes.add(jtsGeometry(PolygonBuilder.polygon(FACTORY, part))); } } } else { - for (BasePolygonBuilder polygon : this.polygons) { + for (PolygonBuilder polygon : this.polygons) { shapes.add(jtsGeometry(polygon.toPolygon(FACTORY))); } } @@ -99,21 +89,4 @@ public class MultiPolygonBuilder extends ShapeBuilder { return new XShapeCollection<>(shapes, SPATIAL_CONTEXT); //note: ShapeCollection is probably faster than a Multi* geom. } - - public static class InternalPolygonBuilder extends BasePolygonBuilder { - - private final MultiPolygonBuilder collection; - - private InternalPolygonBuilder(MultiPolygonBuilder collection, Orientation orientation) { - super(orientation); - this.collection = collection; - this.shell = new Ring<>(this); - } - - @Override - public MultiPolygonBuilder close() { - super.close(); - return collection; - } - } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java index 53c67387e91..d6d62c28b8c 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java @@ -20,7 +20,10 @@ package org.elasticsearch.common.geo.builders; import java.io.IOException; +import java.util.Objects; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import com.spatial4j.core.shape.Point; @@ -30,6 +33,8 @@ public class PointBuilder extends ShapeBuilder { public static final GeoShapeType TYPE = GeoShapeType.POINT; + public static final PointBuilder PROTOTYPE = new PointBuilder(); + private Coordinate coordinate; public PointBuilder coordinate(Coordinate coordinate) { @@ -48,10 +53,10 @@ public class PointBuilder extends ShapeBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FIELD_TYPE, TYPE.shapename); + builder.field(FIELD_TYPE, TYPE.shapeName()); builder.field(FIELD_COORDINATES); toXContent(builder, coordinate); - return builder.endObject(); + return builder.endObject(); } @Override @@ -63,4 +68,31 @@ public class PointBuilder extends ShapeBuilder { public GeoShapeType type() { return TYPE; } + + @Override + public int hashCode() { + return Objects.hash(coordinate); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PointBuilder other = (PointBuilder) obj; + return Objects.equals(coordinate, other.coordinate); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeCoordinateTo(coordinate, out); + } + + @Override + public PointBuilder readFrom(StreamInput in) throws IOException { + return new PointBuilder().coordinate(readCoordinateFrom(in)); + } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PointCollection.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PointCollection.java index de1db188b31..45ce5adb595 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/PointCollection.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PointCollection.java @@ -29,12 +29,11 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import com.vividsolutions.jts.geom.Coordinate; /** - * The {@link PointCollection} is an abstract base implementation for all GeoShapes. It simply handles a set of points. + * The {@link PointCollection} is an abstract base implementation for all GeoShapes. It simply handles a set of points. */ public abstract class PointCollection> extends ShapeBuilder { protected final ArrayList points; - protected boolean translated = false; protected PointCollection() { this(new ArrayList()); @@ -43,7 +42,7 @@ public abstract class PointCollection> extends Shap protected PointCollection(ArrayList points) { this.points = points; } - + @SuppressWarnings("unchecked") private E thisRef() { return (E)this; @@ -57,7 +56,7 @@ public abstract class PointCollection> extends Shap */ public E point(double longitude, double latitude) { return this.point(coordinate(longitude, latitude)); - } + } /** * Add a new point to the collection @@ -71,7 +70,7 @@ public abstract class PointCollection> extends Shap /** * Add a array of points to the collection - * + * * @param coordinates array of {@link Coordinate}s to add * @return this */ @@ -81,7 +80,7 @@ public abstract class PointCollection> extends Shap /** * Add a collection of points to the collection - * + * * @param coordinates array of {@link Coordinate}s to add * @return this */ @@ -92,7 +91,7 @@ public abstract class PointCollection> extends Shap /** * Copy all points to a new Array - * + * * @param closed if set to true the first point of the array is repeated as last element * @return Array of coordinates */ @@ -106,9 +105,9 @@ public abstract class PointCollection> extends Shap /** * builds an array of coordinates to a {@link XContentBuilder} - * - * @param builder builder to use - * @param closed repeat the first point at the end of the array if it's not already defines as last element of the array + * + * @param builder builder to use + * @param closed repeat the first point at the end of the array if it's not already defines as last element of the array * @return the builder */ protected XContentBuilder coordinatesToXcontent(XContentBuilder builder, boolean closed) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java index d7c7fa6abd3..04540df27e9 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java @@ -19,11 +19,41 @@ package org.elasticsearch.common.geo.builders; -import java.util.ArrayList; - +import com.spatial4j.core.exception.InvalidShapeException; +import com.spatial4j.core.shape.Shape; import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.Geometry; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.LinearRing; +import com.vividsolutions.jts.geom.MultiPolygon; +import com.vividsolutions.jts.geom.Polygon; -public class PolygonBuilder extends BasePolygonBuilder { +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The {@link PolygonBuilder} implements the groundwork to create polygons. This contains + * Methods to wrap polygons at the dateline and building shapes from the data held by the + * builder. + */ +public class PolygonBuilder extends ShapeBuilder { + + public static final GeoShapeType TYPE = GeoShapeType.POLYGON; + + // line string defining the shell of the polygon + private LineStringBuilder shell; + + // List of line strings defining the holes of the polygon + private final ArrayList holes = new ArrayList<>(); public PolygonBuilder() { this(new ArrayList(), Orientation.RIGHT); @@ -33,14 +63,604 @@ public class PolygonBuilder extends BasePolygonBuilder { this(new ArrayList(), orientation); } - protected PolygonBuilder(ArrayList points, Orientation orientation) { + public PolygonBuilder(ArrayList points, Orientation orientation) { super(orientation); - this.shell = new Ring<>(this, points); + this.shell = new LineStringBuilder().points(points); + } + + public PolygonBuilder point(double longitude, double latitude) { + shell.point(longitude, latitude); + return this; + } + + /** + * Add a point to the shell of the polygon + * @param coordinate coordinate of the new point + * @return this + */ + public PolygonBuilder point(Coordinate coordinate) { + shell.point(coordinate); + return this; + } + + /** + * Add an array of points to the shell of the polygon + * @param coordinates coordinates of the new points to add + * @return this + */ + public PolygonBuilder points(Coordinate...coordinates) { + shell.points(coordinates); + return this; + } + + /** + * Add a new hole to the polygon + * @param hole linear ring defining the hole + * @return this + */ + public PolygonBuilder hole(LineStringBuilder hole) { + holes.add(hole); + return this; + } + + /** + * Close the shell of the polygon + */ + public PolygonBuilder close() { + shell.close(); + return this; + } + + /** + * Validates only 1 vertex is tangential (shared) between the interior and exterior of a polygon + */ + protected void validateHole(LineStringBuilder shell, LineStringBuilder hole) { + HashSet exterior = Sets.newHashSet(shell.points); + HashSet interior = Sets.newHashSet(hole.points); + exterior.retainAll(interior); + if (exterior.size() >= 2) { + throw new InvalidShapeException("Invalid polygon, interior cannot share more than one point with the exterior"); + } + } + + /** + * The coordinates setup by the builder will be assembled to a polygon. The result will consist of + * a set of polygons. Each of these components holds a list of linestrings defining the polygon: the + * first set of coordinates will be used as the shell of the polygon. The others are defined to holes + * within the polygon. + * This Method also wraps the polygons at the dateline. In order to this fact the result may + * contains more polygons and less holes than defined in the builder it self. + * + * @return coordinates of the polygon + */ + public Coordinate[][][] coordinates() { + int numEdges = shell.points.size()-1; // Last point is repeated + for (int i = 0; i < holes.size(); i++) { + numEdges += holes.get(i).points.size()-1; + validateHole(shell, this.holes.get(i)); + } + + Edge[] edges = new Edge[numEdges]; + Edge[] holeComponents = new Edge[holes.size()]; + final AtomicBoolean translated = new AtomicBoolean(false); + int offset = createEdges(0, orientation, shell, null, edges, 0, translated); + for (int i = 0; i < holes.size(); i++) { + int length = createEdges(i+1, orientation, shell, this.holes.get(i), edges, offset, translated); + holeComponents[i] = edges[offset]; + offset += length; + } + + int numHoles = holeComponents.length; + + numHoles = merge(edges, 0, intersections(+DATELINE, edges), holeComponents, numHoles); + numHoles = merge(edges, 0, intersections(-DATELINE, edges), holeComponents, numHoles); + + return compose(edges, holeComponents, numHoles); } @Override - public PolygonBuilder close() { - super.close(); - return this; + public Shape build() { + return jtsGeometry(buildGeometry(FACTORY, wrapdateline)); + } + + protected XContentBuilder coordinatesArray(XContentBuilder builder, Params params) throws IOException { + shell.coordinatesToXcontent(builder, true); + for(LineStringBuilder hole : holes) { + hole.coordinatesToXcontent(builder, true); + } + return builder; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FIELD_TYPE, TYPE.shapeName()); + builder.startArray(FIELD_COORDINATES); + coordinatesArray(builder, params); + builder.endArray(); + builder.endObject(); + return builder; + } + + public Geometry buildGeometry(GeometryFactory factory, boolean fixDateline) { + if(fixDateline) { + Coordinate[][][] polygons = coordinates(); + return polygons.length == 1 + ? polygon(factory, polygons[0]) + : multipolygon(factory, polygons); + } else { + return toPolygon(factory); + } + } + + public Polygon toPolygon() { + return toPolygon(FACTORY); + } + + protected Polygon toPolygon(GeometryFactory factory) { + final LinearRing shell = linearRing(factory, this.shell.points); + final LinearRing[] holes = new LinearRing[this.holes.size()]; + Iterator iterator = this.holes.iterator(); + for (int i = 0; iterator.hasNext(); i++) { + holes[i] = linearRing(factory, iterator.next().points); + } + return factory.createPolygon(shell, holes); + } + + protected static LinearRing linearRing(GeometryFactory factory, ArrayList coordinates) { + return factory.createLinearRing(coordinates.toArray(new Coordinate[coordinates.size()])); + } + + @Override + public GeoShapeType type() { + return TYPE; + } + + protected static Polygon polygon(GeometryFactory factory, Coordinate[][] polygon) { + LinearRing shell = factory.createLinearRing(polygon[0]); + LinearRing[] holes; + + if(polygon.length > 1) { + holes = new LinearRing[polygon.length-1]; + for (int i = 0; i < holes.length; i++) { + holes[i] = factory.createLinearRing(polygon[i+1]); + } + } else { + holes = null; + } + return factory.createPolygon(shell, holes); + } + + /** + * Create a Multipolygon from a set of coordinates. Each primary array contains a polygon which + * in turn contains an array of linestrings. These line Strings are represented as an array of + * coordinates. The first linestring will be the shell of the polygon the others define holes + * within the polygon. + * + * @param factory {@link GeometryFactory} to use + * @param polygons definition of polygons + * @return a new Multipolygon + */ + protected static MultiPolygon multipolygon(GeometryFactory factory, Coordinate[][][] polygons) { + Polygon[] polygonSet = new Polygon[polygons.length]; + for (int i = 0; i < polygonSet.length; i++) { + polygonSet[i] = polygon(factory, polygons[i]); + } + return factory.createMultiPolygon(polygonSet); + } + + /** + * This method sets the component id of all edges in a ring to a given id and shifts the + * coordinates of this component according to the dateline + * + * @param edge An arbitrary edge of the component + * @param id id to apply to the component + * @param edges a list of edges to which all edges of the component will be added (could be null) + * @return number of edges that belong to this component + */ + private static int component(final Edge edge, final int id, final ArrayList edges) { + // find a coordinate that is not part of the dateline + Edge any = edge; + while(any.coordinate.x == +DATELINE || any.coordinate.x == -DATELINE) { + if((any = any.next) == edge) { + break; + } + } + + double shiftOffset = any.coordinate.x > DATELINE ? DATELINE : (any.coordinate.x < -DATELINE ? -DATELINE : 0); + if (debugEnabled()) { + LOGGER.debug("shift: {[]}", shiftOffset); + } + + // run along the border of the component, collect the + // edges, shift them according to the dateline and + // update the component id + int length = 0, connectedComponents = 0; + // if there are two connected components, splitIndex keeps track of where to split the edge array + // start at 1 since the source coordinate is shared + int splitIndex = 1; + Edge current = edge; + Edge prev = edge; + // bookkeep the source and sink of each visited coordinate + HashMap> visitedEdge = new HashMap<>(); + do { + current.coordinate = shift(current.coordinate, shiftOffset); + current.component = id; + + if (edges != null) { + // found a closed loop - we have two connected components so we need to slice into two distinct components + if (visitedEdge.containsKey(current.coordinate)) { + if (connectedComponents > 0 && current.next != edge) { + throw new InvalidShapeException("Shape contains more than one shared point"); + } + + // a negative id flags the edge as visited for the edges(...) method. + // since we're splitting connected components, we want the edges method to visit + // the newly separated component + final int visitID = -id; + Edge firstAppearance = visitedEdge.get(current.coordinate).v2(); + // correct the graph pointers by correcting the 'next' pointer for both the + // first appearance and this appearance of the edge + Edge temp = firstAppearance.next; + firstAppearance.next = current.next; + current.next = temp; + current.component = visitID; + // backtrack until we get back to this coordinate, setting the visit id to + // a non-visited value (anything positive) + do { + prev.component = visitID; + prev = visitedEdge.get(prev.coordinate).v1(); + ++splitIndex; + } while (!current.coordinate.equals(prev.coordinate)); + ++connectedComponents; + } else { + visitedEdge.put(current.coordinate, new Tuple(prev, current)); + } + edges.add(current); + prev = current; + } + length++; + } while(connectedComponents == 0 && (current = current.next) != edge); + + return (splitIndex != 1) ? length-splitIndex: length; + } + + /** + * Compute all coordinates of a component + * @param component an arbitrary edge of the component + * @param coordinates Array of coordinates to write the result to + * @return the coordinates parameter + */ + private static Coordinate[] coordinates(Edge component, Coordinate[] coordinates) { + for (int i = 0; i < coordinates.length; i++) { + coordinates[i] = (component = component.next).coordinate; + } + return coordinates; + } + + private static Coordinate[][][] buildCoordinates(ArrayList> components) { + Coordinate[][][] result = new Coordinate[components.size()][][]; + for (int i = 0; i < result.length; i++) { + ArrayList component = components.get(i); + result[i] = component.toArray(new Coordinate[component.size()][]); + } + + if(debugEnabled()) { + for (int i = 0; i < result.length; i++) { + LOGGER.debug("Component {[]}:", i); + for (int j = 0; j < result[i].length; j++) { + LOGGER.debug("\t" + Arrays.toString(result[i][j])); + } + } + } + + return result; + } + + private static final Coordinate[][] EMPTY = new Coordinate[0][]; + + private static Coordinate[][] holes(Edge[] holes, int numHoles) { + if (numHoles == 0) { + return EMPTY; + } + final Coordinate[][] points = new Coordinate[numHoles][]; + + for (int i = 0; i < numHoles; i++) { + int length = component(holes[i], -(i+1), null); // mark as visited by inverting the sign + points[i] = coordinates(holes[i], new Coordinate[length+1]); + } + + return points; + } + + private static Edge[] edges(Edge[] edges, int numHoles, ArrayList> components) { + ArrayList mainEdges = new ArrayList<>(edges.length); + + for (int i = 0; i < edges.length; i++) { + if (edges[i].component >= 0) { + int length = component(edges[i], -(components.size()+numHoles+1), mainEdges); + ArrayList component = new ArrayList<>(); + component.add(coordinates(edges[i], new Coordinate[length+1])); + components.add(component); + } + } + + return mainEdges.toArray(new Edge[mainEdges.size()]); + } + + private static Coordinate[][][] compose(Edge[] edges, Edge[] holes, int numHoles) { + final ArrayList> components = new ArrayList<>(); + assign(holes, holes(holes, numHoles), numHoles, edges(edges, numHoles, components), components); + return buildCoordinates(components); + } + + private static void assign(Edge[] holes, Coordinate[][] points, int numHoles, Edge[] edges, ArrayList> components) { + // Assign Hole to related components + // To find the new component the hole belongs to all intersections of the + // polygon edges with a vertical line are calculated. This vertical line + // is an arbitrary point of the hole. The polygon edge next to this point + // is part of the polygon the hole belongs to. + if (debugEnabled()) { + LOGGER.debug("Holes: " + Arrays.toString(holes)); + } + for (int i = 0; i < numHoles; i++) { + final Edge current = new Edge(holes[i].coordinate, holes[i].next); + // the edge intersects with itself at its own coordinate. We need intersect to be set this way so the binary search + // will get the correct position in the edge list and therefore the correct component to add the hole + current.intersect = current.coordinate; + final int intersections = intersections(current.coordinate.x, edges); + // if no intersection is found then the hole is not within the polygon, so + // don't waste time calling a binary search + final int pos; + boolean sharedVertex = false; + if (intersections == 0 || ((pos = Arrays.binarySearch(edges, 0, intersections, current, INTERSECTION_ORDER)) >= 0) + && !(sharedVertex = (edges[pos].intersect.compareTo(current.coordinate) == 0)) ) { + throw new InvalidShapeException("Invalid shape: Hole is not within polygon"); + } + final int index = -((sharedVertex) ? 0 : pos+2); + final int component = -edges[index].component - numHoles - 1; + + if(debugEnabled()) { + LOGGER.debug("\tposition ("+index+") of edge "+current+": " + edges[index]); + LOGGER.debug("\tComponent: " + component); + LOGGER.debug("\tHole intersections ("+current.coordinate.x+"): " + Arrays.toString(edges)); + } + + components.get(component).add(points[i]); + } + } + + private static int merge(Edge[] intersections, int offset, int length, Edge[] holes, int numHoles) { + // Intersections appear pairwise. On the first edge the inner of + // of the polygon is entered. On the second edge the outer face + // is entered. Other kinds of intersections are discard by the + // intersection function + + for (int i = 0; i < length; i += 2) { + Edge e1 = intersections[offset + i + 0]; + Edge e2 = intersections[offset + i + 1]; + + // If two segments are connected maybe a hole must be deleted + // Since Edges of components appear pairwise we need to check + // the second edge only (the first edge is either polygon or + // already handled) + if (e2.component > 0) { + //TODO: Check if we could save the set null step + numHoles--; + holes[e2.component-1] = holes[numHoles]; + holes[numHoles] = null; + } + // only connect edges if intersections are pairwise + // 1. per the comment above, the edge array is sorted by y-value of the intersection + // with the dateline. Two edges have the same y intercept when they cross the + // dateline thus they appear sequentially (pairwise) in the edge array. Two edges + // do not have the same y intercept when we're forming a multi-poly from a poly + // that wraps the dateline (but there are 2 ordered intercepts). + // The connect method creates a new edge for these paired edges in the linked list. + // For boundary conditions (e.g., intersect but not crossing) there is no sibling edge + // to connect. Thus the first logic check enforces the pairwise rule + // 2. the second logic check ensures the two candidate edges aren't already connected by an + // existing edge along the dateline - this is necessary due to a logic change in + // ShapeBuilder.intersection that computes dateline edges as valid intersect points + // in support of OGC standards + if (e1.intersect != Edge.MAX_COORDINATE && e2.intersect != Edge.MAX_COORDINATE + && !(e1.next.next.coordinate.equals3D(e2.coordinate) && Math.abs(e1.next.coordinate.x) == DATELINE + && Math.abs(e2.coordinate.x) == DATELINE) ) { + connect(e1, e2); + } + } + return numHoles; + } + + private static void connect(Edge in, Edge out) { + assert in != null && out != null; + assert in != out; + // Connecting two Edges by inserting the point at + // dateline intersection and connect these by adding + // two edges between this points. One per direction + if(in.intersect != in.next.coordinate) { + // NOTE: the order of the object creation is crucial here! Don't change it! + // first edge has no point on dateline + Edge e1 = new Edge(in.intersect, in.next); + + if(out.intersect != out.next.coordinate) { + // second edge has no point on dateline + Edge e2 = new Edge(out.intersect, out.next); + in.next = new Edge(in.intersect, e2, in.intersect); + } else { + // second edge intersects with dateline + in.next = new Edge(in.intersect, out.next, in.intersect); + } + out.next = new Edge(out.intersect, e1, out.intersect); + } else if (in.next != out && in.coordinate != out.intersect) { + // first edge intersects with dateline + Edge e2 = new Edge(out.intersect, in.next, out.intersect); + + if(out.intersect != out.next.coordinate) { + // second edge has no point on dateline + Edge e1 = new Edge(out.intersect, out.next); + in.next = new Edge(in.intersect, e1, in.intersect); + + } else { + // second edge intersects with dateline + in.next = new Edge(in.intersect, out.next, in.intersect); + } + out.next = e2; + } + } + + private static int createEdges(int component, Orientation orientation, LineStringBuilder shell, + LineStringBuilder hole, Edge[] edges, int offset, final AtomicBoolean translated) { + // inner rings (holes) have an opposite direction than the outer rings + // XOR will invert the orientation for outer ring cases (Truth Table:, T/T = F, T/F = T, F/T = T, F/F = F) + boolean direction = (component == 0 ^ orientation == Orientation.RIGHT); + // set the points array accordingly (shell or hole) + Coordinate[] points = (hole != null) ? hole.coordinates(false) : shell.coordinates(false); + ring(component, direction, orientation == Orientation.LEFT, shell, points, 0, edges, offset, points.length-1, translated); + return points.length-1; + } + + /** + * Create a connected list of a list of coordinates + * + * @param points + * array of point + * @param offset + * index of the first point + * @param length + * number of points + * @return Array of edges + */ + private static Edge[] ring(int component, boolean direction, boolean handedness, LineStringBuilder shell, + Coordinate[] points, int offset, Edge[] edges, int toffset, int length, final AtomicBoolean translated) { + // calculate the direction of the points: + // find the point a the top of the set and check its + // neighbors orientation. So direction is equivalent + // to clockwise/counterclockwise + final int top = top(points, offset, length); + final int prev = (offset + ((top + length - 1) % length)); + final int next = (offset + ((top + 1) % length)); + boolean orientation = points[offset + prev].x > points[offset + next].x; + + // OGC requires shell as ccw (Right-Handedness) and holes as cw (Left-Handedness) + // since GeoJSON doesn't specify (and doesn't need to) GEO core will assume OGC standards + // thus if orientation is computed as cw, the logic will translate points across dateline + // and convert to a right handed system + + // compute the bounding box and calculate range + double[] range = range(points, offset, length); + final double rng = range[1] - range[0]; + // translate the points if the following is true + // 1. shell orientation is cw and range is greater than a hemisphere (180 degrees) but not spanning 2 hemispheres + // (translation would result in a collapsed poly) + // 2. the shell of the candidate hole has been translated (to preserve the coordinate system) + boolean incorrectOrientation = component == 0 && handedness != orientation; + if ( (incorrectOrientation && (rng > DATELINE && rng != 2*DATELINE)) || (translated.get() && component != 0)) { + translate(points); + // flip the translation bit if the shell is being translated + if (component == 0) { + translated.set(true); + } + // correct the orientation post translation (ccw for shell, cw for holes) + if (component == 0 || (component != 0 && handedness == orientation)) { + orientation = !orientation; + } + } + return concat(component, direction ^ orientation, points, offset, edges, toffset, length); + } + + private static final int top(Coordinate[] points, int offset, int length) { + int top = 0; // we start at 1 here since top points to 0 + for (int i = 1; i < length; i++) { + if (points[offset + i].y < points[offset + top].y) { + top = i; + } else if (points[offset + i].y == points[offset + top].y) { + if (points[offset + i].x < points[offset + top].x) { + top = i; + } + } + } + return top; + } + + private static final double[] range(Coordinate[] points, int offset, int length) { + double minX = points[0].x; + double maxX = points[0].x; + double minY = points[0].y; + double maxY = points[0].y; + // compute the bounding coordinates (@todo: cleanup brute force) + for (int i = 1; i < length; ++i) { + if (points[offset + i].x < minX) { + minX = points[offset + i].x; + } + if (points[offset + i].x > maxX) { + maxX = points[offset + i].x; + } + if (points[offset + i].y < minY) { + minY = points[offset + i].y; + } + if (points[offset + i].y > maxY) { + maxY = points[offset + i].y; + } + } + return new double[] {minX, maxX, minY, maxY}; + } + + /** + * Concatenate a set of points to a polygon + * + * @param component + * component id of the polygon + * @param direction + * direction of the ring + * @param points + * list of points to concatenate + * @param pointOffset + * index of the first point + * @param edges + * Array of edges to write the result to + * @param edgeOffset + * index of the first edge in the result + * @param length + * number of points to use + * @return the edges creates + */ + private static Edge[] concat(int component, boolean direction, Coordinate[] points, final int pointOffset, Edge[] edges, final int edgeOffset, + int length) { + assert edges.length >= length+edgeOffset; + assert points.length >= length+pointOffset; + edges[edgeOffset] = new Edge(points[pointOffset], null); + for (int i = 1; i < length; i++) { + if (direction) { + edges[edgeOffset + i] = new Edge(points[pointOffset + i], edges[edgeOffset + i - 1]); + edges[edgeOffset + i].component = component; + } else if(!edges[edgeOffset + i - 1].coordinate.equals(points[pointOffset + i])) { + edges[edgeOffset + i - 1].next = edges[edgeOffset + i] = new Edge(points[pointOffset + i], null); + edges[edgeOffset + i - 1].component = component; + } else { + throw new InvalidShapeException("Provided shape has duplicate consecutive coordinates at: " + points[pointOffset + i]); + } + } + + if (direction) { + edges[edgeOffset].setNext(edges[edgeOffset + length - 1]); + edges[edgeOffset].component = component; + } else { + edges[edgeOffset + length - 1].setNext(edges[edgeOffset]); + edges[edgeOffset + length - 1].component = component; + } + + return edges; + } + + /** + * Transforms coordinates in the eastern hemisphere (-180:0) to a (180:360) range + */ + private static void translate(Coordinate[] points) { + for (Coordinate c : points) { + if (c.x < 0) { + c.x += 2*DATELINE; + } + } } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java index 2380b975bd1..d8689ee737f 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java @@ -26,8 +26,12 @@ import com.spatial4j.core.shape.jts.JtsGeometry; import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Geometry; import com.vividsolutions.jts.geom.GeometryFactory; + import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.unit.DistanceUnit.Distance; @@ -41,9 +45,9 @@ import java.io.IOException; import java.util.*; /** - * Basic class for building GeoJSON shapes like Polygons, Linestrings, etc + * Basic class for building GeoJSON shapes like Polygons, Linestrings, etc */ -public abstract class ShapeBuilder extends ToXContentToBytes { +public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWriteable { protected static final ESLogger LOGGER = ESLoggerFactory.getLogger(ShapeBuilder.class.getName()); @@ -97,122 +101,10 @@ public abstract class ShapeBuilder extends ToXContentToBytes { return jtsGeometry; } - /** - * Create a new point - * - * @param longitude longitude of the point - * @param latitude latitude of the point - * @return a new {@link PointBuilder} - */ - public static PointBuilder newPoint(double longitude, double latitude) { - return newPoint(new Coordinate(longitude, latitude)); - } - - /** - * Create a new {@link PointBuilder} from a {@link Coordinate} - * @param coordinate coordinate defining the position of the point - * @return a new {@link PointBuilder} - */ - public static PointBuilder newPoint(Coordinate coordinate) { - return new PointBuilder().coordinate(coordinate); - } - - /** - * Create a new set of points - * @return new {@link MultiPointBuilder} - */ - public static MultiPointBuilder newMultiPoint() { - return new MultiPointBuilder(); - } - - /** - * Create a new lineString - * @return a new {@link LineStringBuilder} - */ - public static LineStringBuilder newLineString() { - return new LineStringBuilder(); - } - - /** - * Create a new Collection of lineStrings - * @return a new {@link MultiLineStringBuilder} - */ - public static MultiLineStringBuilder newMultiLinestring() { - return new MultiLineStringBuilder(); - } - - /** - * Create a new Polygon - * @return a new {@link PointBuilder} - */ - public static PolygonBuilder newPolygon() { - return new PolygonBuilder(); - } - - /** - * Create a new Polygon - * @return a new {@link PointBuilder} - */ - public static PolygonBuilder newPolygon(Orientation orientation) { - return new PolygonBuilder(orientation); - } - - /** - * Create a new Collection of polygons - * @return a new {@link MultiPolygonBuilder} - */ - public static MultiPolygonBuilder newMultiPolygon() { - return new MultiPolygonBuilder(); - } - - /** - * Create a new Collection of polygons - * @return a new {@link MultiPolygonBuilder} - */ - public static MultiPolygonBuilder newMultiPolygon(Orientation orientation) { - return new MultiPolygonBuilder(orientation); - } - - /** - * Create a new GeometryCollection - * @return a new {@link GeometryCollectionBuilder} - */ - public static GeometryCollectionBuilder newGeometryCollection() { - return new GeometryCollectionBuilder(); - } - - /** - * Create a new GeometryCollection - * @return a new {@link GeometryCollectionBuilder} - */ - public static GeometryCollectionBuilder newGeometryCollection(Orientation orientation) { - return new GeometryCollectionBuilder(orientation); - } - - /** - * create a new Circle - * @return a new {@link CircleBuilder} - */ - public static CircleBuilder newCircleBuilder() { - return new CircleBuilder(); - } - - /** - * create a new rectangle - * @return a new {@link EnvelopeBuilder} - */ - public static EnvelopeBuilder newEnvelope() { return new EnvelopeBuilder(); } - - /** - * create a new rectangle - * @return a new {@link EnvelopeBuilder} - */ - public static EnvelopeBuilder newEnvelope(Orientation orientation) { return new EnvelopeBuilder(orientation); } - /** * Create a new Shape from this builder. Since calling this method could change the * defined shape. (by inserting new coordinates or change the position of points) - * the builder looses its validity. So this method should only be called once on a builder + * the builder looses its validity. So this method should only be called once on a builder * @return new {@link Shape} defined by the builder */ public abstract Shape build(); @@ -220,7 +112,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { /** * Recursive method which parses the arrays of coordinates used to define * Shapes - * + * * @param parser * Parser that will be read from * @return CoordinateNode representing the start of the coordinate tree @@ -232,8 +124,8 @@ public abstract class ShapeBuilder extends ToXContentToBytes { XContentParser.Token token = parser.nextToken(); // Base cases - if (token != XContentParser.Token.START_ARRAY && - token != XContentParser.Token.END_ARRAY && + if (token != XContentParser.Token.START_ARRAY && + token != XContentParser.Token.END_ARRAY && token != XContentParser.Token.VALUE_NULL) { double lon = parser.doubleValue(); token = parser.nextToken(); @@ -285,6 +177,15 @@ public abstract class ShapeBuilder extends ToXContentToBytes { return builder.startArray().value(coordinate.x).value(coordinate.y).endArray(); } + protected static void writeCoordinateTo(Coordinate coordinate, StreamOutput out) throws IOException { + out.writeDouble(coordinate.x); + out.writeDouble(coordinate.y); + } + + protected Coordinate readCoordinateFrom(StreamInput in) throws IOException { + return new Coordinate(in.readDouble(), in.readDouble()); + } + public static Orientation orientationFromString(String orientation) { orientation = orientation.toLowerCase(Locale.ROOT); switch (orientation) { @@ -317,7 +218,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { /** * Calculate the intersection of a line segment and a vertical dateline. - * + * * @param p1 * start-point of the line segment * @param p2 @@ -347,7 +248,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { * Calculate all intersections of line segments and a vertical line. The * Array of edges will be ordered asc by the y-coordinate of the * intersections of edges. - * + * * @param dateline * x-coordinate of the dateline * @param edges @@ -360,7 +261,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { for (int i = 0; i < edges.length; i++) { Coordinate p1 = edges[i].coordinate; Coordinate p2 = edges[i].next.coordinate; - assert !Double.isNaN(p2.x) && !Double.isNaN(p1.x); + assert !Double.isNaN(p2.x) && !Double.isNaN(p1.x); edges[i].intersect = Edge.MAX_COORDINATE; double position = intersection(p1, p2, dateline); @@ -386,7 +287,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { /** * Creates a new leaf CoordinateNode - * + * * @param coordinate * Coordinate for the Node */ @@ -397,7 +298,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { /** * Creates a new parent CoordinateNode - * + * * @param children * Children of the Node */ @@ -427,7 +328,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { /** * This helper class implements a linked list for {@link Coordinate}. It contains - * fields for a dateline intersection and component id + * fields for a dateline intersection and component id */ protected static final class Edge { Coordinate coordinate; // coordinate of the start point @@ -461,153 +362,9 @@ public abstract class ShapeBuilder extends ToXContentToBytes { } } - private static final int top(Coordinate[] points, int offset, int length) { - int top = 0; // we start at 1 here since top points to 0 - for (int i = 1; i < length; i++) { - if (points[offset + i].y < points[offset + top].y) { - top = i; - } else if (points[offset + i].y == points[offset + top].y) { - if (points[offset + i].x < points[offset + top].x) { - top = i; - } - } - } - return top; - } - - private static final double[] range(Coordinate[] points, int offset, int length) { - double minX = points[0].x; - double maxX = points[0].x; - double minY = points[0].y; - double maxY = points[0].y; - // compute the bounding coordinates (@todo: cleanup brute force) - for (int i = 1; i < length; ++i) { - if (points[offset + i].x < minX) { - minX = points[offset + i].x; - } - if (points[offset + i].x > maxX) { - maxX = points[offset + i].x; - } - if (points[offset + i].y < minY) { - minY = points[offset + i].y; - } - if (points[offset + i].y > maxY) { - maxY = points[offset + i].y; - } - } - return new double[] {minX, maxX, minY, maxY}; - } - - /** - * Concatenate a set of points to a polygon - * - * @param component - * component id of the polygon - * @param direction - * direction of the ring - * @param points - * list of points to concatenate - * @param pointOffset - * index of the first point - * @param edges - * Array of edges to write the result to - * @param edgeOffset - * index of the first edge in the result - * @param length - * number of points to use - * @return the edges creates - */ - private static Edge[] concat(int component, boolean direction, Coordinate[] points, final int pointOffset, Edge[] edges, final int edgeOffset, - int length) { - assert edges.length >= length+edgeOffset; - assert points.length >= length+pointOffset; - edges[edgeOffset] = new Edge(points[pointOffset], null); - for (int i = 1; i < length; i++) { - if (direction) { - edges[edgeOffset + i] = new Edge(points[pointOffset + i], edges[edgeOffset + i - 1]); - edges[edgeOffset + i].component = component; - } else if(!edges[edgeOffset + i - 1].coordinate.equals(points[pointOffset + i])) { - edges[edgeOffset + i - 1].next = edges[edgeOffset + i] = new Edge(points[pointOffset + i], null); - edges[edgeOffset + i - 1].component = component; - } else { - throw new InvalidShapeException("Provided shape has duplicate consecutive coordinates at: " + points[pointOffset + i]); - } - } - - if (direction) { - edges[edgeOffset].setNext(edges[edgeOffset + length - 1]); - edges[edgeOffset].component = component; - } else { - edges[edgeOffset + length - 1].setNext(edges[edgeOffset]); - edges[edgeOffset + length - 1].component = component; - } - - return edges; - } - - /** - * Create a connected list of a list of coordinates - * - * @param points - * array of point - * @param offset - * index of the first point - * @param length - * number of points - * @return Array of edges - */ - protected static Edge[] ring(int component, boolean direction, boolean handedness, BaseLineStringBuilder shell, - Coordinate[] points, int offset, Edge[] edges, int toffset, int length) { - // calculate the direction of the points: - // find the point a the top of the set and check its - // neighbors orientation. So direction is equivalent - // to clockwise/counterclockwise - final int top = top(points, offset, length); - final int prev = (offset + ((top + length - 1) % length)); - final int next = (offset + ((top + 1) % length)); - boolean orientation = points[offset + prev].x > points[offset + next].x; - - // OGC requires shell as ccw (Right-Handedness) and holes as cw (Left-Handedness) - // since GeoJSON doesn't specify (and doesn't need to) GEO core will assume OGC standards - // thus if orientation is computed as cw, the logic will translate points across dateline - // and convert to a right handed system - - // compute the bounding box and calculate range - double[] range = range(points, offset, length); - final double rng = range[1] - range[0]; - // translate the points if the following is true - // 1. shell orientation is cw and range is greater than a hemisphere (180 degrees) but not spanning 2 hemispheres - // (translation would result in a collapsed poly) - // 2. the shell of the candidate hole has been translated (to preserve the coordinate system) - boolean incorrectOrientation = component == 0 && handedness != orientation; - if ( (incorrectOrientation && (rng > DATELINE && rng != 2*DATELINE)) || (shell.translated && component != 0)) { - translate(points); - // flip the translation bit if the shell is being translated - if (component == 0) { - shell.translated = true; - } - // correct the orientation post translation (ccw for shell, cw for holes) - if (component == 0 || (component != 0 && handedness == orientation)) { - orientation = !orientation; - } - } - return concat(component, direction ^ orientation, points, offset, edges, toffset, length); - } - - /** - * Transforms coordinates in the eastern hemisphere (-180:0) to a (180:360) range - */ - protected static void translate(Coordinate[] points) { - for (Coordinate c : points) { - if (c.x < 0) { - c.x += 2*DATELINE; - } - } - } - /** * Set the intersection of this line segment to the given position - * + * * @param position * position of the intersection [0..1] * @return the {@link Coordinate} of the intersection @@ -616,7 +373,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { return intersect = position(coordinate, next.coordinate, position); } - public static Coordinate position(Coordinate p1, Coordinate p2, double position) { + protected static Coordinate position(Coordinate p1, Coordinate p2, double position) { if (position == 0) { return p1; } else if (position == 1) { @@ -641,7 +398,6 @@ public abstract class ShapeBuilder extends ToXContentToBytes { public int compare(Edge o1, Edge o2) { return Double.compare(o1.intersect.y, o2.intersect.y); } - } public static enum Orientation { @@ -677,12 +433,16 @@ public abstract class ShapeBuilder extends ToXContentToBytes { ENVELOPE("envelope"), CIRCLE("circle"); - protected final String shapename; + private final String shapename; private GeoShapeType(String shapename) { this.shapename = shapename; } + protected String shapeName() { + return shapename; + } + public static GeoShapeType forName(String geoshapename) { String typename = geoshapename.toLowerCase(Locale.ROOT); for (GeoShapeType type : values()) { @@ -770,7 +530,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { throw new ElasticsearchParseException("shape type [{}] not included", shapeType); } } - + protected static void validatePointNode(CoordinateNode node) { if (node.isEmpty()) { throw new ElasticsearchParseException("invalid number of points (0) provided when expecting a single coordinate ([lat, lng])"); @@ -783,11 +543,11 @@ public abstract class ShapeBuilder extends ToXContentToBytes { protected static PointBuilder parsePoint(CoordinateNode node) { validatePointNode(node); - return newPoint(node.coordinate); + return ShapeBuilders.newPoint(node.coordinate); } protected static CircleBuilder parseCircle(CoordinateNode coordinates, Distance radius) { - return newCircleBuilder().center(coordinates.coordinate).radius(radius); + return ShapeBuilders.newCircleBuilder().center(coordinates.coordinate).radius(radius); } protected static EnvelopeBuilder parseEnvelope(CoordinateNode coordinates, final Orientation orientation) { @@ -804,7 +564,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { uL = new Coordinate(Math.min(uL.x, lR.x), Math.max(uL.y, lR.y)); lR = new Coordinate(Math.max(uLtmp.x, lR.x), Math.min(uLtmp.y, lR.y)); } - return newEnvelope(orientation).topLeft(uL).bottomRight(lR); + return ShapeBuilders.newEnvelope(orientation).topLeft(uL).bottomRight(lR); } protected static void validateMultiPointNode(CoordinateNode coordinates) { @@ -842,7 +602,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { throw new ElasticsearchParseException("invalid number of points in LineString (found [{}] - must be >= 2)", coordinates.children.size()); } - LineStringBuilder line = newLineString(); + LineStringBuilder line = ShapeBuilders.newLineString(); for (CoordinateNode node : coordinates.children) { line.point(node.coordinate); } @@ -850,7 +610,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes { } protected static MultiLineStringBuilder parseMultiLine(CoordinateNode coordinates) { - MultiLineStringBuilder multiline = newMultiLinestring(); + MultiLineStringBuilder multiline = ShapeBuilders.newMultiLinestring(); for (CoordinateNode node : coordinates.children) { multiline.linestring(parseLineString(node)); } @@ -903,13 +663,13 @@ public abstract class ShapeBuilder extends ToXContentToBytes { protected static MultiPolygonBuilder parseMultiPolygon(CoordinateNode coordinates, final Orientation orientation, final boolean coerce) { - MultiPolygonBuilder polygons = newMultiPolygon(orientation); + MultiPolygonBuilder polygons = ShapeBuilders.newMultiPolygon(orientation); for (CoordinateNode node : coordinates.children) { polygons.polygon(parsePolygon(node, orientation, coerce)); } return polygons; } - + /** * Parse the geometries array of a GeometryCollection * @@ -922,17 +682,33 @@ public abstract class ShapeBuilder extends ToXContentToBytes { if (parser.currentToken() != XContentParser.Token.START_ARRAY) { throw new ElasticsearchParseException("geometries must be an array of geojson objects"); } - + XContentParser.Token token = parser.nextToken(); - GeometryCollectionBuilder geometryCollection = newGeometryCollection( (mapper == null) ? Orientation.RIGHT : mapper + GeometryCollectionBuilder geometryCollection = ShapeBuilders.newGeometryCollection( (mapper == null) ? Orientation.RIGHT : mapper .fieldType().orientation()); while (token != XContentParser.Token.END_ARRAY) { ShapeBuilder shapeBuilder = GeoShapeType.parse(parser); geometryCollection.shape(shapeBuilder); token = parser.nextToken(); } - + return geometryCollection; } } + + @Override + public String getWriteableName() { + return type().shapeName(); + } + + // NORELEASE this should be deleted as soon as all shape builders implement writable + @Override + public void writeTo(StreamOutput out) throws IOException { + } + + // NORELEASE this should be deleted as soon as all shape builders implement writable + @Override + public ShapeBuilder readFrom(StreamInput in) throws IOException { + return null; + } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java new file mode 100644 index 00000000000..e294a9d6ef7 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo.builders; + +import com.vividsolutions.jts.geom.Coordinate; + +/** + * A collection of static methods for creating ShapeBuilders. + */ +public class ShapeBuilders { + + /** + * Create a new point + * + * @param longitude longitude of the point + * @param latitude latitude of the point + * @return a new {@link PointBuilder} + */ + public static PointBuilder newPoint(double longitude, double latitude) { + return ShapeBuilders.newPoint(new Coordinate(longitude, latitude)); + } + + /** + * Create a new {@link PointBuilder} from a {@link Coordinate} + * @param coordinate coordinate defining the position of the point + * @return a new {@link PointBuilder} + */ + public static PointBuilder newPoint(Coordinate coordinate) { + return new PointBuilder().coordinate(coordinate); + } + + /** + * Create a new set of points + * @return new {@link MultiPointBuilder} + */ + public static MultiPointBuilder newMultiPoint() { + return new MultiPointBuilder(); + } + + /** + * Create a new lineString + * @return a new {@link LineStringBuilder} + */ + public static LineStringBuilder newLineString() { + return new LineStringBuilder(); + } + + /** + * Create a new Collection of lineStrings + * @return a new {@link MultiLineStringBuilder} + */ + public static MultiLineStringBuilder newMultiLinestring() { + return new MultiLineStringBuilder(); + } + + /** + * Create a new Polygon + * @return a new {@link PointBuilder} + */ + public static PolygonBuilder newPolygon() { + return new PolygonBuilder(); + } + + /** + * Create a new Polygon + * @return a new {@link PointBuilder} + */ + public static PolygonBuilder newPolygon(ShapeBuilder.Orientation orientation) { + return new PolygonBuilder(orientation); + } + + /** + * Create a new Collection of polygons + * @return a new {@link MultiPolygonBuilder} + */ + public static MultiPolygonBuilder newMultiPolygon() { + return new MultiPolygonBuilder(); + } + + /** + * Create a new Collection of polygons + * @return a new {@link MultiPolygonBuilder} + */ + public static MultiPolygonBuilder newMultiPolygon(ShapeBuilder.Orientation orientation) { + return new MultiPolygonBuilder(orientation); + } + + /** + * Create a new GeometryCollection + * @return a new {@link GeometryCollectionBuilder} + */ + public static GeometryCollectionBuilder newGeometryCollection() { + return new GeometryCollectionBuilder(); + } + + /** + * Create a new GeometryCollection + * + * @return a new {@link GeometryCollectionBuilder} + */ + public static GeometryCollectionBuilder newGeometryCollection(ShapeBuilder.Orientation orientation) { + return new GeometryCollectionBuilder(orientation); + } + + /** + * create a new Circle + * + * @return a new {@link CircleBuilder} + */ + public static CircleBuilder newCircleBuilder() { + return new CircleBuilder(); + } + + /** + * create a new rectangle + * + * @return a new {@link EnvelopeBuilder} + */ + public static EnvelopeBuilder newEnvelope() { + return new EnvelopeBuilder(); + } + + /** + * create a new rectangle + * + * @return a new {@link EnvelopeBuilder} + */ + public static EnvelopeBuilder newEnvelope(ShapeBuilder.Orientation orientation) { + return new EnvelopeBuilder(orientation); + } +} diff --git a/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java b/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java index 7fe26ed81d9..01079ecf45a 100644 --- a/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java +++ b/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java @@ -358,7 +358,7 @@ public class HttpDownloadHelper { connection.setConnectTimeout(5000); } connection.setRequestProperty("ES-Version", Version.CURRENT.toString()); - connection.setRequestProperty("ES-Build-Hash", Build.CURRENT.hashShort()); + connection.setRequestProperty("ES-Build-Hash", Build.CURRENT.shortHash()); connection.setRequestProperty("User-Agent", "elasticsearch-plugin-manager"); // connect to the remote site (may take some time) diff --git a/core/src/main/java/org/elasticsearch/common/inject/Key.java b/core/src/main/java/org/elasticsearch/common/inject/Key.java index 3af3b4e1a8a..7344dfe5b41 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/Key.java +++ b/core/src/main/java/org/elasticsearch/common/inject/Key.java @@ -333,7 +333,7 @@ public class Key { * Returns {@code true} if the given annotation type has no attributes. */ static boolean isMarker(Class annotationType) { - return annotationType.getDeclaredMethods().length == 0; + return annotationType.getMethods().length == 0; } /** @@ -345,7 +345,7 @@ public class Key { ensureRetainedAtRuntime(annotationType); ensureIsBindingAnnotation(annotationType); - if (annotationType.getDeclaredMethods().length == 0) { + if (annotationType.getMethods().length == 0) { return new AnnotationTypeStrategy(annotationType, annotation); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/Reflection.java b/core/src/main/java/org/elasticsearch/common/inject/Reflection.java index 22c542bb9ef..667466f751b 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/Reflection.java +++ b/core/src/main/java/org/elasticsearch/common/inject/Reflection.java @@ -41,7 +41,7 @@ class Reflection { @SuppressWarnings("unchecked") static Constructor invalidConstructor() { try { - return (Constructor) InvalidConstructor.class.getDeclaredConstructor(); + return (Constructor) InvalidConstructor.class.getConstructor(); } catch (NoSuchMethodException e) { throw new AssertionError(e); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java b/core/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java index 485fab738a9..e42082817c1 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java +++ b/core/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java @@ -106,7 +106,7 @@ class TypeConverterBindingProcessor extends AbstractProcessor { try { return Class.forName(value); } catch (ClassNotFoundException e) { - throw new RuntimeException(e.getMessage()); + throw new RuntimeException(e); } } @@ -135,7 +135,7 @@ class TypeConverterBindingProcessor extends AbstractProcessor { } catch (IllegalAccessException e) { throw new AssertionError(e); } catch (InvocationTargetException e) { - throw new RuntimeException(e.getTargetException().getMessage()); + throw new RuntimeException(e.getTargetException()); } } diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java index 3837de81b60..0def65b9a13 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java +++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java @@ -212,7 +212,7 @@ public class FactoryProvider implements Provider, HasDependencies { TypeLiteral factoryType, TypeLiteral implementationType) { List> constructors = new ArrayList<>(); - for (Constructor constructor : implementationType.getRawType().getDeclaredConstructors()) { + for (Constructor constructor : implementationType.getRawType().getConstructors()) { if (constructor.getAnnotation(AssistedInject.class) != null) { @SuppressWarnings("unchecked") // the constructor type and implementation type agree AssistedConstructor assistedConstructor = new AssistedConstructor( diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethodsModule.java b/core/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethodsModule.java index 1671b4a8eac..5e45b4990c6 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethodsModule.java +++ b/core/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethodsModule.java @@ -83,7 +83,7 @@ public final class ProviderMethodsModule implements Module { public List> getProviderMethods(Binder binder) { List> result = new ArrayList<>(); for (Class c = delegate.getClass(); c != Object.class; c = c.getSuperclass()) { - for (Method method : c.getDeclaredMethods()) { + for (Method method : c.getMethods()) { if (method.getAnnotation(Provides.class) != null) { result.add(createProviderMethod(binder, method)); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java b/core/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java index 56f0ec0f055..5bc1595be5f 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java +++ b/core/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java @@ -331,6 +331,6 @@ public abstract class Multibinder { NullPointerException npe = new NullPointerException(name); throw new ConfigurationException(singleton( - new Message(emptyList(), npe.toString(), npe))); + new Message(emptyList(), npe))); } } diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/InjectionPoint.java b/core/src/main/java/org/elasticsearch/common/inject/spi/InjectionPoint.java index 17497d5f429..286635b9b65 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/InjectionPoint.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/InjectionPoint.java @@ -188,7 +188,7 @@ public final class InjectionPoint { Errors errors = new Errors(rawType); Constructor injectableConstructor = null; - for (Constructor constructor : rawType.getDeclaredConstructors()) { + for (Constructor constructor : rawType.getConstructors()) { Inject inject = constructor.getAnnotation(Inject.class); if (inject != null) { if (inject.optional()) { @@ -212,7 +212,7 @@ public final class InjectionPoint { // If no annotated constructor is found, look for a no-arg constructor instead. try { - Constructor noArgConstructor = rawType.getDeclaredConstructor(); + Constructor noArgConstructor = rawType.getConstructor(); // Disallow private constructors on non-private classes (unless they have @Inject) if (Modifier.isPrivate(noArgConstructor.getModifiers()) @@ -334,7 +334,7 @@ public final class InjectionPoint { // name. In Scala, fields always get accessor methods (that we need to ignore). See bug 242. if (member instanceof Method) { try { - if (member.getDeclaringClass().getDeclaredField(member.getName()) != null) { + if (member.getDeclaringClass().getField(member.getName()) != null) { return; } } catch (NoSuchFieldException ignore) { @@ -390,7 +390,7 @@ public final class InjectionPoint { Factory FIELDS = new Factory() { @Override public Field[] getMembers(Class type) { - return type.getDeclaredFields(); + return type.getFields(); } @Override @@ -402,7 +402,7 @@ public final class InjectionPoint { Factory METHODS = new Factory() { @Override public Method[] getMembers(Class type) { - return type.getDeclaredMethods(); + return type.getMethods(); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/Message.java b/core/src/main/java/org/elasticsearch/common/inject/spi/Message.java index e5488d07417..5a39b9edf13 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/Message.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/Message.java @@ -58,6 +58,10 @@ public final class Message implements Serializable, Element { this(Collections.singletonList(source), message, null); } + public Message(Object source, Throwable cause) { + this(Collections.singletonList(source), null, cause); + } + public Message(String message) { this(Collections.emptyList(), message, null); } diff --git a/core/src/main/java/org/elasticsearch/common/io/Streams.java b/core/src/main/java/org/elasticsearch/common/io/Streams.java index 8adf0919e50..36b1d9445b0 100644 --- a/core/src/main/java/org/elasticsearch/common/io/Streams.java +++ b/core/src/main/java/org/elasticsearch/common/io/Streams.java @@ -20,6 +20,8 @@ package org.elasticsearch.common.io; import java.nio.charset.StandardCharsets; + +import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.util.Callback; import java.io.BufferedReader; @@ -68,6 +70,7 @@ public abstract class Streams { public static long copy(InputStream in, OutputStream out, byte[] buffer) throws IOException { Objects.requireNonNull(in, "No InputStream specified"); Objects.requireNonNull(out, "No OutputStream specified"); + boolean success = false; try { long byteCount = 0; int bytesRead; @@ -76,17 +79,13 @@ public abstract class Streams { byteCount += bytesRead; } out.flush(); + success = true; return byteCount; } finally { - try { - in.close(); - } catch (IOException ex) { - // do nothing - } - try { - out.close(); - } catch (IOException ex) { - // do nothing + if (success) { + IOUtils.close(in, out); + } else { + IOUtils.closeWhileHandlingException(in, out); } } } @@ -130,6 +129,7 @@ public abstract class Streams { public static int copy(Reader in, Writer out) throws IOException { Objects.requireNonNull(in, "No Reader specified"); Objects.requireNonNull(out, "No Writer specified"); + boolean success = false; try { int byteCount = 0; char[] buffer = new char[BUFFER_SIZE]; @@ -139,17 +139,13 @@ public abstract class Streams { byteCount += bytesRead; } out.flush(); + success = true; return byteCount; } finally { - try { - in.close(); - } catch (IOException ex) { - // do nothing - } - try { - out.close(); - } catch (IOException ex) { - // do nothing + if (success) { + IOUtils.close(in, out); + } else { + IOUtils.closeWhileHandlingException(in, out); } } } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 4b56027e2a2..20859e2716a 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.Version; @@ -52,6 +53,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static org.elasticsearch.ElasticsearchException.readException; import static org.elasticsearch.ElasticsearchException.readStackTrace; @@ -234,6 +236,20 @@ public abstract class StreamInput extends InputStream { return i | ((b & 0x7FL) << 56); } + public long readZLong() throws IOException { + long accumulator = 0L; + int i = 0; + long currentByte; + while (((currentByte = readByte()) & 0x80L) != 0) { + accumulator |= (currentByte & 0x7F) << i; + i += 7; + if (i > 63) { + throw new IOException("variable-length stream is too long"); + } + } + return BitUtil.zigZagDecode(accumulator | (currentByte << i)); + } + @Nullable public Text readOptionalText() throws IOException { int length = readInt(); @@ -517,8 +533,9 @@ public abstract class StreamInput extends InputStream { /** * Serializes a potential null value. */ - public T readOptionalStreamable(T streamable) throws IOException { + public T readOptionalStreamable(Supplier supplier) throws IOException { if (readBoolean()) { + T streamable = supplier.get(); streamable.readFrom(this); return streamable; } else { @@ -602,7 +619,7 @@ public abstract class StreamInput extends InputStream { * Use {@link FilterInputStream} instead which wraps a stream and supports a {@link NamedWriteableRegistry} too. */ C readNamedWriteable(@SuppressWarnings("unused") Class categoryClass) throws IOException { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("can't read named writeable from StreamInput"); } /** diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 71558ff49c4..5f1e7623d28 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.ElasticsearchException; @@ -172,9 +173,9 @@ public abstract class StreamOutput extends OutputStream { } /** - * Writes an long in a variable-length format. Writes between one and nine - * bytes. Smaller values take fewer bytes. Negative numbers are not - * supported. + * Writes a non-negative long in a variable-length format. + * Writes between one and nine bytes. Smaller values take fewer bytes. + * Negative numbers are not supported. */ public void writeVLong(long i) throws IOException { assert i >= 0; @@ -185,6 +186,23 @@ public abstract class StreamOutput extends OutputStream { writeByte((byte) i); } + /** + * Writes a long in a variable-length format. Writes between one and ten bytes. + * Values are remapped by sliding the sign bit into the lsb and then encoded as an unsigned number + * e.g., 0 -;> 0, -1 -;> 1, 1 -;> 2, ..., Long.MIN_VALUE -;> -1, Long.MAX_VALUE -;> -2 + * Numbers with small absolute value will have a small encoding + * If the numbers are known to be non-negative, use {@link #writeVLong(long)} + */ + public void writeZLong(long i) throws IOException { + // zig-zag encoding cf. https://developers.google.com/protocol-buffers/docs/encoding?hl=en + long value = BitUtil.zigZagEncode(i); + while ((value & 0xFFFFFFFFFFFFFF80L) != 0L) { + writeByte((byte)((value & 0x7F) | 0x80)); + value >>>= 7; + } + writeByte((byte) (value & 0x7F)); + } + public void writeOptionalString(@Nullable String str) throws IOException { if (str == null) { writeBoolean(false); diff --git a/core/src/main/java/org/elasticsearch/common/lease/Releasables.java b/core/src/main/java/org/elasticsearch/common/lease/Releasables.java index c91494a235d..e91bc5c0f71 100644 --- a/core/src/main/java/org/elasticsearch/common/lease/Releasables.java +++ b/core/src/main/java/org/elasticsearch/common/lease/Releasables.java @@ -19,8 +19,6 @@ package org.elasticsearch.common.lease; -import org.elasticsearch.ElasticsearchException; - import java.util.Arrays; /** Utility methods to work with {@link Releasable}s. */ diff --git a/core/src/main/java/org/elasticsearch/common/logging/jdk/JdkESLogger.java b/core/src/main/java/org/elasticsearch/common/logging/jdk/JdkESLogger.java index dc6c3f3939c..2db16983e1a 100644 --- a/core/src/main/java/org/elasticsearch/common/logging/jdk/JdkESLogger.java +++ b/core/src/main/java/org/elasticsearch/common/logging/jdk/JdkESLogger.java @@ -50,7 +50,7 @@ public class JdkESLogger extends AbstractESLogger { } else if ("debug".equalsIgnoreCase(level)) { logger.setLevel(Level.FINE); } else if ("trace".equalsIgnoreCase(level)) { - logger.setLevel(Level.FINE); + logger.setLevel(Level.FINEST); } } diff --git a/core/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java b/core/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java index 5e7517efc19..e0d5f15630a 100644 --- a/core/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java +++ b/core/src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.logging.log4j; import org.apache.log4j.PropertyConfigurator; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.env.Environment; @@ -106,8 +107,8 @@ public class LogConfigurator { resolveConfig(environment, settingsBuilder); } settingsBuilder - .putProperties("elasticsearch.", System.getProperties()) - .putProperties("es.", System.getProperties()); + .putProperties("elasticsearch.", BootstrapInfo.getSystemProperties()) + .putProperties("es.", BootstrapInfo.getSystemProperties()); // add custom settings after config was added so that they are not overwritten by config settingsBuilder.put(settings); settingsBuilder.replacePropertyPlaceholders(); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 3aaaf9677b8..16a9796d8b6 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -60,9 +60,9 @@ public class Lucene { public static final Version VERSION = Version.LATEST; public static final Version ANALYZER_VERSION = VERSION; public static final Version QUERYPARSER_VERSION = VERSION; - public static final String LATEST_DOC_VALUES_FORMAT = "Lucene50"; + public static final String LATEST_DOC_VALUES_FORMAT = "Lucene54"; public static final String LATEST_POSTINGS_FORMAT = "Lucene50"; - public static final String LATEST_CODEC = "Lucene53"; + public static final String LATEST_CODEC = "Lucene54"; static { Deprecated annotation = PostingsFormat.forName(LATEST_POSTINGS_FORMAT).getClass().getAnnotation(Deprecated.class); @@ -717,13 +717,6 @@ public class Lucene { } } - /** - * Is it an empty {@link DocIdSet}? - */ - public static boolean isEmpty(@Nullable DocIdSet set) { - return set == null || set == DocIdSet.EMPTY; - } - /** * Given a {@link Scorer}, return a {@link Bits} instance that will match * all documents contained in the set. Note that the returned {@link Bits} diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java index a59af2c7f51..7191c96e33e 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java @@ -64,8 +64,9 @@ public final class AllTermQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1f) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } boolean fieldExists = false; boolean hasPayloads = false; @@ -80,14 +81,10 @@ public final class AllTermQuery extends Query { } } if (fieldExists == false) { - Query rewritten = new MatchNoDocsQuery(); - rewritten.setBoost(getBoost()); - return rewritten; + return new MatchNoDocsQuery(); } if (hasPayloads == false) { - TermQuery rewritten = new TermQuery(term); - rewritten.setBoost(getBoost()); - return rewritten; + return new TermQuery(term); } return this; } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java index 410796497d5..8b1dcd9dfcf 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java @@ -35,10 +35,7 @@ import org.elasticsearch.common.io.FastStringReader; import java.io.IOException; import java.io.Reader; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; /** * @@ -79,29 +76,17 @@ public class MoreLikeThisQuery extends Query { @Override public int hashCode() { - int result = boostTerms ? 1 : 0; - result = 31 * result + Float.floatToIntBits(boostTermsFactor); - result = 31 * result + Arrays.hashCode(likeText); - result = 31 * result + maxDocFreq; - result = 31 * result + maxQueryTerms; - result = 31 * result + maxWordLen; - result = 31 * result + minDocFreq; - result = 31 * result + minTermFrequency; - result = 31 * result + minWordLen; - result = 31 * result + Arrays.hashCode(moreLikeFields); - result = 31 * result + minimumShouldMatch.hashCode(); - result = 31 * result + (stopWords == null ? 0 : stopWords.hashCode()); - result = 31 * result + Float.floatToIntBits(getBoost()); - return result; + return Objects.hash(super.hashCode(), boostTerms, boostTermsFactor, Arrays.hashCode(likeText), + maxDocFreq, maxQueryTerms, maxWordLen, minDocFreq, minTermFrequency, minWordLen, + Arrays.hashCode(moreLikeFields), minimumShouldMatch, stopWords); } @Override public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) + if (super.equals(obj) == false) { return false; + } MoreLikeThisQuery other = (MoreLikeThisQuery) obj; - if (getBoost() != other.getBoost()) - return false; if (!analyzer.equals(other.analyzer)) return false; if (boostTerms != other.boostTerms) @@ -141,6 +126,10 @@ public class MoreLikeThisQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; + } XMoreLikeThis mlt = new XMoreLikeThis(reader, similarity == null ? new DefaultSimilarity() : similarity); mlt.setFieldNames(moreLikeFields); @@ -179,10 +168,7 @@ public class MoreLikeThisQuery extends Query { mltQuery = Queries.applyMinimumShouldMatch((BooleanQuery) mltQuery, minimumShouldMatch); bqBuilder.add(mltQuery, BooleanClause.Occur.SHOULD); } - - BooleanQuery bq = bqBuilder.build(); - bq.setBoost(getBoost()); - return bq; + return bqBuilder.build(); } private void handleUnlike(XMoreLikeThis mlt, String[] unlikeText, Fields[] unlikeFields) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java index 3d870bc0794..662c3294151 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -120,8 +120,9 @@ public class MultiPhrasePrefixQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } if (termArrays.isEmpty()) { return new MatchNoDocsQuery(); @@ -145,7 +146,6 @@ public class MultiPhrasePrefixQuery extends Query { return Queries.newMatchNoDocsQuery(); } query.add(terms.toArray(Term.class), position); - query.setBoost(getBoost()); return query.rewrite(reader); } @@ -233,10 +233,11 @@ public class MultiPhrasePrefixQuery extends Query { */ @Override public boolean equals(Object o) { - if (!(o instanceof MultiPhrasePrefixQuery)) return false; + if (super.equals(o) == false) { + return false; + } MultiPhrasePrefixQuery other = (MultiPhrasePrefixQuery) o; - return this.getBoost() == other.getBoost() - && this.slop == other.slop + return this.slop == other.slop && termArraysEquals(this.termArrays, other.termArrays) && this.positions.equals(other.positions); } @@ -246,11 +247,10 @@ public class MultiPhrasePrefixQuery extends Query { */ @Override public int hashCode() { - return Float.floatToIntBits(getBoost()) + return super.hashCode() ^ slop ^ termArraysHashCode() - ^ positions.hashCode() - ^ 0x4AC65113; + ^ positions.hashCode(); } // Breakout calculation of the termArrays hashcode diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java index 0b7682cd7fd..b7f534d2124 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java @@ -43,12 +43,12 @@ public class Queries { return new BooleanQuery.Builder().build(); } - public static Filter newNestedFilter() { - return new QueryWrapperFilter(new PrefixQuery(new Term(TypeFieldMapper.NAME, new BytesRef("__")))); + public static Query newNestedFilter() { + return new PrefixQuery(new Term(TypeFieldMapper.NAME, new BytesRef("__"))); } - public static Filter newNonNestedFilter() { - return new QueryWrapperFilter(not(newNestedFilter())); + public static Query newNonNestedFilter() { + return not(newNestedFilter()); } public static BooleanQuery filtered(@Nullable Query query, @Nullable Query filter) { @@ -70,7 +70,7 @@ public class Queries { .build(); } - public static boolean isNegativeQuery(Query q) { + private static boolean isNegativeQuery(Query q) { if (!(q instanceof BooleanQuery)) { return false; } @@ -101,15 +101,13 @@ public class Queries { public static boolean isConstantMatchAllQuery(Query query) { if (query instanceof ConstantScoreQuery) { return isConstantMatchAllQuery(((ConstantScoreQuery) query).getQuery()); - } else if (query instanceof QueryWrapperFilter) { - return isConstantMatchAllQuery(((QueryWrapperFilter) query).getQuery()); } else if (query instanceof MatchAllDocsQuery) { return true; } return false; } - public static BooleanQuery applyMinimumShouldMatch(BooleanQuery query, @Nullable String minimumShouldMatch) { + public static Query applyMinimumShouldMatch(BooleanQuery query, @Nullable String minimumShouldMatch) { if (minimumShouldMatch == null) { return query; } @@ -129,10 +127,13 @@ public class Queries { } builder.setMinimumNumberShouldMatch(msm); BooleanQuery bq = builder.build(); - bq.setBoost(query.getBoost()); - query = bq; + if (query.getBoost() != 1f) { + return new BoostQuery(bq, query.getBoost()); + } + return bq; + } else { + return query; } - return query; } private static Pattern spaceAroundLessThanPattern = Pattern.compile("(\\s+<\\s*)|(\\s*<\\s+)"); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java b/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java index 85e1899582c..53159660089 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java @@ -670,14 +670,14 @@ public final class XMoreLikeThis { float bestScore = -1; while ((scoreTerm = q.pop()) != null) { - TermQuery tq = new TermQuery(new Term(scoreTerm.topField, scoreTerm.word)); + Query tq = new TermQuery(new Term(scoreTerm.topField, scoreTerm.word)); if (boost) { if (bestScore == -1) { bestScore = (scoreTerm.score); } float myScore = (scoreTerm.score); - tq.setBoost(boostFactor * myScore / bestScore); + tq = new BoostQuery(tq, boostFactor * myScore / bestScore); } try { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index 210b32d5e42..3da5ae0e4ab 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -123,8 +123,9 @@ public class FiltersFunctionScoreQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } Query newQ = subQuery.rewrite(reader); if (newQ == subQuery) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index 907d66957ac..972fb794fb5 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -71,8 +71,9 @@ public class FunctionScoreQuery extends Query { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } Query newQ = subQuery.rewrite(reader); if (newQ == subQuery) { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java index a715c61a7dd..9013b4b60e0 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java @@ -111,9 +111,9 @@ public class ScriptScoreFunction extends ScoreFunction { exp = ((ExplainableSearchScript) leafScript).explain(subQueryScore); } else { double score = score(docId, subQueryScore.getValue()); - String explanation = "script score function, computed with script:\"" + sScript; + String explanation = "script score function, computed with script:\"" + sScript + "\""; if (sScript.getParams() != null) { - explanation += "\" and parameters: \n" + sScript.getParams().toString(); + explanation += " and parameters: \n" + sScript.getParams().toString(); } Explanation scoreExp = Explanation.match( subQueryScore.getValue(), "_score: ", diff --git a/core/src/main/java/org/elasticsearch/common/metrics/EWMA.java b/core/src/main/java/org/elasticsearch/common/metrics/EWMA.java index b0cfd1bebc6..eefaaf22e7a 100644 --- a/core/src/main/java/org/elasticsearch/common/metrics/EWMA.java +++ b/core/src/main/java/org/elasticsearch/common/metrics/EWMA.java @@ -25,11 +25,12 @@ import java.util.concurrent.TimeUnit; /** * An exponentially-weighted moving average. + * + *

    + * Taken from codahale metric module, changed to use LongAdder * * @see UNIX Load Average Part 1: How It Works * @see UNIX Load Average Part 2: Not Your Average Average - *

    - * Taken from codahale metric module, changed to use LongAdder */ public class EWMA { private static final double M1_ALPHA = 1 - Math.exp(-5 / 60.0); diff --git a/core/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java b/core/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java index 4bd995c4294..8f7b46c355f 100644 --- a/core/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java +++ b/core/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java @@ -30,9 +30,10 @@ import java.util.concurrent.TimeUnit; * A meter metric which measures mean throughput and one-, five-, and * fifteen-minute exponentially-weighted moving average throughputs. * + *

    + * taken from codahale metric module, replaced with LongAdder + * * @see EMA - *

    - * taken from codahale metric module, replaced with LongAdder */ public class MeterMetric implements Metric { private static final long INTERVAL = 5; // seconds diff --git a/core/src/main/java/org/elasticsearch/common/network/Cidrs.java b/core/src/main/java/org/elasticsearch/common/network/Cidrs.java new file mode 100644 index 00000000000..d0557248a68 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/network/Cidrs.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.network; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Objects; + +public final class Cidrs { + private Cidrs() { + } + + /** + * Parses an IPv4 address block in CIDR notation into a pair of + * longs representing the bottom and top of the address block + * + * @param cidr an address block in CIDR notation a.b.c.d/n + * @return array representing the address block + * @throws IllegalArgumentException if the cidr can not be parsed + */ + public static long[] cidrMaskToMinMax(String cidr) { + Objects.requireNonNull(cidr, "cidr"); + String[] fields = cidr.split("/"); + if (fields.length != 2) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; expected [a.b.c.d, e] but was [%s] after splitting on \"/\" in [%s]", Arrays.toString(fields), cidr) + ); + } + // do not try to parse IPv4-mapped IPv6 address + if (fields[0].contains(":")) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; expected [a.b.c.d, e] where a, b, c, d are decimal octets but was [%s] after splitting on \"/\" in [%s]", Arrays.toString(fields), cidr) + ); + } + byte[] addressBytes; + try { + addressBytes = InetAddresses.forString(fields[0]).getAddress(); + } catch (Throwable t) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; unable to parse [%s] as an IP address literal", fields[0]), t + ); + } + long accumulator = + ((addressBytes[0] & 0xFFL) << 24) + + ((addressBytes[1] & 0xFFL) << 16) + + ((addressBytes[2] & 0xFFL) << 8) + + ((addressBytes[3] & 0xFFL)); + int networkMask; + try { + networkMask = Integer.parseInt(fields[1]); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; invalid network mask [%s] in [%s]", fields[1], cidr), + e + ); + } + if (networkMask < 0 || networkMask > 32) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; invalid network mask [%s], out of range in [%s]", fields[1], cidr) + ); + } + + long blockSize = 1L << (32 - networkMask); + // validation + if ((accumulator & (blockSize - 1)) != 0) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "invalid IPv4/CIDR; invalid address/network mask combination in [%s]; perhaps [%s] was intended?", + cidr, + octetsToCIDR(longToOctets(accumulator - (accumulator & (blockSize - 1))), networkMask) + ) + ); + } + return new long[] { accumulator, accumulator + blockSize }; + } + + static int[] longToOctets(long value) { + assert value >= 0 && value <= (1L << 32) : value; + int[] octets = new int[4]; + octets[0] = (int)((value >> 24) & 0xFF); + octets[1] = (int)((value >> 16) & 0xFF); + octets[2] = (int)((value >> 8) & 0xFF); + octets[3] = (int)(value & 0xFF); + return octets; + } + + static String octetsToString(int[] octets) { + assert octets != null; + assert octets.length == 4; + return String.format(Locale.ROOT, "%d.%d.%d.%d", octets[0], octets[1], octets[2], octets[3]); + } + + static String octetsToCIDR(int[] octets, int networkMask) { + assert octets != null; + assert octets.length == 4; + return octetsToString(octets) + "/" + networkMask; + } +} diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java index 6f5d728a6dd..c1f282ac234 100644 --- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java +++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java @@ -26,8 +26,14 @@ import org.elasticsearch.common.inject.AbstractModule; */ public class NetworkModule extends AbstractModule { + private final NetworkService networkService; + + public NetworkModule(NetworkService networkService) { + this.networkService = networkService; + } + @Override protected void configure() { - bind(NetworkService.class).asEagerSingleton(); + bind(NetworkService.class).toInstance(networkService); } } diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java index cd46d1416f4..835a35d2383 100644 --- a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java +++ b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java @@ -20,13 +20,15 @@ package org.elasticsearch.common.network; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; @@ -77,7 +79,6 @@ public class NetworkService extends AbstractComponent { private final List customNameResolvers = new CopyOnWriteArrayList<>(); - @Inject public NetworkService(Settings settings) { super(settings); IfConfig.logIfNecessary(); @@ -90,13 +91,21 @@ public class NetworkService extends AbstractComponent { customNameResolvers.add(customNameResolver); } - public InetAddress[] resolveBindHostAddress(String bindHost) throws IOException { + /** + * Resolves {@code bindHosts} to a list of internet addresses. The list will + * not contain duplicate addresses. + * @param bindHosts list of hosts to bind to. this may contain special pseudo-hostnames + * such as _local_ (see the documentation). if it is null, it will be populated + * based on global default settings. + * @return unique set of internet addresses + */ + public InetAddress[] resolveBindHostAddresses(String bindHosts[]) throws IOException { // first check settings - if (bindHost == null) { - bindHost = settings.get(GLOBAL_NETWORK_BINDHOST_SETTING, settings.get(GLOBAL_NETWORK_HOST_SETTING)); + if (bindHosts == null) { + bindHosts = settings.getAsArray(GLOBAL_NETWORK_BINDHOST_SETTING, settings.getAsArray(GLOBAL_NETWORK_HOST_SETTING, null)); } // next check any registered custom resolvers - if (bindHost == null) { + if (bindHosts == null) { for (CustomNameResolver customNameResolver : customNameResolvers) { InetAddress addresses[] = customNameResolver.resolveDefault(); if (addresses != null) { @@ -105,31 +114,43 @@ public class NetworkService extends AbstractComponent { } } // finally, fill with our default - if (bindHost == null) { - bindHost = DEFAULT_NETWORK_HOST; + if (bindHosts == null) { + bindHosts = new String[] { DEFAULT_NETWORK_HOST }; } - InetAddress addresses[] = resolveInetAddress(bindHost); + InetAddress addresses[] = resolveInetAddresses(bindHosts); // try to deal with some (mis)configuration - if (addresses != null) { - for (InetAddress address : addresses) { - // check if its multicast: flat out mistake - if (address.isMulticastAddress()) { - throw new IllegalArgumentException("bind address: {" + NetworkAddress.format(address) + "} is invalid: multicast address"); - } + for (InetAddress address : addresses) { + // check if its multicast: flat out mistake + if (address.isMulticastAddress()) { + throw new IllegalArgumentException("bind address: {" + NetworkAddress.format(address) + "} is invalid: multicast address"); + } + // check if its a wildcard address: this is only ok if its the only address! + if (address.isAnyLocalAddress() && addresses.length > 1) { + throw new IllegalArgumentException("bind address: {" + NetworkAddress.format(address) + "} is wildcard, but multiple addresses specified: this makes no sense"); } } return addresses; } + /** + * Resolves {@code publishHosts} to a single publish address. The fact that it returns + * only one address is just a current limitation. + *

    + * If {@code publishHosts} resolves to more than one address, then one is selected with magic + * @param publishHosts list of hosts to publish as. this may contain special pseudo-hostnames + * such as _local_ (see the documentation). if it is null, it will be populated + * based on global default settings. + * @return single internet address + */ // TODO: needs to be InetAddress[] - public InetAddress resolvePublishHostAddress(String publishHost) throws IOException { + public InetAddress resolvePublishHostAddresses(String publishHosts[]) throws IOException { // first check settings - if (publishHost == null) { - publishHost = settings.get(GLOBAL_NETWORK_PUBLISHHOST_SETTING, settings.get(GLOBAL_NETWORK_HOST_SETTING)); + if (publishHosts == null) { + publishHosts = settings.getAsArray(GLOBAL_NETWORK_PUBLISHHOST_SETTING, settings.getAsArray(GLOBAL_NETWORK_HOST_SETTING, null)); } // next check any registered custom resolvers - if (publishHost == null) { + if (publishHosts == null) { for (CustomNameResolver customNameResolver : customNameResolvers) { InetAddress addresses[] = customNameResolver.resolveDefault(); if (addresses != null) { @@ -138,30 +159,58 @@ public class NetworkService extends AbstractComponent { } } // finally, fill with our default - if (publishHost == null) { - publishHost = DEFAULT_NETWORK_HOST; + if (publishHosts == null) { + publishHosts = new String[] { DEFAULT_NETWORK_HOST }; } + InetAddress addresses[] = resolveInetAddresses(publishHosts); // TODO: allow publishing multiple addresses - InetAddress address = resolveInetAddress(publishHost)[0]; + // for now... the hack begins - // try to deal with some (mis)configuration - if (address != null) { + // 1. single wildcard address, probably set by network.host: expand to all interface addresses. + if (addresses.length == 1 && addresses[0].isAnyLocalAddress()) { + HashSet all = new HashSet<>(Arrays.asList(NetworkUtils.getAllAddresses())); + addresses = all.toArray(new InetAddress[all.size()]); + } + + // 2. try to deal with some (mis)configuration + for (InetAddress address : addresses) { // check if its multicast: flat out mistake if (address.isMulticastAddress()) { throw new IllegalArgumentException("publish address: {" + NetworkAddress.format(address) + "} is invalid: multicast address"); } - // wildcard address, probably set by network.host + // check if its a wildcard address: this is only ok if its the only address! + // (if it was a single wildcard address, it was replaced by step 1 above) if (address.isAnyLocalAddress()) { - InetAddress old = address; - address = NetworkUtils.getFirstNonLoopbackAddresses()[0]; - logger.warn("publish address: {{}} is a wildcard address, falling back to first non-loopback: {{}}", - NetworkAddress.format(old), NetworkAddress.format(address)); + throw new IllegalArgumentException("publish address: {" + NetworkAddress.format(address) + "} is wildcard, but multiple addresses specified: this makes no sense"); } } - return address; + + // 3. if we end out with multiple publish addresses, select by preference. + // don't warn the user, or they will get confused by bind_host vs publish_host etc. + if (addresses.length > 1) { + List sorted = new ArrayList<>(Arrays.asList(addresses)); + NetworkUtils.sortAddresses(sorted); + addresses = new InetAddress[] { sorted.get(0) }; + } + return addresses[0]; + } + + /** resolves (and deduplicates) host specification */ + private InetAddress[] resolveInetAddresses(String hosts[]) throws IOException { + if (hosts.length == 0) { + throw new IllegalArgumentException("empty host specification"); + } + // deduplicate, in case of resolver misconfiguration + // stuff like https://bugzilla.redhat.com/show_bug.cgi?id=496300 + HashSet set = new HashSet<>(); + for (String host : hosts) { + set.addAll(Arrays.asList(resolveInternal(host))); + } + return set.toArray(new InetAddress[set.size()]); } - private InetAddress[] resolveInetAddress(String host) throws IOException { + /** resolves a single host specification */ + private InetAddress[] resolveInternal(String host) throws IOException { if ((host.startsWith("#") && host.endsWith("#")) || (host.startsWith("_") && host.endsWith("_"))) { host = host.substring(1, host.length() - 1); // allow custom resolvers to have special names @@ -178,12 +227,18 @@ public class NetworkService extends AbstractComponent { return NetworkUtils.filterIPV4(NetworkUtils.getLoopbackAddresses()); case "local:ipv6": return NetworkUtils.filterIPV6(NetworkUtils.getLoopbackAddresses()); - case "non_loopback": - return NetworkUtils.getFirstNonLoopbackAddresses(); - case "non_loopback:ipv4": - return NetworkUtils.filterIPV4(NetworkUtils.getFirstNonLoopbackAddresses()); - case "non_loopback:ipv6": - return NetworkUtils.filterIPV6(NetworkUtils.getFirstNonLoopbackAddresses()); + case "site": + return NetworkUtils.getSiteLocalAddresses(); + case "site:ipv4": + return NetworkUtils.filterIPV4(NetworkUtils.getSiteLocalAddresses()); + case "site:ipv6": + return NetworkUtils.filterIPV6(NetworkUtils.getSiteLocalAddresses()); + case "global": + return NetworkUtils.getGlobalAddresses(); + case "global:ipv4": + return NetworkUtils.filterIPV4(NetworkUtils.getGlobalAddresses()); + case "global:ipv6": + return NetworkUtils.filterIPV6(NetworkUtils.getGlobalAddresses()); default: /* an interface specification */ if (host.endsWith(":ipv4")) { @@ -197,6 +252,6 @@ public class NetworkService extends AbstractComponent { } } } - return NetworkUtils.getAllByName(host); + return InetAddress.getAllByName(host); } } diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java b/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java index 62bc91cfb85..8652d4c5c05 100644 --- a/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java +++ b/core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java @@ -27,12 +27,10 @@ import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.SocketException; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.HashSet; import java.util.List; /** @@ -109,7 +107,8 @@ public abstract class NetworkUtils { * @deprecated remove this when multihoming is really correct */ @Deprecated - static void sortAddresses(List list) { + // only public because of silly multicast + public static void sortAddresses(List list) { Collections.sort(list, new Comparator() { @Override public int compare(InetAddress left, InetAddress right) { @@ -150,34 +149,79 @@ public abstract class NetworkUtils { return Constants.WINDOWS ? false : true; } - /** Returns addresses for all loopback interfaces that are up. */ + /** Returns all interface-local scope (loopback) addresses for interfaces that are up. */ static InetAddress[] getLoopbackAddresses() throws SocketException { List list = new ArrayList<>(); for (NetworkInterface intf : getInterfaces()) { - if (intf.isLoopback() && intf.isUp()) { - list.addAll(Collections.list(intf.getInetAddresses())); + if (intf.isUp()) { + // NOTE: some operating systems (e.g. BSD stack) assign a link local address to the loopback interface + // while technically not a loopback address, some of these treat them as one (e.g. OS X "localhost") so we must too, + // otherwise things just won't work out of box. So we include all addresses from loopback interfaces. + for (InetAddress address : Collections.list(intf.getInetAddresses())) { + if (intf.isLoopback() || address.isLoopbackAddress()) { + list.add(address); + } + } } } if (list.isEmpty()) { - throw new IllegalArgumentException("No up-and-running loopback interfaces found, got " + getInterfaces()); + throw new IllegalArgumentException("No up-and-running loopback addresses found, got " + getInterfaces()); } - sortAddresses(list); return list.toArray(new InetAddress[list.size()]); } - /** Returns addresses for the first non-loopback interface that is up. */ - static InetAddress[] getFirstNonLoopbackAddresses() throws SocketException { + /** Returns all site-local scope (private) addresses for interfaces that are up. */ + static InetAddress[] getSiteLocalAddresses() throws SocketException { List list = new ArrayList<>(); for (NetworkInterface intf : getInterfaces()) { - if (intf.isLoopback() == false && intf.isUp()) { - list.addAll(Collections.list(intf.getInetAddresses())); - break; + if (intf.isUp()) { + for (InetAddress address : Collections.list(intf.getInetAddresses())) { + if (address.isSiteLocalAddress()) { + list.add(address); + } + } } } if (list.isEmpty()) { - throw new IllegalArgumentException("No up-and-running non-loopback interfaces found, got " + getInterfaces()); + throw new IllegalArgumentException("No up-and-running site-local (private) addresses found, got " + getInterfaces()); + } + return list.toArray(new InetAddress[list.size()]); + } + + /** Returns all global scope addresses for interfaces that are up. */ + static InetAddress[] getGlobalAddresses() throws SocketException { + List list = new ArrayList<>(); + for (NetworkInterface intf : getInterfaces()) { + if (intf.isUp()) { + for (InetAddress address : Collections.list(intf.getInetAddresses())) { + if (address.isLoopbackAddress() == false && + address.isSiteLocalAddress() == false && + address.isLinkLocalAddress() == false) { + list.add(address); + } + } + } + } + if (list.isEmpty()) { + throw new IllegalArgumentException("No up-and-running global-scope (public) addresses found, got " + getInterfaces()); + } + return list.toArray(new InetAddress[list.size()]); + } + + /** Returns all addresses (any scope) for interfaces that are up. + * This is only used to pick a publish address, when the user set network.host to a wildcard */ + static InetAddress[] getAllAddresses() throws SocketException { + List list = new ArrayList<>(); + for (NetworkInterface intf : getInterfaces()) { + if (intf.isUp()) { + for (InetAddress address : Collections.list(intf.getInetAddresses())) { + list.add(address); + } + } + } + if (list.isEmpty()) { + throw new IllegalArgumentException("No up-and-running addresses found, got " + getInterfaces()); } - sortAddresses(list); return list.toArray(new InetAddress[list.size()]); } @@ -194,20 +238,9 @@ public abstract class NetworkUtils { if (list.isEmpty()) { throw new IllegalArgumentException("Interface '" + name + "' has no internet addresses"); } - sortAddresses(list); return list.toArray(new InetAddress[list.size()]); } - /** Returns addresses for the given host, sorted by order of preference */ - static InetAddress[] getAllByName(String host) throws UnknownHostException { - InetAddress addresses[] = InetAddress.getAllByName(host); - // deduplicate, in case of resolver misconfiguration - // stuff like https://bugzilla.redhat.com/show_bug.cgi?id=496300 - List unique = new ArrayList<>(new HashSet<>(Arrays.asList(addresses))); - sortAddresses(unique); - return unique.toArray(new InetAddress[unique.size()]); - } - /** Returns only the IPV4 addresses in {@code addresses} */ static InetAddress[] filterIPV4(InetAddress addresses[]) { List list = new ArrayList<>(); diff --git a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java index 2bee8273667..704468f7533 100644 --- a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java +++ b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java @@ -21,9 +21,7 @@ package org.elasticsearch.common.path; import org.elasticsearch.common.Strings; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; @@ -34,26 +32,15 @@ import static java.util.Collections.unmodifiableMap; */ public class PathTrie { - public static interface Decoder { + public interface Decoder { String decode(String value); } - public static final Decoder NO_DECODER = new Decoder() { - @Override - public String decode(String value) { - return value; - } - }; - private final Decoder decoder; private final TrieNode root; private final char separator; private T rootValue; - public PathTrie() { - this('/', "*", NO_DECODER); - } - public PathTrie(Decoder decoder) { this('/', "*", decoder); } @@ -198,7 +185,7 @@ public class PathTrie { private void put(Map params, TrieNode node, String value) { if (params != null && node.isNamedWildcard()) { - params.put(node.namedWildcard(), value); + params.put(node.namedWildcard(), decoder.decode(value)); } } @@ -230,7 +217,7 @@ public class PathTrie { if (path.length() == 0) { return rootValue; } - String[] strings = splitPath(decoder.decode(path)); + String[] strings = Strings.splitStringToArray(path, separator); if (strings.length == 0) { return rootValue; } @@ -241,50 +228,4 @@ public class PathTrie { } return root.retrieve(strings, index, params); } - - /* - Splits up the url path up by '/' and is aware of - index name expressions that appear between '<' and '>'. - */ - String[] splitPath(final String path) { - if (path == null || path.length() == 0) { - return Strings.EMPTY_ARRAY; - } - int count = 1; - boolean splitAllowed = true; - for (int i = 0; i < path.length(); i++) { - final char currentC = path.charAt(i); - if ('<' == currentC) { - splitAllowed = false; - } else if (currentC == '>') { - splitAllowed = true; - } else if (splitAllowed && currentC == separator) { - count++; - } - } - - final List result = new ArrayList<>(count); - final StringBuilder builder = new StringBuilder(); - - splitAllowed = true; - for (int i = 0; i < path.length(); i++) { - final char currentC = path.charAt(i); - if ('<' == currentC) { - splitAllowed = false; - } else if (currentC == '>') { - splitAllowed = true; - } else if (splitAllowed && currentC == separator) { - if (builder.length() > 0) { - result.add(builder.toString()); - builder.setLength(0); - } - continue; - } - builder.append(currentC); - } - if (builder.length() > 0) { - result.add(builder.toString()); - } - return result.toArray(new String[result.size()]); - } } diff --git a/core/src/main/java/org/elasticsearch/common/regex/Regex.java b/core/src/main/java/org/elasticsearch/common/regex/Regex.java index f5c3094e31d..061ad6c26c0 100644 --- a/core/src/main/java/org/elasticsearch/common/regex/Regex.java +++ b/core/src/main/java/org/elasticsearch/common/regex/Regex.java @@ -150,7 +150,7 @@ public class Regex { pFlags |= Pattern.LITERAL; } else if ("COMMENTS".equals(s)) { pFlags |= Pattern.COMMENTS; - } else if ("UNICODE_CHAR_CLASS".equals(s)) { + } else if (("UNICODE_CHAR_CLASS".equals(s)) || ("UNICODE_CHARACTER_CLASS".equals(s))) { pFlags |= UNICODE_CHARACTER_CLASS; } else { throw new IllegalArgumentException("Unknown regex flag [" + s + "]"); diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index 663abd746f0..5e083a9e740 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -46,13 +46,14 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Dictionary; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Properties; +import java.util.Objects; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -1028,9 +1029,9 @@ public final class Settings implements ToXContent { /** * Sets all the provided settings. */ - public Builder put(Properties properties) { - for (Map.Entry entry : properties.entrySet()) { - map.put((String) entry.getKey(), (String) entry.getValue()); + public Builder put(Dictionary properties) { + for (Object key : Collections.list(properties.keys())) { + map.put(Objects.toString(key), Objects.toString(properties.get(key))); } return this; } @@ -1096,10 +1097,10 @@ public final class Settings implements ToXContent { * @param properties The properties to put * @return The builder */ - public Builder putProperties(String prefix, Properties properties) { - for (Object key1 : properties.keySet()) { - String key = (String) key1; - String value = properties.getProperty(key); + public Builder putProperties(String prefix, Dictionary properties) { + for (Object key1 : Collections.list(properties.keys())) { + String key = Objects.toString(key1); + String value = Objects.toString(properties.get(key)); if (key.startsWith(prefix)) { map.put(key.substring(prefix.length()), value); } @@ -1114,10 +1115,10 @@ public final class Settings implements ToXContent { * @param properties The properties to put * @return The builder */ - public Builder putProperties(String prefix, Properties properties, String[] ignorePrefixes) { - for (Object key1 : properties.keySet()) { - String key = (String) key1; - String value = properties.getProperty(key); + public Builder putProperties(String prefix, Dictionary properties, String[] ignorePrefixes) { + for (Object key1 : Collections.list(properties.keys())) { + String key = Objects.toString(key1); + String value = Objects.toString(properties.get(key)); if (key.startsWith(prefix)) { boolean ignore = false; for (String ignorePrefix : ignorePrefixes) { diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java index 421e0081998..11fbe65cf01 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java +++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java @@ -20,7 +20,6 @@ package org.elasticsearch.common.settings; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.rest.RestRequest; @@ -35,7 +34,7 @@ import java.util.concurrent.CopyOnWriteArrayList; /** * */ -public class SettingsFilter extends AbstractComponent { +public final class SettingsFilter extends AbstractComponent { /** * Can be used to specify settings filter that will be used to filter out matching settings in toXContent method */ @@ -43,7 +42,6 @@ public class SettingsFilter extends AbstractComponent { private final CopyOnWriteArrayList patterns = new CopyOnWriteArrayList<>(); - @Inject public SettingsFilter(Settings settings) { super(settings); } diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java index 20e65760245..2ae4799d9f3 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java +++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java @@ -29,14 +29,16 @@ import org.elasticsearch.common.inject.AbstractModule; public class SettingsModule extends AbstractModule { private final Settings settings; + private final SettingsFilter settingsFilter; - public SettingsModule(Settings settings) { + public SettingsModule(Settings settings, SettingsFilter settingsFilter) { this.settings = settings; + this.settingsFilter = settingsFilter; } @Override protected void configure() { bind(Settings.class).toInstance(settings); - bind(SettingsFilter.class).asEagerSingleton(); + bind(SettingsFilter.class).toInstance(settingsFilter); } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java index a269d23c0ce..d4390febaaf 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java +++ b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java @@ -259,7 +259,7 @@ public class ByteSizeValue implements Streamable { @Override public int hashCode() { - int result = (int) (size ^ (size >>> 32)); + int result = Long.hashCode(size); result = 31 * result + (sizeUnit != null ? sizeUnit.hashCode() : 0); return result; } diff --git a/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java b/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java index 9abce6989fd..feebd93c5ab 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java +++ b/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java @@ -225,7 +225,7 @@ public enum DistanceUnit { * @param in {@link StreamInput} to read the {@link DistanceUnit} from * @return {@link DistanceUnit} read from the {@link StreamInput} * @throws IOException if no unit can be read from the {@link StreamInput} - * @thrown ElasticsearchIllegalArgumentException if no matching {@link DistanceUnit} can be found + * @throws IllegalArgumentException if no matching {@link DistanceUnit} can be found */ public static DistanceUnit readDistanceUnit(StreamInput in) throws IOException { byte b = in.readByte(); diff --git a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java index fcbcff3c3d7..3958b19309e 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java +++ b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java @@ -228,7 +228,7 @@ public class SizeValue implements Streamable { @Override public int hashCode() { - int result = (int) (size ^ (size >>> 32)); + int result = Long.hashCode(size); result = 31 * result + (sizeUnit != null ? sizeUnit.hashCode() : 0); return result; } diff --git a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java index 5f9eb953a42..ee6371605ee 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java +++ b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java @@ -231,7 +231,7 @@ public class TimeValue implements Streamable { public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, String settingName) { settingName = Objects.requireNonNull(settingName); - assert settingName.startsWith("index.") == false || MetaDataIndexUpgradeService.INDEX_TIME_SETTINGS.contains(settingName); + assert settingName.startsWith("index.") == false || MetaDataIndexUpgradeService.INDEX_TIME_SETTINGS.contains(settingName) : settingName; if (sValue == null) { return defaultValue; } @@ -311,7 +311,7 @@ public class TimeValue implements Streamable { @Override public int hashCode() { long normalized = timeUnit.toNanos(duration); - return (int) (normalized ^ (normalized >>> 32)); + return Long.hashCode(normalized); } public static long nsecToMSec(long ns) { diff --git a/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java b/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java index b8c5ba09b9c..a605d66e80d 100644 --- a/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java +++ b/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.common.util; +import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Nullable; @@ -84,7 +85,7 @@ public class CancellableThreads { RuntimeException throwable = null; try { interruptable.run(); - } catch (InterruptedException e) { + } catch (InterruptedException | ThreadInterruptedException e) { // assume this is us and ignore } catch (RuntimeException t) { throwable = t; diff --git a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java b/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java index 056142a48c7..d25113a54bb 100644 --- a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java +++ b/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java @@ -123,7 +123,7 @@ public abstract class ExtensionPoint { public static final class SelectedType extends ClassMap { public SelectedType(String name, Class extensionClass) { - super(name, extensionClass, Collections.EMPTY_SET); + super(name, extensionClass, Collections.emptySet()); } /** @@ -191,7 +191,8 @@ public abstract class ExtensionPoint { protected final void bindExtensions(Binder binder) { Multibinder allocationMultibinder = Multibinder.newSetBinder(binder, extensionClass); for (Class clazz : extensions) { - allocationMultibinder.addBinding().to(clazz).asEagerSingleton(); + binder.bind(clazz).asEagerSingleton(); + allocationMultibinder.addBinding().to(clazz); } } } diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java index 42022b14913..b63cc2e31ca 100644 --- a/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java +++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java @@ -35,9 +35,6 @@ public abstract class AbstractRunnable implements Runnable { public final void run() { try { doRun(); - } catch (InterruptedException ex) { - Thread.interrupted(); - onFailure(ex); } catch (Throwable t) { onFailure(t); } finally { diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java index 95def1161c4..e5ba66300f3 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java @@ -214,8 +214,16 @@ public final class ObjectParser implements BiFunction List parseArray(XContentParser parser, IOSupplier supplier) throws IOException { List list = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - list.add(supplier.get()); + if (parser.currentToken().isValue()) { + list.add(supplier.get()); // single value + } else { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + if (parser.currentToken().isValue()) { + list.add(supplier.get()); + } else { + throw new IllegalStateException("expected value but got [" + parser.currentToken() + "]"); + } + } } return list; } @@ -224,6 +232,19 @@ public final class ObjectParser implements BiFunction consumer.accept(v, objectParser.apply(p, c)), field, ValueType.OBJECT); } + public void declareObjectOrDefault(BiConsumer consumer, BiFunction objectParser, Supplier defaultValue, ParseField field) { + declareField((p, v, c) -> { + if (p.currentToken() == XContentParser.Token.VALUE_BOOLEAN) { + if (p.booleanValue()) { + consumer.accept(v, defaultValue.get()); + } + } else { + consumer.accept(v, objectParser.apply(p, c)); + } + }, field, ValueType.OBJECT_OR_BOOLEAN); + } + + public void declareFloat(BiConsumer consumer, ParseField field) { declareField((p, v, c) -> consumer.accept(v, p.floatValue()), field, ValueType.FLOAT); } @@ -240,6 +261,10 @@ public final class ObjectParser implements BiFunction consumer.accept(v, p.intValue()), field, ValueType.INT); } + public void declareValue(BiConsumer consumer, ParseField field) { + declareField((p, v, c) -> consumer.accept(v, p), field, ValueType.VALUE); + } + public void declareString(BiConsumer consumer, ParseField field) { declareField((p, v, c) -> consumer.accept(v, p.text()), field, ValueType.STRING); } @@ -296,13 +321,15 @@ public final class ObjectParser implements BiFunction tokens; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java index 101098d67a4..50c04930901 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java @@ -45,11 +45,6 @@ public interface XContent { */ XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException; - /** - * Creates a new generator using the provided writer. - */ - XContentGenerator createGenerator(Writer writer) throws IOException; - /** * Creates a parser over the provided string content. */ diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index d5480cd4e3a..af8e7534692 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -920,23 +920,18 @@ public final class XContentBuilder implements BytesStream, Releasable { return this; } - public XContentBuilder rawField(String fieldName, byte[] content) throws IOException { - generator.writeRawField(fieldName, content, bos); - return this; - } - - public XContentBuilder rawField(String fieldName, byte[] content, int offset, int length) throws IOException { - generator.writeRawField(fieldName, content, offset, length, bos); - return this; - } - public XContentBuilder rawField(String fieldName, InputStream content) throws IOException { - generator.writeRawField(fieldName, content, bos); + generator.writeRawField(fieldName, content); return this; } public XContentBuilder rawField(String fieldName, BytesReference content) throws IOException { - generator.writeRawField(fieldName, content, bos); + generator.writeRawField(fieldName, content); + return this; + } + + public XContentBuilder rawValue(BytesReference content) throws IOException { + generator.writeRawValue(content); return this; } @@ -1202,24 +1197,12 @@ public final class XContentBuilder implements BytesStream, Releasable { return this.generator; } - public OutputStream stream() { - return this.bos; - } - @Override public BytesReference bytes() { close(); return ((BytesStream) bos).bytes(); } - /** - * Returns the actual stream used. - */ - public BytesStream bytesStream() throws IOException { - close(); - return (BytesStream) bos; - } - /** * Returns a string representation of the builder (only applicable for text based xcontent). */ diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java index c9e38f1d94e..f835b8ce8b1 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.dataformat.smile.SmileConstants; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.cbor.CborXContent; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.smile.SmileXContent; @@ -33,7 +34,6 @@ import org.elasticsearch.common.xcontent.yaml.YamlXContent; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Arrays; /** * A one stop to use {@link org.elasticsearch.common.xcontent.XContent} and {@link XContentBuilder}. @@ -216,65 +216,9 @@ public class XContentFactory { } si.mark(GUESS_HEADER_LENGTH); try { - final int firstInt = si.read(); // this must be an int since we need to respect the method contract - if (firstInt == -1) { - return null; - } - - final int secondInt = si.read(); // this must be an int since we need to respect the method contract - if (secondInt == -1) { - return null; - } - final byte first = (byte) (0xff & firstInt); - final byte second = (byte) (0xff & secondInt); - if (first == SmileConstants.HEADER_BYTE_1 && second == SmileConstants.HEADER_BYTE_2) { - int third = si.read(); - if (third == SmileConstants.HEADER_BYTE_3) { - return XContentType.SMILE; - } - } - if (first == '{' || second == '{') { - return XContentType.JSON; - } - if (first == '-' && second == '-') { - int third = si.read(); - if (third == '-') { - return XContentType.YAML; - } - } - // CBOR logic similar to CBORFactory#hasCBORFormat - if (first == CBORConstants.BYTE_OBJECT_INDEFINITE){ - return XContentType.CBOR; - } - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first)) { - // Actually, specific "self-describe tag" is a very good indicator - int third = si.read(); - if (third == -1) { - return null; - } - if (first == (byte) 0xD9 && second == (byte) 0xD9 && third == (byte) 0xF7) { - return XContentType.CBOR; - } - } - // for small objects, some encoders just encode as major type object, we can safely - // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, first)) { - return XContentType.CBOR; - } - - for (int i = 2; i < GUESS_HEADER_LENGTH; i++) { - int val = si.read(); - if (val == -1) { - return null; - } - if (val == '{') { - return XContentType.JSON; - } - if (Character.isWhitespace(val) == false) { - break; - } - } - return null; + final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH]; + final int read = Streams.readFully(si, firstBytes); + return xContentType(new BytesArray(firstBytes, 0, read)); } finally { si.reset(); } @@ -329,8 +273,14 @@ public class XContentFactory { return XContentType.CBOR; } + int jsonStart = 0; + // JSON may be preceded by UTF-8 BOM + if (length > 3 && first == (byte) 0xEF && bytes.get(1) == (byte) 0xBB && bytes.get(2) == (byte) 0xBF) { + jsonStart = 3; + } + // a last chance for JSON - for (int i = 0; i < length; i++) { + for (int i = jsonStart; i < length; i++) { byte b = bytes.get(i); if (b == '{') { return XContentType.JSON; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index a17ef93f65d..11a42e37279 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -21,14 +21,14 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.bytes.BytesReference; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; /** * */ -public interface XContentGenerator { +public interface XContentGenerator extends Closeable { XContentType contentType(); @@ -111,13 +111,11 @@ public interface XContentGenerator { void writeObjectFieldStart(XContentString fieldName) throws IOException; - void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException; + void writeRawField(String fieldName, InputStream content) throws IOException; - void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException; + void writeRawField(String fieldName, BytesReference content) throws IOException; - void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException; - - void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException; + void writeRawValue(BytesReference content) throws IOException; void copyCurrentStructure(XContentParser parser) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 9fa14a3cc8d..4466d295735 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.ToXContent.Params; import java.io.BufferedInputStream; @@ -102,9 +101,7 @@ public class XContentHelper { BytesArray bytesArray = bytes.toBytesArray(); return new String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), StandardCharsets.UTF_8); } - XContentParser parser = null; - try { - parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput()); + try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput())) { parser.nextToken(); XContentBuilder builder = XContentFactory.jsonBuilder(); if (prettyPrint) { @@ -112,10 +109,6 @@ public class XContentHelper { } builder.copyCurrentStructure(parser); return builder.string(); - } finally { - if (parser != null) { - parser.close(); - } } } @@ -128,9 +121,7 @@ public class XContentHelper { if (xContentType == XContentType.JSON && !reformatJson) { return new String(data, offset, length, StandardCharsets.UTF_8); } - XContentParser parser = null; - try { - parser = XContentFactory.xContent(xContentType).createParser(data, offset, length); + try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(data, offset, length)) { parser.nextToken(); XContentBuilder builder = XContentFactory.jsonBuilder(); if (prettyPrint) { @@ -138,10 +129,6 @@ public class XContentHelper { } builder.copyCurrentStructure(parser); return builder.string(); - } finally { - if (parser != null) { - parser.close(); - } } } @@ -378,38 +365,6 @@ public class XContentHelper { } } - /** - * Directly writes the source to the output builder - */ - public static void writeDirect(BytesReference source, XContentBuilder rawBuilder, ToXContent.Params params) throws IOException { - Compressor compressor = CompressorFactory.compressor(source); - if (compressor != null) { - InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); - if (compressedStreamInput.markSupported() == false) { - compressedStreamInput = new BufferedInputStream(compressedStreamInput); - } - XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - if (contentType == rawBuilder.contentType()) { - Streams.copy(compressedStreamInput, rawBuilder.stream()); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput)) { - parser.nextToken(); - rawBuilder.copyCurrentStructure(parser); - } - } - } else { - XContentType contentType = XContentFactory.xContentType(source); - if (contentType == rawBuilder.contentType()) { - source.writeTo(rawBuilder.stream()); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(source)) { - parser.nextToken(); - rawBuilder.copyCurrentStructure(parser); - } - } - } - } - /** * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}. @@ -418,30 +373,9 @@ public class XContentHelper { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); - if (compressedStreamInput.markSupported() == false) { - compressedStreamInput = new BufferedInputStream(compressedStreamInput); - } - XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - if (contentType == builder.contentType()) { - builder.rawField(field, compressedStreamInput); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput)) { - parser.nextToken(); - builder.field(field); - builder.copyCurrentStructure(parser); - } - } + builder.rawField(field, compressedStreamInput); } else { - XContentType contentType = XContentFactory.xContentType(source); - if (contentType == builder.contentType()) { - builder.rawField(field, source); - } else { - try (XContentParser parser = XContentFactory.xContent(contentType).createParser(source)) { - parser.nextToken(); - builder.field(field); - builder.copyCurrentStructure(parser); - } - } + builder.rawField(field, source); } } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index b68d3e11f14..d647c5f0134 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -178,12 +178,6 @@ public interface XContentParser extends Releasable { NumberType numberType() throws IOException; - /** - * Is the number type estimated or not (i.e. an int might actually be a long, its just low enough - * to be an int). - */ - boolean estimatedNumberType(); - short shortValue(boolean coerce) throws IOException; int intValue(boolean coerce) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java index 5210a82527e..5f8dddc5b56 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -20,15 +20,11 @@ package org.elasticsearch.common.xcontent.cbor; import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.cbor.CBORFactory; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -63,27 +59,14 @@ public class CborXContent implements XContent { throw new ElasticsearchParseException("cbor does not support stream parsing..."); } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new CborXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8)); + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator cborGenerator = new FilteringJsonGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new CborXContentGenerator(cborGenerator); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(cborFactory.createGenerator(writer)); + return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java index 70b92b0708c..517266b8170 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java @@ -19,14 +19,10 @@ package org.elasticsearch.common.xcontent.cbor; -import com.fasterxml.jackson.dataformat.cbor.CBORParser; -import org.elasticsearch.common.bytes.BytesReference; +import com.fasterxml.jackson.core.JsonGenerator; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; -import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -34,8 +30,8 @@ import java.io.OutputStream; */ public class CborXContentGenerator extends JsonXContentGenerator { - public CborXContentGenerator(BaseJsonGenerator generator) { - super(generator); + public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { + super(jsonGenerator, os, filters); } @Override @@ -49,46 +45,7 @@ public class CborXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (CBORParser parser = CborXContent.cborFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (CBORParser parser = CborXContent.cborFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - CBORParser parser; - if (content.hasArray()) { - parser = CborXContent.cborFactory.createParser(content.array(), content.arrayOffset(), content.length()); - } else { - parser = CborXContent.cborFactory.createParser(content.streamInput()); - } - try { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } finally { - parser.close(); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (CBORParser parser = CborXContent.cborFactory.createParser(content, offset, length)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } + protected boolean supportsRawWrites() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/BaseJsonGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/BaseJsonGenerator.java deleted file mode 100644 index 0b485508c32..00000000000 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/BaseJsonGenerator.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.json; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.base.GeneratorBase; -import com.fasterxml.jackson.core.util.JsonGeneratorDelegate; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class BaseJsonGenerator extends JsonGeneratorDelegate { - - protected final GeneratorBase base; - - public BaseJsonGenerator(JsonGenerator generator, JsonGenerator base) { - super(generator, true); - if (base instanceof GeneratorBase) { - this.base = (GeneratorBase) base; - } else { - this.base = null; - } - } - - public BaseJsonGenerator(JsonGenerator generator) { - this(generator, generator); - } - - protected void writeStartRaw(String fieldName) throws IOException { - writeFieldName(fieldName); - writeRaw(':'); - } - - public void writeEndRaw() { - assert base != null : "JsonGenerator should be of instance GeneratorBase but was: " + delegate.getClass(); - if (base != null) { - base.getOutputContext().writeValue(); - } - } - - protected void writeRawValue(byte[] content, OutputStream bos) throws IOException { - flush(); - bos.write(content); - } - - protected void writeRawValue(byte[] content, int offset, int length, OutputStream bos) throws IOException { - flush(); - bos.write(content, offset, length); - } - - protected void writeRawValue(InputStream content, OutputStream bos) throws IOException { - flush(); - Streams.copy(content, bos); - } - - protected void writeRawValue(BytesReference content, OutputStream bos) throws IOException { - flush(); - content.writeTo(bos); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index 47da7934939..86f5bc28a7d 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -25,9 +25,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -65,27 +63,14 @@ public class JsonXContent implements XContent { return '\n'; } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new JsonXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8)); + return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator jsonGenerator = new FilteringJsonGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new JsonXContentGenerator(jsonGenerator); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(jsonFactory.createGenerator(writer)); + return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index b4a39cc42af..0854f7a6d2b 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -19,12 +19,20 @@ package org.elasticsearch.common.xcontent.json; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonStreamContext; +import com.fasterxml.jackson.core.base.GeneratorBase; +import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate; import com.fasterxml.jackson.core.io.SerializedString; import com.fasterxml.jackson.core.util.DefaultIndenter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.support.filtering.FilterPathBasedFilter; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -34,13 +42,45 @@ import java.io.OutputStream; */ public class JsonXContentGenerator implements XContentGenerator { - protected final BaseJsonGenerator generator; + /** Generator used to write content **/ + protected final JsonGenerator generator; + + /** + * Reference to base generator because + * writing raw values needs a specific method call. + */ + private final GeneratorBase base; + + /** + * Reference to filtering generator because + * writing an empty object '{}' when everything is filtered + * out needs a specific treatment + */ + private final FilteringGeneratorDelegate filter; + + private final OutputStream os; + private boolean writeLineFeedAtEnd; private static final SerializedString LF = new SerializedString("\n"); - private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter(" ", LF.getValue()); + private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter(" ", LF.getValue()); + private boolean prettyPrint = false; - public JsonXContentGenerator(BaseJsonGenerator generator) { - this.generator = generator; + public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { + if (jsonGenerator instanceof GeneratorBase) { + this.base = (GeneratorBase) jsonGenerator; + } else { + this.base = null; + } + + if (CollectionUtils.isEmpty(filters)) { + this.generator = jsonGenerator; + this.filter = null; + } else { + this.filter = new FilteringGeneratorDelegate(jsonGenerator, new FilterPathBasedFilter(filters), true, true); + this.generator = this.filter; + } + + this.os = os; } @Override @@ -51,6 +91,7 @@ public class JsonXContentGenerator implements XContentGenerator { @Override public final void usePrettyPrint() { generator.setPrettyPrinter(new DefaultPrettyPrinter().withObjectIndenter(INDENTER)); + prettyPrint = true; } @Override @@ -68,13 +109,35 @@ public class JsonXContentGenerator implements XContentGenerator { generator.writeEndArray(); } + protected boolean isFiltered() { + return filter != null; + } + + protected boolean inRoot() { + if (isFiltered()) { + JsonStreamContext context = filter.getFilterContext(); + return ((context != null) && (context.inRoot() && context.getCurrentName() == null)); + } + return false; + } + @Override public void writeStartObject() throws IOException { + if (isFiltered() && inRoot()) { + // Bypass generator to always write the root start object + filter.getDelegate().writeStartObject(); + return; + } generator.writeStartObject(); } @Override public void writeEndObject() throws IOException { + if (isFiltered() && inRoot()) { + // Bypass generator to always write the root end object + filter.getDelegate().writeEndObject(); + return; + } generator.writeEndObject(); } @@ -253,52 +316,103 @@ public class JsonXContentGenerator implements XContentGenerator { generator.writeStartObject(); } - @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, bos); - generator.writeEndRaw(); + private void writeStartRaw(String fieldName) throws IOException { + writeFieldName(fieldName); + generator.writeRaw(':'); } - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, offset, length, bos); - generator.writeEndRaw(); - } - - @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, bos); - generator.writeEndRaw(); - } - - @Override - public final void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException { - XContentType contentType = XContentFactory.xContentType(content); - if (contentType != null) { - writeObjectRaw(fieldName, content, bos); - } else { - writeFieldName(fieldName); - // we could potentially optimize this to not rely on exception logic... - String sValue = content.toUtf8(); - try { - writeNumber(Long.parseLong(sValue)); - } catch (NumberFormatException e) { - try { - writeNumber(Double.parseDouble(sValue)); - } catch (NumberFormatException e1) { - writeString(sValue); - } - } + public void writeEndRaw() { + assert base != null : "JsonGenerator should be of instance GeneratorBase but was: " + generator.getClass(); + if (base != null) { + base.getOutputContext().writeValue(); } } - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - generator.writeStartRaw(fieldName); - generator.writeRawValue(content, bos); - generator.writeEndRaw(); + @Override + public void writeRawField(String fieldName, InputStream content) throws IOException { + if (content.markSupported() == false) { + // needed for the XContentFactory.xContentType call + content = new BufferedInputStream(content); + } + XContentType contentType = XContentFactory.xContentType(content); + if (contentType == null) { + throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); + } + if (mayWriteRawData(contentType) == false) { + try (XContentParser parser = XContentFactory.xContent(contentType).createParser(content)) { + parser.nextToken(); + writeFieldName(fieldName); + copyCurrentStructure(parser); + } + } else { + writeStartRaw(fieldName); + flush(); + Streams.copy(content, os); + writeEndRaw(); + } + } + + @Override + public final void writeRawField(String fieldName, BytesReference content) throws IOException { + XContentType contentType = XContentFactory.xContentType(content); + if (contentType == null) { + throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); + } + if (mayWriteRawData(contentType) == false) { + writeFieldName(fieldName); + copyRawValue(content, contentType.xContent()); + } else { + writeStartRaw(fieldName); + flush(); + content.writeTo(os); + writeEndRaw(); + } + } + + public final void writeRawValue(BytesReference content) throws IOException { + XContentType contentType = XContentFactory.xContentType(content); + if (contentType == null) { + throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); + } + if (mayWriteRawData(contentType) == false) { + copyRawValue(content, contentType.xContent()); + } else { + flush(); + content.writeTo(os); + writeEndRaw(); + } + } + + private boolean mayWriteRawData(XContentType contentType) { + // When the current generator is filtered (ie filter != null) + // or the content is in a different format than the current generator, + // we need to copy the whole structure so that it will be correctly + // filtered or converted + return supportsRawWrites() + && isFiltered() == false + && contentType == contentType() + && prettyPrint == false; + } + + /** Whether this generator supports writing raw data directly */ + protected boolean supportsRawWrites() { + return true; + } + + protected void copyRawValue(BytesReference content, XContent xContent) throws IOException { + XContentParser parser = null; + try { + if (content.hasArray()) { + parser = xContent.createParser(content.array(), content.arrayOffset(), content.length()); + } else { + parser = xContent.createParser(content.streamInput()); + } + copyCurrentStructure(parser); + } finally { + if (parser != null) { + parser.close(); + } + } } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index 787c28324de..c3aca7626b7 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -68,11 +68,6 @@ public class JsonXContentParser extends AbstractXContentParser { return convertNumberType(parser.getNumberType()); } - @Override - public boolean estimatedNumberType() { - return true; - } - @Override public String currentName() throws IOException { return parser.getCurrentName(); diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java index 8a21ce1d93a..51b85900f99 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java @@ -20,15 +20,11 @@ package org.elasticsearch.common.xcontent.smile; import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -64,27 +60,14 @@ public class SmileXContent implements XContent { return (byte) 0xFF; } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new SmileXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8)); + return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator smileGenerator = new FilteringJsonGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new SmileXContentGenerator(smileGenerator); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(smileFactory.createGenerator(writer)); + return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java index b8c1b3dad65..451abab33d5 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java @@ -19,14 +19,10 @@ package org.elasticsearch.common.xcontent.smile; -import com.fasterxml.jackson.dataformat.smile.SmileParser; -import org.elasticsearch.common.bytes.BytesReference; +import com.fasterxml.jackson.core.JsonGenerator; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; -import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -34,8 +30,8 @@ import java.io.OutputStream; */ public class SmileXContentGenerator extends JsonXContentGenerator { - public SmileXContentGenerator(BaseJsonGenerator generator) { - super(generator); + public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { + super(jsonGenerator, os, filters); } @Override @@ -49,46 +45,7 @@ public class SmileXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (SmileParser parser = SmileXContent.smileFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (SmileParser parser = SmileXContent.smileFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - SmileParser parser; - if (content.hasArray()) { - parser = SmileXContent.smileFactory.createParser(content.array(), content.arrayOffset(), content.length()); - } else { - parser = SmileXContent.smileFactory.createParser(content.streamInput()); - } - try { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } finally { - parser.close(); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (SmileParser parser = SmileXContent.smileFactory.createParser(content, offset, length)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } + protected boolean supportsRawWrites() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentGenerator.java deleted file mode 100644 index e400a488304..00000000000 --- a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentGenerator.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.support; - -import org.elasticsearch.common.xcontent.XContentGenerator; - -import java.io.IOException; - -/** - * - */ -public abstract class AbstractXContentGenerator implements XContentGenerator { - - @Override - public void writeStringField(String fieldName, String value) throws IOException { - writeFieldName(fieldName); - writeString(value); - } - - @Override - public void writeBooleanField(String fieldName, boolean value) throws IOException { - writeFieldName(fieldName); - writeBoolean(value); - } - - @Override - public void writeNullField(String fieldName) throws IOException { - writeFieldName(fieldName); - writeNull(); - } - - @Override - public void writeNumberField(String fieldName, int value) throws IOException { - writeFieldName(fieldName); - writeNumber(value); - } - - @Override - public void writeNumberField(String fieldName, long value) throws IOException { - writeFieldName(fieldName); - writeNumber(value); - } - - @Override - public void writeNumberField(String fieldName, double value) throws IOException { - writeFieldName(fieldName); - writeNumber(value); - } - - @Override - public void writeNumberField(String fieldName, float value) throws IOException { - writeFieldName(fieldName); - writeNumber(value); - } - - @Override - public void writeBinaryField(String fieldName, byte[] data) throws IOException { - writeFieldName(fieldName); - writeBinary(data); - } - - @Override - public void writeArrayFieldStart(String fieldName) throws IOException { - writeFieldName(fieldName); - writeStartArray(); - } - - @Override - public void writeObjectFieldStart(String fieldName) throws IOException { - writeFieldName(fieldName); - writeStartObject(); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterContext.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterContext.java deleted file mode 100644 index 66f20cce435..00000000000 --- a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterContext.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.support.filtering; - -import com.fasterxml.jackson.core.JsonGenerator; -import org.elasticsearch.common.regex.Regex; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * A FilterContext contains the description of a field about to be written by a JsonGenerator. - */ -public class FilterContext { - - /** - * The field/property name to be write - */ - private String property; - - /** - * List of XContentFilter matched by the current filtering context - */ - private List matchings; - - /** - * Flag to indicate if the field/property must be written - */ - private Boolean write = null; - - /** - * Flag to indicate if the field/property match a filter - */ - private boolean match = false; - - /** - * Points to the parent context - */ - private FilterContext parent; - - /** - * Type of the field/property - */ - private Type type = Type.VALUE; - - protected enum Type { - VALUE, - OBJECT, - ARRAY, - ARRAY_OF_OBJECT - } - - public FilterContext(String property, FilterContext parent) { - this.property = property; - this.parent = parent; - } - - public void reset(String property) { - this.property = property; - this.write = null; - if (matchings != null) { - matchings.clear(); - } - this.match = false; - this.type = Type.VALUE; - } - - public void reset(String property, FilterContext parent) { - reset(property); - this.parent = parent; - if (parent.isMatch()) { - match = true; - } - } - - public FilterContext parent() { - return parent; - } - - public List matchings() { - return matchings; - } - - public void addMatching(String[] matching) { - if (matchings == null) { - matchings = new ArrayList<>(); - } - matchings.add(matching); - } - - public boolean isRoot() { - return parent == null; - } - - public boolean isArray() { - return Type.ARRAY.equals(type); - } - - public void initArray() { - this.type = Type.ARRAY; - } - - public boolean isObject() { - return Type.OBJECT.equals(type); - } - - public void initObject() { - this.type = Type.OBJECT; - } - - public boolean isArrayOfObject() { - return Type.ARRAY_OF_OBJECT.equals(type); - } - - public void initArrayOfObject() { - this.type = Type.ARRAY_OF_OBJECT; - } - - public boolean isMatch() { - return match; - } - - /** - * This method contains the logic to check if a field/property must be included - * or not. - */ - public boolean include() { - if (write == null) { - if (parent != null) { - // the parent context matches the end of a filter list: - // by default we include all the sub properties so we - // don't need to check if the sub properties also match - if (parent.isMatch()) { - write = true; - match = true; - return write; - } - - if (parent.matchings() != null) { - - // Iterates over the filters matched by the parent context - // and checks if the current context also match - for (String[] matcher : parent.matchings()) { - if (matcher.length > 0) { - String field = matcher[0]; - - if ("**".equals(field)) { - addMatching(matcher); - } - - if ((field != null) && (Regex.simpleMatch(field, property))) { - int remaining = matcher.length - 1; - - // the current context matches the end of a filter list: - // it must be written and it is flagged as a direct match - if (remaining == 0) { - write = true; - match = true; - return write; - } else { - String[] submatching = new String[remaining]; - System.arraycopy(matcher, 1, submatching, 0, remaining); - addMatching(submatching); - } - } - } - } - } - } else { - // Root object is always written - write = true; - } - - if (write == null) { - write = false; - } - } - return write; - } - - /** - * Ensure that the full path to the current field is write by the JsonGenerator - */ - public void writePath(JsonGenerator generator) throws IOException { - if (parent != null) { - parent.writePath(generator); - } - - if ((write == null) || (!write)) { - write = true; - - if (property == null) { - generator.writeStartObject(); - } else { - generator.writeFieldName(property); - if (isArray()) { - generator.writeStartArray(); - } else if (isObject() || isArrayOfObject()) { - generator.writeStartObject(); - } - } - } - } -} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java new file mode 100644 index 00000000000..9d7961ec0b5 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.elasticsearch.common.xcontent.support.filtering; + +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +public class FilterPath { + + static final FilterPath EMPTY = new FilterPath(); + + private final String filter; + private final String segment; + private final FilterPath next; + private final boolean simpleWildcard; + private final boolean doubleWildcard; + + protected FilterPath(String filter, String segment, FilterPath next) { + this.filter = filter; + this.segment = segment; + this.next = next; + this.simpleWildcard = (segment != null) && (segment.length() == 1) && (segment.charAt(0) == '*'); + this.doubleWildcard = (segment != null) && (segment.length() == 2) && (segment.charAt(0) == '*') && (segment.charAt(1) == '*'); + } + + private FilterPath() { + this("", "", null); + } + + public FilterPath matchProperty(String name) { + if ((next != null) && (simpleWildcard || doubleWildcard || Regex.simpleMatch(segment, name))) { + return next; + } + return null; + } + + public boolean matches() { + return next == null; + } + + boolean isDoubleWildcard() { + return doubleWildcard; + } + + boolean isSimpleWildcard() { + return simpleWildcard; + } + + String getSegment() { + return segment; + } + + FilterPath getNext() { + return next; + } + + public static FilterPath[] compile(String... filters) { + if (CollectionUtils.isEmpty(filters)) { + return null; + } + + List paths = new ArrayList<>(); + for (String filter : filters) { + if (filter != null) { + filter = filter.trim(); + if (filter.length() > 0) { + paths.add(parse(filter, filter)); + } + } + } + return paths.toArray(new FilterPath[paths.size()]); + } + + private static FilterPath parse(final String filter, final String segment) { + int end = segment.length(); + + for (int i = 0; i < end; ) { + char c = segment.charAt(i); + + if (c == '.') { + String current = segment.substring(0, i).replaceAll("\\\\.", "."); + return new FilterPath(filter, current, parse(filter, segment.substring(i + 1))); + } + ++i; + if ((c == '\\') && (i < end) && (segment.charAt(i) == '.')) { + ++i; + } + } + return new FilterPath(filter, segment.replaceAll("\\\\.", "."), EMPTY); + } + + @Override + public String toString() { + return "FilterPath [filter=" + filter + ", segment=" + segment + "]"; + } +} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java new file mode 100644 index 00000000000..d2f28611bae --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support.filtering; + +import com.fasterxml.jackson.core.filter.TokenFilter; +import org.elasticsearch.common.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +public class FilterPathBasedFilter extends TokenFilter { + + /** + * Marker value that should be used to indicate that a property name + * or value matches one of the filter paths. + */ + private static final TokenFilter MATCHING = new TokenFilter(){}; + + /** + * Marker value that should be used to indicate that none of the + * property names/values matches one of the filter paths. + */ + private static final TokenFilter NO_MATCHING = new TokenFilter(){}; + + private final FilterPath[] filters; + + public FilterPathBasedFilter(FilterPath[] filters) { + if (CollectionUtils.isEmpty(filters)) { + throw new IllegalArgumentException("filters cannot be null or empty"); + } + this.filters = filters; + } + + public FilterPathBasedFilter(String[] filters) { + this(FilterPath.compile(filters)); + } + + /** + * Evaluates if a property name matches one of the given filter paths. + */ + private TokenFilter evaluate(String name, FilterPath[] filters) { + if (filters != null) { + List nextFilters = null; + + for (FilterPath filter : filters) { + FilterPath next = filter.matchProperty(name); + if (next != null) { + if (next.matches()) { + return MATCHING; + } else { + if (nextFilters == null) { + nextFilters = new ArrayList<>(); + } + if (filter.isDoubleWildcard()) { + nextFilters.add(filter); + } + nextFilters.add(next); + } + } + } + + if ((nextFilters != null) && (nextFilters.isEmpty() == false)) { + return new FilterPathBasedFilter(nextFilters.toArray(new FilterPath[nextFilters.size()])); + } + } + return NO_MATCHING; + } + + @Override + public TokenFilter includeProperty(String name) { + TokenFilter include = evaluate(name, filters); + if (include == MATCHING) { + return TokenFilter.INCLUDE_ALL; + } + if (include == NO_MATCHING) { + return null; + } + return include; + } + + @Override + protected boolean _includeScalar() { + for (FilterPath filter : filters) { + if (filter.matches()) { + return true; + } + } + return false; + } +} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGenerator.java deleted file mode 100644 index b70a1ae9365..00000000000 --- a/core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGenerator.java +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.support.filtering; - -import com.fasterxml.jackson.core.Base64Variant; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.SerializableString; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Queue; - -/** - * A FilteringJsonGenerator uses antpath-like filters to include/exclude fields when writing XContent streams. - * - * When writing a XContent stream, this class instantiates (or reuses) a FilterContext instance for each - * field (or property) that must be generated. This filter context is used to check if the field/property must be - * written according to the current list of XContentFilter filters. - */ -public class FilteringJsonGenerator extends BaseJsonGenerator { - - /** - * List of previous contexts - * (MAX_CONTEXTS contexts are kept around in order to be reused) - */ - private Queue contexts = new ArrayDeque<>(); - private static final int MAX_CONTEXTS = 10; - - /** - * Current filter context - */ - private FilterContext context; - - public FilteringJsonGenerator(JsonGenerator generator, String[] filters) { - super(generator); - - List builder = new ArrayList<>(); - if (filters != null) { - for (String filter : filters) { - String[] matcher = Strings.delimitedListToStringArray(filter, "."); - if (matcher != null) { - builder.add(matcher); - } - } - } - - // Creates a root context that matches all filtering rules - this.context = get(null, null, Collections.unmodifiableList(builder)); - } - - /** - * Get a new context instance (and reset it if needed) - */ - private FilterContext get(String property, FilterContext parent) { - FilterContext ctx = contexts.poll(); - if (ctx == null) { - ctx = new FilterContext(property, parent); - } else { - ctx.reset(property, parent); - } - return ctx; - } - - /** - * Get a new context instance (and reset it if needed) - */ - private FilterContext get(String property, FilterContext context, List matchings) { - FilterContext ctx = get(property, context); - if (matchings != null) { - for (String[] matching : matchings) { - ctx.addMatching(matching); - } - } - return ctx; - } - - /** - * Adds a context instance to the pool in order to reuse it if needed - */ - private void put(FilterContext ctx) { - if (contexts.size() <= MAX_CONTEXTS) { - contexts.offer(ctx); - } - } - - @Override - public void writeStartArray() throws IOException { - context.initArray(); - if (context.include()) { - super.writeStartArray(); - } - } - - @Override - public void writeStartArray(int size) throws IOException { - context.initArray(); - if (context.include()) { - super.writeStartArray(size); - } - } - - @Override - public void writeEndArray() throws IOException { - // Case of array of objects - if (context.isArrayOfObject()) { - // Release current context and go one level up - FilterContext parent = context.parent(); - put(context); - context = parent; - } - - if (context.include()) { - super.writeEndArray(); - } - } - - @Override - public void writeStartObject() throws IOException { - // Case of array of objects - if (context.isArray()) { - // Get a context for the anonymous object - context = get(null, context, context.matchings()); - context.initArrayOfObject(); - } - - if (!context.isArrayOfObject()) { - context.initObject(); - } - - if (context.include()) { - super.writeStartObject(); - } - - context = get(null, context); - } - - @Override - public void writeEndObject() throws IOException { - if (!context.isRoot()) { - // Release current context and go one level up - FilterContext parent = context.parent(); - put(context); - context = parent; - } - - if (context.include()) { - super.writeEndObject(); - } - } - - @Override - public void writeFieldName(String name) throws IOException { - context.reset(name); - - if (context.include()) { - // Ensure that the full path to the field is written - context.writePath(delegate); - super.writeFieldName(name); - } - } - - @Override - public void writeFieldName(SerializableString name) throws IOException { - context.reset(name.getValue()); - - if (context.include()) { - // Ensure that the full path to the field is written - context.writePath(delegate); - super.writeFieldName(name); - } - } - - @Override - public void writeString(String text) throws IOException { - if (context.include()) { - super.writeString(text); - } - } - - @Override - public void writeString(char[] text, int offset, int len) throws IOException { - if (context.include()) { - super.writeString(text, offset, len); - } - } - - @Override - public void writeString(SerializableString text) throws IOException { - if (context.include()) { - super.writeString(text); - } - } - - @Override - public void writeRawUTF8String(byte[] text, int offset, int length) throws IOException { - if (context.include()) { - super.writeRawUTF8String(text, offset, length); - } - } - - @Override - public void writeUTF8String(byte[] text, int offset, int length) throws IOException { - if (context.include()) { - super.writeUTF8String(text, offset, length); - } - } - - @Override - public void writeRaw(String text) throws IOException { - if (context.include()) { - super.writeRaw(text); - } - } - - @Override - public void writeRaw(String text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRaw(text, offset, len); - } - } - - @Override - public void writeRaw(SerializableString raw) throws IOException { - if (context.include()) { - super.writeRaw(raw); - } - } - - @Override - public void writeRaw(char[] text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRaw(text, offset, len); - } - } - - @Override - public void writeRaw(char c) throws IOException { - if (context.include()) { - super.writeRaw(c); - } - } - - @Override - public void writeRawValue(String text) throws IOException { - if (context.include()) { - super.writeRawValue(text); - } - } - - @Override - public void writeRawValue(String text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRawValue(text, offset, len); - } - } - - @Override - public void writeRawValue(char[] text, int offset, int len) throws IOException { - if (context.include()) { - super.writeRawValue(text, offset, len); - } - } - - @Override - public void writeBinary(Base64Variant b64variant, byte[] data, int offset, int len) throws IOException { - if (context.include()) { - super.writeBinary(b64variant, data, offset, len); - } - } - - @Override - public int writeBinary(Base64Variant b64variant, InputStream data, int dataLength) throws IOException { - if (context.include()) { - return super.writeBinary(b64variant, data, dataLength); - } - return 0; - } - - @Override - public void writeNumber(short v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(int v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(long v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(BigInteger v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(double v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(float v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(BigDecimal v) throws IOException { - if (context.include()) { - super.writeNumber(v); - } - } - - @Override - public void writeNumber(String encodedValue) throws IOException, UnsupportedOperationException { - if (context.include()) { - super.writeNumber(encodedValue); - } - } - - @Override - public void writeBoolean(boolean state) throws IOException { - if (context.include()) { - super.writeBoolean(state); - } - } - - @Override - public void writeNull() throws IOException { - if (context.include()) { - super.writeNull(); - } - } - - @Override - public void copyCurrentEvent(JsonParser jp) throws IOException { - if (context.include()) { - super.copyCurrentEvent(jp); - } - } - - @Override - public void copyCurrentStructure(JsonParser jp) throws IOException { - if (context.include()) { - super.copyCurrentStructure(jp); - } - } - - @Override - protected void writeRawValue(byte[] content, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, bos); - } - } - - @Override - protected void writeRawValue(byte[] content, int offset, int length, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, offset, length, bos); - } - } - - @Override - protected void writeRawValue(InputStream content, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, bos); - } - } - - @Override - protected void writeRawValue(BytesReference content, OutputStream bos) throws IOException { - if (context.include()) { - super.writeRawValue(content, bos); - } - } - - @Override - public void close() throws IOException { - contexts.clear(); - super.close(); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java index 388cd992e2b..c24ddb7f296 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java @@ -20,15 +20,11 @@ package org.elasticsearch.common.xcontent.yaml; import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; -import org.elasticsearch.common.xcontent.support.filtering.FilteringJsonGenerator; import java.io.*; @@ -62,27 +58,14 @@ public class YamlXContent implements XContent { throw new ElasticsearchParseException("yaml does not support stream parsing..."); } - private XContentGenerator newXContentGenerator(JsonGenerator jsonGenerator) { - return new YamlXContentGenerator(new BaseJsonGenerator(jsonGenerator)); - } - @Override public XContentGenerator createGenerator(OutputStream os) throws IOException { - return newXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8)); + return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os); } @Override public XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException { - if (CollectionUtils.isEmpty(filters)) { - return createGenerator(os); - } - FilteringJsonGenerator yamlGenerator = new FilteringJsonGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), filters); - return new YamlXContentGenerator(yamlGenerator); - } - - @Override - public XContentGenerator createGenerator(Writer writer) throws IOException { - return newXContentGenerator(yamlFactory.createGenerator(writer)); + return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os, filters); } @Override diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java index 62967247a82..dcb2155c82e 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java @@ -19,14 +19,10 @@ package org.elasticsearch.common.xcontent.yaml; -import com.fasterxml.jackson.dataformat.yaml.YAMLParser; -import org.elasticsearch.common.bytes.BytesReference; +import com.fasterxml.jackson.core.JsonGenerator; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.BaseJsonGenerator; import org.elasticsearch.common.xcontent.json.JsonXContentGenerator; -import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -34,8 +30,8 @@ import java.io.OutputStream; */ public class YamlXContentGenerator extends JsonXContentGenerator { - public YamlXContentGenerator(BaseJsonGenerator generator) { - super(generator); + public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) { + super(jsonGenerator, os, filters); } @Override @@ -49,46 +45,7 @@ public class YamlXContentGenerator extends JsonXContentGenerator { } @Override - public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } - } - - @Override - protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException { - writeFieldName(fieldName); - YAMLParser parser; - if (content.hasArray()) { - parser = YamlXContent.yamlFactory.createParser(content.array(), content.arrayOffset(), content.length()); - } else { - parser = YamlXContent.yamlFactory.createParser(content.streamInput()); - } - try { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } finally { - parser.close(); - } - } - - @Override - public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException { - writeFieldName(fieldName); - try (YAMLParser parser = YamlXContent.yamlFactory.createParser(content, offset, length)) { - parser.nextToken(); - generator.copyCurrentStructure(parser); - } + protected boolean supportsRawWrites() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/discovery/Discovery.java b/core/src/main/java/org/elasticsearch/discovery/Discovery.java index 13eb86f1ce4..980543d45e6 100644 --- a/core/src/main/java/org/elasticsearch/discovery/Discovery.java +++ b/core/src/main/java/org/elasticsearch/discovery/Discovery.java @@ -87,4 +87,10 @@ public interface Discovery extends LifecycleComponent { super(msg, cause, args); } } + + /** + * @return stats about the discovery + */ + DiscoveryStats stats(); + } diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryService.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryService.java index 9a8b19e371f..a82099658ea 100644 --- a/core/src/main/java/org/elasticsearch/discovery/DiscoveryService.java +++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryService.java @@ -23,6 +23,7 @@ import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; @@ -39,6 +40,7 @@ import java.util.concurrent.TimeUnit; public class DiscoveryService extends AbstractLifecycleComponent { public static final String SETTING_INITIAL_STATE_TIMEOUT = "discovery.initial_state_timeout"; + public static final String SETTING_DISCOVERY_SEED = "discovery.id.seed"; private static class InitialStateListener implements InitialStateDiscoveryListener { @@ -112,14 +114,6 @@ public class DiscoveryService extends AbstractLifecycleComponenttrue if the initial state was received within the timeout waiting for it - * on {@link #doStart()}. - */ - public boolean initialStateReceived() { - return initialStateListener.initialStateReceived; - } - public String nodeDescription() { return discovery.nodeDescription(); } @@ -138,10 +132,7 @@ public class DiscoveryService extends AbstractLifecycleComponent implem } // reroute here, so we eagerly remove dead nodes from the routing ClusterState updatedState = ClusterState.builder(currentState).nodes(newNodes).build(); - RoutingAllocation.Result routingResult = master.routingService.getAllocationService().reroute(ClusterState.builder(updatedState).build()); + RoutingAllocation.Result routingResult = master.routingService.getAllocationService().reroute( + ClusterState.builder(updatedState).build(), "elected as master"); return ClusterState.builder(updatedState).routingResult(routingResult).build(); } @@ -316,6 +317,11 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem } } + @Override + public DiscoveryStats stats() { + return new DiscoveryStats(null); + } + private LocalDiscovery[] members() { ClusterGroup clusterGroup = clusterGroups.get(clusterName); if (clusterGroup == null) { diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java index 64f88d5443d..9cec672ad43 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -86,7 +87,7 @@ public class NodeJoinController extends AbstractComponent { @Override void onClose() { if (electionContext.compareAndSet(this, null)) { - stopAccumulatingJoins(); + stopAccumulatingJoins("election closed"); } else { assert false : "failed to remove current election context"; } @@ -132,7 +133,7 @@ public class NodeJoinController extends AbstractComponent { /** utility method to fail the given election context under the cluster state thread */ private void failContext(final ElectionContext context, final String reason, final Throwable throwable) { - clusterService.submitStateUpdateTask("zen-disco-join(failure [" + reason + "])", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("zen-disco-join(failure [" + reason + "])", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public boolean runOnlyOnMaster() { @@ -156,7 +157,7 @@ public class NodeJoinController extends AbstractComponent { /** * Accumulates any future incoming join request. Pending join requests will be processed in the final steps of becoming a - * master or when {@link #stopAccumulatingJoins()} is called. + * master or when {@link #stopAccumulatingJoins(String)} is called. */ public void startAccumulatingJoins() { logger.trace("starting to accumulate joins"); @@ -166,14 +167,14 @@ public class NodeJoinController extends AbstractComponent { } /** Stopped accumulating joins. All pending joins will be processed. Future joins will be processed immediately */ - public void stopAccumulatingJoins() { - logger.trace("stopping join accumulation"); + public void stopAccumulatingJoins(String reason) { + logger.trace("stopping join accumulation ([{}])", reason); assert electionContext.get() == null : "stopAccumulatingJoins() called, but there is an ongoing election context"; boolean b = accumulateJoins.getAndSet(false); assert b : "stopAccumulatingJoins() called but not accumulating"; synchronized (pendingJoinRequests) { if (pendingJoinRequests.size() > 0) { - processJoins("stopping to accumulate joins"); + processJoins("pending joins after accumulation stop [" + reason + "]"); } } } @@ -210,7 +211,7 @@ public class NodeJoinController extends AbstractComponent { return; } - int pendingMasterJoins=0; + int pendingMasterJoins = 0; synchronized (pendingJoinRequests) { for (DiscoveryNode node : pendingJoinRequests.keySet()) { if (node.isMasterNode()) { @@ -219,7 +220,9 @@ public class NodeJoinController extends AbstractComponent { } } if (pendingMasterJoins < context.requiredMasterJoins) { - logger.trace("not enough joins for election. Got [{}], required [{}]", pendingMasterJoins, context.requiredMasterJoins); + if (context.pendingSetAsMasterTask.get() == false) { + logger.trace("not enough joins for election. Got [{}], required [{}]", pendingMasterJoins, context.requiredMasterJoins); + } return; } if (context.pendingSetAsMasterTask.getAndSet(true)) { @@ -228,7 +231,7 @@ public class NodeJoinController extends AbstractComponent { } final String source = "zen-disco-join(elected_as_master, [" + pendingMasterJoins + "] joins received)"; - clusterService.submitStateUpdateTask(source, Priority.IMMEDIATE, new ProcessJoinsTask() { + clusterService.submitStateUpdateTask(source, new ProcessJoinsTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) { // Take into account the previous known nodes, if they happen not to be available @@ -246,7 +249,7 @@ public class NodeJoinController extends AbstractComponent { currentState = ClusterState.builder(currentState).nodes(builder).blocks(clusterBlocks).build(); // reroute now to remove any dead nodes (master may have stepped down when they left and didn't update the routing table) - RoutingAllocation.Result result = routingService.getAllocationService().reroute(currentState); + RoutingAllocation.Result result = routingService.getAllocationService().reroute(currentState, "nodes joined"); if (result.changed()) { currentState = ClusterState.builder(currentState).routingResult(result).build(); } @@ -277,7 +280,7 @@ public class NodeJoinController extends AbstractComponent { /** process all pending joins */ private void processJoins(String reason) { - clusterService.submitStateUpdateTask("zen-disco-join(" + reason + ")", Priority.URGENT, new ProcessJoinsTask()); + clusterService.submitStateUpdateTask("zen-disco-join(" + reason + ")", new ProcessJoinsTask(Priority.URGENT)); } @@ -353,6 +356,10 @@ public class NodeJoinController extends AbstractComponent { private final List joinCallbacksToRespondTo = new ArrayList<>(); private boolean nodeAdded = false; + public ProcessJoinsTask(Priority priority) { + super(priority); + } + @Override public ClusterState execute(ClusterState currentState) { DiscoveryNodes.Builder nodesBuilder; diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index 7255aa812f8..03111d141ef 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoverySettings; +import org.elasticsearch.discovery.DiscoveryStats; import org.elasticsearch.discovery.InitialStateDiscoveryListener; import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.discovery.zen.fd.MasterFaultDetection; @@ -51,6 +52,7 @@ import org.elasticsearch.discovery.zen.membership.MembershipAction; import org.elasticsearch.discovery.zen.ping.PingContextProvider; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingService; +import org.elasticsearch.discovery.zen.publish.PendingClusterStateStats; import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction; import org.elasticsearch.node.service.NodeService; import org.elasticsearch.node.settings.NodeSettingsService; @@ -58,10 +60,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -321,7 +320,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen } catch (FailedToCommitClusterStateException t) { // cluster service logs a WARN message logger.debug("failed to publish cluster state version [{}] (not enough nodes acknowledged, min master nodes [{}])", clusterChangedEvent.state().version(), electMaster.minimumMasterNodes()); - clusterService.submitStateUpdateTask("zen-disco-failed-to-publish", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("zen-disco-failed-to-publish", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) { return rejoin(currentState, "failed to publish to min_master_nodes"); @@ -337,6 +336,12 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen } } + @Override + public DiscoveryStats stats() { + PendingClusterStateStats queueStats = publishClusterState.pendingStatesQueue().stats(); + return new DiscoveryStats(queueStats); + } + /** * returns true if zen discovery is started and there is a currently a background thread active for (re)joining * the cluster used for testing. @@ -393,7 +398,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen ); } else { // process any incoming joins (they will fail because we are not the master) - nodeJoinController.stopAccumulatingJoins(); + nodeJoinController.stopAccumulatingJoins("not master"); // send join request final boolean success = joinElectedMaster(masterNode); @@ -493,7 +498,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen return; } if (localNodeMaster()) { - clusterService.submitStateUpdateTask("zen-disco-node_left(" + node + ")", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("zen-disco-node_left(" + node + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(currentState.nodes()).remove(node.id()); @@ -503,7 +508,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen return rejoin(currentState, "not enough master nodes"); } // eagerly run reroute to remove dead nodes from routing table - RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(ClusterState.builder(currentState).build()); + RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute( + ClusterState.builder(currentState).build(), + "[" + node + "] left"); return ClusterState.builder(currentState).routingResult(routingResult).build(); } @@ -518,7 +525,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen } }); } else if (node.equals(nodes().masterNode())) { - handleMasterGone(node, "shut_down"); + handleMasterGone(node, null, "shut_down"); } } @@ -531,7 +538,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen // nothing to do here... return; } - clusterService.submitStateUpdateTask("zen-disco-node_failed(" + node + "), reason " + reason, Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("zen-disco-node_failed(" + node + "), reason " + reason, new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) { if (currentState.nodes().get(node.id()) == null) { @@ -546,7 +553,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen return rejoin(currentState, "not enough master nodes"); } // eagerly run reroute to remove dead nodes from routing table - RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(ClusterState.builder(currentState).build()); + RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute( + ClusterState.builder(currentState).build(), + "[" + node + "] failed"); return ClusterState.builder(currentState).routingResult(routingResult).build(); } @@ -578,7 +587,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen // We only set the new value. If the master doesn't see enough nodes it will revoke it's mastership. return; } - clusterService.submitStateUpdateTask("zen-disco-minimum_master_nodes_changed", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("zen-disco-minimum_master_nodes_changed", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) { // check if we have enough master nodes, if not, we need to move into joining the cluster again @@ -606,7 +615,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen }); } - private void handleMasterGone(final DiscoveryNode masterNode, final String reason) { + private void handleMasterGone(final DiscoveryNode masterNode, final Throwable cause, final String reason) { if (lifecycleState() != Lifecycle.State.STARTED) { // not started, ignore a master failure return; @@ -616,9 +625,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen return; } - logger.info("master_left [{}], reason [{}]", masterNode, reason); + logger.info("master_left [{}], reason [{}]", cause, masterNode, reason); - clusterService.submitStateUpdateTask("zen-disco-master_failed (" + masterNode + ")", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("zen-disco-master_failed (" + masterNode + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public boolean runOnlyOnMaster() { @@ -685,7 +694,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen } void processNextPendingClusterState(String reason) { - clusterService.submitStateUpdateTask("zen-disco-receive(from master [" + reason + "])", Priority.URGENT, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("zen-disco-receive(from master [" + reason + "])", new ClusterStateUpdateTask(Priority.URGENT) { @Override public boolean runOnlyOnMaster() { return false; @@ -742,9 +751,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen // if its not the same version, only copy over new indices or ones that changed the version MetaData.Builder metaDataBuilder = MetaData.builder(newClusterState.metaData()).removeAllIndices(); for (IndexMetaData indexMetaData : newClusterState.metaData()) { - IndexMetaData currentIndexMetaData = currentState.metaData().index(indexMetaData.index()); - if (currentIndexMetaData != null && currentIndexMetaData.isSameUUID(indexMetaData.indexUUID()) && - currentIndexMetaData.version() == indexMetaData.version()) { + IndexMetaData currentIndexMetaData = currentState.metaData().index(indexMetaData.getIndex()); + if (currentIndexMetaData != null && currentIndexMetaData.isSameUUID(indexMetaData.getIndexUUID()) && + currentIndexMetaData.getVersion() == indexMetaData.getVersion()) { // safe to reuse metaDataBuilder.put(currentIndexMetaData, false); } else { @@ -1050,7 +1059,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen return; } logger.debug("got a ping from another master {}. resolving who should rejoin. current ping count: [{}]", pingRequest.masterNode(), pingsWhileMaster.get()); - clusterService.submitStateUpdateTask("ping from another master", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("ping from another master", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -1069,8 +1078,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen private class MasterNodeFailureListener implements MasterFaultDetection.Listener { @Override - public void onMasterFailure(DiscoveryNode masterNode, String reason) { - handleMasterGone(masterNode, reason); + public void onMasterFailure(DiscoveryNode masterNode, Throwable cause, String reason) { + handleMasterGone(masterNode, cause, reason); } } @@ -1105,7 +1114,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen class RejoinClusterRequestHandler implements TransportRequestHandler { @Override public void messageReceived(final RejoinClusterRequest request, final TransportChannel channel) throws Exception { - clusterService.submitStateUpdateTask("received a request to rejoin the cluster from [" + request.fromNodeId + "]", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("received a request to rejoin the cluster from [" + request.fromNodeId + "]", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public boolean runOnlyOnMaster() { diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java index 3163e061692..8842bafb116 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.NotMasterException; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; @@ -39,8 +39,6 @@ import java.io.IOException; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.transport.TransportRequestOptions.options; - /** * A fault detection that pings the master periodically to see if its alive. */ @@ -51,7 +49,7 @@ public class MasterFaultDetection extends FaultDetection { public static interface Listener { /** called when pinging the master failed, like a timeout, transport disconnects etc */ - void onMasterFailure(DiscoveryNode masterNode, String reason); + void onMasterFailure(DiscoveryNode masterNode, Throwable cause, String reason); } @@ -119,7 +117,7 @@ public class MasterFaultDetection extends FaultDetection { transportService.connectToNode(masterNode); } catch (final Exception e) { // notify master failure (which stops also) and bail.. - notifyMasterFailure(masterNode, "failed to perform initial connect [" + e.getMessage() + "]"); + notifyMasterFailure(masterNode, e, "failed to perform initial connect "); return; } if (masterPinger != null) { @@ -178,22 +176,22 @@ public class MasterFaultDetection extends FaultDetection { threadPool.schedule(TimeValue.timeValueMillis(0), ThreadPool.Names.SAME, masterPinger); } catch (Exception e) { logger.trace("[master] [{}] transport disconnected (with verified connect)", masterNode); - notifyMasterFailure(masterNode, "transport disconnected (with verified connect)"); + notifyMasterFailure(masterNode, null, "transport disconnected (with verified connect)"); } } else { logger.trace("[master] [{}] transport disconnected", node); - notifyMasterFailure(node, "transport disconnected"); + notifyMasterFailure(node, null, "transport disconnected"); } } } - private void notifyMasterFailure(final DiscoveryNode masterNode, final String reason) { + private void notifyMasterFailure(final DiscoveryNode masterNode, final Throwable cause, final String reason) { if (notifiedMasterFailure.compareAndSet(false, true)) { threadPool.generic().execute(new Runnable() { @Override public void run() { for (Listener listener : listeners) { - listener.onMasterFailure(masterNode, reason); + listener.onMasterFailure(masterNode, cause, reason); } } }); @@ -222,7 +220,7 @@ public class MasterFaultDetection extends FaultDetection { return; } final MasterPingRequest request = new MasterPingRequest(clusterService.localNode().id(), masterToPing.id(), clusterName); - final TransportRequestOptions options = options().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout); + final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout).build(); transportService.sendRequest(masterToPing, MASTER_PING_ACTION_NAME, request, options, new BaseTransportResponseHandler() { @Override @@ -257,15 +255,15 @@ public class MasterFaultDetection extends FaultDetection { return; } else if (exp.getCause() instanceof NotMasterException) { logger.debug("[master] pinging a master {} that is no longer a master", masterNode); - notifyMasterFailure(masterToPing, "no longer master"); + notifyMasterFailure(masterToPing, exp, "no longer master"); return; } else if (exp.getCause() instanceof ThisIsNotTheMasterYouAreLookingForException) { logger.debug("[master] pinging a master {} that is not the master", masterNode); - notifyMasterFailure(masterToPing, "not master"); + notifyMasterFailure(masterToPing, exp,"not master"); return; } else if (exp.getCause() instanceof NodeDoesNotExistOnMasterException) { logger.debug("[master] pinging a master {} but we do not exists on it, act as if its master failure", masterNode); - notifyMasterFailure(masterToPing, "do not exists on master, act as master failure"); + notifyMasterFailure(masterToPing, exp,"do not exists on master, act as master failure"); return; } @@ -274,7 +272,7 @@ public class MasterFaultDetection extends FaultDetection { if (retryCount >= pingRetryCount) { logger.debug("[master] failed to ping [{}], tried [{}] times, each with maximum [{}] timeout", masterNode, pingRetryCount, pingRetryTimeout); // not good, failure - notifyMasterFailure(masterToPing, "failed to ping, tried [" + pingRetryCount + "] times, each with maximum [" + pingRetryTimeout + "] timeout"); + notifyMasterFailure(masterToPing, null, "failed to ping, tried [" + pingRetryCount + "] times, each with maximum [" + pingRetryTimeout + "] timeout"); } else { // resend the request, not reschedule, rely on send timeout transportService.sendRequest(masterToPing, MASTER_PING_ACTION_NAME, request, options, this); diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java index 5619b58dc53..2abe730b1e8 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; @@ -34,7 +35,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; -import static org.elasticsearch.transport.TransportRequestOptions.options; /** * A fault detection of multiple nodes. @@ -42,7 +42,7 @@ import static org.elasticsearch.transport.TransportRequestOptions.options; public class NodesFaultDetection extends FaultDetection { public static final String PING_ACTION_NAME = "internal:discovery/zen/fd/ping"; - + public abstract static class Listener { public void onNodeFailure(DiscoveryNode node, String reason) {} @@ -146,14 +146,18 @@ public class NodesFaultDetection extends FaultDetection { } private void notifyNodeFailure(final DiscoveryNode node, final String reason) { - threadPool.generic().execute(new Runnable() { - @Override - public void run() { - for (Listener listener : listeners) { - listener.onNodeFailure(node, reason); + try { + threadPool.generic().execute(new Runnable() { + @Override + public void run() { + for (Listener listener : listeners) { + listener.onNodeFailure(node, reason); + } } - } - }); + }); + } catch (EsRejectedExecutionException ex) { + logger.trace("[node ] [{}] ignoring node failure (reason [{}]). Local node is shutting down", ex, node, reason); + } } private void notifyPingReceived(final PingRequest pingRequest) { @@ -189,7 +193,7 @@ public class NodesFaultDetection extends FaultDetection { return; } final PingRequest pingRequest = new PingRequest(node.id(), clusterName, localNode, clusterStateVersion); - final TransportRequestOptions options = options().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout); + final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout).build(); transportService.sendRequest(node, PING_ACTION_NAME, pingRequest, options, new BaseTransportResponseHandler() { @Override public PingResponse newInstance() { diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java index c9c4d298cc4..99feb4b7f72 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java @@ -437,7 +437,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent implemen private void sendPingRequestToNode(final int id, final TimeValue timeout, final UnicastPingRequest pingRequest, final CountDownLatch latch, final DiscoveryNode node, final DiscoveryNode nodeToSend) { logger.trace("[{}] sending to {}", id, nodeToSend); - transportService.sendRequest(nodeToSend, ACTION_NAME, pingRequest, TransportRequestOptions.options().withTimeout((long) (timeout.millis() * 1.25)), new BaseTransportResponseHandler() { + transportService.sendRequest(nodeToSend, ACTION_NAME, pingRequest, TransportRequestOptions.builder().withTimeout((long) (timeout.millis() * 1.25)).build(), new BaseTransportResponseHandler() { @Override public UnicastPingResponse newInstance() { diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java new file mode 100644 index 00000000000..44265b0e481 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStateStats.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.zen.publish; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; + +import java.io.IOException; + +/** + * Class encapsulating stats about the PendingClusterStatsQueue + */ +public class PendingClusterStateStats implements Streamable, ToXContent { + + private int total; + private int pending; + private int committed; + + public PendingClusterStateStats() { + + } + + public PendingClusterStateStats(int total, int pending, int committed) { + this.total = total; + this.pending = pending; + this.committed = committed; + } + + public int getCommitted() { + return committed; + } + + public int getPending() { + return pending; + } + + public int getTotal() { + return total; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.QUEUE); + builder.field(Fields.TOTAL, total); + builder.field(Fields.PENDING, pending); + builder.field(Fields.COMMITTED, committed); + builder.endObject(); + return builder; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + total = in.readVInt(); + pending = in.readVInt(); + committed = in.readVInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(total); + out.writeVInt(pending); + out.writeVInt(committed); + } + + static final class Fields { + static final XContentBuilderString QUEUE = new XContentBuilderString("cluster_state_queue"); + static final XContentBuilderString TOTAL = new XContentBuilderString("total"); + static final XContentBuilderString PENDING = new XContentBuilderString("pending"); + static final XContentBuilderString COMMITTED = new XContentBuilderString("committed"); + } + + @Override + public String toString() { + return "PendingClusterStateStats(total=" + total + ", pending=" + pending + ", committed=" + committed + ")"; + } +} diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java index e3550e657fc..2f444f50288 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java @@ -283,4 +283,17 @@ public class PendingClusterStatesQueue { } } + public synchronized PendingClusterStateStats stats() { + + // calculate committed cluster state + int committed = 0; + for (ClusterStateContext clusterStatsContext : pendingStates) { + if (clusterStatsContext.committed()) { + committed += 1; + } + } + + return new PendingClusterStateStats(pendingStates.size(), pendingStates.size() - committed, committed); + } + } diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index a8c29523011..91fd622023f 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -140,9 +140,9 @@ public class PublishClusterStateAction extends AbstractComponent { throw t; } catch (Throwable t) { // try to fail committing, in cause it's still on going - if (sendingController.markAsFailed("unexpected error [" + t.getMessage() + "]")) { + if (sendingController.markAsFailed("unexpected error", t)) { // signal the change should be rejected - throw new Discovery.FailedToCommitClusterStateException("unexpected error [{}]", t, t.getMessage()); + throw new Discovery.FailedToCommitClusterStateException("unexpected error", t); } else { throw t; } @@ -248,7 +248,7 @@ public class PublishClusterStateAction extends AbstractComponent { // -> no need to put a timeout on the options here, because we want the response to eventually be received // and not log an error if it arrives after the timeout // -> no need to compress, we already compressed the bytes - TransportRequestOptions options = TransportRequestOptions.options().withType(TransportRequestOptions.Type.STATE).withCompress(false); + TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withCompress(false).build(); transportService.sendRequest(node, SEND_ACTION_NAME, new BytesTransportRequest(bytes, node.version()), options, @@ -282,7 +282,7 @@ public class PublishClusterStateAction extends AbstractComponent { private void sendCommitToNode(final DiscoveryNode node, final ClusterState clusterState, final SendingController sendingController) { try { logger.trace("sending commit for cluster state (uuid: [{}], version [{}]) to [{}]", clusterState.stateUUID(), clusterState.version(), node); - TransportRequestOptions options = TransportRequestOptions.options().withType(TransportRequestOptions.Type.STATE); + TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).build(); // no need to put a timeout on the options here, because we want the response to eventually be received // and not log an error if it arrives after the timeout transportService.sendRequest(node, COMMIT_ACTION_NAME, @@ -583,6 +583,21 @@ public class PublishClusterStateAction extends AbstractComponent { return true; } + /** + * tries marking the publishing as failed, if a decision wasn't made yet + * + * @return true if the publishing was failed and the cluster state is *not* committed + **/ + synchronized private boolean markAsFailed(String details, Throwable reason) { + if (committedOrFailed()) { + return committed == false; + } + logger.trace("failed to commit version [{}]. {}", reason, clusterState.version(), details); + committed = false; + committedOrFailedLatch.countDown(); + return true; + } + /** * tries marking the publishing as failed, if a decision wasn't made yet * diff --git a/core/src/main/java/org/elasticsearch/env/ESFileStore.java b/core/src/main/java/org/elasticsearch/env/ESFileStore.java index d74432c591a..abd9d2e072f 100644 --- a/core/src/main/java/org/elasticsearch/env/ESFileStore.java +++ b/core/src/main/java/org/elasticsearch/env/ESFileStore.java @@ -84,7 +84,18 @@ class ESFileStore extends FileStore { return getFileStoreWindows(path, fileStores); } - FileStore store = Files.getFileStore(path); + final FileStore store; + try { + store = Files.getFileStore(path); + } catch (IOException unexpected) { + // give a better error message if a filestore cannot be retrieved from inside a FreeBSD jail. + if (Constants.FREE_BSD) { + throw new IOException("Unable to retrieve mount point data for " + path + + ". If you are running within a jail, set enforce_statfs=1. See jail(8)", unexpected); + } else { + throw unexpected; + } + } try { String mount = getMountPointLinux(store); diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java index a82dab995e5..7982c2f35ea 100644 --- a/core/src/main/java/org/elasticsearch/env/Environment.java +++ b/core/src/main/java/org/elasticsearch/env/Environment.java @@ -58,6 +58,8 @@ public class Environment { private final Path pluginsFile; + private final Path modulesFile; + private final Path sharedDataFile; /** location of bin/, used by plugin manager */ @@ -157,6 +159,7 @@ public class Environment { binFile = homeFile.resolve("bin"); libFile = homeFile.resolve("lib"); + modulesFile = homeFile.resolve("modules"); } /** @@ -275,6 +278,10 @@ public class Environment { return libFile; } + public Path modulesFile() { + return modulesFile; + } + public Path logsFile() { return logsFile; } @@ -304,6 +311,7 @@ public class Environment { * no permissions to the actual mount point are required. *

  • Exception handling has the same semantics as {@link Files#getFileStore(Path)}. *
  • Works around https://bugs.openjdk.java.net/browse/JDK-8034057. + *
  • Gives a better exception when filestore cannot be retrieved from inside a FreeBSD jail. * */ public static FileStore getFileStore(Path path) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 8c599995b9b..3a1b430f98b 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -21,12 +21,7 @@ package org.elasticsearch.env; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.Lock; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.store.NativeFSLockFactory; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.*; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -38,7 +33,7 @@ import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.monitor.fs.FsInfo; @@ -46,21 +41,8 @@ import org.elasticsearch.monitor.fs.FsProbe; import java.io.Closeable; import java.io.IOException; -import java.nio.file.AtomicMoveNotSupportedException; -import java.nio.file.DirectoryStream; -import java.nio.file.FileStore; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.nio.file.*; +import java.util.*; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -309,9 +291,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * @param shardId the id of the shard to delete to delete * @throws IOException if an IOException occurs */ - public void deleteShardDirectorySafe(ShardId shardId, @IndexSettings Settings indexSettings) throws IOException { - // This is to ensure someone doesn't use Settings.EMPTY - assert indexSettings != Settings.EMPTY; + public void deleteShardDirectorySafe(ShardId shardId, IndexSettings indexSettings) throws IOException { final Path[] paths = availableShardPaths(shardId); logger.trace("deleting shard {} directory, paths: [{}]", shardId, paths); try (ShardLock lock = shardLock(shardId)) { @@ -326,7 +306,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * * @throws LockObtainFailedException if any of the locks could not be acquired */ - public static void acquireFSLockForPaths(@IndexSettings Settings indexSettings, Path... shardPaths) throws IOException { + public static void acquireFSLockForPaths(IndexSettings indexSettings, Path... shardPaths) throws IOException { Lock[] locks = new Lock[shardPaths.length]; Directory[] dirs = new Directory[shardPaths.length]; try { @@ -360,15 +340,14 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * @throws IOException if an IOException occurs * @throws ElasticsearchException if the write.lock is not acquirable */ - public void deleteShardDirectoryUnderLock(ShardLock lock, @IndexSettings Settings indexSettings) throws IOException { - assert indexSettings != Settings.EMPTY; + public void deleteShardDirectoryUnderLock(ShardLock lock, IndexSettings indexSettings) throws IOException { final ShardId shardId = lock.getShardId(); assert isShardLocked(shardId) : "shard " + shardId + " is not locked"; final Path[] paths = availableShardPaths(shardId); logger.trace("acquiring locks for {}, paths: [{}]", shardId, paths); acquireFSLockForPaths(indexSettings, paths); IOUtils.rm(paths); - if (hasCustomDataPath(indexSettings)) { + if (indexSettings.hasCustomDataPath()) { Path customLocation = resolveCustomLocation(indexSettings, shardId); logger.trace("acquiring lock for {}, custom path: [{}]", shardId, customLocation); acquireFSLockForPaths(indexSettings, customLocation); @@ -398,9 +377,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * @param indexSettings settings for the index being deleted * @throws IOException if any of the shards data directories can't be locked or deleted */ - public void deleteIndexDirectorySafe(Index index, long lockTimeoutMS, @IndexSettings Settings indexSettings) throws IOException { - // This is to ensure someone doesn't use Settings.EMPTY - assert indexSettings != Settings.EMPTY; + public void deleteIndexDirectorySafe(Index index, long lockTimeoutMS, IndexSettings indexSettings) throws IOException { final List locks = lockAllForIndex(index, indexSettings, lockTimeoutMS); try { deleteIndexDirectoryUnderLock(index, indexSettings); @@ -416,13 +393,11 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * @param index the index to delete * @param indexSettings settings for the index being deleted */ - public void deleteIndexDirectoryUnderLock(Index index, @IndexSettings Settings indexSettings) throws IOException { - // This is to ensure someone doesn't use Settings.EMPTY - assert indexSettings != Settings.EMPTY; + public void deleteIndexDirectoryUnderLock(Index index, IndexSettings indexSettings) throws IOException { final Path[] indexPaths = indexPaths(index); logger.trace("deleting index {} directory, paths({}): [{}]", index, indexPaths.length, indexPaths); IOUtils.rm(indexPaths); - if (hasCustomDataPath(indexSettings)) { + if (indexSettings.hasCustomDataPath()) { Path customLocation = resolveCustomLocation(indexSettings, index.name()); logger.trace("deleting custom index {} directory [{}]", index, customLocation); IOUtils.rm(customLocation); @@ -439,9 +414,9 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * @return the {@link ShardLock} instances for this index. * @throws IOException if an IOException occurs. */ - public List lockAllForIndex(Index index, @IndexSettings Settings settings, long lockTimeoutMS) throws IOException { - final Integer numShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null); - if (numShards == null || numShards <= 0) { + public List lockAllForIndex(Index index, IndexSettings settings, long lockTimeoutMS) throws IOException { + final int numShards = settings.getNumberOfShards(); + if (numShards <= 0) { throw new IllegalArgumentException("settings must contain a non-null > 0 number of shards"); } logger.trace("locking all shards for index {} - [{}]", index, numShards); @@ -638,8 +613,8 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * Returns all shard paths excluding custom shard path. Note: Shards are only allocated on one of the * returned paths. The returned array may contain paths to non-existing directories. * - * @see #hasCustomDataPath(org.elasticsearch.common.settings.Settings) - * @see #resolveCustomLocation(org.elasticsearch.common.settings.Settings, org.elasticsearch.index.shard.ShardId) + * @see IndexSettings#hasCustomDataPath() + * @see #resolveCustomLocation(IndexSettings, ShardId) * */ public Path[] availableShardPaths(ShardId shardId) { @@ -781,14 +756,6 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { return settings; } - /** - * @param indexSettings settings for an index - * @return true if the index has a custom data path - */ - public static boolean hasCustomDataPath(@IndexSettings Settings indexSettings) { - return indexSettings.get(IndexMetaData.SETTING_DATA_PATH) != null; - } - /** * Resolve the custom path for a index's shard. * Uses the {@code IndexMetaData.SETTING_DATA_PATH} setting to determine @@ -796,9 +763,8 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * * @param indexSettings settings for the index */ - private Path resolveCustomLocation(@IndexSettings Settings indexSettings) { - assert indexSettings != Settings.EMPTY; - String customDataDir = indexSettings.get(IndexMetaData.SETTING_DATA_PATH); + private Path resolveCustomLocation(IndexSettings indexSettings) { + String customDataDir = indexSettings.customDataPath(); if (customDataDir != null) { // This assert is because this should be caught by MetaDataCreateIndexService assert sharedDataPath != null; @@ -820,7 +786,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * @param indexSettings settings for the index * @param indexName index to resolve the path for */ - private Path resolveCustomLocation(@IndexSettings Settings indexSettings, final String indexName) { + private Path resolveCustomLocation(IndexSettings indexSettings, final String indexName) { return resolveCustomLocation(indexSettings).resolve(indexName); } @@ -832,7 +798,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { * @param indexSettings settings for the index * @param shardId shard to resolve the path to */ - public Path resolveCustomLocation(@IndexSettings Settings indexSettings, final ShardId shardId) { + public Path resolveCustomLocation(IndexSettings indexSettings, final ShardId shardId) { return resolveCustomLocation(indexSettings, shardId.index().name()).resolve(Integer.toString(shardId.id())); } diff --git a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java index 0ebba04a9fa..d2bbeafed4d 100644 --- a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java +++ b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java @@ -120,8 +120,8 @@ public class DanglingIndicesState extends AbstractComponent { IndexMetaData indexMetaData = metaStateService.loadIndexState(indexName); if (indexMetaData != null) { logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, auto import to cluster state", indexName); - if (!indexMetaData.index().equals(indexName)) { - logger.info("dangled index directory name is [{}], state name is [{}], renaming to directory name", indexName, indexMetaData.index()); + if (!indexMetaData.getIndex().equals(indexName)) { + logger.info("dangled index directory name is [{}], state name is [{}], renaming to directory name", indexName, indexMetaData.getIndex()); indexMetaData = IndexMetaData.builder(indexMetaData).index(indexName).build(); } newIndices.put(indexName, indexMetaData); diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java index 8405aec8267..e89cd6c8577 100644 --- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java +++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import java.nio.file.Path; -import java.util.Arrays; /** * @@ -96,7 +95,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener { electedGlobalState = nodeState.metaData(); } for (ObjectCursor cursor : nodeState.metaData().indices().values()) { - indices.addTo(cursor.value.index(), 1); + indices.addTo(cursor.value.getIndex(), 1); } } if (found < requiredAllocation) { @@ -123,7 +122,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener { } if (electedIndexMetaData == null) { electedIndexMetaData = indexMetaData; - } else if (indexMetaData.version() > electedIndexMetaData.version()) { + } else if (indexMetaData.getVersion() > electedIndexMetaData.getVersion()) { electedIndexMetaData = indexMetaData; } indexMetaDataCount++; diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java index 50a77e197f0..0850064f320 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java @@ -113,10 +113,6 @@ public class GatewayAllocator extends AbstractComponent { } public boolean allocateUnassigned(final RoutingAllocation allocation) { - // Take a snapshot of the current time and tell the RoutingService - // about it, so it will use a consistent timestamp for delays - long lastAllocateUnassignedRun = System.currentTimeMillis(); - this.routingService.setUnassignedShardsAllocatedTimestamp(lastAllocateUnassignedRun); boolean changed = false; RoutingNodes.UnassignedShards unassigned = allocation.routingNodes().unassigned(); @@ -124,7 +120,7 @@ public class GatewayAllocator extends AbstractComponent { changed |= primaryShardAllocator.allocateUnassigned(allocation); changed |= replicaShardAllocator.processExistingRecoveries(allocation); - changed |= replicaShardAllocator.allocateUnassigned(allocation, lastAllocateUnassignedRun); + changed |= replicaShardAllocator.allocateUnassigned(allocation); return changed; } diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index cb462fbbd78..117a0c6959b 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -131,11 +131,11 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL Set newPreviouslyWrittenIndices = new HashSet<>(previouslyWrittenIndices.size()); for (IndexMetaData indexMetaData : newMetaData) { IndexMetaData indexMetaDataOnDisk = null; - if (indexMetaData.state().equals(IndexMetaData.State.CLOSE)) { - indexMetaDataOnDisk = metaStateService.loadIndexState(indexMetaData.index()); + if (indexMetaData.getState().equals(IndexMetaData.State.CLOSE)) { + indexMetaDataOnDisk = metaStateService.loadIndexState(indexMetaData.getIndex()); } if (indexMetaDataOnDisk != null) { - newPreviouslyWrittenIndices.add(indexMetaDataOnDisk.index()); + newPreviouslyWrittenIndices.add(indexMetaDataOnDisk.getIndex()); } } newPreviouslyWrittenIndices.addAll(previouslyWrittenIndices); @@ -274,8 +274,8 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL String writeReason = null; if (previouslyWrittenIndices.contains(index) == false || previousIndexMetaData == null) { writeReason = "freshly created"; - } else if (previousIndexMetaData.version() != newIndexMetaData.version()) { - writeReason = "version changed from [" + previousIndexMetaData.version() + "] to [" + newIndexMetaData.version() + "]"; + } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) { + writeReason = "version changed from [" + previousIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]"; } if (writeReason != null) { indicesToWrite.add(new GatewayMetaState.IndexMetaWriteInfo(newIndexMetaData, previousIndexMetaData, writeReason)); @@ -295,12 +295,12 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL } // we have to check the meta data also: closed indices will not appear in the routing table, but we must still write the state if we have it written on disk previously for (IndexMetaData indexMetaData : state.metaData()) { - boolean isOrWasClosed = indexMetaData.state().equals(IndexMetaData.State.CLOSE); + boolean isOrWasClosed = indexMetaData.getState().equals(IndexMetaData.State.CLOSE); // if the index is open we might still have to write the state if it just transitioned from closed to open // so we have to check for that as well. IndexMetaData previousMetaData = previousState.metaData().getIndices().get(indexMetaData.getIndex()); if (previousMetaData != null) { - isOrWasClosed = isOrWasClosed || previousMetaData.state().equals(IndexMetaData.State.CLOSE); + isOrWasClosed = isOrWasClosed || previousMetaData.getState().equals(IndexMetaData.State.CLOSE); } if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && isOrWasClosed) { indices.add(indexMetaData.getIndex()); diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java index 855b6ce2bba..e83ec695a96 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -20,12 +20,7 @@ package org.elasticsearch.gateway; import com.carrotsearch.hppc.cursors.ObjectCursor; - -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -43,7 +38,6 @@ import org.elasticsearch.discovery.DiscoveryService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -108,16 +102,27 @@ public class GatewayService extends AbstractLifecycleComponent i @Override protected void doStart() { clusterService.addLast(this); - // if we received initial state, see if we can recover within the start phase, so we hold the - // node from starting until we recovered properly - if (discoveryService.initialStateReceived()) { - ClusterState clusterState = clusterService.state(); - if (clusterState.nodes().localNodeMaster() && clusterState.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { - checkStateMeetsSettingsAndMaybeRecover(clusterState, false); + // check we didn't miss any cluster state that came in until now / during the addition + clusterService.submitStateUpdateTask("gateway_initial_state_recovery", new ClusterStateUpdateTask() { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + checkStateMeetsSettingsAndMaybeRecover(currentState); + return currentState; } - } else { - logger.debug("can't wait on start for (possibly) reading state from gateway, will do it asynchronously"); - } + + @Override + public boolean runOnlyOnMaster() { + // It's OK to run on non masters as checkStateMeetsSettingsAndMaybeRecover checks for this + // we return false to avoid unneeded failure logs + return false; + } + + @Override + public void onFailure(String source, Throwable t) { + logger.warn("unexpected failure while checking if state can be recovered. another attempt will be made with the next cluster state change", t); + } + }); } @Override @@ -134,12 +139,19 @@ public class GatewayService extends AbstractLifecycleComponent i if (lifecycle.stoppedOrClosed()) { return; } - if (event.localNodeMaster() && event.state().blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { - checkStateMeetsSettingsAndMaybeRecover(event.state(), true); - } + checkStateMeetsSettingsAndMaybeRecover(event.state()); } - protected void checkStateMeetsSettingsAndMaybeRecover(ClusterState state, boolean asyncRecovery) { + protected void checkStateMeetsSettingsAndMaybeRecover(ClusterState state) { + if (state.nodes().localNodeMaster() == false) { + // not our job to recover + return; + } + if (state.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false) { + // already recovered + return; + } + DiscoveryNodes nodes = state.nodes(); if (state.blocks().hasGlobalBlock(discoveryService.getNoMasterBlock())) { logger.debug("not recovering from gateway, no master elected yet"); @@ -171,51 +183,32 @@ public class GatewayService extends AbstractLifecycleComponent i reason = "expecting [" + expectedMasterNodes + "] master nodes, but only have [" + nodes.masterNodes().size() + "]"; } } - performStateRecovery(asyncRecovery, enforceRecoverAfterTime, reason); + performStateRecovery(enforceRecoverAfterTime, reason); } } - private void performStateRecovery(boolean asyncRecovery, boolean enforceRecoverAfterTime, String reason) { - final Gateway.GatewayStateRecoveredListener recoveryListener = new GatewayRecoveryListener(new CountDownLatch(1)); + private void performStateRecovery(boolean enforceRecoverAfterTime, String reason) { + final Gateway.GatewayStateRecoveredListener recoveryListener = new GatewayRecoveryListener(); if (enforceRecoverAfterTime && recoverAfterTime != null) { if (scheduledRecovery.compareAndSet(false, true)) { logger.info("delaying initial state recovery for [{}]. {}", recoverAfterTime, reason); - threadPool.schedule(recoverAfterTime, ThreadPool.Names.GENERIC, new Runnable() { - @Override - public void run() { - if (recovered.compareAndSet(false, true)) { - logger.info("recover_after_time [{}] elapsed. performing state recovery...", recoverAfterTime); - gateway.performStateRecovery(recoveryListener); - } + threadPool.schedule(recoverAfterTime, ThreadPool.Names.GENERIC, () -> { + if (recovered.compareAndSet(false, true)) { + logger.info("recover_after_time [{}] elapsed. performing state recovery...", recoverAfterTime); + gateway.performStateRecovery(recoveryListener); } }); } } else { if (recovered.compareAndSet(false, true)) { - if (asyncRecovery) { - threadPool.generic().execute(new Runnable() { - @Override - public void run() { - gateway.performStateRecovery(recoveryListener); - } - }); - } else { - logger.trace("performing state recovery..."); - gateway.performStateRecovery(recoveryListener); - } + threadPool.generic().execute(() -> gateway.performStateRecovery(recoveryListener)); } } } class GatewayRecoveryListener implements Gateway.GatewayStateRecoveredListener { - private final CountDownLatch latch; - - GatewayRecoveryListener(CountDownLatch latch) { - this.latch = latch; - } - @Override public void onSuccess(final ClusterState recoveredState) { logger.trace("successful state recovery, importing cluster state..."); @@ -258,7 +251,9 @@ public class GatewayService extends AbstractLifecycleComponent i routingTableBuilder.version(0); // now, reroute - RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build()); + RoutingAllocation.Result routingResult = allocationService.reroute( + ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(), + "state recovered"); return ClusterState.builder(updatedState).routingResult(routingResult).build(); } @@ -266,12 +261,12 @@ public class GatewayService extends AbstractLifecycleComponent i @Override public void onFailure(String source, Throwable t) { logger.error("unexpected failure during [{}]", t, source); + GatewayRecoveryListener.this.onFailure("failed to updated cluster state"); } @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { logger.info("recovered [{}] indices into cluster_state", newState.metaData().indices().size()); - latch.countDown(); } }); } @@ -283,6 +278,7 @@ public class GatewayService extends AbstractLifecycleComponent i // don't remove the block here, we don't want to allow anything in such a case logger.info("metadata state not restored, reason: {}", message); } + } // used for testing diff --git a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java index b8491b9db43..0e151cec5e5 100644 --- a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java +++ b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java @@ -116,7 +116,7 @@ public class LocalAllocateDangledIndices extends AbstractComponent { public void messageReceived(final AllocateDangledRequest request, final TransportChannel channel) throws Exception { String[] indexNames = new String[request.indices.length]; for (int i = 0; i < request.indices.length; i++) { - indexNames[i] = request.indices[i].index(); + indexNames[i] = request.indices[i].getIndex(); } clusterService.submitStateUpdateTask("allocation dangled indices " + Arrays.toString(indexNames), new ClusterStateUpdateTask() { @Override @@ -131,12 +131,12 @@ public class LocalAllocateDangledIndices extends AbstractComponent { boolean importNeeded = false; StringBuilder sb = new StringBuilder(); for (IndexMetaData indexMetaData : request.indices) { - if (currentState.metaData().hasIndex(indexMetaData.index())) { + if (currentState.metaData().hasIndex(indexMetaData.getIndex())) { continue; } - if (currentState.metaData().hasAlias(indexMetaData.index())) { + if (currentState.metaData().hasAlias(indexMetaData.getIndex())) { logger.warn("ignoring dangled index [{}] on node [{}] due to an existing alias with the same name", - indexMetaData.index(), request.fromNode); + indexMetaData.getIndex(), request.fromNode); continue; } importNeeded = true; @@ -149,15 +149,15 @@ public class LocalAllocateDangledIndices extends AbstractComponent { } catch (Exception ex) { // upgrade failed - adding index as closed logger.warn("found dangled index [{}] on node [{}]. This index cannot be upgraded to the latest version, adding as closed", ex, - indexMetaData.index(), request.fromNode); - upgradedIndexMetaData = IndexMetaData.builder(indexMetaData).state(IndexMetaData.State.CLOSE).version(indexMetaData.version() + 1).build(); + indexMetaData.getIndex(), request.fromNode); + upgradedIndexMetaData = IndexMetaData.builder(indexMetaData).state(IndexMetaData.State.CLOSE).version(indexMetaData.getVersion() + 1).build(); } metaData.put(upgradedIndexMetaData, false); blocks.addBlocks(upgradedIndexMetaData); if (upgradedIndexMetaData.getState() == IndexMetaData.State.OPEN) { routingTableBuilder.addAsFromDangling(upgradedIndexMetaData); } - sb.append("[").append(upgradedIndexMetaData.index()).append("/").append(upgradedIndexMetaData.state()).append("]"); + sb.append("[").append(upgradedIndexMetaData.getIndex()).append("/").append(upgradedIndexMetaData.getState()).append("]"); } if (!importNeeded) { return currentState; @@ -168,7 +168,8 @@ public class LocalAllocateDangledIndices extends AbstractComponent { ClusterState updatedState = ClusterState.builder(currentState).metaData(metaData).blocks(blocks).routingTable(routingTable).build(); // now, reroute - RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(routingTable).build()); + RoutingAllocation.Result routingResult = allocationService.reroute( + ClusterState.builder(updatedState).routingTable(routingTable).build(), "dangling indices allocated"); return ClusterState.builder(updatedState).routingResult(routingResult).build(); } diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java index aa89f787653..43f9ffbe26f 100644 --- a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -130,13 +130,13 @@ public class MetaStateService extends AbstractComponent { * Writes the index state. */ void writeIndex(String reason, IndexMetaData indexMetaData, @Nullable IndexMetaData previousIndexMetaData) throws Exception { - logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason); + logger.trace("[{}] writing state, reason [{}]", indexMetaData.getIndex(), reason); try { - indexStateFormat.write(indexMetaData, indexMetaData.version(), - nodeEnv.indexPaths(new Index(indexMetaData.index()))); + indexStateFormat.write(indexMetaData, indexMetaData.getVersion(), + nodeEnv.indexPaths(new Index(indexMetaData.getIndex()))); } catch (Throwable ex) { - logger.warn("[{}]: failed to write index state", ex, indexMetaData.index()); - throw new IOException("failed to write state for [" + indexMetaData.index() + "]", ex); + logger.warn("[{}]: failed to write index state", ex, indexMetaData.getIndex()); + throw new IOException("failed to write state for [" + indexMetaData.getIndex() + "]", ex); } } diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java index b3bda4ec139..e560b4458b7 100644 --- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java @@ -30,15 +30,8 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.Decision; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * The primary shard allocator allocates primary shard that were not created as @@ -72,13 +65,15 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { AsyncShardFetch.FetchResult shardState = fetchData(shard, allocation); if (shardState.hasData() == false) { logger.trace("{}: ignoring allocation, still fetching shard started state", shard); + allocation.setHasPendingAsyncFetch(); unassignedIterator.removeAndIgnore(); continue; } IndexMetaData indexMetaData = metaData.index(shard.getIndex()); + Settings indexSettings = Settings.builder().put(settings).put(indexMetaData.getSettings()).build(); - NodesAndVersions nodesAndVersions = buildNodesAndVersions(shard, recoverOnAnyNode(indexMetaData.settings()), allocation.getIgnoreNodes(shard.shardId()), shardState); + NodesAndVersions nodesAndVersions = buildNodesAndVersions(shard, recoverOnAnyNode(indexSettings), allocation.getIgnoreNodes(shard.shardId()), shardState); logger.debug("[{}][{}] found {} allocations of {}, highest version: [{}]", shard.index(), shard.id(), nodesAndVersions.allocationsFound, shard, nodesAndVersions.highestVersion); if (isEnoughAllocationsFound(shard, indexMetaData, nodesAndVersions) == false) { @@ -135,22 +130,22 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { // if we restore from a repository one copy is more then enough if (shard.restoreSource() == null) { try { - String initialShards = indexMetaData.settings().get(INDEX_RECOVERY_INITIAL_SHARDS, settings.get(INDEX_RECOVERY_INITIAL_SHARDS, this.initialShards)); + String initialShards = indexMetaData.getSettings().get(INDEX_RECOVERY_INITIAL_SHARDS, settings.get(INDEX_RECOVERY_INITIAL_SHARDS, this.initialShards)); if ("quorum".equals(initialShards)) { - if (indexMetaData.numberOfReplicas() > 1) { - requiredAllocation = ((1 + indexMetaData.numberOfReplicas()) / 2) + 1; + if (indexMetaData.getNumberOfReplicas() > 1) { + requiredAllocation = ((1 + indexMetaData.getNumberOfReplicas()) / 2) + 1; } } else if ("quorum-1".equals(initialShards) || "half".equals(initialShards)) { - if (indexMetaData.numberOfReplicas() > 2) { - requiredAllocation = ((1 + indexMetaData.numberOfReplicas()) / 2); + if (indexMetaData.getNumberOfReplicas() > 2) { + requiredAllocation = ((1 + indexMetaData.getNumberOfReplicas()) / 2); } } else if ("one".equals(initialShards)) { requiredAllocation = 1; } else if ("full".equals(initialShards) || "all".equals(initialShards)) { - requiredAllocation = indexMetaData.numberOfReplicas() + 1; + requiredAllocation = indexMetaData.getNumberOfReplicas() + 1; } else if ("full-1".equals(initialShards) || "all-1".equals(initialShards)) { - if (indexMetaData.numberOfReplicas() > 1) { - requiredAllocation = indexMetaData.numberOfReplicas(); + if (indexMetaData.getNumberOfReplicas() > 1) { + requiredAllocation = indexMetaData.getNumberOfReplicas(); } } else { requiredAllocation = Integer.parseInt(initialShards); @@ -263,7 +258,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { * Return {@code true} if the index is configured to allow shards to be * recovered on any node */ - private boolean recoverOnAnyNode(@IndexSettings Settings idxSettings) { + private boolean recoverOnAnyNode(Settings idxSettings) { return IndexMetaData.isOnSharedFilesystem(idxSettings) && idxSettings.getAsBoolean(IndexMetaData.SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false); } diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java index 03772f74630..c87f4d94755 100644 --- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java @@ -24,7 +24,6 @@ import com.carrotsearch.hppc.ObjectLongMap; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; @@ -101,7 +100,8 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { // we found a better match that has a full sync id match, the existing allocation is not fully synced // so we found a better one, cancel this one it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA, - "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]")); + "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]", + null, allocation.getCurrentNanoTime(), System.currentTimeMillis())); changed = true; } } @@ -111,10 +111,6 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { } public boolean allocateUnassigned(RoutingAllocation allocation) { - return allocateUnassigned(allocation, System.currentTimeMillis()); - } - - public boolean allocateUnassigned(RoutingAllocation allocation, long allocateUnassignedTimestapm) { boolean changed = false; final RoutingNodes routingNodes = allocation.routingNodes(); final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator(); @@ -139,6 +135,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { AsyncShardFetch.FetchResult shardStores = fetchData(shard, allocation); if (shardStores.hasData() == false) { logger.trace("{}: ignoring allocation, still fetching shard stores", shard); + allocation.setHasPendingAsyncFetch(); unassignedIterator.removeAndIgnore(); continue; // still fetching } @@ -172,27 +169,40 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { unassignedIterator.initialize(nodeWithHighestMatch.nodeId(), shard.version(), allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE)); } } else if (matchingNodes.hasAnyData() == false) { - // if we didn't manage to find *any* data (regardless of matching sizes), check if the allocation - // of the replica shard needs to be delayed, and if so, add it to the ignore unassigned list - // note: we only care about replica in delayed allocation, since if we have an unassigned primary it - // will anyhow wait to find an existing copy of the shard to be allocated - // note: the other side of the equation is scheduling a reroute in a timely manner, which happens in the RoutingService - IndexMetaData indexMetaData = allocation.metaData().index(shard.getIndex()); - long delay = shard.unassignedInfo().getDelayAllocationExpirationIn(allocateUnassignedTimestapm, settings, indexMetaData.getSettings()); - if (delay > 0) { - logger.debug("[{}][{}]: delaying allocation of [{}] for [{}]", shard.index(), shard.id(), shard, TimeValue.timeValueMillis(delay)); - /** - * mark it as changed, since we want to kick a publishing to schedule future allocation, - * see {@link org.elasticsearch.cluster.routing.RoutingService#clusterChanged(ClusterChangedEvent)}). - */ - changed = true; - unassignedIterator.removeAndIgnore(); - } + // if we didn't manage to find *any* data (regardless of matching sizes), check if the allocation of the replica shard needs to be delayed + changed |= ignoreUnassignedIfDelayed(unassignedIterator, shard); } } return changed; } + /** + * Check if the allocation of the replica is to be delayed. Compute the delay and if it is delayed, add it to the ignore unassigned list + * Note: we only care about replica in delayed allocation, since if we have an unassigned primary it + * will anyhow wait to find an existing copy of the shard to be allocated + * Note: the other side of the equation is scheduling a reroute in a timely manner, which happens in the RoutingService + * + * PUBLIC FOR TESTS! + * + * @param unassignedIterator iterator over unassigned shards + * @param shard the shard which might be delayed + * @return true iff allocation is delayed for this shard + */ + public boolean ignoreUnassignedIfDelayed(RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) { + // calculate delay and store it in UnassignedInfo to be used by RoutingService + long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos(); + if (delay > 0) { + logger.debug("[{}][{}]: delaying allocation of [{}] for [{}]", shard.index(), shard.id(), shard, TimeValue.timeValueNanos(delay)); + /** + * mark it as changed, since we want to kick a publishing to schedule future allocation, + * see {@link org.elasticsearch.cluster.routing.RoutingService#clusterChanged(ClusterChangedEvent)}). + */ + unassignedIterator.removeAndIgnore(); + return true; + } + return false; + } + /** * Can the shard be allocated on at least one node based on the allocation deciders. */ diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java index 63194d24ed2..d91b4bd8cdd 100644 --- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java @@ -39,6 +39,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.shard.ShardStateMetaData; @@ -48,6 +50,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -128,7 +131,8 @@ public class TransportNodesListGatewayStartedShards extends TransportNodesAction if (metaData != null) { ShardPath shardPath = null; try { - shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, metaData.settings()); + IndexSettings indexSettings = new IndexSettings(metaData, settings, Collections.emptyList()); + shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings); if (shardPath == null) { throw new IllegalStateException(shardId + " no shard path found"); } diff --git a/core/src/main/java/org/elasticsearch/http/BindHttpException.java b/core/src/main/java/org/elasticsearch/http/BindHttpException.java index ebfc7e768eb..f8cac9f2a97 100644 --- a/core/src/main/java/org/elasticsearch/http/BindHttpException.java +++ b/core/src/main/java/org/elasticsearch/http/BindHttpException.java @@ -28,6 +28,10 @@ import java.io.IOException; */ public class BindHttpException extends HttpException { + public BindHttpException(String message) { + super(message); + } + public BindHttpException(String message, Throwable cause) { super(message, cause); } diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java index f3b8c3f0a4b..9971ce7722d 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java @@ -189,7 +189,7 @@ public class HttpServer extends AbstractLifecycleComponent { sitePath = null; // If a trailing / is missing, we redirect to the right page #2654 String redirectUrl = request.rawPath() + "/"; - BytesRestResponse restResponse = new BytesRestResponse(RestStatus.MOVED_PERMANENTLY, "text/html", ""); + BytesRestResponse restResponse = new BytesRestResponse(RestStatus.MOVED_PERMANENTLY, "text/html", ""); restResponse.addHeader("Location", redirectUrl); channel.sendResponse(restResponse); return; diff --git a/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java b/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java index 1a42ce8997d..2eb57f187fa 100644 --- a/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java +++ b/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java @@ -40,7 +40,7 @@ public class HttpRequestHandler extends SimpleChannelUpstreamHandler { public HttpRequestHandler(NettyHttpServerTransport serverTransport, boolean detailedErrorsEnabled) { this.serverTransport = serverTransport; - this.corsPattern = RestUtils.getCorsSettingRegex(serverTransport.settings()); + this.corsPattern = RestUtils.checkCorsSettingForRegex(serverTransport.settings().get(NettyHttpServerTransport.SETTING_CORS_ALLOW_ORIGIN)); this.httpPipeliningEnabled = serverTransport.pipelining; this.detailedErrorsEnabled = detailedErrorsEnabled; } diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java index a794f52dfaf..ebc655ae4a5 100644 --- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java +++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java @@ -20,6 +20,7 @@ package org.elasticsearch.http.netty; import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.netty.NettyUtils; @@ -50,6 +51,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; @@ -80,6 +82,7 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent boundAddresses = new ArrayList<>(hostAddresses.length); @@ -256,16 +261,28 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent listeners = new CopyOnWriteArrayList<>(); + private final List listeners; + private final ESLogger logger; - @Inject - public InternalIndicesLifecycle(Settings settings) { - super(settings); - } - @Override - public void addListener(Listener listener) { - listeners.add(listener); + CompositeIndexEventListener(IndexSettings indexSettings, Collection listeners) { + for (IndexEventListener listener : listeners) { + if (listener == null) { + throw new IllegalArgumentException("listeners must be non-null"); + } + } + this.listeners = Collections.unmodifiableList(new ArrayList<>(listeners)); + this.logger = Loggers.getLogger(getClass(), indexSettings.getSettings(), indexSettings.getIndex()); } @Override - public void removeListener(Listener listener) { - listeners.remove(listener); - } - public void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting oldRouting, ShardRouting newRouting) { - for (Listener listener : listeners) { + for (IndexEventListener listener : listeners) { try { listener.shardRoutingChanged(indexShard, oldRouting, newRouting); } catch (Throwable t) { - logger.warn("{} failed to invoke shard touring changed callback", t, indexShard.shardId()); - } - } - } - - public void beforeIndexAddedToCluster(Index index, @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { - try { - listener.beforeIndexAddedToCluster(index, indexSettings); - } catch (Throwable t) { - logger.warn("[{}] failed to invoke before index added to cluster callback", t, index.name()); - throw t; - } - } - } - - public void beforeIndexCreated(Index index, @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { - try { - listener.beforeIndexCreated(index, indexSettings); - } catch (Throwable t) { - logger.warn("[{}] failed to invoke before index created callback", t, index.name()); - throw t; - } - } - } - - public void afterIndexCreated(IndexService indexService) { - for (Listener listener : listeners) { - try { - listener.afterIndexCreated(indexService); - } catch (Throwable t) { - logger.warn("[{}] failed to invoke after index created callback", t, indexService.index().name()); - throw t; - } - } - } - - public void beforeIndexShardCreated(ShardId shardId, @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { - try { - listener.beforeIndexShardCreated(shardId, indexSettings); - } catch (Throwable t) { - logger.warn("{} failed to invoke before shard created callback", t, shardId); - throw t; + logger.warn("[{}] failed to invoke shard touring changed callback", t, indexShard.shardId().getId()); } } } + @Override public void afterIndexShardCreated(IndexShard indexShard) { - for (Listener listener : listeners) { + for (IndexEventListener listener : listeners) { try { listener.afterIndexShardCreated(indexShard); } catch (Throwable t) { - logger.warn("{} failed to invoke after shard created callback", t, indexShard.shardId()); + logger.warn("[{}] failed to invoke after shard created callback", t, indexShard.shardId().getId()); throw t; } } } - + @Override public void afterIndexShardStarted(IndexShard indexShard) { - for (Listener listener : listeners) { + for (IndexEventListener listener : listeners) { try { listener.afterIndexShardStarted(indexShard); } catch (Throwable t) { - logger.warn("{} failed to invoke after shard started callback", t, indexShard.shardId()); - throw t; - } - } - } - - public void beforeIndexClosed(IndexService indexService) { - for (Listener listener : listeners) { - try { - listener.beforeIndexClosed(indexService); - } catch (Throwable t) { - logger.warn("[{}] failed to invoke before index closed callback", t, indexService.index().name()); - throw t; - } - } - } - - public void beforeIndexDeleted(IndexService indexService) { - for (Listener listener : listeners) { - try { - listener.beforeIndexDeleted(indexService); - } catch (Throwable t) { - logger.warn("[{}] failed to invoke before index deleted callback", t, indexService.index().name()); - throw t; - } - } - } - - public void afterIndexDeleted(Index index, @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { - try { - listener.afterIndexDeleted(index, indexSettings); - } catch (Throwable t) { - logger.warn("[{}] failed to invoke after index deleted callback", t, index.name()); - throw t; - } - } - } - - public void afterIndexClosed(Index index, @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { - try { - listener.afterIndexClosed(index, indexSettings); - } catch (Throwable t) { - logger.warn("[{}] failed to invoke after index closed callback", t, index.name()); + logger.warn("[{}] failed to invoke after shard started callback", t, indexShard.shardId().getId()); throw t; } } } + @Override public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { + Settings indexSettings) { + for (IndexEventListener listener : listeners) { try { listener.beforeIndexShardClosed(shardId, indexShard, indexSettings); } catch (Throwable t) { - logger.warn("{} failed to invoke before shard closed callback", t, shardId); + logger.warn("[{}] failed to invoke before shard closed callback", t, shardId.getId()); throw t; } } } + @Override public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { + Settings indexSettings) { + for (IndexEventListener listener : listeners) { try { listener.afterIndexShardClosed(shardId, indexShard, indexSettings); } catch (Throwable t) { - logger.warn("{} failed to invoke after shard closed callback", t, shardId); - throw t; - } - } - } - - public void beforeIndexShardDeleted(ShardId shardId, - @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { - try { - listener.beforeIndexShardDeleted(shardId, indexSettings); - } catch (Throwable t) { - logger.warn("{} failed to invoke before shard deleted callback", t, shardId); - throw t; - } - } - } - - public void afterIndexShardDeleted(ShardId shardId, - @IndexSettings Settings indexSettings) { - for (Listener listener : listeners) { - try { - listener.afterIndexShardDeleted(shardId, indexSettings); - } catch (Throwable t) { - logger.warn("{} failed to invoke after shard deleted callback", t, shardId); - throw t; - } - } - } - - public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, @Nullable String reason) { - for (Listener listener : listeners) { - try { - listener.indexShardStateChanged(indexShard, previousState, indexShard.state(), reason); - } catch (Throwable t) { - logger.warn("{} failed to invoke index shard state changed callback", t, indexShard.shardId()); + logger.warn("[{}] failed to invoke after shard closed callback", t, shardId.getId()); throw t; } } } + @Override public void onShardInactive(IndexShard indexShard) { - for (Listener listener : listeners) { + for (IndexEventListener listener : listeners) { try { listener.onShardInactive(indexShard); } catch (Throwable t) { - logger.warn("{} failed to invoke on shard inactive callback", t, indexShard.shardId()); + logger.warn("[{}] failed to invoke on shard inactive callback", t, indexShard.shardId().getId()); + throw t; + } + } + } + + @Override + public void onShardActive(IndexShard indexShard) { + for (IndexEventListener listener : listeners) { + try { + listener.onShardActive(indexShard); + } catch (Throwable t) { + logger.warn("[{}] failed to invoke on shard active callback", t, indexShard.shardId().getId()); + throw t; + } + } + } + + @Override + public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) { + for (IndexEventListener listener : listeners) { + try { + listener.indexShardStateChanged(indexShard, previousState, indexShard.state(), reason); + } catch (Throwable t) { + logger.warn("[{}] failed to invoke index shard state changed callback", t, indexShard.shardId().getId()); + throw t; + } + } + } + + @Override + public void beforeIndexCreated(Index index, Settings indexSettings) { + for (IndexEventListener listener : listeners) { + try { + listener.beforeIndexCreated(index, indexSettings); + } catch (Throwable t) { + logger.warn("failed to invoke before index created callback", t); + throw t; + } + } + } + + @Override + public void afterIndexCreated(IndexService indexService) { + for (IndexEventListener listener : listeners) { + try { + listener.afterIndexCreated(indexService); + } catch (Throwable t) { + logger.warn("failed to invoke after index created callback", t); + throw t; + } + } + } + + @Override + public void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) { + for (IndexEventListener listener : listeners) { + try { + listener.beforeIndexShardCreated(shardId, indexSettings); + } catch (Throwable t) { + logger.warn("[{}] failed to invoke before shard created callback", t, shardId); + throw t; + } + } + } + + @Override + public void beforeIndexClosed(IndexService indexService) { + for (IndexEventListener listener : listeners) { + try { + listener.beforeIndexClosed(indexService); + } catch (Throwable t) { + logger.warn("failed to invoke before index closed callback", t); + throw t; + } + } + } + + @Override + public void beforeIndexDeleted(IndexService indexService) { + for (IndexEventListener listener : listeners) { + try { + listener.beforeIndexDeleted(indexService); + } catch (Throwable t) { + logger.warn("failed to invoke before index deleted callback", t); + throw t; + } + } + } + + @Override + public void afterIndexDeleted(Index index, Settings indexSettings) { + for (IndexEventListener listener : listeners) { + try { + listener.afterIndexDeleted(index, indexSettings); + } catch (Throwable t) { + logger.warn("failed to invoke after index deleted callback", t); + throw t; + } + } + } + + @Override + public void afterIndexClosed(Index index, Settings indexSettings) { + for (IndexEventListener listener : listeners) { + try { + listener.afterIndexClosed(index, indexSettings); + } catch (Throwable t) { + logger.warn("failed to invoke after index closed callback", t); + throw t; + } + } + } + + @Override + public void beforeIndexShardDeleted(ShardId shardId, + Settings indexSettings) { + for (IndexEventListener listener : listeners) { + try { + listener.beforeIndexShardDeleted(shardId, indexSettings); + } catch (Throwable t) { + logger.warn("[{}] failed to invoke before shard deleted callback", t, shardId.getId()); + throw t; + } + } + } + + @Override + public void afterIndexShardDeleted(ShardId shardId, + Settings indexSettings) { + for (IndexEventListener listener : listeners) { + try { + listener.afterIndexShardDeleted(shardId, indexSettings); + } catch (Throwable t) { + logger.warn("[{}] failed to invoke after shard deleted callback", t, shardId.getId()); + throw t; + } + } + } + + @Override + public void beforeIndexAddedToCluster(Index index, Settings indexSettings) { + for (IndexEventListener listener : listeners) { + try { + listener.beforeIndexAddedToCluster(index, indexSettings); + } catch (Throwable t) { + logger.warn("failed to invoke before index added to cluster callback", t); throw t; } } diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java index dc637cfd5e9..231baaefaab 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java @@ -19,43 +19,246 @@ package org.elasticsearch.index; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.util.Providers; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.cache.query.QueryCache; +import org.elasticsearch.index.cache.query.index.IndexQueryCache; +import org.elasticsearch.index.cache.query.none.NoneQueryCache; import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.engine.InternalEngineFactory; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexSearcherWrapper; +import org.elasticsearch.index.similarity.BM25SimilarityProvider; +import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.index.store.IndexStore; +import org.elasticsearch.index.store.IndexStoreConfig; +import org.elasticsearch.indices.cache.query.IndicesQueryCache; +import org.elasticsearch.indices.mapper.MapperRegistry; + +import java.io.IOException; +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Consumer; /** - * + * IndexModule represents the central extension point for index level custom implementations like: + *
      + *
    • {@link SimilarityProvider} - New {@link SimilarityProvider} implementations can be registered through {@link #addSimilarity(String, BiFunction)} + * while existing Providers can be referenced through Settings under the {@link IndexModule#SIMILARITY_SETTINGS_PREFIX} prefix + * along with the "type" value. For example, to reference the {@link BM25SimilarityProvider}, the configuration + * "index.similarity.my_similarity.type : "BM25" can be used.
      • + *
    • {@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link #addIndexStore(String, BiFunction)}
      • + *
    • {@link IndexEventListener} - Custom {@link IndexEventListener} instances can be registered via {@link #addIndexEventListener(IndexEventListener)}
      • + *
    • Settings update listener - Custom settings update listener can be registered via {@link #addIndexSettingsListener(Consumer)}
      • + *
    */ -public class IndexModule extends AbstractModule { +public final class IndexModule { - private final IndexMetaData indexMetaData; + public static final String STORE_TYPE = "index.store.type"; + public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity"; + public static final String INDEX_QUERY_CACHE = "index"; + public static final String NONE_QUERY_CACHE = "none"; + public static final String QUERY_CACHE_TYPE = "index.queries.cache.type"; + // for test purposes only + public static final String QUERY_CACHE_EVERYTHING = "index.queries.cache.everything"; + private final IndexSettings indexSettings; + private final IndexStoreConfig indexStoreConfig; + private final AnalysisRegistry analysisRegistry; // pkg private so tests can mock - Class engineFactoryImpl = InternalEngineFactory.class; - Class indexSearcherWrapper = null; + final SetOnce engineFactory = new SetOnce<>(); + private SetOnce indexSearcherWrapper = new SetOnce<>(); + private final Set> settingsConsumers = new HashSet<>(); + private final Set indexEventListeners = new HashSet<>(); + private IndexEventListener listener; + private final Map> similarities = new HashMap<>(); + private final Map> storeTypes = new HashMap<>(); + private final Map> queryCaches = new HashMap<>(); - public IndexModule(IndexMetaData indexMetaData) { - this.indexMetaData = indexMetaData; + + public IndexModule(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig, AnalysisRegistry analysisRegistry) { + this.indexStoreConfig = indexStoreConfig; + this.indexSettings = indexSettings; + this.analysisRegistry = analysisRegistry; + registerQueryCache(INDEX_QUERY_CACHE, IndexQueryCache::new); + registerQueryCache(NONE_QUERY_CACHE, (a, b) -> new NoneQueryCache(a)); } - @Override - protected void configure() { - bind(EngineFactory.class).to(engineFactoryImpl).asEagerSingleton(); - if (indexSearcherWrapper == null) { - bind(IndexSearcherWrapper.class).toProvider(Providers.of(null)); - } else { - bind(IndexSearcherWrapper.class).to(indexSearcherWrapper).asEagerSingleton(); + /** + * Adds a settings consumer for this index + */ + public void addIndexSettingsListener(Consumer listener) { + if (listener == null) { + throw new IllegalArgumentException("listener must not be null"); } - bind(IndexMetaData.class).toInstance(indexMetaData); - bind(IndexService.class).asEagerSingleton(); - bind(IndexServicesProvider.class).asEagerSingleton(); - bind(MapperService.class).asEagerSingleton(); - bind(IndexFieldDataService.class).asEagerSingleton(); + + if (settingsConsumers.contains(listener)) { + throw new IllegalStateException("listener already registered"); + } + settingsConsumers.add(listener); + } + + /** + * Returns the index {@link Settings} for this index + */ + public Settings getSettings() { + return indexSettings.getSettings(); + } + + /** + * Returns the index this module is associated with + */ + public Index getIndex() { + return indexSettings.getIndex(); + } + + /** + * Adds an {@link IndexEventListener} for this index. All listeners added here + * are maintained for the entire index lifecycle on this node. Once an index is closed or deleted these + * listeners go out of scope. + *

    + * Note: an index might be created on a node multiple times. For instance if the last shard from an index is + * relocated to another node the internal representation will be destroyed which includes the registered listeners. + * Once the node holds at least one shard of an index all modules are reloaded and listeners are registered again. + * Listeners can't be unregistered they will stay alive for the entire time the index is allocated on a node. + *

    + */ + public void addIndexEventListener(IndexEventListener listener) { + if (this.listener != null) { + throw new IllegalStateException("can't add listener after listeners are frozen"); + } + if (listener == null) { + throw new IllegalArgumentException("listener must not be null"); + } + if (indexEventListeners.contains(listener)) { + throw new IllegalArgumentException("listener already added"); + } + + this.indexEventListeners.add(listener); + } + + /** + * Adds an {@link IndexStore} type to this index module. Typically stores are registered with a refrence to + * it's constructor: + * 
    +     *     indexModule.addIndexStore("my_store_type", MyStore::new);
+     *
    + * + * @param type the type to register + * @param provider the instance provider / factory method + */ + public void addIndexStore(String type, BiFunction provider) { + if (storeTypes.containsKey(type)) { + throw new IllegalArgumentException("key [" + type +"] already registerd"); + } + storeTypes.put(type, provider); } + /** + * Registers the given {@link SimilarityProvider} with the given name + * + * @param name Name of the SimilarityProvider + * @param similarity SimilarityProvider to register + */ + public void addSimilarity(String name, BiFunction similarity) { + if (similarities.containsKey(name) || SimilarityService.BUILT_IN.containsKey(name)) { + throw new IllegalArgumentException("similarity for name: [" + name + " is already registered"); + } + similarities.put(name, similarity); + } + + /** + * Registers a {@link QueryCache} provider for a given name + * @param name the providers / caches name + * @param provider the provider instance + */ + public void registerQueryCache(String name, BiFunction provider) { + if (provider == null) { + throw new IllegalArgumentException("provider must not be null"); + } + if (queryCaches.containsKey(name)) { + throw new IllegalArgumentException("Can't register the same [query_cache] more than once for [" + name + "]"); + } + queryCaches.put(name, provider); + } + + /** + * Sets a {@link org.elasticsearch.index.IndexModule.IndexSearcherWrapperFactory} that is called once the IndexService is fully constructed. + * Note: this method can only be called once per index. Multiple wrappers are not supported. + */ + public void setSearcherWrapper(IndexSearcherWrapperFactory indexSearcherWrapperFactory) { + this.indexSearcherWrapper.set(indexSearcherWrapperFactory); + } + + public IndexEventListener freeze() { + // TODO somehow we need to make this pkg private... + if (listener == null) { + listener = new CompositeIndexEventListener(indexSettings, indexEventListeners); + } + return listener; + } + + private static boolean isBuiltinType(String storeType) { + for (Type type : Type.values()) { + if (type.match(storeType)) { + return true; + } + } + return false; + } + + public enum Type { + NIOFS, + MMAPFS, + SIMPLEFS, + FS, + DEFAULT; + + public String getSettingsKey() { + return this.name().toLowerCase(Locale.ROOT); + } + /** + * Returns true iff this settings matches the type. + */ + public boolean match(String setting) { + return getSettingsKey().equals(setting); + } + } + + /** + * Factory for creating new {@link IndexSearcherWrapper} instances + */ + public interface IndexSearcherWrapperFactory { + /** + * Returns a new IndexSearcherWrapper. This method is called once per index per node + */ + IndexSearcherWrapper newWrapper(final IndexService indexService); + } + + public IndexService newIndexService(NodeEnvironment environment, IndexService.ShardStoreDeleter shardStoreDeleter, NodeServicesProvider servicesProvider, MapperRegistry mapperRegistry) throws IOException { + final IndexSettings settings = indexSettings.newWithListener(settingsConsumers); + IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get(); + IndexEventListener eventListener = freeze(); + final String storeType = settings.getSettings().get(STORE_TYPE); + final IndexStore store; + if (storeType == null || isBuiltinType(storeType)) { + store = new IndexStore(settings, indexStoreConfig); + } else { + BiFunction factory = storeTypes.get(storeType); + if (factory == null) { + throw new IllegalArgumentException("Unknown store type [" + storeType + "]"); + } + store = factory.apply(settings, indexStoreConfig); + if (store == null) { + throw new IllegalStateException("store must not be null"); + } + } + final String queryCacheType = settings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE); + final BiFunction queryCacheProvider = queryCaches.get(queryCacheType); + final QueryCache queryCache = queryCacheProvider.apply(settings, servicesProvider.getIndicesQueryCache()); + return new IndexService(settings, environment, new SimilarityService(settings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(), + servicesProvider, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry); + } } diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index 9c932e2468a..92ca00231b5 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -24,7 +24,6 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.ShardRouting; @@ -36,30 +35,26 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; +import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.cache.query.QueryCache; +import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.ParsedQuery; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShadowIndexShard; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardNotFoundException; -import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.shard.*; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.AliasFilterParsingException; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.InternalIndicesLifecycle; import org.elasticsearch.indices.InvalidAliasNameException; +import org.elasticsearch.indices.mapper.MapperRegistry; import java.io.Closeable; import java.io.IOException; @@ -78,53 +73,63 @@ import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; /** * */ -public class IndexService extends AbstractIndexComponent implements IndexComponent, Iterable { +public final class IndexService extends AbstractIndexComponent implements IndexComponent, Iterable{ - private final InternalIndicesLifecycle indicesLifecycle; + private final IndexEventListener eventListener; private final AnalysisService analysisService; private final IndexFieldDataService indexFieldData; private final BitsetFilterCache bitsetFilterCache; - private final IndexSettingsService settingsService; private final NodeEnvironment nodeEnv; - private final IndicesService indicesServices; - private final IndexServicesProvider indexServicesProvider; + private final ShardStoreDeleter shardStoreDeleter; + private final NodeServicesProvider nodeServicesProvider; private final IndexStore indexStore; + private final IndexSearcherWrapper searcherWrapper; + private final IndexCache indexCache; + private final MapperService mapperService; + private final SimilarityService similarityService; + private final EngineFactory engineFactory; private volatile Map shards = emptyMap(); private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean deleted = new AtomicBoolean(false); - private volatile IndexMetaData indexMetaData; + private final IndexSettings indexSettings; @Inject - public IndexService(Index index, IndexMetaData indexMetaData, NodeEnvironment nodeEnv, - AnalysisService analysisService, - IndexSettingsService settingsService, - IndexFieldDataService indexFieldData, - BitsetFilterCache bitSetFilterCache, - IndicesService indicesServices, - IndexServicesProvider indexServicesProvider, - IndexStore indexStore) { - super(index, settingsService.indexSettings()); - assert indexMetaData != null; - this.analysisService = analysisService; - this.indexFieldData = indexFieldData; - this.settingsService = settingsService; - this.bitsetFilterCache = bitSetFilterCache; - this.indicesServices = indicesServices; - this.indicesLifecycle = (InternalIndicesLifecycle) indexServicesProvider.getIndicesLifecycle(); + public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv, + SimilarityService similarityService, + ShardStoreDeleter shardStoreDeleter, + AnalysisRegistry registry, + @Nullable EngineFactory engineFactory, + NodeServicesProvider nodeServicesProvider, + QueryCache queryCache, + IndexStore indexStore, + IndexEventListener eventListener, + IndexModule.IndexSearcherWrapperFactory wrapperFactory, + MapperRegistry mapperRegistry) throws IOException { + super(indexSettings); + this.indexSettings = indexSettings; + this.analysisService = registry.build(indexSettings); + this.similarityService = similarityService; + this.mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry); + this.indexFieldData = new IndexFieldDataService(indexSettings, nodeServicesProvider.getIndicesFieldDataCache(), nodeServicesProvider.getCircuitBreakerService(), mapperService); + this.shardStoreDeleter = shardStoreDeleter; + this.eventListener = eventListener; this.nodeEnv = nodeEnv; - this.indexServicesProvider = indexServicesProvider; + this.nodeServicesProvider = nodeServicesProvider; this.indexStore = indexStore; - this.indexMetaData = indexMetaData; indexFieldData.setListener(new FieldDataCacheListener(this)); - bitSetFilterCache.setListener(new BitsetCacheListener(this)); + this.bitsetFilterCache = new BitsetFilterCache(indexSettings, nodeServicesProvider.getWarmer(), new BitsetCacheListener(this)); + this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache); + this.engineFactory = engineFactory; + // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE + this.searcherWrapper = wrapperFactory.newWrapper(this); } public int numberOfShards() { return shards.size(); } - public InternalIndicesLifecycle indicesLifecycle() { - return this.indicesLifecycle; + public IndexEventListener getIndexEventListener() { + return this.eventListener; } @Override @@ -150,64 +155,52 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone public IndexShard getShard(int shardId) { IndexShard indexShard = getShardOrNull(shardId); if (indexShard == null) { - throw new ShardNotFoundException(new ShardId(index, shardId)); + throw new ShardNotFoundException(new ShardId(index(), shardId)); } return indexShard; } - public Set shardIds() { - return shards.keySet(); - } - - public IndexSettingsService settingsService() { - return this.settingsService; - } + public Set shardIds() { return shards.keySet(); } public IndexCache cache() { - return indexServicesProvider.getIndexCache(); + return indexCache; } - public IndexFieldDataService fieldData() { - return indexFieldData; - } - - public BitsetFilterCache bitsetFilterCache() { - return bitsetFilterCache; - } + public IndexFieldDataService fieldData() { return indexFieldData; } public AnalysisService analysisService() { return this.analysisService; } public MapperService mapperService() { - return indexServicesProvider.getMapperService(); - } - - public IndexQueryParserService queryParserService() { - return indexServicesProvider.getQueryParserService(); + return mapperService; } public SimilarityService similarityService() { - return indexServicesProvider.getSimilarityService(); + return similarityService; } - public synchronized void close(final String reason, boolean delete) { + public synchronized void close(final String reason, boolean delete) throws IOException { if (closed.compareAndSet(false, true)) { deleted.compareAndSet(false, delete); - final Set shardIds = shardIds(); - for (final int shardId : shardIds) { - try { - removeShard(shardId, reason); - } catch (Throwable t) { - logger.warn("failed to close shard", t); + try { + final Set shardIds = shardIds(); + for (final int shardId : shardIds) { + try { + removeShard(shardId, reason); + } catch (Throwable t) { + logger.warn("failed to close shard", t); + } } + } finally { + IOUtils.close(bitsetFilterCache, indexCache, mapperService, indexFieldData, analysisService); } } } public String indexUUID() { - return indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); + return indexSettings.getUUID(); } // NOTE: O(numShards) cost, but numShards should be smallish? @@ -225,7 +218,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } - public synchronized IndexShard createShard(int sShardId, ShardRouting routing) { + public synchronized IndexShard createShard(ShardRouting routing) throws IOException { final boolean primary = routing.primary(); /* * TODO: we execute this in parallel but it's a synced method. Yet, we might @@ -233,25 +226,24 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone * keep it synced. */ if (closed.get()) { - throw new IllegalStateException("Can't create shard [" + index.name() + "][" + sShardId + "], closed"); + throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed"); } - final Settings indexSettings = settingsService.getSettings(); - final ShardId shardId = new ShardId(index, sShardId); - ShardLock lock = null; + final Settings indexSettings = this.indexSettings.getSettings(); + final ShardId shardId = routing.shardId(); boolean success = false; Store store = null; IndexShard indexShard = null; + final ShardLock lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5)); try { - lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5)); - indicesLifecycle.beforeIndexShardCreated(shardId, indexSettings); + eventListener.beforeIndexShardCreated(shardId, indexSettings); ShardPath path; try { - path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings); + path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings); } catch (IllegalStateException ex) { logger.warn("{} failed to load shard path, trying to remove leftover", shardId); try { - ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, indexSettings); - path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings); + ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings); + path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings); } catch (Throwable t) { t.addSuppressed(ex); throw t; @@ -271,7 +263,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } dataPathToShardCount.put(dataPath, curCount+1); } - path = ShardPath.selectNewPathForShard(nodeEnv, shardId, indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), + path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount); logger.debug("{} creating using a new path [{}]", shardId, path); } else { @@ -286,27 +278,23 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary. final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false || (primary && IndexMetaData.isOnSharedFilesystem(indexSettings)); - store = new Store(shardId, indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> indexServicesProvider.getIndicesQueryCache().onClose(shardId))); + store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> nodeServicesProvider.getIndicesQueryCache().onClose(shardId))); if (useShadowEngine(primary, indexSettings)) { - indexShard = new ShadowIndexShard(shardId, indexSettings, path, store, indexServicesProvider); + indexShard = new ShadowIndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, nodeServicesProvider); } else { - indexShard = new IndexShard(shardId, indexSettings, path, store, indexServicesProvider); + indexShard = new IndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, nodeServicesProvider); } - indicesLifecycle.indexShardStateChanged(indexShard, null, "shard created"); - indicesLifecycle.afterIndexShardCreated(indexShard); - settingsService.addListener(indexShard); + eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); + eventListener.afterIndexShardCreated(indexShard); + indexShard.updateRoutingEntry(routing, true); shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap(); success = true; return indexShard; - } catch (IOException e) { - ElasticsearchException ex = new ElasticsearchException("failed to create shard", e); - ex.setShard(shardId); - throw ex; } finally { if (success == false) { IOUtils.closeWhileHandlingException(lock); - closeShard("initialization failed", shardId, indexShard, store); + closeShard("initialization failed", shardId, indexShard, store, eventListener); } } } @@ -316,7 +304,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } public synchronized void removeShard(int shardId, String reason) { - final ShardId sId = new ShardId(index, shardId); + final ShardId sId = new ShardId(index(), shardId); final IndexShard indexShard; if (shards.containsKey(shardId) == false) { return; @@ -325,21 +313,20 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone HashMap newShards = new HashMap<>(shards); indexShard = newShards.remove(shardId); shards = unmodifiableMap(newShards); - closeShard(reason, sId, indexShard, indexShard.store()); + closeShard(reason, sId, indexShard, indexShard.store(), indexShard.getIndexEventListener()); logger.debug("[{}] closed (reason: [{}])", shardId, reason); } - private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store store) { + private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store store, IndexEventListener listener) { final int shardId = sId.id(); - final Settings indexSettings = settingsService.getSettings(); + final Settings indexSettings = this.getIndexSettings().getSettings(); try { try { - indicesLifecycle.beforeIndexShardClosed(sId, indexShard, indexSettings); + listener.beforeIndexShardClosed(sId, indexShard, indexSettings); } finally { // this logic is tricky, we want to close the engine so we rollback the changes done to it // and close the shard so no operations are allowed to it if (indexShard != null) { - settingsService.removeListener(indexShard); try { final boolean flushEngine = deleted.get() == false && closed.get(); // only flush we are we closed (closed index or shutdown) and if we are not deleted indexShard.close(reason, flushEngine); @@ -349,7 +336,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } // call this before we close the store, so we can release resources for it - indicesLifecycle.afterIndexShardClosed(sId, indexShard, indexSettings); + listener.afterIndexShardClosed(sId, indexShard, indexSettings); } } finally { try { @@ -363,25 +350,32 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone private void onShardClose(ShardLock lock, boolean ownsShard) { if (deleted.get()) { // we remove that shards content if this index has been deleted - final Settings indexSettings = settingsService.getSettings(); try { if (ownsShard) { try { - indicesLifecycle.beforeIndexShardDeleted(lock.getShardId(), indexSettings); + eventListener.beforeIndexShardDeleted(lock.getShardId(), indexSettings.getSettings()); } finally { - indicesServices.deleteShardStore("delete index", lock, indexSettings); - indicesLifecycle.afterIndexShardDeleted(lock.getShardId(), indexSettings); + shardStoreDeleter.deleteShardStore("delete index", lock, indexSettings); + eventListener.afterIndexShardDeleted(lock.getShardId(), indexSettings.getSettings()); } } } catch (IOException e) { - indicesServices.addPendingDelete(lock.getShardId(), indexSettings); + shardStoreDeleter.addPendingDelete(lock.getShardId(), indexSettings); logger.debug("[{}] failed to delete shard content - scheduled a retry", e, lock.getShardId().id()); } } } - public IndexServicesProvider getIndexServices() { - return indexServicesProvider; + public NodeServicesProvider getIndexServices() { + return nodeServicesProvider; + } + + public IndexSettings getIndexSettings() { + return indexSettings; + } + + public QueryShardContext getQueryShardContext() { + return new QueryShardContext(indexSettings, nodeServicesProvider.getClient(), indexCache.bitsetFilterCache(), indexFieldData, mapperService(), similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry()); } private class StoreCloseListener implements Store.OnClose { @@ -411,10 +405,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } - public Settings getIndexSettings() { - return settingsService.getSettings(); - } - private static final class BitsetCacheListener implements BitsetFilterCache.Listener { final IndexService indexService; @@ -478,19 +468,18 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone * The list of filtering aliases should be obtained by calling MetaData.filteringAliases. * Returns null if no filtering is required.

    */ - public Query aliasFilter(String... aliasNames) { + public Query aliasFilter(QueryShardContext context, String... aliasNames) { if (aliasNames == null || aliasNames.length == 0) { return null; } - final IndexQueryParserService indexQueryParser = queryParserService(); - final ImmutableOpenMap aliases = this.indexMetaData.getAliases(); + final ImmutableOpenMap aliases = indexSettings.getIndexMetaData().getAliases(); if (aliasNames.length == 1) { AliasMetaData alias = aliases.get(aliasNames[0]); if (alias == null) { // This shouldn't happen unless alias disappeared after filteringAliases was called. - throw new InvalidAliasNameException(index, aliasNames[0], "Unknown alias name was passed to alias Filter"); + throw new InvalidAliasNameException(index(), aliasNames[0], "Unknown alias name was passed to alias Filter"); } - return parse(alias, indexQueryParser); + return parse(alias, context); } else { // we need to bench here a bit, to see maybe it makes sense to use OrFilter BooleanQuery.Builder combined = new BooleanQuery.Builder(); @@ -498,9 +487,9 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone AliasMetaData alias = aliases.get(aliasName); if (alias == null) { // This shouldn't happen unless alias disappeared after filteringAliases was called. - throw new InvalidAliasNameException(indexQueryParser.index(), aliasNames[0], "Unknown alias name was passed to alias Filter"); + throw new InvalidAliasNameException(indexSettings.getIndex(), aliasNames[0], "Unknown alias name was passed to alias Filter"); } - Query parsedFilter = parse(alias, indexQueryParser); + Query parsedFilter = parse(alias, context); if (parsedFilter != null) { combined.add(parsedFilter, BooleanClause.Occur.SHOULD); } else { @@ -512,27 +501,60 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } - private Query parse(AliasMetaData alias, IndexQueryParserService indexQueryParser) { + private Query parse(AliasMetaData alias, QueryShardContext parseContext) { if (alias.filter() == null) { return null; } try { byte[] filterSource = alias.filter().uncompressed(); try (XContentParser parser = XContentFactory.xContent(filterSource).createParser(filterSource)) { - ParsedQuery parsedFilter = indexQueryParser.parseInnerFilter(parser); + ParsedQuery parsedFilter = parseContext.parseInnerFilter(parser); return parsedFilter == null ? null : parsedFilter.query(); } } catch (IOException ex) { - throw new AliasFilterParsingException(indexQueryParser.index(), alias.getAlias(), "Invalid alias filter", ex); + throw new AliasFilterParsingException(parseContext.index(), alias.getAlias(), "Invalid alias filter", ex); } } public IndexMetaData getMetaData() { - return indexMetaData; + return indexSettings.getIndexMetaData(); } - public void updateMetaData(IndexMetaData metadata) { - this.indexMetaData = metadata; + public synchronized void updateMetaData(final IndexMetaData metadata) { + if (indexSettings.updateIndexMetaData(metadata)) { + final Settings settings = indexSettings.getSettings(); + for (final IndexShard shard : this.shards.values()) { + try { + shard.onRefreshSettings(settings); + } catch (Exception e) { + logger.warn("[{}] failed to refresh shard settings", e, shard.shardId().id()); + } + } + try { + indexStore.onRefreshSettings(settings); + } catch (Exception e) { + logger.warn("failed to refresh index store settings", e); + } + } } + public interface ShardStoreDeleter { + void deleteShardStore(String reason, ShardLock lock, IndexSettings indexSettings) throws IOException; + + void addPendingDelete(ShardId shardId, IndexSettings indexSettings); + } + + final EngineFactory getEngineFactory() { + return engineFactory; + } // pkg private for testing + + final IndexSearcherWrapper getSearcherWrapper() { + return searcherWrapper; + } // pkg private for testing + + final IndexStore getIndexStore() { + return indexStore; + } // pkg private for testing + + } diff --git a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java deleted file mode 100644 index 53b9f061abf..00000000000 --- a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.cache.IndexCache; -import org.elasticsearch.index.codec.CodecService; -import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; -import org.elasticsearch.index.shard.IndexSearcherWrapper; -import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.index.termvectors.TermVectorsService; -import org.elasticsearch.indices.IndicesLifecycle; -import org.elasticsearch.indices.IndicesWarmer; -import org.elasticsearch.indices.cache.query.IndicesQueryCache; -import org.elasticsearch.indices.memory.IndexingMemoryController; -import org.elasticsearch.threadpool.ThreadPool; - -/** - * Simple provider class that holds the Index and Node level services used by - * a shard. - * This is just a temporary solution until we cleaned up index creation and removed injectors on that level as well. - */ -public final class IndexServicesProvider { - - private final IndicesLifecycle indicesLifecycle; - private final ThreadPool threadPool; - private final MapperService mapperService; - private final IndexQueryParserService queryParserService; - private final IndexCache indexCache; - private final IndicesQueryCache indicesQueryCache; - private final CodecService codecService; - private final TermVectorsService termVectorsService; - private final IndexFieldDataService indexFieldDataService; - private final IndicesWarmer warmer; - private final SimilarityService similarityService; - private final EngineFactory factory; - private final BigArrays bigArrays; - private final IndexSearcherWrapper indexSearcherWrapper; - private final IndexingMemoryController indexingMemoryController; - - @Inject - public IndexServicesProvider(IndicesLifecycle indicesLifecycle, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays, @Nullable IndexSearcherWrapper indexSearcherWrapper, IndexingMemoryController indexingMemoryController) { - this.indicesLifecycle = indicesLifecycle; - this.threadPool = threadPool; - this.mapperService = mapperService; - this.queryParserService = queryParserService; - this.indexCache = indexCache; - this.indicesQueryCache = indicesQueryCache; - this.codecService = codecService; - this.termVectorsService = termVectorsService; - this.indexFieldDataService = indexFieldDataService; - this.warmer = warmer; - this.similarityService = similarityService; - this.factory = factory; - this.bigArrays = bigArrays; - this.indexSearcherWrapper = indexSearcherWrapper; - this.indexingMemoryController = indexingMemoryController; - } - - public IndicesLifecycle getIndicesLifecycle() { - return indicesLifecycle; - } - - public ThreadPool getThreadPool() { - return threadPool; - } - - public MapperService getMapperService() { - return mapperService; - } - - public IndexQueryParserService getQueryParserService() { - return queryParserService; - } - - public IndexCache getIndexCache() { - return indexCache; - } - - public IndicesQueryCache getIndicesQueryCache() { - return indicesQueryCache; - } - - public CodecService getCodecService() { - return codecService; - } - - public TermVectorsService getTermVectorsService() { - return termVectorsService; - } - - public IndexFieldDataService getIndexFieldDataService() { - return indexFieldDataService; - } - - public IndicesWarmer getWarmer() { - return warmer; - } - - public SimilarityService getSimilarityService() { - return similarityService; - } - - public EngineFactory getFactory() { - return factory; - } - - public BigArrays getBigArrays() { - return bigArrays; - } - - public IndexSearcherWrapper getIndexSearcherWrapper() { - return indexSearcherWrapper; - } - - public IndexingMemoryController getIndexingMemoryController() { - return indexingMemoryController; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java new file mode 100644 index 00000000000..f0e06ea0bc7 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -0,0 +1,307 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.internal.AllFieldMapper; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Predicate; + +/** + * This class encapsulates all index level settings and handles settings updates. + * It's created per index and available to all index level classes and allows them to retrieve + * the latest updated settings instance. Classes that need to listen to settings updates can register + * a settings consumer at index creation via {@link IndexModule#addIndexSettingsListener(Consumer)} that will + * be called for each settings update. + */ +public final class IndexSettings { + + public static final String DEFAULT_FIELD = "index.query.default_field"; + public static final String QUERY_STRING_LENIENT = "index.query_string.lenient"; + public static final String QUERY_STRING_ANALYZE_WILDCARD = "indices.query.query_string.analyze_wildcard"; + public static final String QUERY_STRING_ALLOW_LEADING_WILDCARD = "indices.query.query_string.allowLeadingWildcard"; + public static final String ALLOW_UNMAPPED = "index.query.parse.allow_unmapped_fields"; + private final String uuid; + private final List> updateListeners; + private final Index index; + private final Version version; + private final ESLogger logger; + private final String nodeName; + private final Settings nodeSettings; + private final int numberOfShards; + private final boolean isShadowReplicaIndex; + private final ParseFieldMatcher parseFieldMatcher; + // volatile fields are updated via #updateIndexMetaData(IndexMetaData) under lock + private volatile Settings settings; + private volatile IndexMetaData indexMetaData; + private final String defaultField; + private final boolean queryStringLenient; + private final boolean queryStringAnalyzeWildcard; + private final boolean queryStringAllowLeadingWildcard; + private final boolean defaultAllowUnmappedFields; + private final Predicate indexNameMatcher; + + /** + * Returns the default search field for this index. + */ + public String getDefaultField() { + return defaultField; + } + + /** + * Returns true if query string parsing should be lenient. The default is false + */ + public boolean isQueryStringLenient() { + return queryStringLenient; + } + + /** + * Returns true if the query string should analyze wildcards. The default is false + */ + public boolean isQueryStringAnalyzeWildcard() { + return queryStringAnalyzeWildcard; + } + + /** + * Returns true if the query string parser should allow leading wildcards. The default is true + */ + public boolean isQueryStringAllowLeadingWildcard() { + return queryStringAllowLeadingWildcard; + } + + /** + * Returns true if queries should be lenient about unmapped fields. The default is true + */ + public boolean isDefaultAllowUnmappedFields() { + return defaultAllowUnmappedFields; + } + + /** + * Creates a new {@link IndexSettings} instance. The given node settings will be merged with the settings in the metadata + * while index level settings will overwrite node settings. + * + * @param indexMetaData the index metadata this settings object is associated with + * @param nodeSettings the nodes settings this index is allocated on. + * @param updateListeners a collection of listeners / consumers that should be notified if one or more settings are updated + */ + public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSettings, final Collection> updateListeners) { + this(indexMetaData, nodeSettings, updateListeners, (index) -> Regex.simpleMatch(index, indexMetaData.getIndex())); + } + + /** + * Creates a new {@link IndexSettings} instance. The given node settings will be merged with the settings in the metadata + * while index level settings will overwrite node settings. + * + * @param indexMetaData the index metadata this settings object is associated with + * @param nodeSettings the nodes settings this index is allocated on. + * @param updateListeners a collection of listeners / consumers that should be notified if one or more settings are updated + * @param indexNameMatcher a matcher that can resolve an expression to the index name or index alias + */ + public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSettings, final Collection> updateListeners, final Predicate indexNameMatcher) { + this.nodeSettings = nodeSettings; + this.settings = Settings.builder().put(nodeSettings).put(indexMetaData.getSettings()).build(); + this.updateListeners = Collections.unmodifiableList(new ArrayList<>(updateListeners)); + this.index = new Index(indexMetaData.getIndex()); + version = Version.indexCreated(settings); + uuid = settings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); + logger = Loggers.getLogger(getClass(), settings, index); + nodeName = settings.get("name", ""); + this.indexMetaData = indexMetaData; + numberOfShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null); + isShadowReplicaIndex = IndexMetaData.isIndexUsingShadowReplicas(settings); + + this.defaultField = settings.get(DEFAULT_FIELD, AllFieldMapper.NAME); + this.queryStringLenient = settings.getAsBoolean(QUERY_STRING_LENIENT, false); + this.queryStringAnalyzeWildcard = settings.getAsBoolean(QUERY_STRING_ANALYZE_WILDCARD, false); + this.queryStringAllowLeadingWildcard = settings.getAsBoolean(QUERY_STRING_ALLOW_LEADING_WILDCARD, true); + this.parseFieldMatcher = new ParseFieldMatcher(settings); + this.defaultAllowUnmappedFields = settings.getAsBoolean(ALLOW_UNMAPPED, true); + this.indexNameMatcher = indexNameMatcher; + assert indexNameMatcher.test(indexMetaData.getIndex()); + } + + + /** + * Creates a new {@link IndexSettings} instance adding the given listeners to the settings + */ + IndexSettings newWithListener(final Collection> updateListeners) { + ArrayList> newUpdateListeners = new ArrayList<>(updateListeners); + newUpdateListeners.addAll(this.updateListeners); + return new IndexSettings(indexMetaData, nodeSettings, newUpdateListeners, indexNameMatcher); + } + + /** + * Returns the settings for this index. These settings contain the node and index level settings where + * settings that are specified on both index and node level are overwritten by the index settings. + */ + public Settings getSettings() { return settings; } + + /** + * Returns the index this settings object belongs to + */ + public Index getIndex() { + return index; + } + + /** + * Returns the indexes UUID + */ + public String getUUID() { + return uuid; + } + + /** + * Returns true if the index has a custom data path + */ + public boolean hasCustomDataPath() { + return customDataPath() != null; + } + + /** + * Returns the customDataPath for this index, if configured. null o.w. + */ + public String customDataPath() { + return settings.get(IndexMetaData.SETTING_DATA_PATH); + } + + /** + * Returns true iff the given settings indicate that the index + * associated with these settings allocates it's shards on a shared + * filesystem. + */ + public boolean isOnSharedFilesystem() { + return IndexMetaData.isOnSharedFilesystem(getSettings()); + } + + /** + * Returns true iff the given settings indicate that the index associated + * with these settings uses shadow replicas. Otherwise false. The default + * setting for this is false. + */ + public boolean isIndexUsingShadowReplicas() { + return IndexMetaData.isOnSharedFilesystem(getSettings()); + } + + /** + * Returns the version the index was created on. + * @see Version#indexCreated(Settings) + */ + public Version getIndexVersionCreated() { + return version; + } + + /** + * Returns the current node name + */ + public String getNodeName() { + return nodeName; + } + + /** + * Returns the current IndexMetaData for this index + */ + public IndexMetaData getIndexMetaData() { + return indexMetaData; + } + + /** + * Returns the number of shards this index has. + */ + public int getNumberOfShards() { return numberOfShards; } + + /** + * Returns the number of replicas this index has. + */ + public int getNumberOfReplicas() { return settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, null); } + + /** + * Returns true iff this index uses shadow replicas. + * @see IndexMetaData#isIndexUsingShadowReplicas(Settings) + */ + public boolean isShadowReplicaIndex() { return isShadowReplicaIndex; } + + /** + * Returns the node settings. The settings retured from {@link #getSettings()} are a merged version of the + * index settings and the node settings where node settings are overwritten by index settings. + */ + public Settings getNodeSettings() { + return nodeSettings; + } + + /** + * Returns a {@link ParseFieldMatcher} for this index. + */ + public ParseFieldMatcher getParseFieldMatcher() { return parseFieldMatcher; } + + /** + * Returns true if the given expression matches the index name or one of it's aliases + */ + public boolean matchesIndexName(String expression) { + return indexNameMatcher.test(expression); + } + + /** + * Updates the settings and index metadata and notifies all registered settings consumers with the new settings iff at least one setting has changed. + * + * @return true iff any setting has been updated otherwise false. + */ + synchronized boolean updateIndexMetaData(IndexMetaData indexMetaData) { + final Settings newSettings = indexMetaData.getSettings(); + if (Version.indexCreated(newSettings) != version) { + throw new IllegalArgumentException("version mismatch on settings update expected: " + version + " but was: " + Version.indexCreated(newSettings)); + } + final String newUUID = newSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); + if (newUUID.equals(getUUID()) == false) { + throw new IllegalArgumentException("uuid mismatch on settings update expected: " + uuid + " but was: " + newUUID); + } + this.indexMetaData = indexMetaData; + final Settings existingSettings = this.settings; + if (existingSettings.getByPrefix(IndexMetaData.INDEX_SETTING_PREFIX).getAsMap().equals(newSettings.getByPrefix(IndexMetaData.INDEX_SETTING_PREFIX).getAsMap())) { + // nothing to update, same settings + return false; + } + final Settings mergedSettings = this.settings = Settings.builder().put(nodeSettings).put(newSettings).build(); + for (final Consumer consumer : updateListeners) { + try { + consumer.accept(mergedSettings); + } catch (Exception e) { + logger.warn("failed to refresh index settings for [{}]", e, mergedSettings); + } + } + return true; + } + + /** + * Returns all settings update consumers + */ + List> getUpdateListeners() { // for testing + return updateListeners; + } +} diff --git a/core/src/main/java/org/elasticsearch/index/NodeServicesProvider.java b/core/src/main/java/org/elasticsearch/index/NodeServicesProvider.java new file mode 100644 index 00000000000..84e93b219e3 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/NodeServicesProvider.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.termvectors.TermVectorsService; +import org.elasticsearch.indices.IndicesWarmer; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.cache.query.IndicesQueryCache; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.indices.memory.IndexingMemoryController; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; + +/** + * Simple provider class that holds the Index and Node level services used by + * a shard. + * This is just a temporary solution until we cleaned up index creation and removed injectors on that level as well. + */ +public final class NodeServicesProvider { + + private final ThreadPool threadPool; + private final IndicesQueryCache indicesQueryCache; + private final TermVectorsService termVectorsService; + private final IndicesWarmer warmer; + private final BigArrays bigArrays; + private final Client client; + private final IndicesQueriesRegistry indicesQueriesRegistry; + private final ScriptService scriptService; + private final IndicesFieldDataCache indicesFieldDataCache; + private final CircuitBreakerService circuitBreakerService; + private final IndexingMemoryController indexingMemoryController; + + @Inject + public NodeServicesProvider(ThreadPool threadPool, IndicesQueryCache indicesQueryCache, TermVectorsService termVectorsService, @Nullable IndicesWarmer warmer, BigArrays bigArrays, Client client, ScriptService scriptService, IndicesQueriesRegistry indicesQueriesRegistry, IndicesFieldDataCache indicesFieldDataCache, CircuitBreakerService circuitBreakerService, IndexingMemoryController indexingMemoryController) { + this.threadPool = threadPool; + this.indicesQueryCache = indicesQueryCache; + this.termVectorsService = termVectorsService; + this.warmer = warmer; + this.bigArrays = bigArrays; + this.client = client; + this.indicesQueriesRegistry = indicesQueriesRegistry; + this.scriptService = scriptService; + this.indicesFieldDataCache = indicesFieldDataCache; + this.circuitBreakerService = circuitBreakerService; + this.indexingMemoryController = indexingMemoryController; + } + + public ThreadPool getThreadPool() { + return threadPool; + } + + public IndicesQueryCache getIndicesQueryCache() { + return indicesQueryCache; + } + + public TermVectorsService getTermVectorsService() { + return termVectorsService; + } + + public IndicesWarmer getWarmer() { + return warmer; + } + + public BigArrays getBigArrays() { return bigArrays; } + + public Client getClient() { + return client; + } + + public IndicesQueriesRegistry getIndicesQueriesRegistry() { + return indicesQueriesRegistry; + } + + public ScriptService getScriptService() { + return scriptService; + } + + public IndicesFieldDataCache getIndicesFieldDataCache() { + return indicesFieldDataCache; + } + + public CircuitBreakerService getCircuitBreakerService() { + return circuitBreakerService; + } + + public IndexingMemoryController getIndexingMemoryController() { + return indexingMemoryController; + } +} diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java index 81862618c2f..4638a37de17 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java @@ -21,11 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** * Factory for ASCIIFoldingFilter. @@ -33,9 +31,8 @@ import org.elasticsearch.index.settings.IndexSettings; public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory { private final boolean preserveOriginal; - @Inject - public ASCIIFoldingTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public ASCIIFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); preserveOriginal = settings.getAsBoolean("preserve_original", false); } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AbstractCharFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/AbstractCharFilterFactory.java index 29ff6822d02..2e728386ab8 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AbstractCharFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AbstractCharFilterFactory.java @@ -19,10 +19,8 @@ package org.elasticsearch.index.analysis; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -31,8 +29,8 @@ public abstract class AbstractCharFilterFactory extends AbstractIndexComponent i private final String name; - public AbstractCharFilterFactory(Index index, @IndexSettings Settings indexSettings, String name) { - super(index, indexSettings); + public AbstractCharFilterFactory(IndexSettings indexSettings, String name) { + super(indexSettings); this.name = name; } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AbstractIndexAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/AbstractIndexAnalyzerProvider.java index ce050ca2a55..c0406cb806e 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AbstractIndexAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AbstractIndexAnalyzerProvider.java @@ -23,8 +23,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -38,14 +37,13 @@ public abstract class AbstractIndexAnalyzerProvider extends /** * Constructs a new analyzer component, with the index name and its settings and the analyzer name. * - * @param index The index name - * @param indexSettings The index settings + * @param indexSettings the settings and the name of the index * @param name The analyzer name */ - public AbstractIndexAnalyzerProvider(Index index, @IndexSettings Settings indexSettings, String name, Settings settings) { - super(index, indexSettings); + public AbstractIndexAnalyzerProvider(IndexSettings indexSettings, String name, Settings settings) { + super(indexSettings); this.name = name; - this.version = Analysis.parseAnalysisVersion(indexSettings, settings, logger); + this.version = Analysis.parseAnalysisVersion(this.indexSettings.getSettings(), settings, logger); } /** diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java index f4efeda4d94..16096ca8f3f 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java @@ -22,8 +22,7 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.util.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -34,10 +33,10 @@ public abstract class AbstractTokenFilterFactory extends AbstractIndexComponent protected final Version version; - public AbstractTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, String name, Settings settings) { - super(index, indexSettings); + public AbstractTokenFilterFactory(IndexSettings indexSettings, String name, Settings settings) { + super(indexSettings); this.name = name; - this.version = Analysis.parseAnalysisVersion(indexSettings, settings, logger); + this.version = Analysis.parseAnalysisVersion(this.indexSettings.getSettings(), settings, logger); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java index 94c80bdd763..dafc4b87730 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java @@ -22,8 +22,7 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.util.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,10 +34,10 @@ public abstract class AbstractTokenizerFactory extends AbstractIndexComponent im protected final Version version; - public AbstractTokenizerFactory(Index index, @IndexSettings Settings indexSettings, String name, Settings settings) { - super(index, indexSettings); + public AbstractTokenizerFactory(IndexSettings indexSettings, String name, Settings settings) { + super(indexSettings); this.name = name; - this.version = Analysis.parseAnalysisVersion(indexSettings, settings, logger); + this.version = Analysis.parseAnalysisVersion(this.indexSettings.getSettings(), settings, logger); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java index 861f0705b38..a2c65c6441d 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java @@ -63,21 +63,13 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.settings.IndexSettings; import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; import static java.util.Collections.unmodifiableMap; @@ -86,7 +78,7 @@ import static java.util.Collections.unmodifiableMap; */ public class Analysis { - public static Version parseAnalysisVersion(@IndexSettings Settings indexSettings, Settings settings, ESLogger logger) { + public static Version parseAnalysisVersion(Settings indexSettings, Settings settings, ESLogger logger) { // check for explicit version on the specific analyzer component String sVersion = settings.get("version"); if (sVersion != null) { @@ -243,8 +235,8 @@ public class Analysis { try (BufferedReader reader = FileSystemUtils.newBufferedReader(wordListFile.toUri().toURL(), StandardCharsets.UTF_8)) { return loadWordList(reader, "#"); } catch (IOException ioe) { - String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage()); - throw new IllegalArgumentException(message); + String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix); + throw new IllegalArgumentException(message, ioe); } } @@ -290,8 +282,8 @@ public class Analysis { try { return FileSystemUtils.newBufferedReader(path.toUri().toURL(), StandardCharsets.UTF_8); } catch (IOException ioe) { - String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage()); - throw new IllegalArgumentException(message); + String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix); + throw new IllegalArgumentException(message, ioe); } } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java deleted file mode 100644 index bfa71eefe13..00000000000 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java +++ /dev/null @@ -1,507 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.Scopes; -import org.elasticsearch.common.inject.assistedinject.FactoryProvider; -import org.elasticsearch.common.inject.multibindings.MapBinder; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; -import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.Objects; - -/** - * - */ -public class AnalysisModule extends AbstractModule { - - public static class AnalysisBinderProcessor { - - public void processCharFilters(CharFiltersBindings charFiltersBindings) { - - } - - public static class CharFiltersBindings { - private final Map> charFilters = new HashMap<>(); - - public CharFiltersBindings() { - } - - public void processCharFilter(String name, Class charFilterFactory) { - charFilters.put(name, charFilterFactory); - } - } - - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - - } - - public static class TokenFiltersBindings { - private final Map> tokenFilters = new HashMap<>(); - - public TokenFiltersBindings() { - } - - public void processTokenFilter(String name, Class tokenFilterFactory) { - tokenFilters.put(name, tokenFilterFactory); - } - } - - public void processTokenizers(TokenizersBindings tokenizersBindings) { - - } - - public static class TokenizersBindings { - private final Map> tokenizers = new HashMap<>(); - - public TokenizersBindings() { - } - - public void processTokenizer(String name, Class tokenizerFactory) { - tokenizers.put(name, tokenizerFactory); - } - } - - public void processAnalyzers(AnalyzersBindings analyzersBindings) { - - } - - public static class AnalyzersBindings { - private final Map> analyzers = new HashMap<>(); - - public AnalyzersBindings() { - } - - public void processAnalyzer(String name, Class analyzerProvider) { - analyzers.put(name, analyzerProvider); - } - } - } - - private final Settings settings; - - private final IndicesAnalysisService indicesAnalysisService; - - private final LinkedList processors = new LinkedList<>(); - - private final Map> charFilters = new HashMap<>(); - private final Map> tokenFilters = new HashMap<>(); - private final Map> tokenizers = new HashMap<>(); - private final Map> analyzers = new HashMap<>(); - - public AnalysisModule(Settings settings, IndicesAnalysisService indicesAnalysisService) { - Objects.requireNonNull(indicesAnalysisService); - this.settings = settings; - this.indicesAnalysisService = indicesAnalysisService; - processors.add(new DefaultProcessor()); - try { - processors.add(new ExtendedProcessor()); - } catch (Throwable t) { - // ignore. no extended ones - } - } - - public AnalysisModule addProcessor(AnalysisBinderProcessor processor) { - processors.addFirst(processor); - return this; - } - - public AnalysisModule addCharFilter(String name, Class charFilter) { - charFilters.put(name, charFilter); - return this; - } - - public AnalysisModule addTokenFilter(String name, Class tokenFilter) { - tokenFilters.put(name, tokenFilter); - return this; - } - - public AnalysisModule addTokenizer(String name, Class tokenizer) { - tokenizers.put(name, tokenizer); - return this; - } - - public AnalysisModule addAnalyzer(String name, Class analyzer) { - analyzers.put(name, analyzer); - return this; - } - - @Override - protected void configure() { - MapBinder charFilterBinder - = MapBinder.newMapBinder(binder(), String.class, CharFilterFactoryFactory.class); - - // CHAR FILTERS - - AnalysisBinderProcessor.CharFiltersBindings charFiltersBindings = new AnalysisBinderProcessor.CharFiltersBindings(); - for (AnalysisBinderProcessor processor : processors) { - processor.processCharFilters(charFiltersBindings); - } - charFiltersBindings.charFilters.putAll(charFilters); - - Map charFiltersSettings = settings.getGroups("index.analysis.char_filter"); - for (Map.Entry entry : charFiltersSettings.entrySet()) { - String charFilterName = entry.getKey(); - Settings charFilterSettings = entry.getValue(); - - String typeName = charFilterSettings.get("type"); - if (typeName == null) { - throw new IllegalArgumentException("CharFilter [" + charFilterName + "] must have a type associated with it"); - } - Class type = charFiltersBindings.charFilters.get(typeName); - if (type == null) { - throw new IllegalArgumentException("Unknown CharFilter type [" + typeName + "] for [" + charFilterName + "]"); - } - charFilterBinder.addBinding(charFilterName).toProvider(FactoryProvider.newFactory(CharFilterFactoryFactory.class, type)).in(Scopes.SINGLETON); - } - // go over the char filters in the bindings and register the ones that are not configured - for (Map.Entry> entry : charFiltersBindings.charFilters.entrySet()) { - String charFilterName = entry.getKey(); - Class clazz = entry.getValue(); - // we don't want to re-register one that already exists - if (charFiltersSettings.containsKey(charFilterName)) { - continue; - } - // check, if it requires settings, then don't register it, we know default has no settings... - if (clazz.getAnnotation(AnalysisSettingsRequired.class) != null) { - continue; - } - // register if it's not builtin - if (indicesAnalysisService.hasCharFilter(charFilterName) == false) { - charFilterBinder.addBinding(charFilterName).toProvider(FactoryProvider.newFactory(CharFilterFactoryFactory.class, clazz)).in(Scopes.SINGLETON); - } - } - - - // TOKEN FILTERS - - MapBinder tokenFilterBinder - = MapBinder.newMapBinder(binder(), String.class, TokenFilterFactoryFactory.class); - - // initial default bindings - AnalysisBinderProcessor.TokenFiltersBindings tokenFiltersBindings = new AnalysisBinderProcessor.TokenFiltersBindings(); - for (AnalysisBinderProcessor processor : processors) { - processor.processTokenFilters(tokenFiltersBindings); - } - tokenFiltersBindings.tokenFilters.putAll(tokenFilters); - - Map tokenFiltersSettings = settings.getGroups("index.analysis.filter"); - for (Map.Entry entry : tokenFiltersSettings.entrySet()) { - String tokenFilterName = entry.getKey(); - Settings tokenFilterSettings = entry.getValue(); - - String typeName = tokenFilterSettings.get("type"); - if (typeName == null) { - throw new IllegalArgumentException("TokenFilter [" + tokenFilterName + "] must have a type associated with it"); - } - Class type = tokenFiltersBindings.tokenFilters.get(typeName); - if (type == null) { - throw new IllegalArgumentException("Unknown TokenFilter type [" + typeName + "] for [" + tokenFilterName + "]"); - } - tokenFilterBinder.addBinding(tokenFilterName).toProvider(FactoryProvider.newFactory(TokenFilterFactoryFactory.class, type)).in(Scopes.SINGLETON); - } - // go over the filters in the bindings and register the ones that are not configured - for (Map.Entry> entry : tokenFiltersBindings.tokenFilters.entrySet()) { - String tokenFilterName = entry.getKey(); - Class clazz = entry.getValue(); - // we don't want to re-register one that already exists - if (tokenFiltersSettings.containsKey(tokenFilterName)) { - continue; - } - // check, if it requires settings, then don't register it, we know default has no settings... - if (clazz.getAnnotation(AnalysisSettingsRequired.class) != null) { - continue; - } - // register if it's not builtin - if (indicesAnalysisService.hasTokenFilter(tokenFilterName) == false) { - tokenFilterBinder.addBinding(tokenFilterName).toProvider(FactoryProvider.newFactory(TokenFilterFactoryFactory.class, clazz)).in(Scopes.SINGLETON); - } - } - - // TOKENIZER - - MapBinder tokenizerBinder - = MapBinder.newMapBinder(binder(), String.class, TokenizerFactoryFactory.class); - - // initial default bindings - AnalysisBinderProcessor.TokenizersBindings tokenizersBindings = new AnalysisBinderProcessor.TokenizersBindings(); - for (AnalysisBinderProcessor processor : processors) { - processor.processTokenizers(tokenizersBindings); - } - tokenizersBindings.tokenizers.putAll(tokenizers); - - Map tokenizersSettings = settings.getGroups("index.analysis.tokenizer"); - for (Map.Entry entry : tokenizersSettings.entrySet()) { - String tokenizerName = entry.getKey(); - Settings tokenizerSettings = entry.getValue(); - - String typeName = tokenizerSettings.get("type"); - if (typeName == null) { - throw new IllegalArgumentException("Tokenizer [" + tokenizerName + "] must have a type associated with it"); - } - Class type = tokenizersBindings.tokenizers.get(typeName); - if (type == null) { - throw new IllegalArgumentException("Unknown Tokenizer type [" + typeName + "] for [" + tokenizerName + "]"); - } - tokenizerBinder.addBinding(tokenizerName).toProvider(FactoryProvider.newFactory(TokenizerFactoryFactory.class, type)).in(Scopes.SINGLETON); - } - // go over the tokenizers in the bindings and register the ones that are not configured - for (Map.Entry> entry : tokenizersBindings.tokenizers.entrySet()) { - String tokenizerName = entry.getKey(); - Class clazz = entry.getValue(); - // we don't want to re-register one that already exists - if (tokenizersSettings.containsKey(tokenizerName)) { - continue; - } - // check, if it requires settings, then don't register it, we know default has no settings... - if (clazz.getAnnotation(AnalysisSettingsRequired.class) != null) { - continue; - } - // register if it's not builtin - if (indicesAnalysisService.hasTokenizer(tokenizerName) == false) { - tokenizerBinder.addBinding(tokenizerName).toProvider(FactoryProvider.newFactory(TokenizerFactoryFactory.class, clazz)).in(Scopes.SINGLETON); - } - } - - // ANALYZER - - MapBinder analyzerBinder - = MapBinder.newMapBinder(binder(), String.class, AnalyzerProviderFactory.class); - - // initial default bindings - AnalysisBinderProcessor.AnalyzersBindings analyzersBindings = new AnalysisBinderProcessor.AnalyzersBindings(); - for (AnalysisBinderProcessor processor : processors) { - processor.processAnalyzers(analyzersBindings); - } - analyzersBindings.analyzers.putAll(analyzers); - - Map analyzersSettings = settings.getGroups("index.analysis.analyzer"); - for (Map.Entry entry : analyzersSettings.entrySet()) { - String analyzerName = entry.getKey(); - Settings analyzerSettings = entry.getValue(); - - String typeName = analyzerSettings.get("type"); - Class type; - if (typeName == null) { - if (analyzerSettings.get("tokenizer") != null) { - // custom analyzer, need to add it - type = CustomAnalyzerProvider.class; - } else { - throw new IllegalArgumentException("Analyzer [" + analyzerName + "] must have a type associated with it"); - } - } else if (typeName.equals("custom")) { - type = CustomAnalyzerProvider.class; - } else { - type = analyzersBindings.analyzers.get(typeName); - if (type == null) { - throw new IllegalArgumentException("Unknown Analyzer type [" + typeName + "] for [" + analyzerName + "]"); - } - } - - analyzerBinder.addBinding(analyzerName).toProvider(FactoryProvider.newFactory(AnalyzerProviderFactory.class, type)).in(Scopes.SINGLETON); - } - - // go over the analyzers in the bindings and register the ones that are not configured - for (Map.Entry> entry : analyzersBindings.analyzers.entrySet()) { - String analyzerName = entry.getKey(); - Class clazz = entry.getValue(); - // we don't want to re-register one that already exists - if (analyzersSettings.containsKey(analyzerName)) { - continue; - } - // check, if it requires settings, then don't register it, we know default has no settings... - if (clazz.getAnnotation(AnalysisSettingsRequired.class) != null) { - continue; - } - // register if it's not builtin - if (indicesAnalysisService.hasAnalyzer(analyzerName) == false) { - analyzerBinder.addBinding(analyzerName).toProvider(FactoryProvider.newFactory(AnalyzerProviderFactory.class, clazz)).in(Scopes.SINGLETON); - } - } - - bind(AnalysisService.class).in(Scopes.SINGLETON); - } - - private static class DefaultProcessor extends AnalysisBinderProcessor { - - @Override - public void processCharFilters(CharFiltersBindings charFiltersBindings) { - charFiltersBindings.processCharFilter("html_strip", HtmlStripCharFilterFactory.class); - charFiltersBindings.processCharFilter("pattern_replace", PatternReplaceCharFilterFactory.class); - } - - @Override - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - tokenFiltersBindings.processTokenFilter("stop", StopTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("reverse", ReverseTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("asciifolding", ASCIIFoldingTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("length", LengthTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("lowercase", LowerCaseTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("uppercase", UpperCaseTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("porter_stem", PorterStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("kstem", KStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("standard", StandardTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("nGram", NGramTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("ngram", NGramTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("edgeNGram", EdgeNGramTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("edge_ngram", EdgeNGramTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("shingle", ShingleTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("unique", UniqueTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("truncate", TruncateTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("trim", TrimTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("limit", LimitTokenCountFilterFactory.class); - tokenFiltersBindings.processTokenFilter("common_grams", CommonGramsTokenFilterFactory.class); - } - - @Override - public void processTokenizers(TokenizersBindings tokenizersBindings) { - tokenizersBindings.processTokenizer("standard", StandardTokenizerFactory.class); - tokenizersBindings.processTokenizer("uax_url_email", UAX29URLEmailTokenizerFactory.class); - tokenizersBindings.processTokenizer("path_hierarchy", PathHierarchyTokenizerFactory.class); - tokenizersBindings.processTokenizer("keyword", KeywordTokenizerFactory.class); - tokenizersBindings.processTokenizer("letter", LetterTokenizerFactory.class); - tokenizersBindings.processTokenizer("lowercase", LowerCaseTokenizerFactory.class); - tokenizersBindings.processTokenizer("whitespace", WhitespaceTokenizerFactory.class); - - tokenizersBindings.processTokenizer("nGram", NGramTokenizerFactory.class); - tokenizersBindings.processTokenizer("ngram", NGramTokenizerFactory.class); - tokenizersBindings.processTokenizer("edgeNGram", EdgeNGramTokenizerFactory.class); - tokenizersBindings.processTokenizer("edge_ngram", EdgeNGramTokenizerFactory.class); - } - - @Override - public void processAnalyzers(AnalyzersBindings analyzersBindings) { - analyzersBindings.processAnalyzer("default", StandardAnalyzerProvider.class); - analyzersBindings.processAnalyzer("standard", StandardAnalyzerProvider.class); - analyzersBindings.processAnalyzer("standard_html_strip", StandardHtmlStripAnalyzerProvider.class); - analyzersBindings.processAnalyzer("simple", SimpleAnalyzerProvider.class); - analyzersBindings.processAnalyzer("stop", StopAnalyzerProvider.class); - analyzersBindings.processAnalyzer("whitespace", WhitespaceAnalyzerProvider.class); - analyzersBindings.processAnalyzer("keyword", KeywordAnalyzerProvider.class); - } - } - - private static class ExtendedProcessor extends AnalysisBinderProcessor { - @Override - public void processCharFilters(CharFiltersBindings charFiltersBindings) { - charFiltersBindings.processCharFilter("mapping", MappingCharFilterFactory.class); - } - - @Override - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - tokenFiltersBindings.processTokenFilter("snowball", SnowballTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("stemmer", StemmerTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("word_delimiter", WordDelimiterTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("delimited_payload_filter", DelimitedPayloadTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("synonym", SynonymTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("elision", ElisionTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("keep", KeepWordFilterFactory.class); - tokenFiltersBindings.processTokenFilter("keep_types", KeepTypesFilterFactory.class); - - tokenFiltersBindings.processTokenFilter("pattern_capture", PatternCaptureGroupTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("pattern_replace", PatternReplaceTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("dictionary_decompounder", DictionaryCompoundWordTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("hyphenation_decompounder", HyphenationCompoundWordTokenFilterFactory.class); - - tokenFiltersBindings.processTokenFilter("arabic_stem", ArabicStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("brazilian_stem", BrazilianStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("czech_stem", CzechStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("dutch_stem", DutchStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("french_stem", FrenchStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("german_stem", GermanStemTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("russian_stem", RussianStemTokenFilterFactory.class); - - tokenFiltersBindings.processTokenFilter("keyword_marker", KeywordMarkerTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("stemmer_override", StemmerOverrideTokenFilterFactory.class); - - tokenFiltersBindings.processTokenFilter("arabic_normalization", ArabicNormalizationFilterFactory.class); - tokenFiltersBindings.processTokenFilter("german_normalization", GermanNormalizationFilterFactory.class); - tokenFiltersBindings.processTokenFilter("hindi_normalization", HindiNormalizationFilterFactory.class); - tokenFiltersBindings.processTokenFilter("indic_normalization", IndicNormalizationFilterFactory.class); - tokenFiltersBindings.processTokenFilter("sorani_normalization", SoraniNormalizationFilterFactory.class); - tokenFiltersBindings.processTokenFilter("persian_normalization", PersianNormalizationFilterFactory.class); - tokenFiltersBindings.processTokenFilter("scandinavian_normalization", ScandinavianNormalizationFilterFactory.class); - tokenFiltersBindings.processTokenFilter("scandinavian_folding", ScandinavianFoldingFilterFactory.class); - tokenFiltersBindings.processTokenFilter("serbian_normalization", SerbianNormalizationFilterFactory.class); - - tokenFiltersBindings.processTokenFilter("hunspell", HunspellTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("cjk_bigram", CJKBigramFilterFactory.class); - tokenFiltersBindings.processTokenFilter("cjk_width", CJKWidthFilterFactory.class); - - tokenFiltersBindings.processTokenFilter("apostrophe", ApostropheFilterFactory.class); - tokenFiltersBindings.processTokenFilter("classic", ClassicFilterFactory.class); - tokenFiltersBindings.processTokenFilter("decimal_digit", DecimalDigitFilterFactory.class); - } - - @Override - public void processTokenizers(TokenizersBindings tokenizersBindings) { - tokenizersBindings.processTokenizer("pattern", PatternTokenizerFactory.class); - tokenizersBindings.processTokenizer("classic", ClassicTokenizerFactory.class); - tokenizersBindings.processTokenizer("thai", ThaiTokenizerFactory.class); - } - - @Override - public void processAnalyzers(AnalyzersBindings analyzersBindings) { - analyzersBindings.processAnalyzer("pattern", PatternAnalyzerProvider.class); - analyzersBindings.processAnalyzer("snowball", SnowballAnalyzerProvider.class); - - analyzersBindings.processAnalyzer("arabic", ArabicAnalyzerProvider.class); - analyzersBindings.processAnalyzer("armenian", ArmenianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("basque", BasqueAnalyzerProvider.class); - analyzersBindings.processAnalyzer("brazilian", BrazilianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("bulgarian", BulgarianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("catalan", CatalanAnalyzerProvider.class); - analyzersBindings.processAnalyzer("chinese", ChineseAnalyzerProvider.class); - analyzersBindings.processAnalyzer("cjk", CjkAnalyzerProvider.class); - analyzersBindings.processAnalyzer("czech", CzechAnalyzerProvider.class); - analyzersBindings.processAnalyzer("danish", DanishAnalyzerProvider.class); - analyzersBindings.processAnalyzer("dutch", DutchAnalyzerProvider.class); - analyzersBindings.processAnalyzer("english", EnglishAnalyzerProvider.class); - analyzersBindings.processAnalyzer("finnish", FinnishAnalyzerProvider.class); - analyzersBindings.processAnalyzer("french", FrenchAnalyzerProvider.class); - analyzersBindings.processAnalyzer("galician", GalicianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("german", GermanAnalyzerProvider.class); - analyzersBindings.processAnalyzer("greek", GreekAnalyzerProvider.class); - analyzersBindings.processAnalyzer("hindi", HindiAnalyzerProvider.class); - analyzersBindings.processAnalyzer("hungarian", HungarianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("indonesian", IndonesianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("irish", IrishAnalyzerProvider.class); - analyzersBindings.processAnalyzer("italian", ItalianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("latvian", LatvianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("lithuanian", LithuanianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("norwegian", NorwegianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("persian", PersianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("portuguese", PortugueseAnalyzerProvider.class); - analyzersBindings.processAnalyzer("romanian", RomanianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("russian", RussianAnalyzerProvider.class); - analyzersBindings.processAnalyzer("sorani", SoraniAnalyzerProvider.class); - analyzersBindings.processAnalyzer("spanish", SpanishAnalyzerProvider.class); - analyzersBindings.processAnalyzer("swedish", SwedishAnalyzerProvider.class); - analyzersBindings.processAnalyzer("turkish", TurkishAnalyzerProvider.class); - analyzersBindings.processAnalyzer("thai", ThaiAnalyzerProvider.class); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java new file mode 100644 index 00000000000..86c06dbe54f --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -0,0 +1,461 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; +import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; +import org.elasticsearch.indices.analysis.*; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * An internal registry for tokenizer, token filter, char filter and analyzer. + * This class exists per node and allows to create per-index {@link AnalysisService} via {@link #build(IndexSettings)} + */ +public final class AnalysisRegistry implements Closeable { + private final Map> charFilters; + private final Map> tokenFilters; + private final Map> tokenizers; + private final Map> analyzers; + private final Map cachedAnalyzer = new ConcurrentHashMap<>(); + private final PrebuiltAnalysis prebuiltAnalysis; + private final HunspellService hunspellService; + private final Environment environemnt; + + public AnalysisRegistry(HunspellService hunspellService, Environment environment) { + this(hunspellService, environment, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + } + + public AnalysisRegistry(HunspellService hunspellService, Environment environment, + Map> charFilters, + Map> tokenFilters, + Map> tokenizers, + Map> analyzers) { + prebuiltAnalysis = new PrebuiltAnalysis(); + this.hunspellService = hunspellService; + this.environemnt = environment; + final Map> charFilterBuilder = new HashMap<>(charFilters); + final Map> tokenFilterBuilder = new HashMap<>(tokenFilters); + final Map> tokenizerBuilder = new HashMap<>(tokenizers); + final Map> analyzerBuilder= new HashMap<>(analyzers); + registerBuiltInAnalyzer(analyzerBuilder); + registerBuiltInCharFilter(charFilterBuilder); + registerBuiltInTokenizer(tokenizerBuilder); + registerBuiltInTokenFilters(tokenFilterBuilder); + this.tokenFilters = Collections.unmodifiableMap(tokenFilterBuilder); + this.tokenizers = Collections.unmodifiableMap(tokenizerBuilder); + this.charFilters = Collections.unmodifiableMap(charFilterBuilder); + this.analyzers = Collections.unmodifiableMap(analyzerBuilder); + } + + /** + * Returns a registered {@link TokenizerFactory} provider by name or null if the tokenizer was not registered + */ + public AnalysisModule.AnalysisProvider getTokenizerProvider(String tokenizer) { + return tokenizers.getOrDefault(tokenizer, this.prebuiltAnalysis.getTokenizerFactory(tokenizer)); + } + + /** + * Returns a registered {@link TokenFilterFactory} provider by name or null if the token filter was not registered + */ + public AnalysisModule.AnalysisProvider getTokenFilterProvider(String tokenFilter) { + return tokenFilters.getOrDefault(tokenFilter, this.prebuiltAnalysis.getTokenFilterFactory(tokenFilter)); + } + + /** + * Returns a registered {@link CharFilterFactory} provider by name or null if the char filter was not registered + */ + public AnalysisModule.AnalysisProvider getCharFilterProvider(String charFilter) { + return charFilters.getOrDefault(charFilter, this.prebuiltAnalysis.getCharFilterFactory(charFilter)); + } + + /** + * Returns a registered {@link Analyzer} provider by name or null if the analyzer was not registered + */ + public Analyzer getAnalyzer(String analyzer) throws IOException { + AnalysisModule.AnalysisProvider analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer); + if (analyzerProvider == null) { + AnalysisModule.AnalysisProvider provider = analyzers.get(analyzer); + return provider == null ? null : cachedAnalyzer.computeIfAbsent(analyzer, (key) -> { + try { + return provider.get(environemnt, key).get(); + } catch (IOException ex) { + throw new ElasticsearchException("failed to load analyzer for name " + key, ex); + }} + ); + } + return analyzerProvider.get(environemnt, analyzer).get(); + } + + @Override + public void close() throws IOException { + try { + prebuiltAnalysis.close(); + } finally { + IOUtils.close(cachedAnalyzer.values()); + } + } + + /** + * Creates an index-level {@link AnalysisService} from this registry using the given index settings + */ + public AnalysisService build(IndexSettings indexSettings) throws IOException { + final Map charFiltersSettings = indexSettings.getSettings().getGroups("index.analysis.char_filter"); + final Map tokenFiltersSettings = indexSettings.getSettings().getGroups("index.analysis.filter"); + final Map tokenizersSettings = indexSettings.getSettings().getGroups("index.analysis.tokenizer"); + final Map analyzersSettings = indexSettings.getSettings().getGroups("index.analysis.analyzer"); + + final Map charFilterFactories = buildMapping(false, "charfilter", indexSettings, charFiltersSettings, charFilters, prebuiltAnalysis.charFilterFactories); + final Map tokenizerFactories = buildMapping(false, "tokenizer", indexSettings, tokenizersSettings, tokenizers, prebuiltAnalysis.tokenizerFactories); + + Map> tokenFilters = new HashMap<>(this.tokenFilters); + /* + * synonym is different than everything else since it needs access to the tokenizer factories for this index. + * instead of building the infrastructure for plugins we rather make it a real exception to not pollute the general interface and + * hide internal data-structures as much as possible. + */ + tokenFilters.put("synonym", requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, tokenizerFactories, name, settings))); + final Map tokenFilterFactories = buildMapping(false, "tokenfilter", indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories); + final Map analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories); + return new AnalysisService(indexSettings, analyzierFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories); + } + + + private static AnalysisModule.AnalysisProvider requriesAnalysisSettings(AnalysisModule.AnalysisProvider provider) { + return new AnalysisModule.AnalysisProvider() { + @Override + public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { + return provider.get(indexSettings, environment, name, settings); + } + @Override + public boolean requiresAnalysisSettings() { + return true; + } + }; + } + + private void registerBuiltInCharFilter(Map> charFilters) { + charFilters.put("html_strip", HtmlStripCharFilterFactory::new); + charFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new)); + charFilters.put("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new)); + } + + private void registerBuiltInTokenizer(Map> tokenizers) { + tokenizers.put("standard", StandardTokenizerFactory::new); + tokenizers.put("uax_url_email", UAX29URLEmailTokenizerFactory::new); + tokenizers.put("path_hierarchy", PathHierarchyTokenizerFactory::new); + tokenizers.put("keyword", KeywordTokenizerFactory::new); + tokenizers.put("letter", LetterTokenizerFactory::new); + tokenizers.put("lowercase", LowerCaseTokenizerFactory::new); + tokenizers.put("whitespace", WhitespaceTokenizerFactory::new); + tokenizers.put("nGram", NGramTokenizerFactory::new); + tokenizers.put("ngram", NGramTokenizerFactory::new); + tokenizers.put("edgeNGram", EdgeNGramTokenizerFactory::new); + tokenizers.put("edge_ngram", EdgeNGramTokenizerFactory::new); + tokenizers.put("pattern", PatternTokenizerFactory::new); + tokenizers.put("classic", ClassicTokenizerFactory::new); + tokenizers.put("thai", ThaiTokenizerFactory::new); + } + + private void registerBuiltInTokenFilters(Map> tokenFilters) { + tokenFilters.put("stop", StopTokenFilterFactory::new); + tokenFilters.put("reverse", ReverseTokenFilterFactory::new); + tokenFilters.put("asciifolding", ASCIIFoldingTokenFilterFactory::new); + tokenFilters.put("length", LengthTokenFilterFactory::new); + tokenFilters.put("lowercase", LowerCaseTokenFilterFactory::new); + tokenFilters.put("uppercase", UpperCaseTokenFilterFactory::new); + tokenFilters.put("porter_stem", PorterStemTokenFilterFactory::new); + tokenFilters.put("kstem", KStemTokenFilterFactory::new); + tokenFilters.put("standard", StandardTokenFilterFactory::new); + tokenFilters.put("nGram", NGramTokenFilterFactory::new); + tokenFilters.put("ngram", NGramTokenFilterFactory::new); + tokenFilters.put("edgeNGram", EdgeNGramTokenFilterFactory::new); + tokenFilters.put("edge_ngram", EdgeNGramTokenFilterFactory::new); + tokenFilters.put("shingle", ShingleTokenFilterFactory::new); + tokenFilters.put("unique", UniqueTokenFilterFactory::new); + tokenFilters.put("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new)); + tokenFilters.put("trim", TrimTokenFilterFactory::new); + tokenFilters.put("limit", LimitTokenCountFilterFactory::new); + tokenFilters.put("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new)); + tokenFilters.put("snowball", SnowballTokenFilterFactory::new); + tokenFilters.put("stemmer", StemmerTokenFilterFactory::new); + tokenFilters.put("word_delimiter", WordDelimiterTokenFilterFactory::new); + tokenFilters.put("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new); + tokenFilters.put("elision", ElisionTokenFilterFactory::new); + tokenFilters.put("keep", requriesAnalysisSettings(KeepWordFilterFactory::new)); + tokenFilters.put("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new)); + tokenFilters.put("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new)); + tokenFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new)); + tokenFilters.put("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new)); + tokenFilters.put("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new)); + tokenFilters.put("arabic_stem", ArabicStemTokenFilterFactory::new); + tokenFilters.put("brazilian_stem", BrazilianStemTokenFilterFactory::new); + tokenFilters.put("czech_stem", CzechStemTokenFilterFactory::new); + tokenFilters.put("dutch_stem", DutchStemTokenFilterFactory::new); + tokenFilters.put("french_stem", FrenchStemTokenFilterFactory::new); + tokenFilters.put("german_stem", GermanStemTokenFilterFactory::new); + tokenFilters.put("russian_stem", RussianStemTokenFilterFactory::new); + tokenFilters.put("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new)); + tokenFilters.put("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new)); + tokenFilters.put("arabic_normalization", ArabicNormalizationFilterFactory::new); + tokenFilters.put("german_normalization", GermanNormalizationFilterFactory::new); + tokenFilters.put("hindi_normalization", HindiNormalizationFilterFactory::new); + tokenFilters.put("indic_normalization", IndicNormalizationFilterFactory::new); + tokenFilters.put("sorani_normalization", SoraniNormalizationFilterFactory::new); + tokenFilters.put("persian_normalization", PersianNormalizationFilterFactory::new); + tokenFilters.put("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new); + tokenFilters.put("scandinavian_folding", ScandinavianFoldingFilterFactory::new); + tokenFilters.put("serbian_normalization", SerbianNormalizationFilterFactory::new); + + if (hunspellService != null) { + tokenFilters.put("hunspell", requriesAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService))); + } + tokenFilters.put("cjk_bigram", CJKBigramFilterFactory::new); + tokenFilters.put("cjk_width", CJKWidthFilterFactory::new); + + tokenFilters.put("apostrophe", ApostropheFilterFactory::new); + tokenFilters.put("classic", ClassicFilterFactory::new); + tokenFilters.put("decimal_digit", DecimalDigitFilterFactory::new); + } + + private void registerBuiltInAnalyzer(Map> analyzers) { + analyzers.put("default", StandardAnalyzerProvider::new); + analyzers.put("standard", StandardAnalyzerProvider::new); + analyzers.put("standard_html_strip", StandardHtmlStripAnalyzerProvider::new); + analyzers.put("simple", SimpleAnalyzerProvider::new); + analyzers.put("stop", StopAnalyzerProvider::new); + analyzers.put("whitespace", WhitespaceAnalyzerProvider::new); + analyzers.put("keyword", KeywordAnalyzerProvider::new); + analyzers.put("pattern", PatternAnalyzerProvider::new); + analyzers.put("snowball", SnowballAnalyzerProvider::new); + analyzers.put("arabic", ArabicAnalyzerProvider::new); + analyzers.put("armenian", ArmenianAnalyzerProvider::new); + analyzers.put("basque", BasqueAnalyzerProvider::new); + analyzers.put("brazilian", BrazilianAnalyzerProvider::new); + analyzers.put("bulgarian", BulgarianAnalyzerProvider::new); + analyzers.put("catalan", CatalanAnalyzerProvider::new); + analyzers.put("chinese", ChineseAnalyzerProvider::new); + analyzers.put("cjk", CjkAnalyzerProvider::new); + analyzers.put("czech", CzechAnalyzerProvider::new); + analyzers.put("danish", DanishAnalyzerProvider::new); + analyzers.put("dutch", DutchAnalyzerProvider::new); + analyzers.put("english", EnglishAnalyzerProvider::new); + analyzers.put("finnish", FinnishAnalyzerProvider::new); + analyzers.put("french", FrenchAnalyzerProvider::new); + analyzers.put("galician", GalicianAnalyzerProvider::new); + analyzers.put("german", GermanAnalyzerProvider::new); + analyzers.put("greek", GreekAnalyzerProvider::new); + analyzers.put("hindi", HindiAnalyzerProvider::new); + analyzers.put("hungarian", HungarianAnalyzerProvider::new); + analyzers.put("indonesian", IndonesianAnalyzerProvider::new); + analyzers.put("irish", IrishAnalyzerProvider::new); + analyzers.put("italian", ItalianAnalyzerProvider::new); + analyzers.put("latvian", LatvianAnalyzerProvider::new); + analyzers.put("lithuanian", LithuanianAnalyzerProvider::new); + analyzers.put("norwegian", NorwegianAnalyzerProvider::new); + analyzers.put("persian", PersianAnalyzerProvider::new); + analyzers.put("portuguese", PortugueseAnalyzerProvider::new); + analyzers.put("romanian", RomanianAnalyzerProvider::new); + analyzers.put("russian", RussianAnalyzerProvider::new); + analyzers.put("sorani", SoraniAnalyzerProvider::new); + analyzers.put("spanish", SpanishAnalyzerProvider::new); + analyzers.put("swedish", SwedishAnalyzerProvider::new); + analyzers.put("turkish", TurkishAnalyzerProvider::new); + analyzers.put("thai", ThaiAnalyzerProvider::new); + } + + private Map buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map settingsMap, Map> providerMap, Map> defaultInstance) throws IOException { + Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build(); + Map factories = new HashMap<>(); + for (Map.Entry entry : settingsMap.entrySet()) { + String name = entry.getKey(); + Settings currentSettings = entry.getValue(); + String typeName = currentSettings.get("type"); + if (analyzer) { + T factory; + if (typeName == null) { + if (currentSettings.get("tokenizer") != null) { + factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings); + } else { + throw new IllegalArgumentException(toBuild + " [" + name + "] must have a type associated with it"); + } + } else if (typeName.equals("custom")) { + factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings); + } else { + AnalysisModule.AnalysisProvider type = providerMap.get(typeName); + if (type == null) { + throw new IllegalArgumentException("Unknown " + toBuild + " type [" + typeName + "] for [" + name + "]"); + } + factory = type.get(settings, environemnt, name, currentSettings); + } + factories.put(name, factory); + } else { + if (typeName == null) { + throw new IllegalArgumentException(toBuild + " [" + name + "] must have a type associated with it"); + } + AnalysisModule.AnalysisProvider type = providerMap.get(typeName); + if (type == null) { + throw new IllegalArgumentException("Unknown " + toBuild + " type [" + typeName + "] for [" + name + "]"); + } + final T factory = type.get(settings, environemnt, name, currentSettings); + factories.put(name, factory); + } + + } + // go over the char filters in the bindings and register the ones that are not configured + for (Map.Entry> entry : providerMap.entrySet()) { + String name = entry.getKey(); + AnalysisModule.AnalysisProvider provider = entry.getValue(); + // we don't want to re-register one that already exists + if (settingsMap.containsKey(name)) { + continue; + } + // check, if it requires settings, then don't register it, we know default has no settings... + if (provider.requiresAnalysisSettings()) { + continue; + } + AnalysisModule.AnalysisProvider defaultProvider = defaultInstance.get(name); + final T instance; + if (defaultProvider == null) { + instance = provider.get(settings, environemnt, name, defaultSettings); + } else { + instance = defaultProvider.get(settings, environemnt, name, defaultSettings); + } + factories.put(name, instance); + String camelCase = Strings.toCamelCase(name); + if (providerMap.containsKey(camelCase) == false && factories.containsKey(camelCase) == false) { + factories.put(camelCase, instance); + } + } + + for (Map.Entry> entry : defaultInstance.entrySet()) { + final String name = entry.getKey(); + final AnalysisModule.AnalysisProvider provider = entry.getValue(); + final String camelCase = Strings.toCamelCase(name); + if (factories.containsKey(name) == false || (defaultInstance.containsKey(camelCase) == false && factories.containsKey(camelCase) == false)) { + final T instance = provider.get(settings, environemnt, name, defaultSettings); + if (factories.containsKey(name) == false) { + factories.put(name, instance); + } + if ((defaultInstance.containsKey(camelCase) == false && factories.containsKey(camelCase) == false)) { + factories.put(camelCase, instance); + } + } + } + return factories; + } + + private static class PrebuiltAnalysis implements Closeable { + + final Map> analyzerProviderFactories; + final Map> tokenizerFactories; + final Map> tokenFilterFactories; + final Map> charFilterFactories; + + private PrebuiltAnalysis() { + Map analyzerProviderFactories = new HashMap<>(); + Map tokenizerFactories = new HashMap<>(); + Map tokenFilterFactories = new HashMap<>(); + Map charFilterFactories = new HashMap<>(); + // Analyzers + for (PreBuiltAnalyzers preBuiltAnalyzerEnum : PreBuiltAnalyzers.values()) { + String name = preBuiltAnalyzerEnum.name().toLowerCase(Locale.ROOT); + analyzerProviderFactories.put(name, new PreBuiltAnalyzerProviderFactory(name, AnalyzerScope.INDICES, preBuiltAnalyzerEnum.getAnalyzer(Version.CURRENT))); + } + + // Tokenizers + for (PreBuiltTokenizers preBuiltTokenizer : PreBuiltTokenizers.values()) { + String name = preBuiltTokenizer.name().toLowerCase(Locale.ROOT); + tokenizerFactories.put(name, new PreBuiltTokenizerFactoryFactory(preBuiltTokenizer.getTokenizerFactory(Version.CURRENT))); + } + + // Tokenizer aliases + tokenizerFactories.put("nGram", new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.NGRAM.getTokenizerFactory(Version.CURRENT))); + tokenizerFactories.put("edgeNGram", new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.EDGE_NGRAM.getTokenizerFactory(Version.CURRENT))); + + + // Token filters + for (PreBuiltTokenFilters preBuiltTokenFilter : PreBuiltTokenFilters.values()) { + String name = preBuiltTokenFilter.name().toLowerCase(Locale.ROOT); + tokenFilterFactories.put(name, new PreBuiltTokenFilterFactoryFactory(preBuiltTokenFilter.getTokenFilterFactory(Version.CURRENT))); + } + // Token filter aliases + tokenFilterFactories.put("nGram", new PreBuiltTokenFilterFactoryFactory(PreBuiltTokenFilters.NGRAM.getTokenFilterFactory(Version.CURRENT))); + tokenFilterFactories.put("edgeNGram", new PreBuiltTokenFilterFactoryFactory(PreBuiltTokenFilters.EDGE_NGRAM.getTokenFilterFactory(Version.CURRENT))); + + + // Char Filters + for (PreBuiltCharFilters preBuiltCharFilter : PreBuiltCharFilters.values()) { + String name = preBuiltCharFilter.name().toLowerCase(Locale.ROOT); + charFilterFactories.put(name, new PreBuiltCharFilterFactoryFactory(preBuiltCharFilter.getCharFilterFactory(Version.CURRENT))); + } + // Char filter aliases + charFilterFactories.put("htmlStrip", new PreBuiltCharFilterFactoryFactory(PreBuiltCharFilters.HTML_STRIP.getCharFilterFactory(Version.CURRENT))); + this.analyzerProviderFactories = Collections.unmodifiableMap(analyzerProviderFactories); + this.charFilterFactories = Collections.unmodifiableMap(charFilterFactories); + this.tokenFilterFactories = Collections.unmodifiableMap(tokenFilterFactories); + this.tokenizerFactories = Collections.unmodifiableMap(tokenizerFactories); + } + + public AnalysisModule.AnalysisProvider getCharFilterFactory(String name) { + return charFilterFactories.get(name); + } + + public AnalysisModule.AnalysisProvider getTokenFilterFactory(String name) { + return tokenFilterFactories.get(name); + } + + public AnalysisModule.AnalysisProvider getTokenizerFactory(String name) { + return tokenizerFactories.get(name); + } + + public AnalysisModule.AnalysisProvider getAnalyzerProvider(String name) { + return analyzerProviderFactories.get(name); + } + + Analyzer analyzer(String name) { + PreBuiltAnalyzerProviderFactory analyzerProviderFactory = (PreBuiltAnalyzerProviderFactory) analyzerProviderFactories.get(name); + if (analyzerProviderFactory == null) { + return null; + } + return analyzerProviderFactory.analyzer(); + } + + @Override + public void close() throws IOException { + IOUtils.close(analyzerProviderFactories.values().stream().map((a) -> ((PreBuiltAnalyzerProviderFactory)a).analyzer()).collect(Collectors.toList())); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java index ba8b1e083af..261add4e220 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java @@ -21,18 +21,16 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.core.StringFieldMapper; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; import java.io.Closeable; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -52,159 +50,19 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable private final NamedAnalyzer defaultSearchAnalyzer; private final NamedAnalyzer defaultSearchQuoteAnalyzer; - - public AnalysisService(Index index, Settings indexSettings) { - this(index, indexSettings, null, null, null, null, null); - } - - @Inject - public AnalysisService(Index index, @IndexSettings Settings indexSettings, @Nullable IndicesAnalysisService indicesAnalysisService, - @Nullable Map analyzerFactoryFactories, - @Nullable Map tokenizerFactoryFactories, - @Nullable Map charFilterFactoryFactories, - @Nullable Map tokenFilterFactoryFactories) { - super(index, indexSettings); - Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.indexCreated(indexSettings)).build(); - Map tokenizers = new HashMap<>(); - if (tokenizerFactoryFactories != null) { - Map tokenizersSettings = indexSettings.getGroups("index.analysis.tokenizer"); - for (Map.Entry entry : tokenizerFactoryFactories.entrySet()) { - String tokenizerName = entry.getKey(); - TokenizerFactoryFactory tokenizerFactoryFactory = entry.getValue(); - - Settings tokenizerSettings = tokenizersSettings.get(tokenizerName); - if (tokenizerSettings == null) { - tokenizerSettings = defaultSettings; - } - - TokenizerFactory tokenizerFactory = tokenizerFactoryFactory.create(tokenizerName, tokenizerSettings); - tokenizers.put(tokenizerName, tokenizerFactory); - tokenizers.put(Strings.toCamelCase(tokenizerName), tokenizerFactory); - } - } - - if (indicesAnalysisService != null) { - for (Map.Entry entry : indicesAnalysisService.tokenizerFactories().entrySet()) { - String name = entry.getKey(); - if (!tokenizers.containsKey(name)) { - tokenizers.put(name, entry.getValue().create(name, defaultSettings)); - } - name = Strings.toCamelCase(entry.getKey()); - if (!name.equals(entry.getKey())) { - if (!tokenizers.containsKey(name)) { - tokenizers.put(name, entry.getValue().create(name, defaultSettings)); - } - } - } - } - - this.tokenizers = unmodifiableMap(tokenizers); - - Map charFilters = new HashMap<>(); - if (charFilterFactoryFactories != null) { - Map charFiltersSettings = indexSettings.getGroups("index.analysis.char_filter"); - for (Map.Entry entry : charFilterFactoryFactories.entrySet()) { - String charFilterName = entry.getKey(); - CharFilterFactoryFactory charFilterFactoryFactory = entry.getValue(); - - Settings charFilterSettings = charFiltersSettings.get(charFilterName); - if (charFilterSettings == null) { - charFilterSettings = defaultSettings; - } - - CharFilterFactory tokenFilterFactory = charFilterFactoryFactory.create(charFilterName, charFilterSettings); - charFilters.put(charFilterName, tokenFilterFactory); - charFilters.put(Strings.toCamelCase(charFilterName), tokenFilterFactory); - } - } - - if (indicesAnalysisService != null) { - for (Map.Entry entry : indicesAnalysisService.charFilterFactories().entrySet()) { - String name = entry.getKey(); - if (!charFilters.containsKey(name)) { - charFilters.put(name, entry.getValue().create(name, defaultSettings)); - } - name = Strings.toCamelCase(entry.getKey()); - if (!name.equals(entry.getKey())) { - if (!charFilters.containsKey(name)) { - charFilters.put(name, entry.getValue().create(name, defaultSettings)); - } - } - } - } - - this.charFilters = unmodifiableMap(charFilters); - - Map tokenFilters = new HashMap<>(); - if (tokenFilterFactoryFactories != null) { - Map tokenFiltersSettings = indexSettings.getGroups("index.analysis.filter"); - for (Map.Entry entry : tokenFilterFactoryFactories.entrySet()) { - String tokenFilterName = entry.getKey(); - TokenFilterFactoryFactory tokenFilterFactoryFactory = entry.getValue(); - - Settings tokenFilterSettings = tokenFiltersSettings.get(tokenFilterName); - if (tokenFilterSettings == null) { - tokenFilterSettings = defaultSettings; - } - - TokenFilterFactory tokenFilterFactory = tokenFilterFactoryFactory.create(tokenFilterName, tokenFilterSettings); - tokenFilters.put(tokenFilterName, tokenFilterFactory); - tokenFilters.put(Strings.toCamelCase(tokenFilterName), tokenFilterFactory); - } - } - - // pre initialize the globally registered ones into the map - if (indicesAnalysisService != null) { - for (Map.Entry entry : indicesAnalysisService.tokenFilterFactories().entrySet()) { - String name = entry.getKey(); - if (!tokenFilters.containsKey(name)) { - tokenFilters.put(name, entry.getValue().create(name, defaultSettings)); - } - name = Strings.toCamelCase(entry.getKey()); - if (!name.equals(entry.getKey())) { - if (!tokenFilters.containsKey(name)) { - tokenFilters.put(name, entry.getValue().create(name, defaultSettings)); - } - } - } - } - this.tokenFilters = unmodifiableMap(tokenFilters); - - Map analyzerProviders = new HashMap<>(); - if (analyzerFactoryFactories != null) { - Map analyzersSettings = indexSettings.getGroups("index.analysis.analyzer"); - for (Map.Entry entry : analyzerFactoryFactories.entrySet()) { - String analyzerName = entry.getKey(); - AnalyzerProviderFactory analyzerFactoryFactory = entry.getValue(); - - Settings analyzerSettings = analyzersSettings.get(analyzerName); - if (analyzerSettings == null) { - analyzerSettings = defaultSettings; - } - - AnalyzerProvider analyzerFactory = analyzerFactoryFactory.create(analyzerName, analyzerSettings); - analyzerProviders.put(analyzerName, analyzerFactory); - } - } - if (indicesAnalysisService != null) { - for (Map.Entry entry : indicesAnalysisService.analyzerProviderFactories().entrySet()) { - String name = entry.getKey(); - Version indexVersion = Version.indexCreated(indexSettings); - if (!analyzerProviders.containsKey(name)) { - analyzerProviders.put(name, entry.getValue().create(name, Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build())); - } - String camelCaseName = Strings.toCamelCase(name); - if (!camelCaseName.equals(entry.getKey()) && !analyzerProviders.containsKey(camelCaseName)) { - analyzerProviders.put(camelCaseName, entry.getValue().create(name, Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build())); - } - } - } + public AnalysisService(IndexSettings indexSettings, + Map analyzerProviders, + Map tokenizerFactoryFactories, + Map charFilterFactoryFactories, + Map tokenFilterFactoryFactories) { + super(indexSettings); + this.tokenizers = unmodifiableMap(tokenizerFactoryFactories); + this.charFilters = unmodifiableMap(charFilterFactoryFactories); + this.tokenFilters = unmodifiableMap(tokenFilterFactoryFactories); + analyzerProviders = new HashMap<>(analyzerProviders); if (!analyzerProviders.containsKey("default")) { - analyzerProviders.put("default", new StandardAnalyzerProvider(index, indexSettings, null, "default", Settings.Builder.EMPTY_SETTINGS)); - } - if (!analyzerProviders.containsKey("default_index")) { - analyzerProviders.put("default_index", analyzerProviders.get("default")); + analyzerProviders.put("default", new StandardAnalyzerProvider(indexSettings, null, "default", Settings.Builder.EMPTY_SETTINGS)); } if (!analyzerProviders.containsKey("default_search")) { analyzerProviders.put("default_search", analyzerProviders.get("default")); @@ -214,14 +72,16 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable } Map analyzers = new HashMap<>(); - for (AnalyzerProvider analyzerFactory : analyzerProviders.values()) { + for (Map.Entry entry : analyzerProviders.entrySet()) { + AnalyzerProvider analyzerFactory = entry.getValue(); + String name = entry.getKey(); /* * Lucene defaults positionIncrementGap to 0 in all analyzers but * Elasticsearch defaults them to 0 only before version 2.0 * and 100 afterwards so we override the positionIncrementGap if it * doesn't match here. */ - int overridePositionIncrementGap = StringFieldMapper.Defaults.positionIncrementGap(Version.indexCreated(indexSettings)); + int overridePositionIncrementGap = StringFieldMapper.Defaults.positionIncrementGap(indexSettings.getIndexVersionCreated()); if (analyzerFactory instanceof CustomAnalyzerProvider) { ((CustomAnalyzerProvider) analyzerFactory).build(this); /* @@ -246,17 +106,19 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable analyzer = new NamedAnalyzer(analyzer, overridePositionIncrementGap); } } else { - analyzer = new NamedAnalyzer(analyzerFactory.name(), analyzerFactory.scope(), analyzerF, overridePositionIncrementGap); + analyzer = new NamedAnalyzer(name, analyzerFactory.scope(), analyzerF, overridePositionIncrementGap); } - analyzers.put(analyzerFactory.name(), analyzer); - analyzers.put(Strings.toCamelCase(analyzerFactory.name()), analyzer); - String strAliases = indexSettings.get("index.analysis.analyzer." + analyzerFactory.name() + ".alias"); + if (analyzers.containsKey(name)) { + throw new IllegalStateException("already registered analyzer with name: " + name); + } + analyzers.put(name, analyzer); + String strAliases = this.indexSettings.getSettings().get("index.analysis.analyzer." + analyzerFactory.name() + ".alias"); if (strAliases != null) { for (String alias : Strings.commaDelimitedListToStringArray(strAliases)) { analyzers.put(alias, analyzer); } } - String[] aliases = indexSettings.getAsArray("index.analysis.analyzer." + analyzerFactory.name() + ".alias"); + String[] aliases = this.indexSettings.getSettings().getAsArray("index.analysis.analyzer." + analyzerFactory.name() + ".alias"); for (String alias : aliases) { analyzers.put(alias, analyzer); } @@ -267,11 +129,11 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable throw new IllegalArgumentException("no default analyzer configured"); } if (analyzers.containsKey("default_index")) { - final Version createdVersion = Version.indexCreated(indexSettings); + final Version createdVersion = indexSettings.getIndexVersionCreated(); if (createdVersion.onOrAfter(Version.V_3_0_0)) { - throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use [index.analysis.analyzer.default] instead for index [" + index.getName() + "]"); + throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use [index.analysis.analyzer.default] instead for index [" + index().getName() + "]"); } else { - deprecationLogger.deprecated("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] instead for index [{}]", index.getName()); + deprecationLogger.deprecated("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] instead for index [{}]", index().getName()); } } defaultIndexAnalyzer = analyzers.containsKey("default_index") ? analyzers.get("default_index") : defaultAnalyzer; diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ApostropheFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ApostropheFilterFactory.java index 614e860735a..0ab84f7caf6 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/ApostropheFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/ApostropheFilterFactory.java @@ -20,20 +20,17 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tr.ApostropheFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** * Factory for {@link ApostropheFilter} */ public class ApostropheFilterFactory extends AbstractTokenFilterFactory { - @Inject - public ApostropheFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public ApostropheFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java index c532204b164..5a1754a02fe 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.ar.ArabicAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,9 +32,8 @@ public class ArabicAnalyzerProvider extends AbstractIndexAnalyzerProvider scripts = new HashSet<>(Arrays.asList("han", "hiragana", "katakana", "hangul")); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java index d1aa0c25fa3..d7b6ab02511 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java @@ -21,15 +21,14 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.cjk.CJKWidthFilter; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory { - @Inject - public CJKWidthFilterFactory(Index index, Settings indexSettings, String name, Settings settings) { - super(index, indexSettings, name, settings); + public CJKWidthFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java index 4b354f09a42..04c068a4371 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.ca.CatalanAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,9 +32,8 @@ public class CatalanAnalyzerProvider extends AbstractIndexAnalyzerProvider escapedTags; - @Inject - public HtmlStripCharFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name); + public HtmlStripCharFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name); String[] escapedTags = settings.getAsArray("escaped_tags"); if (escapedTags.length > 0) { this.escapedTags = unmodifiableSet(newHashSet(escapedTags)); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java index 8f6eef17b7f..751ef0094f6 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.hu.HungarianAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,9 +32,8 @@ public class HungarianAnalyzerProvider extends AbstractIndexAnalyzerProvider{@value #KEEP_TYPES_KEY} the array of words / tokens to keep.
    • * */ -@AnalysisSettingsRequired public class KeepTypesFilterFactory extends AbstractTokenFilterFactory { private final Set keepTypes; private static final String KEEP_TYPES_KEY = "types"; - @Inject - public KeepTypesFilterFactory(Index index, @IndexSettings Settings indexSettings, - Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KeepTypesFilterFactory(IndexSettings indexSettings, + Environment env, String name, Settings settings) { + super(indexSettings, name, settings); final String[] arrayKeepTypes = settings.getAsArray(KEEP_TYPES_KEY, null); if ((arrayKeepTypes == null)) { diff --git a/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java index 2c082288edc..56a62624af9 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java @@ -24,12 +24,9 @@ import org.apache.lucene.analysis.miscellaneous.KeepWordFilter; import org.apache.lucene.analysis.miscellaneous.Lucene43KeepWordFilter; import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.util.Version; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; /** * A {@link TokenFilterFactory} for {@link KeepWordFilter}. This filter only @@ -52,7 +49,6 @@ import org.elasticsearch.index.settings.IndexSettings; * * @see StopTokenFilterFactory */ -@AnalysisSettingsRequired public class KeepWordFilterFactory extends AbstractTokenFilterFactory { private final CharArraySet keepWords; private final boolean enablePositionIncrements; @@ -61,10 +57,9 @@ public class KeepWordFilterFactory extends AbstractTokenFilterFactory { private static final String KEEP_WORDS_CASE_KEY = KEEP_WORDS_KEY + "_case"; // for javadoc private static final String ENABLE_POS_INC_KEY = "enable_position_increments"; - @Inject - public KeepWordFilterFactory(Index index, @IndexSettings Settings indexSettings, - Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KeepWordFilterFactory(IndexSettings indexSettings, + Environment env, String name, Settings settings) { + super(indexSettings, name, settings); final String[] arrayKeepWords = settings.getAsArray(KEEP_WORDS_KEY, null); final String keepWordsPath = settings.get(KEEP_WORDS_PATH_KEY, null); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/KeywordAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/KeywordAnalyzerProvider.java index 2de249d7ecf..0bf134cb380 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/KeywordAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/KeywordAnalyzerProvider.java @@ -20,11 +20,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.core.KeywordAnalyzer; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** * @@ -33,9 +31,8 @@ public class KeywordAnalyzerProvider extends AbstractIndexAnalyzerProvider rules = Analysis.getWordSet(env, settings, "keywords"); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/KeywordTokenizerFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/KeywordTokenizerFactory.java index 44ed001c2d4..ac44f106f51 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/KeywordTokenizerFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/KeywordTokenizerFactory.java @@ -21,11 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** * @@ -34,9 +32,8 @@ public class KeywordTokenizerFactory extends AbstractTokenizerFactory { private final int bufferSize; - @Inject - public KeywordTokenizerFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KeywordTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); bufferSize = settings.getAsInt("buffer_size", 256); } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java index 236676e4b5b..01865c17d14 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.lv.LatvianAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,9 +32,8 @@ public class LatvianAnalyzerProvider extends AbstractIndexAnalyzerProvider rules = Analysis.getWordList(env, settings, "mappings"); if (rules == null) { diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenFilterFactory.java index 226c41e67a5..80e0aeb32eb 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenFilterFactory.java @@ -20,14 +20,12 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.ngram.NGramTokenFilter; import org.apache.lucene.analysis.ngram.Lucene43NGramTokenFilter; +import org.apache.lucene.analysis.ngram.NGramTokenFilter; import org.apache.lucene.util.Version; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** @@ -40,9 +38,8 @@ public class NGramTokenFilterFactory extends AbstractTokenFilterFactory { private final int maxGram; - @Inject - public NGramTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public NGramTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); this.minGram = settings.getAsInt("min_gram", NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE); this.maxGram = settings.getAsInt("max_gram", NGramTokenFilter.DEFAULT_MAX_NGRAM_SIZE); } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenizerFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenizerFactory.java index f1ad1d59a42..84da43497ac 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenizerFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/NGramTokenizerFactory.java @@ -23,11 +23,9 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.ngram.Lucene43NGramTokenizer; import org.apache.lucene.analysis.ngram.NGramTokenizer; import org.apache.lucene.util.Version; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -89,13 +87,12 @@ public class NGramTokenizerFactory extends AbstractTokenizerFactory { return builder.build(); } - @Inject - public NGramTokenizerFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public NGramTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); this.minGram = settings.getAsInt("min_gram", NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE); this.maxGram = settings.getAsInt("max_gram", NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE); this.matcher = parseTokenChars(settings.getAsArray("token_chars")); - this.esVersion = org.elasticsearch.Version.indexCreated(indexSettings); + this.esVersion = indexSettings.getIndexVersionCreated(); } @SuppressWarnings("deprecation") diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java index 7e1b303e4c6..1b136bfcef9 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.no.NorwegianAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,9 +32,8 @@ public class NorwegianAnalyzerProvider extends AbstractIndexAnalyzerProvider { private final PreBuiltAnalyzerProvider analyzerProvider; @@ -35,7 +40,6 @@ public class PreBuiltAnalyzerProviderFactory implements AnalyzerProviderFactory analyzerProvider = new PreBuiltAnalyzerProvider(name, scope, analyzer); } - @Override public AnalyzerProvider create(String name, Settings settings) { Version indexVersion = Version.indexCreated(settings); if (!Version.CURRENT.equals(indexVersion)) { @@ -49,6 +53,11 @@ public class PreBuiltAnalyzerProviderFactory implements AnalyzerProviderFactory return analyzerProvider; } + @Override + public AnalyzerProvider get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { + return create(name, settings); + } + public Analyzer analyzer() { return analyzerProvider.get(); } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactory.java index f88b904436b..62a8ff1ff3e 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactory.java @@ -21,9 +21,14 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.PreBuiltCharFilters; -public class PreBuiltCharFilterFactoryFactory implements CharFilterFactoryFactory { +import java.io.IOException; + +public class PreBuiltCharFilterFactoryFactory implements AnalysisModule.AnalysisProvider { private final CharFilterFactory charFilterFactory; @@ -32,7 +37,7 @@ public class PreBuiltCharFilterFactoryFactory implements CharFilterFactoryFactor } @Override - public CharFilterFactory create(String name, Settings settings) { + public CharFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { Version indexVersion = Version.indexCreated(settings); if (!Version.CURRENT.equals(indexVersion)) { PreBuiltCharFilters preBuiltCharFilters = PreBuiltCharFilters.getOrDefault(name, null); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactory.java index 4430204d47e..52c9f2851a2 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactory.java @@ -21,9 +21,14 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.PreBuiltTokenFilters; -public class PreBuiltTokenFilterFactoryFactory implements TokenFilterFactoryFactory { +import java.io.IOException; + +public class PreBuiltTokenFilterFactoryFactory implements AnalysisModule.AnalysisProvider { private final TokenFilterFactory tokenFilterFactory; @@ -32,7 +37,7 @@ public class PreBuiltTokenFilterFactoryFactory implements TokenFilterFactoryFact } @Override - public TokenFilterFactory create(String name, Settings settings) { + public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { Version indexVersion = Version.indexCreated(settings); if (!Version.CURRENT.equals(indexVersion)) { PreBuiltTokenFilters preBuiltTokenFilters = PreBuiltTokenFilters.getOrDefault(name, null); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactory.java index 64213f7d400..02218bd7ceb 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactory.java @@ -21,9 +21,14 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.PreBuiltTokenizers; -public class PreBuiltTokenizerFactoryFactory implements TokenizerFactoryFactory { +import java.io.IOException; + +public class PreBuiltTokenizerFactoryFactory implements AnalysisModule.AnalysisProvider { private final TokenizerFactory tokenizerFactory; @@ -31,8 +36,7 @@ public class PreBuiltTokenizerFactoryFactory implements TokenizerFactoryFactory this.tokenizerFactory = tokenizerFactory; } - @Override - public TokenizerFactory create(String name, Settings settings) { + public TokenizerFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { Version indexVersion = Version.indexCreated(settings); if (!Version.CURRENT.equals(indexVersion)) { PreBuiltTokenizers preBuiltTokenizers = PreBuiltTokenizers.getOrDefault(name, null); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ReverseTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ReverseTokenFilterFactory.java index 43842aed86f..8fad0a14c7f 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/ReverseTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/ReverseTokenFilterFactory.java @@ -21,20 +21,17 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.reverse.ReverseStringFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** * */ public class ReverseTokenFilterFactory extends AbstractTokenFilterFactory { - @Inject - public ReverseTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public ReverseTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java index 0cd1ab1b233..a455cef3ad1 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.ro.RomanianAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,9 +32,8 @@ public class RomanianAnalyzerProvider extends AbstractIndexAnalyzerProvider rules = Analysis.getWordList(env, settings, "rules"); if (rules == null) { diff --git a/core/src/main/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactory.java index 84ebe4087af..7f8b65676bf 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactory.java @@ -54,11 +54,9 @@ import org.apache.lucene.analysis.snowball.SnowballFilter; import org.apache.lucene.analysis.sv.SwedishLightStemFilter; import org.elasticsearch.Version; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; import org.tartarus.snowball.ext.*; /** @@ -67,15 +65,14 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory { private String language; - @Inject - public StemmerTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public StemmerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter"))); } @Override public TokenStream create(TokenStream tokenStream) { - final Version indexVersion = Version.indexCreated(indexSettings); + final Version indexVersion = indexSettings.getIndexVersionCreated(); if ("arabic".equalsIgnoreCase(language)) { return new ArabicStemFilter(tokenStream); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java index ed09590fa69..cb1c4b8f5c5 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.core.StopAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -35,9 +32,8 @@ public class StopAnalyzerProvider extends AbstractIndexAnalyzerProvider tokenizerFactories, - @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, Map tokenizerFactories, + String name, Settings settings) throws IOException { + super(indexSettings, name, settings); Reader rulesReader = null; if (settings.getAsArray("synonyms", null) != null) { @@ -70,17 +65,11 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { boolean expand = settings.getAsBoolean("expand", true); String tokenizerName = settings.get("tokenizer", "whitespace"); - - TokenizerFactoryFactory tokenizerFactoryFactory = tokenizerFactories.get(tokenizerName); - if (tokenizerFactoryFactory == null) { - tokenizerFactoryFactory = indicesAnalysisService.tokenizerFactoryFactory(tokenizerName); - } - if (tokenizerFactoryFactory == null) { + final TokenizerFactory tokenizerFactory = tokenizerFactories.get(tokenizerName); + if (tokenizerFactory == null) { throw new IllegalArgumentException("failed to find tokenizer [" + tokenizerName + "] for synonym token filter"); } - final TokenizerFactory tokenizerFactory = tokenizerFactoryFactory.create(tokenizerName, Settings.builder().put(indexSettings).put(settings).build()); - Analyzer analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java index 2b75ae8591d..cf4b9dbdb1e 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java @@ -20,12 +20,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.th.ThaiAnalyzer; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -34,9 +31,8 @@ public class ThaiAnalyzerProvider extends AbstractIndexAnalyzerProvider DIGIT diff --git a/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java index b9c296df22b..b67aebd5024 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java @@ -21,14 +21,11 @@ package org.elasticsearch.index.analysis.compound; import org.apache.lucene.analysis.compound.CompoundWordTokenFilterBase; import org.apache.lucene.analysis.util.CharArraySet; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.Analysis; -import org.elasticsearch.index.settings.IndexSettings; /** * Contains the common configuration settings between subclasses of this class. @@ -41,9 +38,8 @@ public abstract class AbstractCompoundWordTokenFilterFactory extends AbstractTok protected final boolean onlyLongestMatch; protected final CharArraySet wordList; - @Inject - public AbstractCompoundWordTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public AbstractCompoundWordTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); minWordSize = settings.getAsInt("min_word_size", CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE); minSubwordSize = settings.getAsInt("min_subword_size", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java index 55c1b4e3df1..d0388205b1b 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java @@ -23,14 +23,9 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter; import org.apache.lucene.analysis.compound.Lucene43DictionaryCompoundWordTokenFilter; import org.apache.lucene.util.Version; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.analysis.AnalysisSettingsRequired; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; /** @@ -38,12 +33,10 @@ import org.elasticsearch.index.settings.IndexSettings; * * @see org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter */ -@AnalysisSettingsRequired public class DictionaryCompoundWordTokenFilterFactory extends AbstractCompoundWordTokenFilterFactory { - @Inject - public DictionaryCompoundWordTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, env, name, settings); + public DictionaryCompoundWordTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, env, name, settings); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java index a0c7ef58dc2..841ca7966d9 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java @@ -24,17 +24,11 @@ import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter; import org.apache.lucene.analysis.compound.Lucene43HyphenationCompoundWordTokenFilter; import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree; import org.apache.lucene.util.Version; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.analysis.AnalysisSettingsRequired; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.xml.sax.InputSource; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -43,14 +37,12 @@ import java.nio.file.Path; * * @see org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter */ -@AnalysisSettingsRequired public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundWordTokenFilterFactory { private final HyphenationTree hyphenationTree; - @Inject - public HyphenationCompoundWordTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, env, name, settings); + public HyphenationCompoundWordTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, env, name, settings); String hyphenationPatternsPath = settings.get("hyphenation_patterns_path", null); if (hyphenationPatternsPath == null) { @@ -62,7 +54,7 @@ public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundW try { hyphenationTree = HyphenationCompoundWordTokenFilter.getHyphenationTree(new InputSource(Files.newInputStream(hyphenationPatternsFile))); } catch (Exception e) { - throw new IllegalArgumentException("Exception while reading hyphenation_patterns_path: " + e.getMessage()); + throw new IllegalArgumentException("Exception while reading hyphenation_patterns_path.", e); } } diff --git a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java index 0f2ace8c28b..61733f24695 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java @@ -20,13 +20,10 @@ package org.elasticsearch.index.cache; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.cache.query.QueryCache; -import org.elasticsearch.index.settings.IndexSettings; import java.io.Closeable; import java.io.IOException; @@ -39,9 +36,8 @@ public class IndexCache extends AbstractIndexComponent implements Closeable { private final QueryCache queryCache; private final BitsetFilterCache bitsetFilterCache; - @Inject - public IndexCache(Index index, @IndexSettings Settings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) { - super(index, indexSettings); + public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) { + super(indexSettings); this.queryCache = queryCache; this.bitsetFilterCache = bitsetFilterCache; } diff --git a/core/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java b/core/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java deleted file mode 100644 index 86e20490fa1..00000000000 --- a/core/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.ExtensionPoint; -import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.cache.query.QueryCache; -import org.elasticsearch.index.cache.query.index.IndexQueryCache; -import org.elasticsearch.index.cache.query.none.NoneQueryCache; - -public class IndexCacheModule extends AbstractModule { - - public static final String INDEX_QUERY_CACHE = "index"; - public static final String NONE_QUERY_CACHE = "none"; - public static final String QUERY_CACHE_TYPE = "index.queries.cache.type"; - // for test purposes only - public static final String QUERY_CACHE_EVERYTHING = "index.queries.cache.everything"; - - private final Settings indexSettings; - private final ExtensionPoint.SelectedType queryCaches; - - public IndexCacheModule(Settings settings) { - this.indexSettings = settings; - this.queryCaches = new ExtensionPoint.SelectedType<>("query_cache", QueryCache.class); - - registerQueryCache(INDEX_QUERY_CACHE, IndexQueryCache.class); - registerQueryCache(NONE_QUERY_CACHE, NoneQueryCache.class); - } - - public void registerQueryCache(String name, Class clazz) { - queryCaches.registerExtension(name, clazz); - } - - @Override - protected void configure() { - queryCaches.bindType(binder(), indexSettings, QUERY_CACHE_TYPE, INDEX_QUERY_CACHE); - bind(BitsetFilterCache.class).asEagerSingleton(); - bind(IndexCache.class).asEagerSingleton(); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index f2b7ba8e131..ceac3ca15c5 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -32,27 +32,23 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.BitSet; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.cache.RemovalListener; import org.elasticsearch.common.cache.RemovalNotification; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.IndicesWarmer.TerminationHandle; -import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; import java.io.IOException; @@ -70,54 +66,29 @@ import java.util.concurrent.Executor; * and require that it should always be around should use this cache, otherwise the * {@link org.elasticsearch.index.cache.query.QueryCache} should be used instead. */ -public class BitsetFilterCache extends AbstractIndexComponent implements LeafReader.CoreClosedListener, RemovalListener>, Closeable { +public final class BitsetFilterCache extends AbstractIndexComponent implements LeafReader.CoreClosedListener, RemovalListener>, Closeable { public static final String LOAD_RANDOM_ACCESS_FILTERS_EAGERLY = "index.load_fixed_bitset_filters_eagerly"; - private static final Listener DEFAULT_NOOP_LISTENER = new Listener() { - @Override - public void onCache(ShardId shardId, Accountable accountable) { - } - - @Override - public void onRemoval(ShardId shardId, Accountable accountable) { - } - }; private final boolean loadRandomAccessFiltersEagerly; private final Cache> loadedFilters; - private volatile Listener listener = DEFAULT_NOOP_LISTENER; + private final Listener listener; private final BitSetProducerWarmer warmer; + private final IndicesWarmer indicesWarmer; - private IndicesWarmer indicesWarmer; - - @Inject - public BitsetFilterCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings); - this.loadRandomAccessFiltersEagerly = indexSettings.getAsBoolean(LOAD_RANDOM_ACCESS_FILTERS_EAGERLY, true); - this.loadedFilters = CacheBuilder.>builder().removalListener(this).build(); - this.warmer = new BitSetProducerWarmer(); - } - - - @Inject(optional = true) - public void setIndicesWarmer(IndicesWarmer indicesWarmer) { - this.indicesWarmer = indicesWarmer; - indicesWarmer.addListener(warmer); - } - - /** - * Sets a listener that is invoked for all subsequent cache and removal events. - * @throws IllegalStateException if the listener is set more than once - */ - public void setListener(Listener listener) { + public BitsetFilterCache(IndexSettings indexSettings, IndicesWarmer indicesWarmer, Listener listener) { + super(indexSettings); if (listener == null) { throw new IllegalArgumentException("listener must not be null"); } - if (this.listener != DEFAULT_NOOP_LISTENER) { - throw new IllegalStateException("can't set listener more than once"); - } + this.loadRandomAccessFiltersEagerly = this.indexSettings.getSettings().getAsBoolean(LOAD_RANDOM_ACCESS_FILTERS_EAGERLY, true); + this.loadedFilters = CacheBuilder.>builder().removalListener(this).build(); + this.warmer = new BitSetProducerWarmer(); + this.indicesWarmer = indicesWarmer; + indicesWarmer.addListener(warmer); this.listener = listener; } + public BitSetProducer getBitSetProducer(Query query) { @@ -131,10 +102,11 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea @Override public void close() { - if (indicesWarmer != null) { + try { indicesWarmer.removeListener(warmer); + } finally { + clear("close"); } - clear("close"); } public void clear(String reason) { @@ -231,10 +203,10 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea } } - final class BitSetProducerWarmer extends IndicesWarmer.Listener { + final class BitSetProducerWarmer implements IndicesWarmer.Listener { @Override - public IndicesWarmer.TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, IndicesWarmer.WarmerContext context, ThreadPool threadPool) { + public IndicesWarmer.TerminationHandle warmNewReaders(final IndexShard indexShard, final Engine.Searcher searcher) { if (!loadRandomAccessFiltersEagerly) { return TerminationHandle.NO_WAIT; } @@ -260,9 +232,9 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea warmUp.add(Queries.newNonNestedFilter()); } - final Executor executor = threadPool.executor(executor()); - final CountDownLatch latch = new CountDownLatch(context.searcher().reader().leaves().size() * warmUp.size()); - for (final LeafReaderContext ctx : context.searcher().reader().leaves()) { + final Executor executor = indicesWarmer.getExecutor(); + final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size() * warmUp.size()); + for (final LeafReaderContext ctx : searcher.reader().leaves()) { for (final Query filterToWarm : warmUp) { executor.execute(() -> { try { @@ -283,7 +255,7 @@ public class BitsetFilterCache extends AbstractIndexComponent implements LeafRea } @Override - public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, IndicesWarmer.WarmerContext context, ThreadPool threadPool) { + public TerminationHandle warmTopReader(IndexShard indexShard, Engine.Searcher searcher) { return TerminationHandle.NO_WAIT; } diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/ShardBitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/ShardBitsetFilterCache.java index 6e0551a8905..86a00fdbcf7 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/bitset/ShardBitsetFilterCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/ShardBitsetFilterCache.java @@ -20,8 +20,7 @@ package org.elasticsearch.index.cache.bitset; import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; @@ -31,7 +30,7 @@ public class ShardBitsetFilterCache extends AbstractIndexShardComponent { private final CounterMetric totalMetric = new CounterMetric(); - public ShardBitsetFilterCache(ShardId shardId, @IndexSettings Settings indexSettings) { + public ShardBitsetFilterCache(ShardId shardId, IndexSettings indexSettings) { super(shardId, indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/index/cache/query/index/IndexQueryCache.java b/core/src/main/java/org/elasticsearch/index/cache/query/index/IndexQueryCache.java index af3d2ea5cb9..04f66290a5e 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/query/index/IndexQueryCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/query/index/IndexQueryCache.java @@ -23,11 +23,9 @@ import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.Weight; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.query.QueryCache; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.indices.cache.query.IndicesQueryCache; /** @@ -39,8 +37,8 @@ public class IndexQueryCache extends AbstractIndexComponent implements QueryCach final IndicesQueryCache indicesQueryCache; @Inject - public IndexQueryCache(Index index, @IndexSettings Settings indexSettings, IndicesQueryCache indicesQueryCache) { - super(index, indexSettings); + public IndexQueryCache(IndexSettings indexSettings, IndicesQueryCache indicesQueryCache) { + super(indexSettings); this.indicesQueryCache = indicesQueryCache; } @@ -52,7 +50,7 @@ public class IndexQueryCache extends AbstractIndexComponent implements QueryCach @Override public void clear(String reason) { logger.debug("full cache clear, reason [{}]", reason); - indicesQueryCache.clearIndex(index.getName()); + indicesQueryCache.clearIndex(index().getName()); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/cache/query/none/NoneQueryCache.java b/core/src/main/java/org/elasticsearch/index/cache/query/none/NoneQueryCache.java index 9d88940ff89..67408655726 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/query/none/NoneQueryCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/query/none/NoneQueryCache.java @@ -22,11 +22,9 @@ package org.elasticsearch.index.cache.query.none; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.Weight; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.query.QueryCache; -import org.elasticsearch.index.settings.IndexSettings; /** * @@ -34,8 +32,8 @@ import org.elasticsearch.index.settings.IndexSettings; public class NoneQueryCache extends AbstractIndexComponent implements QueryCache { @Inject - public NoneQueryCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings); + public NoneQueryCache(IndexSettings indexSettings) { + super(indexSettings); logger.debug("Using no query cache"); } diff --git a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java index 0f594d2faca..5e9c8156046 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java @@ -22,8 +22,7 @@ package org.elasticsearch.index.cache.request; import org.elasticsearch.common.cache.RemovalListener; import org.elasticsearch.common.cache.RemovalNotification; import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.cache.request.IndicesRequestCache; @@ -37,7 +36,7 @@ public class ShardRequestCache extends AbstractIndexShardComponent implements Re final CounterMetric hitCount = new CounterMetric(); final CounterMetric missCount = new CounterMetric(); - public ShardRequestCache(ShardId shardId, @IndexSettings Settings indexSettings) { + public ShardRequestCache(ShardId shardId, IndexSettings indexSettings) { super(shardId, indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java index 62ff583220c..e4d86be1bda 100644 --- a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -21,15 +21,11 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode; -import org.apache.lucene.codecs.lucene53.Lucene53Codec; +import org.apache.lucene.codecs.lucene54.Lucene54Codec; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; - import java.util.Map; /** @@ -37,11 +33,9 @@ import java.util.Map; * codec layer that allows to use use-case specific file formats & * data-structures per field. Elasticsearch exposes the full * {@link Codec} capabilities through this {@link CodecService}. - * */ -public class CodecService extends AbstractIndexComponent { +public class CodecService { - private final MapperService mapperService; private final Map codecs; public final static String DEFAULT_CODEC = "default"; @@ -49,22 +43,11 @@ public class CodecService extends AbstractIndexComponent { /** the raw unfiltered lucene default. useful for testing */ public final static String LUCENE_DEFAULT_CODEC = "lucene_default"; - public CodecService(Index index) { - this(index, Settings.Builder.EMPTY_SETTINGS); - } - - public CodecService(Index index, @IndexSettings Settings indexSettings) { - this(index, indexSettings, null); - } - - @Inject - public CodecService(Index index, @IndexSettings Settings indexSettings, MapperService mapperService) { - super(index, indexSettings); - this.mapperService = mapperService; - MapBuilder codecs = MapBuilder.newMapBuilder(); + public CodecService(@Nullable MapperService mapperService, ESLogger logger) { + final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene53Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene53Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene54Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene54Codec(Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); @@ -78,10 +61,6 @@ public class CodecService extends AbstractIndexComponent { this.codecs = codecs.immutableMap(); } - public MapperService mapperService() { - return mapperService; - } - public Codec codec(String name) { Codec codec = codecs.get(name); if (codec == null) { diff --git a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java index b8e44bdadb6..2c23f947475 100644 --- a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.codecs.lucene53.Lucene53Codec; +import org.apache.lucene.codecs.lucene54.Lucene54Codec; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.mapper.MappedFieldType; @@ -38,7 +38,7 @@ import org.elasticsearch.index.mapper.core.CompletionFieldMapper; * configured for a specific field the default postings format is used. */ // LUCENE UPGRADE: make sure to move to a new codec depending on the lucene version -public class PerFieldMappingPostingFormatCodec extends Lucene53Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene54Codec { private final ESLogger logger; private final MapperService mapperService; @@ -58,10 +58,7 @@ public class PerFieldMappingPostingFormatCodec extends Lucene53Codec { if (indexName == null) { logger.warn("no index mapper found for field: [{}] returning default postings format", field); } else if (indexName instanceof CompletionFieldMapper.CompletionFieldType) { - // CompletionFieldMapper needs a special postings format - final CompletionFieldMapper.CompletionFieldType fieldType = (CompletionFieldMapper.CompletionFieldType) indexName; - final PostingsFormat defaultFormat = super.getPostingsFormatForField(field); - return fieldType.postingsFormat(defaultFormat); + return CompletionFieldMapper.CompletionFieldType.postingsFormat(); } return super.getPostingsFormatForField(field); } diff --git a/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java b/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java index 94d0e00e029..050c23701f8 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java +++ b/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java @@ -62,7 +62,7 @@ public final class CommitStats implements Streamable, ToXContent { } public static CommitStats readOptionalCommitStatsFrom(StreamInput in) throws IOException { - return in.readOptionalStreamable(new CommitStats()); + return in.readOptionalStreamable(CommitStats::new); } diff --git a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java index 0566e80a37d..32ffbf371f4 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java +++ b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.merge.OnGoingMerge; import org.elasticsearch.index.shard.MergeSchedulerConfig; @@ -62,11 +63,11 @@ class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler { private final Set readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges); private final MergeSchedulerConfig config; - public ElasticsearchConcurrentMergeScheduler(ShardId shardId, Settings indexSettings, MergeSchedulerConfig config) { + public ElasticsearchConcurrentMergeScheduler(ShardId shardId, IndexSettings indexSettings, MergeSchedulerConfig config) { this.config = config; this.shardId = shardId; - this.indexSettings = indexSettings; - this.logger = Loggers.getLogger(getClass(), indexSettings, shardId); + this.indexSettings = indexSettings.getSettings(); + this.logger = Loggers.getLogger(getClass(), this.indexSettings, shardId); refreshConfig(); } diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index 62e7ec81f46..d6dd31481db 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -73,13 +73,25 @@ public abstract class Engine implements Closeable { protected final EngineConfig engineConfig; protected final Store store; protected final AtomicBoolean isClosed = new AtomicBoolean(false); - protected final FailedEngineListener failedEngineListener; + protected final EventListener eventListener; protected final SnapshotDeletionPolicy deletionPolicy; protected final ReentrantLock failEngineLock = new ReentrantLock(); protected final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); protected final ReleasableLock readLock = new ReleasableLock(rwl.readLock()); protected final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock()); protected volatile Throwable failedEngine = null; + /* + * on lastWriteNanos we use System.nanoTime() to initialize this since: + * - we use the value for figuring out if the shard / engine is active so if we startup and no write has happened yet we still consider it active + * for the duration of the configured active to inactive period. If we initialize to 0 or Long.MAX_VALUE we either immediately or never mark it + * inactive if no writes at all happen to the shard. + * - we also use this to flush big-ass merges on an inactive engine / shard but if we we initialize 0 or Long.MAX_VALUE we either immediately or never + * commit merges even though we shouldn't from a user perspective (this can also have funky sideeffects in tests when we open indices with lots of segments + * and suddenly merges kick in. + * NOTE: don't use this value for anything accurate it's a best effort for freeing up diskspace after merges and on a shard level to reduce index buffer sizes on + * inactive shards. + */ + protected volatile long lastWriteNanos = System.nanoTime(); protected Engine(EngineConfig engineConfig) { Objects.requireNonNull(engineConfig.getStore(), "Store must be provided to the engine"); @@ -89,8 +101,8 @@ public abstract class Engine implements Closeable { this.shardId = engineConfig.getShardId(); this.store = engineConfig.getStore(); this.logger = Loggers.getLogger(Engine.class, // we use the engine class directly here to make sure all subclasses have the same logger name - engineConfig.getIndexSettings(), engineConfig.getShardId()); - this.failedEngineListener = engineConfig.getFailedEngineListener(); + engineConfig.getIndexSettings().getSettings(), engineConfig.getShardId()); + this.eventListener = engineConfig.getEventListener(); this.deletionPolicy = engineConfig.getDeletionPolicy(); } @@ -487,7 +499,7 @@ public abstract class Engine implements Closeable { public abstract CommitId flush() throws EngineException; /** - * Optimizes to 1 segment + * Force merges to 1 segment */ public void forceMerge(boolean flush) throws IOException { forceMerge(flush, 1, false, false, false); @@ -536,7 +548,7 @@ public abstract class Engine implements Closeable { logger.warn("Couldn't mark store corrupted", e); } } - failedEngineListener.onFailedEngine(shardId, reason, failure); + eventListener.onFailedEngine(reason, failure); } } catch (Throwable t) { // don't bubble up these exceptions up @@ -561,19 +573,12 @@ public abstract class Engine implements Closeable { return false; } - /** Wrap a Throwable in an {@code EngineClosedException} if the engine is already closed */ - protected Throwable wrapIfClosed(Throwable t) { - if (isClosed.get()) { - if (t != failedEngine && failedEngine != null) { - t.addSuppressed(failedEngine); - } - return new EngineClosedException(shardId, t); - } - return t; - } - public interface FailedEngineListener { - void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable t); + public interface EventListener { + /** + * Called when a fatal exception occurred + */ + default void onFailedEngine(String reason, @Nullable Throwable t) {} } public static class Searcher implements Releasable { @@ -992,11 +997,6 @@ public abstract class Engine implements Closeable { } } - /** - * Returns true the internal writer has any uncommitted changes. Otherwise false - */ - public abstract boolean hasUncommittedChanges(); - public static class CommitId implements Writeable { private final byte[] id; @@ -1056,4 +1056,29 @@ public abstract class Engine implements Closeable { public void onSettingsChanged() { } + + /** + * Returns the timestamp of the last write in nanoseconds. + * Note: this time might not be absolutely accurate since the {@link Operation#startTime()} is used which might be + * slightly inaccurate. + * @see System#nanoTime() + * @see Operation#startTime() + */ + public long getLastWriteNanos() { + return this.lastWriteNanos; + } + + /** + * Called for each new opened engine searcher to warm new segments + * @see EngineConfig#getWarmer() + */ + public interface Warmer { + /** + * Called once a new Searcher is opened. + * @param searcher the searcer to warm + * @param isTopLevelReader true iff the searcher is build from a top-level reader. + * Otherwise the searcher might be build from a leaf reader to warm in isolation + */ + void warm(Engine.Searcher searcher, boolean isTopLevelReader); + } } diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index abca9cc875e..98572c39f4e 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -25,21 +25,18 @@ import org.apache.lucene.index.SnapshotDeletionPolicy; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.indexing.ShardIndexingService; -import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.TranslogRecoveryPerformer; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.TranslogConfig; -import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.threadpool.ThreadPool; @@ -53,16 +50,16 @@ import java.util.concurrent.TimeUnit; public final class EngineConfig { private final ShardId shardId; private final TranslogRecoveryPerformer translogRecoveryPerformer; - private final Settings indexSettings; + private final IndexSettings indexSettings; private final ByteSizeValue indexingBufferSize; private volatile boolean compoundOnFlush = true; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private volatile boolean enableGcDeletes = true; + private final TimeValue flushMergesAfter; private final String codecName; private final ThreadPool threadPool; private final ShardIndexingService indexingService; - @Nullable - private final IndicesWarmer warmer; + private final Engine.Warmer warmer; private final Store store; private final SnapshotDeletionPolicy deletionPolicy; private final MergePolicy mergePolicy; @@ -70,11 +67,10 @@ public final class EngineConfig { private final Analyzer analyzer; private final Similarity similarity; private final CodecService codecService; - private final Engine.FailedEngineListener failedEngineListener; + private final Engine.EventListener eventListener; private final boolean forceNewTranslog; private final QueryCache queryCache; private final QueryCachingPolicy queryCachingPolicy; - private final SetOnce searcherWrapper = new SetOnce<>(); /** * Index setting for compound file on flush. This setting is realtime updateable. @@ -112,15 +108,16 @@ public final class EngineConfig { * Creates a new {@link org.elasticsearch.index.engine.EngineConfig} */ public EngineConfig(ShardId shardId, ThreadPool threadPool, ShardIndexingService indexingService, - Settings indexSettings, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy, + IndexSettings indexSettings, Engine.Warmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy, MergePolicy mergePolicy, MergeSchedulerConfig mergeSchedulerConfig, Analyzer analyzer, - Similarity similarity, CodecService codecService, Engine.FailedEngineListener failedEngineListener, - TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, TranslogConfig translogConfig) { + Similarity similarity, CodecService codecService, Engine.EventListener eventListener, + TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, TranslogConfig translogConfig, TimeValue flushMergesAfter) { this.shardId = shardId; + final Settings settings = indexSettings.getSettings(); this.indexSettings = indexSettings; this.threadPool = threadPool; this.indexingService = indexingService; - this.warmer = warmer; + this.warmer = warmer == null ? (a,b) -> {} : warmer; this.store = store; this.deletionPolicy = deletionPolicy; this.mergePolicy = mergePolicy; @@ -128,18 +125,19 @@ public final class EngineConfig { this.analyzer = analyzer; this.similarity = similarity; this.codecService = codecService; - this.failedEngineListener = failedEngineListener; - this.compoundOnFlush = indexSettings.getAsBoolean(INDEX_COMPOUND_ON_FLUSH, compoundOnFlush); - codecName = indexSettings.get(INDEX_CODEC_SETTING, DEFAULT_CODEC_NAME); + this.eventListener = eventListener; + this.compoundOnFlush = settings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush); + codecName = settings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME); // We give IndexWriter a huge buffer, so it won't flush on its own. Instead, IndexingMemoryController periodically checks - // and refreshes the most heap-consuming shards when total indexing heap usage is too high: + // and refreshes the most heap-consuming shards when total indexing heap usage across all shards is too high: indexingBufferSize = new ByteSizeValue(256, ByteSizeUnit.MB); - gcDeletesInMillis = indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, DEFAULT_GC_DELETES).millis(); + gcDeletesInMillis = settings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis(); this.translogRecoveryPerformer = translogRecoveryPerformer; - this.forceNewTranslog = indexSettings.getAsBoolean(INDEX_FORCE_NEW_TRANSLOG, false); + this.forceNewTranslog = settings.getAsBoolean(INDEX_FORCE_NEW_TRANSLOG, false); this.queryCache = queryCache; this.queryCachingPolicy = queryCachingPolicy; this.translogConfig = translogConfig; + this.flushMergesAfter = flushMergesAfter; } /** if true the engine will start even if the translog id in the commit point can not be found */ @@ -202,7 +200,7 @@ public final class EngineConfig { /** * Returns a thread-pool mainly used to get estimated time stamps from {@link org.elasticsearch.threadpool.ThreadPool#estimatedTimeInMillis()} and to schedule - * async force merge calls on the {@link org.elasticsearch.threadpool.ThreadPool.Names#OPTIMIZE} thread-pool + * async force merge calls on the {@link org.elasticsearch.threadpool.ThreadPool.Names#FORCE_MERGE} thread-pool */ public ThreadPool getThreadPool() { return threadPool; @@ -221,11 +219,9 @@ public final class EngineConfig { } /** - * Returns an {@link org.elasticsearch.indices.IndicesWarmer} used to warm new searchers before they are used for searching. - * Note: This method might retrun null + * Returns an {@link org.elasticsearch.index.engine.Engine.Warmer} used to warm new searchers before they are used for searching. */ - @Nullable - public IndicesWarmer getWarmer() { + public Engine.Warmer getWarmer() { return warmer; } @@ -266,14 +262,14 @@ public final class EngineConfig { /** * Returns a listener that should be called on engine failure */ - public Engine.FailedEngineListener getFailedEngineListener() { - return failedEngineListener; + public Engine.EventListener getEventListener() { + return eventListener; } /** - * Returns the latest index settings directly from the index settings service. + * Returns the index settings for this index. */ - public Settings getIndexSettings() { + public IndexSettings getIndexSettings() { return indexSettings; } @@ -355,4 +351,12 @@ public final class EngineConfig { public boolean isCreate() { return create; } + + /** + * Returns a {@link TimeValue} at what time interval after the last write modification to the engine finished merges + * should be automatically flushed. This is used to free up transient disk usage of potentially large segments that + * are written after the engine became inactive from an indexing perspective. + */ + public TimeValue getFlushMergesAfter() { return flushMergesAfter; } + } diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 089ddfc9998..2aed653d02e 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -42,10 +42,10 @@ import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.lucene.index.ElasticsearchLeafReader; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.math.MathUtils; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ReleasableLock; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.indexing.ShardIndexingService; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.merge.MergeStats; @@ -57,7 +57,6 @@ import org.elasticsearch.index.shard.TranslogRecoveryPerformer; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.index.translog.TranslogCorruptedException; -import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; @@ -79,8 +78,7 @@ public class InternalEngine extends Engine { private volatile long lastDeleteVersionPruneTimeMSec; private final ShardIndexingService indexingService; - @Nullable - private final IndicesWarmer warmer; + private final Engine.Warmer warmer; private final Translog translog; private final ElasticsearchConcurrentMergeScheduler mergeScheduler; @@ -351,6 +349,7 @@ public class InternalEngine extends Engine { private boolean innerIndex(Index index) throws IOException { synchronized (dirtyLock(index.uid())) { + lastWriteNanos = index.startTime(); final long currentVersion; final boolean deleted; VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes()); @@ -434,6 +433,7 @@ public class InternalEngine extends Engine { private void innerDelete(Delete delete) throws IOException { synchronized (dirtyLock(delete.uid())) { + lastWriteNanos = delete.startTime(); final long currentVersion; final boolean deleted; VersionValue versionValue = versionMap.getUnderLock(delete.uid().bytes()); @@ -539,6 +539,29 @@ public class InternalEngine extends Engine { } } + final boolean tryRenewSyncCommit() { + boolean renewed = false; + try (ReleasableLock lock = writeLock.acquire()) { + ensureOpen(); + String syncId = lastCommittedSegmentInfos.getUserData().get(SYNC_COMMIT_ID); + if (syncId != null && translog.totalOperations() == 0 && indexWriter.hasUncommittedChanges()) { + logger.trace("start renewing sync commit [{}]", syncId); + commitIndexWriter(indexWriter, translog, syncId); + logger.debug("successfully sync committed. sync id [{}].", syncId); + lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo(); + renewed = true; + } + } catch (IOException ex) { + maybeFailEngine("renew sync commit", ex); + throw new EngineException(shardId, "failed to renew sync commit", ex); + } + if (renewed) { // refresh outside of the write lock + refresh("renew sync commit"); + } + + return renewed; + } + @Override public CommitId flush() throws EngineException { return flush(false, false); @@ -728,10 +751,14 @@ public class InternalEngine extends Engine { // we need to fail the engine. it might have already been failed before // but we are double-checking it's failed and closed if (indexWriter.isOpen() == false && indexWriter.getTragicException() != null) { - failEngine("already closed by tragic event", indexWriter.getTragicException()); + failEngine("already closed by tragic event on the index writer", indexWriter.getTragicException()); + } else if (translog.isOpen() == false && translog.getTragicException() != null) { + failEngine("already closed by tragic event on the translog", translog.getTragicException()); } return true; - } else if (t != null && indexWriter.isOpen() == false && indexWriter.getTragicException() == t) { + } else if (t != null && + ((indexWriter.isOpen() == false && indexWriter.getTragicException() == t) + || (translog.isOpen() == false && translog.getTragicException() == t))) { // this spot on - we are handling the tragic event exception here so we have to fail the engine // right away failEngine(source, t); @@ -816,11 +843,6 @@ public class InternalEngine extends Engine { } } - @Override - public boolean hasUncommittedChanges() { - return indexWriter.hasUncommittedChanges(); - } - @Override protected SearcherManager getSearcherManager() { return searcherManager; @@ -880,8 +902,7 @@ public class InternalEngine extends Engine { assert isMergedSegment(esLeafReader); if (warmer != null) { final Engine.Searcher searcher = new Searcher("warmer", searcherFactory.newSearcher(esLeafReader, null)); - final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher); - warmer.warmNewReaders(context); + warmer.warm(searcher, false); } } catch (Throwable t) { // Don't fail a merge if the warm-up failed @@ -905,7 +926,7 @@ public class InternalEngine extends Engine { /** Extended SearcherFactory that warms the segments if needed when acquiring a new searcher */ final static class SearchFactory extends EngineSearcherFactory { - private final IndicesWarmer warmer; + private final Engine.Warmer warmer; private final ShardId shardId; private final ESLogger logger; private final AtomicBoolean isEngineClosed; @@ -964,11 +985,10 @@ public class InternalEngine extends Engine { } if (newSearcher != null) { - IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, new Searcher("new_reader_warming", newSearcher)); - warmer.warmNewReaders(context); + warmer.warm(new Searcher("new_reader_warming", newSearcher), false); } assert searcher.getIndexReader() instanceof ElasticsearchDirectoryReader : "this class needs an ElasticsearchDirectoryReader but got: " + searcher.getIndexReader().getClass(); - warmer.warmTopReader(new IndicesWarmer.WarmerContext(shardId, new Searcher("top_reader_warming", searcher))); + warmer.warm(new Searcher("top_reader_warming", searcher), true); } catch (Throwable e) { if (isEngineClosed.get() == false) { logger.warn("failed to prepare/warm", e); @@ -1004,7 +1024,7 @@ public class InternalEngine extends Engine { private final AtomicInteger numMergesInFlight = new AtomicInteger(0); private final AtomicBoolean isThrottling = new AtomicBoolean(); - EngineMergeScheduler(ShardId shardId, Settings indexSettings, MergeSchedulerConfig config) { + EngineMergeScheduler(ShardId shardId, IndexSettings indexSettings, MergeSchedulerConfig config) { super(shardId, indexSettings, config); } @@ -1030,6 +1050,32 @@ public class InternalEngine extends Engine { deactivateThrottling(); } } + if (indexWriter.hasPendingMerges() == false && System.nanoTime() - lastWriteNanos >= engineConfig.getFlushMergesAfter().nanos()) { + // NEVER do this on a merge thread since we acquire some locks blocking here and if we concurrently rollback the writer + // we deadlock on engine#close for instance. + engineConfig.getThreadPool().executor(ThreadPool.Names.FLUSH).execute(new AbstractRunnable() { + @Override + public void onFailure(Throwable t) { + if (isClosed.get() == false) { + logger.warn("failed to flush after merge has finished"); + } + } + + @Override + protected void doRun() throws Exception { + // if we have no pending merges and we are supposed to flush once merges have finished + // we try to renew a sync commit which is the case when we are having a big merge after we + // are inactive. If that didn't work we go and do a real flush which is ok since it only doesn't work + // if we either have records in the translog or if we don't have a sync ID at all... + // maybe even more important, we flush after all merges finish and we are inactive indexing-wise to + // free up transient disk usage of the (presumably biggish) segments that were just merged + if (tryRenewSyncCommit() == false) { + flush(); + } + } + }); + + } } @Override diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java index 82aee8340fd..dad1c5e09f2 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java @@ -68,7 +68,7 @@ public class ShadowEngine extends Engine { public ShadowEngine(EngineConfig engineConfig) { super(engineConfig); SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig); - final long nonexistentRetryTime = engineConfig.getIndexSettings() + final long nonexistentRetryTime = engineConfig.getIndexSettings().getSettings() .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT) .getMillis(); try { @@ -202,8 +202,6 @@ public class ShadowEngine extends Engine { throw new UnsupportedOperationException("Can not take snapshot from a shadow engine"); } - - @Override protected SearcherManager getSearcherManager() { return searcherManager; @@ -223,11 +221,6 @@ public class ShadowEngine extends Engine { } } - @Override - public boolean hasUncommittedChanges() { - return false; - } - @Override protected SegmentInfos getLastCommittedSegmentInfos() { return lastCommittedSegmentInfos; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java index e549eb32dd6..7d2689dc157 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java @@ -20,24 +20,22 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparatorSource; -import org.apache.lucene.search.Filter; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexComponent; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; @@ -123,9 +121,9 @@ public interface IndexFieldData extends IndexCompone public static class Nested { private final BitSetProducer rootFilter; - private final Filter innerFilter; + private final Weight innerFilter; - public Nested(BitSetProducer rootFilter, Filter innerFilter) { + public Nested(BitSetProducer rootFilter, Weight innerFilter) { this.rootFilter = rootFilter; this.innerFilter = innerFilter; } @@ -140,8 +138,8 @@ public interface IndexFieldData extends IndexCompone /** * Get a {@link DocIdSet} that matches the inner documents. */ - public DocIdSet innerDocs(LeafReaderContext ctx) throws IOException { - return innerFilter.getDocIdSet(ctx, null); + public DocIdSetIterator innerDocs(LeafReaderContext ctx) throws IOException { + return innerFilter.scorer(ctx); } } @@ -232,7 +230,7 @@ public interface IndexFieldData extends IndexCompone interface Builder { - IndexFieldData build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index de3adbcf2e7..80947260442 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -23,19 +23,14 @@ import org.apache.lucene.util.Accountable; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData; import org.elasticsearch.index.fielddata.plain.DisabledIndexFieldData; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; -import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData; -import org.elasticsearch.index.fielddata.plain.FloatArrayIndexFieldData; -import org.elasticsearch.index.fielddata.plain.GeoPointBinaryDVIndexFieldData; -import org.elasticsearch.index.fielddata.plain.GeoPointDoubleArrayIndexFieldData; +import org.elasticsearch.index.fielddata.plain.AbstractGeoPointDVIndexFieldData; +import org.elasticsearch.index.fielddata.plain.GeoPointArrayIndexFieldData; import org.elasticsearch.index.fielddata.plain.IndexIndexFieldData; -import org.elasticsearch.index.fielddata.plain.PackedArrayIndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -44,11 +39,12 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.BooleanFieldMapper; import org.elasticsearch.index.mapper.internal.IndexFieldMapper; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import java.io.Closeable; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -59,14 +55,20 @@ import static java.util.Collections.unmodifiableMap; /** */ -public class IndexFieldDataService extends AbstractIndexComponent { +public class IndexFieldDataService extends AbstractIndexComponent implements Closeable { public static final String FIELDDATA_CACHE_KEY = "index.fielddata.cache"; public static final String FIELDDATA_CACHE_VALUE_NODE = "node"; + private static final IndexFieldData.Builder MISSING_DOC_VALUES_BUILDER = (indexProperties, fieldType, cache, breakerService, mapperService1) -> { + throw new IllegalStateException("Can't load fielddata on [" + fieldType.names().fullName() + + "] of index [" + indexProperties.getIndex().getName() + "] because fielddata is unsupported on fields of type [" + + fieldType.fieldDataType().getType() + "]. Use doc values instead."); + }; + + private static final String ARRAY_FORMAT = "array"; private static final String DISABLED_FORMAT = "disabled"; private static final String DOC_VALUES_FORMAT = "doc_values"; - private static final String ARRAY_FORMAT = "array"; private static final String PAGED_BYTES_FORMAT = "paged_bytes"; private final static Map buildersByType; @@ -77,19 +79,18 @@ public class IndexFieldDataService extends AbstractIndexComponent { static { Map buildersByTypeBuilder = new HashMap<>(); buildersByTypeBuilder.put("string", new PagedBytesIndexFieldData.Builder()); - buildersByTypeBuilder.put("float", new FloatArrayIndexFieldData.Builder()); - buildersByTypeBuilder.put("double", new DoubleArrayIndexFieldData.Builder()); - buildersByTypeBuilder.put("byte", new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BYTE)); - buildersByTypeBuilder.put("short", new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.SHORT)); - buildersByTypeBuilder.put("int", new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.INT)); - buildersByTypeBuilder.put("long", new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.LONG)); - buildersByTypeBuilder.put("geo_point", new GeoPointDoubleArrayIndexFieldData.Builder()); + buildersByTypeBuilder.put("float", MISSING_DOC_VALUES_BUILDER); + buildersByTypeBuilder.put("double", MISSING_DOC_VALUES_BUILDER); + buildersByTypeBuilder.put("byte", MISSING_DOC_VALUES_BUILDER); + buildersByTypeBuilder.put("short", MISSING_DOC_VALUES_BUILDER); + buildersByTypeBuilder.put("int", MISSING_DOC_VALUES_BUILDER); + buildersByTypeBuilder.put("long", MISSING_DOC_VALUES_BUILDER); + buildersByTypeBuilder.put("geo_point", new GeoPointArrayIndexFieldData.Builder()); buildersByTypeBuilder.put(ParentFieldMapper.NAME, new ParentChildIndexFieldData.Builder()); buildersByTypeBuilder.put(IndexFieldMapper.NAME, new IndexIndexFieldData.Builder()); buildersByTypeBuilder.put("binary", new DisabledIndexFieldData.Builder()); - buildersByTypeBuilder.put(BooleanFieldMapper.CONTENT_TYPE, - new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BOOLEAN)); - buildersByType = unmodifiableMap(buildersByTypeBuilder); + buildersByTypeBuilder.put(BooleanFieldMapper.CONTENT_TYPE, MISSING_DOC_VALUES_BUILDER); + buildersByType = unmodifiableMap(buildersByTypeBuilder); docValuesBuildersByType = MapBuilder.newMapBuilder() @@ -100,7 +101,7 @@ public class IndexFieldDataService extends AbstractIndexComponent { .put("short", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT)) .put("int", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT)) .put("long", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG)) - .put("geo_point", new GeoPointBinaryDVIndexFieldData.Builder()) + .put("geo_point", new AbstractGeoPointDVIndexFieldData.Builder()) .put("binary", new BytesBinaryDVIndexFieldData.Builder()) .put(BooleanFieldMapper.CONTENT_TYPE, new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BOOLEAN)) .immutableMap(); @@ -110,38 +111,31 @@ public class IndexFieldDataService extends AbstractIndexComponent { .put(Tuple.tuple("string", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder()) .put(Tuple.tuple("string", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("float", ARRAY_FORMAT), new FloatArrayIndexFieldData.Builder()) .put(Tuple.tuple("float", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FLOAT)) .put(Tuple.tuple("float", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("double", ARRAY_FORMAT), new DoubleArrayIndexFieldData.Builder()) .put(Tuple.tuple("double", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.DOUBLE)) .put(Tuple.tuple("double", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("byte", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BYTE)) .put(Tuple.tuple("byte", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BYTE)) .put(Tuple.tuple("byte", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("short", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.SHORT)) .put(Tuple.tuple("short", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT)) .put(Tuple.tuple("short", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("int", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.INT)) .put(Tuple.tuple("int", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT)) .put(Tuple.tuple("int", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("long", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.LONG)) .put(Tuple.tuple("long", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG)) .put(Tuple.tuple("long", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple("geo_point", ARRAY_FORMAT), new GeoPointDoubleArrayIndexFieldData.Builder()) - .put(Tuple.tuple("geo_point", DOC_VALUES_FORMAT), new GeoPointBinaryDVIndexFieldData.Builder()) + .put(Tuple.tuple("geo_point", ARRAY_FORMAT), new GeoPointArrayIndexFieldData.Builder()) + .put(Tuple.tuple("geo_point", DOC_VALUES_FORMAT), new AbstractGeoPointDVIndexFieldData.Builder()) .put(Tuple.tuple("geo_point", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) .put(Tuple.tuple("binary", DOC_VALUES_FORMAT), new BytesBinaryDVIndexFieldData.Builder()) .put(Tuple.tuple("binary", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) - .put(Tuple.tuple(BooleanFieldMapper.CONTENT_TYPE, ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BOOLEAN)) .put(Tuple.tuple(BooleanFieldMapper.CONTENT_TYPE, DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BOOLEAN)) .put(Tuple.tuple(BooleanFieldMapper.CONTENT_TYPE, DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) @@ -164,22 +158,15 @@ public class IndexFieldDataService extends AbstractIndexComponent { private volatile IndexFieldDataCache.Listener listener = DEFAULT_NOOP_LISTENER; - // We need to cache fielddata on the _parent field because of 1.x indices. - // When we don't support 1.x anymore (3.0) then remove this caching - // This variable needs to be read/written under lock - private IndexFieldData parentIndexFieldData; - - @Inject - public IndexFieldDataService(Index index, @IndexSettings Settings indexSettings, IndicesFieldDataCache indicesFieldDataCache, + public IndexFieldDataService(IndexSettings indexSettings, IndicesFieldDataCache indicesFieldDataCache, CircuitBreakerService circuitBreakerService, MapperService mapperService) { - super(index, indexSettings); + super(indexSettings); this.indicesFieldDataCache = indicesFieldDataCache; this.circuitBreakerService = circuitBreakerService; this.mapperService = mapperService; } public synchronized void clear() { - parentIndexFieldData = null; List exceptions = new ArrayList<>(0); final Collection fieldDataCacheValues = fieldDataCaches.values(); for (IndexFieldDataCache cache : fieldDataCacheValues) { @@ -194,9 +181,6 @@ public class IndexFieldDataService extends AbstractIndexComponent { } public synchronized void clearField(final String fieldName) { - if (ParentFieldMapper.NAME.equals(fieldName)) { - parentIndexFieldData = null; - } List exceptions = new ArrayList<>(0); final IndexFieldDataCache cache = fieldDataCaches.remove(fieldName); if (cache != null) { @@ -218,7 +202,7 @@ public class IndexFieldDataService extends AbstractIndexComponent { } final boolean docValues = fieldType.hasDocValues(); IndexFieldData.Builder builder = null; - String format = type.getFormat(indexSettings); + String format = type.getFormat(indexSettings.getSettings()); if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) { logger.warn("field [" + fieldNames.fullName() + "] has no doc values, will use default field data format"); format = null; @@ -245,9 +229,9 @@ public class IndexFieldDataService extends AbstractIndexComponent { if (cache == null) { // we default to node level cache, which in turn defaults to be unbounded // this means changing the node level settings is simple, just set the bounds there - String cacheType = type.getSettings().get("cache", indexSettings.get(FIELDDATA_CACHE_KEY, FIELDDATA_CACHE_VALUE_NODE)); + String cacheType = type.getSettings().get("cache", indexSettings.getSettings().get(FIELDDATA_CACHE_KEY, FIELDDATA_CACHE_VALUE_NODE)); if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { - cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index, fieldNames, type); + cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldNames, type); } else if ("none".equals(cacheType)){ cache = new IndexFieldDataCache.None(); } else { @@ -257,7 +241,7 @@ public class IndexFieldDataService extends AbstractIndexComponent { } } - return (IFD) builder.build(index, indexSettings, fieldType, cache, circuitBreakerService, mapperService); + return (IFD) builder.build(indexSettings, fieldType, cache, circuitBreakerService, mapperService); } /** @@ -276,4 +260,8 @@ public class IndexFieldDataService extends AbstractIndexComponent { this.listener = listener; } + @Override + public void close() throws IOException { + clear(); + } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java index 8fccda94d80..e646364ef13 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java @@ -21,14 +21,10 @@ package org.elasticsearch.index.fielddata; import com.carrotsearch.hppc.ObjectLongHashMap; import org.apache.lucene.util.Accountable; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; import java.util.Map; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java index ff87cc00022..883444fabc7 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java @@ -37,7 +37,7 @@ final class SingletonMultiGeoPointValues extends MultiGeoPointValues { @Override public void setDocument(int docID) { value = in.get(docID); - if (value.lat() == 0 && value.lon() == 0 && docsWithField != null && !docsWithField.get(docID)) { + if (value.lat() == Double.NaN && value.lon() == Double.NaN || (docsWithField != null && !docsWithField.get(docID))) { count = 0; } else { count = 1; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 3d5959a054a..1789c3537e6 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -23,7 +23,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; @@ -95,7 +95,7 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat selectedValues = sortMode.select(values); } else { final BitSet rootDocs = nested.rootDocs(context); - final DocIdSet innerDocs = nested.innerDocs(context); + final DocIdSetIterator innerDocs = nested.innerDocs(context); selectedValues = sortMode.select(values, rootDocs, innerDocs); } if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) { @@ -125,7 +125,7 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat selectedValues = sortMode.select(values, nonNullMissingBytes); } else { final BitSet rootDocs = nested.rootDocs(context); - final DocIdSet innerDocs = nested.innerDocs(context); + final DocIdSetIterator innerDocs = nested.innerDocs(context); selectedValues = sortMode.select(values, nonNullMissingBytes, rootDocs, innerDocs, context.reader().maxDoc()); } return selectedValues; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index 4ea2eaed7de..5391345e793 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -21,7 +21,7 @@ package org.elasticsearch.index.fielddata.fieldcomparator; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; @@ -79,7 +79,7 @@ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparato selectedValues = sortMode.select(values, dMissingValue); } else { final BitSet rootDocs = nested.rootDocs(context); - final DocIdSet innerDocs = nested.innerDocs(context); + final DocIdSetIterator innerDocs = nested.innerDocs(context); selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc()); } return selectedValues.getRawDoubleValues(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 40c0e83ddca..15628513e80 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.fielddata.fieldcomparator; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSet; @@ -71,7 +71,7 @@ public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparator selectedValues = sortMode.select(values, dMissingValue); } else { final BitSet rootDocs = nested.rootDocs(context); - final DocIdSet innerDocs = nested.innerDocs(context); + final DocIdSetIterator innerDocs = nested.innerDocs(context); selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc()); } return selectedValues.getRawFloatValues(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index 2e596c463c7..15961ffedce 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSet; @@ -70,7 +71,7 @@ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorS selectedValues = sortMode.select(values, dMissingValue); } else { final BitSet rootDocs = nested.rootDocs(context); - final DocIdSet innerDocs = nested.innerDocs(context); + final DocIdSetIterator innerDocs = nested.innerDocs(context); selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc()); } return selectedValues; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index a7f25d4f8f5..e6f1d24f1e3 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -19,19 +19,24 @@ package org.elasticsearch.index.fielddata.ordinals; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiDocValues.OrdinalMap; import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.util.Accountable; import org.apache.lucene.util.packed.PackedInts; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData; import org.elasticsearch.indices.breaker.CircuitBreakerService; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; /** * Utility class to build global ordinals. @@ -42,7 +47,7 @@ public enum GlobalOrdinalsBuilder { /** * Build global ordinals for the provided {@link IndexReader}. */ - public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, Settings settings, CircuitBreakerService breakerService, ESLogger logger) throws IOException { + public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, IndexSettings indexSettings, CircuitBreakerService breakerService, ESLogger logger) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); @@ -64,9 +69,43 @@ public enum GlobalOrdinalsBuilder { TimeValue.nsecToMSec(System.nanoTime() - startTimeNS) ); } - return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(), + return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldNames(), indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes ); } + public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData) throws IOException { + assert indexReader.leaves().size() > 1; + + final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; + final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; + for (int i = 0; i < indexReader.leaves().size(); ++i) { + atomicFD[i] = new AbstractAtomicOrdinalsFieldData() { + @Override + public RandomAccessOrds getOrdinalsValues() { + return DocValues.emptySortedSet(); + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() { + } + }; + subs[i] = atomicFD[i].getOrdinalsValues(); + } + final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); + return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldNames(), + indexFieldData.getFieldDataType(), atomicFD, ordinalMap, 0 + ); + } + } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java index 0d6c129d73c..4a8bd78bb4e 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java @@ -19,19 +19,16 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.Accountable; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.MultiValueMode; @@ -47,8 +44,8 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen private final FieldDataType fieldDataType; private final long memorySizeInBytes; - protected GlobalOrdinalsIndexFieldData(Index index, Settings settings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, long memorySizeInBytes) { - super(index, settings); + protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, long memorySizeInBytes) { + super(indexSettings); this.fieldNames = fieldNames; this.fieldDataType = fieldDataType; this.memorySizeInBytes = memorySizeInBytes; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java index b91d98f4a62..fc1b6db9758 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java @@ -22,12 +22,10 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues.OrdinalMap; import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.util.Accountable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import java.util.Collection; @@ -39,8 +37,8 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel private final Atomic[] atomicReaders; - InternalGlobalOrdinalsIndexFieldData(Index index, Settings settings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, AtomicOrdinalsFieldData[] segmentAfd, OrdinalMap ordinalMap, long memorySizeInBytes) { - super(index, settings, fieldNames, fieldDataType, memorySizeInBytes); + InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, AtomicOrdinalsFieldData[] segmentAfd, OrdinalMap ordinalMap, long memorySizeInBytes) { + super(indexSettings, fieldNames, fieldDataType, memorySizeInBytes); this.atomicReaders = new Atomic[segmentAfd.length]; for (int i = 0; i < segmentAfd.length; i++) { atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java index fde583e68dc..175f041bd66 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java @@ -30,7 +30,7 @@ import java.util.Collections; /** */ -abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData { +public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData { @Override public final SortedBinaryDocValues getBytesValues() { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java new file mode 100644 index 00000000000..0b86b17f211 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.DocValues; +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MappedFieldType.Names; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.MultiValueMode; + +import java.io.IOException; + +public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { + + AbstractGeoPointDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType) { + super(index, fieldNames, fieldDataType); + } + + @Override + public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { + throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); + } + + /** + * Lucene 5.4 GeoPointFieldType + */ + public static class GeoPointDVIndexFieldData extends AbstractGeoPointDVIndexFieldData { + final boolean indexCreatedBefore2x; + + public GeoPointDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType, final boolean indexCreatedBefore2x) { + super(index, fieldNames, fieldDataType); + this.indexCreatedBefore2x = indexCreatedBefore2x; + } + + @Override + public AtomicGeoPointFieldData load(LeafReaderContext context) { + try { + if (indexCreatedBefore2x) { + return new GeoPointLegacyDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName())); + } + return new GeoPointDVAtomicFieldData(DocValues.getSortedNumeric(context.reader(), fieldNames.indexName())); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { + return load(context); + } + } + + public static class Builder implements IndexFieldData.Builder { + @Override + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + CircuitBreakerService breakerService, MapperService mapperService) { + // Ignore breaker + return new GeoPointDVIndexFieldData(indexSettings.getIndex(), fieldType.names(), fieldType.fieldDataType(), + indexSettings.getIndexVersionCreated().before(Version.V_2_2_0)); + } + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java index 34b11c47cc8..8f0f2798c05 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java @@ -19,18 +19,15 @@ package org.elasticsearch.index.fielddata.plain; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.settings.IndexSettings; import java.io.IOException; @@ -42,8 +39,8 @@ public abstract class AbstractIndexFieldData extends protected final FieldDataType fieldDataType; protected final IndexFieldDataCache cache; - public AbstractIndexFieldData(Index index, @IndexSettings Settings indexSettings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) { - super(index, indexSettings); + public AbstractIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) { + super(indexSettings); this.fieldNames = fieldNames; this.fieldDataType = fieldDataType; this.cache = cache; @@ -67,7 +64,10 @@ public abstract class AbstractIndexFieldData extends @Override public FD load(LeafReaderContext context) { if (context.reader().getFieldInfos().fieldInfo(fieldNames.indexName()) == null) { - // If the field doesn't exist, then don't bother with loading and adding an empty instance to the field data cache + // Some leaf readers may be wrapped and report different set of fields and use the same cache key. + // If a field can't be found then it doesn't mean it isn't there, + // so if a field doesn't exist then we don't cache it and just return an empty field data instance. + // The next time the field is found, we do cache. return empty(context.reader().maxDoc()); } @@ -78,7 +78,7 @@ public abstract class AbstractIndexFieldData extends if (e instanceof ElasticsearchException) { throw (ElasticsearchException) e; } else { - throw new ElasticsearchException(e.getMessage(), e); + throw new ElasticsearchException(e); } } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java index 480cbfa2baf..3b1629f7882 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java @@ -22,10 +22,10 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.MappedFieldType.Names; @@ -34,15 +34,34 @@ import org.elasticsearch.search.MultiValueMode; import java.io.IOException; abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData implements IndexGeoPointFieldData { + protected abstract static class BaseGeoPointTermsEnum { + protected final BytesRefIterator termsEnum; - protected static class GeoPointEnum { + protected BaseGeoPointTermsEnum(BytesRefIterator termsEnum) { + this.termsEnum = termsEnum; + } + } - private final BytesRefIterator termsEnum; + protected static class GeoPointTermsEnum extends BaseGeoPointTermsEnum { + protected GeoPointTermsEnum(BytesRefIterator termsEnum) { + super(termsEnum); + } + + public Long next() throws IOException { + final BytesRef term = termsEnum.next(); + if (term == null) { + return null; + } + return NumericUtils.prefixCodedToLong(term); + } + } + + protected static class GeoPointTermsEnumLegacy extends BaseGeoPointTermsEnum { private final GeoPoint next; private final CharsRefBuilder spare; - protected GeoPointEnum(BytesRefIterator termsEnum) { - this.termsEnum = termsEnum; + protected GeoPointTermsEnumLegacy(BytesRefIterator termsEnum) { + super(termsEnum); next = new GeoPoint(); spare = new CharsRefBuilder(); } @@ -68,11 +87,10 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData groups = fieldDataType.getSettings().getGroups("filter"); frequency = groups.get("frequency"); regex = groups.get("regex"); @@ -64,13 +64,31 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD // ordinals are already global return this; } + boolean fieldFound = false; + for (LeafReaderContext context : indexReader.leaves()) { + if (context.reader().getFieldInfos().fieldInfo(getFieldNames().indexName()) != null) { + fieldFound = true; + break; + } + } + if (fieldFound == false) { + // Some directory readers may be wrapped and report different set of fields and use the same cache key. + // If a field can't be found then it doesn't mean it isn't there, + // so if a field doesn't exist then we don't cache it and just return an empty field data instance. + // The next time the field is found, we do cache. + try { + return GlobalOrdinalsBuilder.buildEmpty(indexSettings, indexReader, this); + } catch (IOException e) { + throw new RuntimeException(e); + } + } try { return cache.load(indexReader, this); } catch (Throwable e) { if (e instanceof ElasticsearchException) { throw (ElasticsearchException) e; } else { - throw new ElasticsearchException(e.getMessage(), e); + throw new ElasticsearchException(e); } } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java deleted file mode 100644 index 86966617fcc..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.store.ByteArrayDataInput; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.AtomicNumericFieldData; -import org.elasticsearch.index.fielddata.FieldDataType; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; -import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; -import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; -import org.elasticsearch.index.mapper.MappedFieldType.Names; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData implements IndexNumericFieldData { - - private final NumericType numericType; - - public BinaryDVNumericIndexFieldData(Index index, Names fieldNames, NumericType numericType, FieldDataType fieldDataType) { - super(index, fieldNames, fieldDataType); - if (numericType == null) { - throw new IllegalArgumentException("numericType must be non-null"); - } - this.numericType = numericType; - } - - @Override - public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(final Object missingValue, final MultiValueMode sortMode, Nested nested) { - switch (numericType) { - case FLOAT: - return new FloatValuesComparatorSource(this, missingValue, sortMode, nested); - case DOUBLE: - return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); - default: - assert !numericType.isFloatingPoint(); - return new LongValuesComparatorSource(this, missingValue, sortMode, nested); - } - } - - @Override - public AtomicNumericFieldData load(LeafReaderContext context) { - try { - final BinaryDocValues values = DocValues.getBinary(context.reader(), fieldNames.indexName()); - if (numericType.isFloatingPoint()) { - return new AtomicDoubleFieldData(-1) { - - @Override - public SortedNumericDoubleValues getDoubleValues() { - switch (numericType) { - case FLOAT: - return new BinaryAsSortedNumericFloatValues(values); - case DOUBLE: - return new BinaryAsSortedNumericDoubleValues(values); - default: - throw new IllegalArgumentException("" + numericType); - } - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - }; - } else { - return new AtomicLongFieldData(0) { - - @Override - public SortedNumericDocValues getLongValues() { - return new BinaryAsSortedNumericDocValues(values); - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - }; - } - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - } - - @Override - public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Exception { - return load(context); - } - - @Override - public NumericType getNumericType() { - return numericType; - } - - private static class BinaryAsSortedNumericDocValues extends SortedNumericDocValues { - - private final BinaryDocValues values; - private BytesRef bytes; - private final ByteArrayDataInput in = new ByteArrayDataInput(); - private long[] longs = new long[1]; - private int count = 0; - - BinaryAsSortedNumericDocValues(BinaryDocValues values) { - this.values = values; - } - - @Override - public void setDocument(int docId) { - bytes = values.get(docId); - in.reset(bytes.bytes, bytes.offset, bytes.length); - if (!in.eof()) { - // first value uses vLong on top of zig-zag encoding, then deltas are encoded using vLong - long previousValue = longs[0] = ByteUtils.zigZagDecode(ByteUtils.readVLong(in)); - count = 1; - while (!in.eof()) { - longs = ArrayUtil.grow(longs, count + 1); - previousValue = longs[count++] = previousValue + ByteUtils.readVLong(in); - } - } else { - count = 0; - } - } - - @Override - public int count() { - return count; - } - - @Override - public long valueAt(int index) { - return longs[index]; - } - - } - - private static class BinaryAsSortedNumericDoubleValues extends SortedNumericDoubleValues { - - private final BinaryDocValues values; - private BytesRef bytes; - private int valueCount = 0; - - BinaryAsSortedNumericDoubleValues(BinaryDocValues values) { - this.values = values; - } - - @Override - public void setDocument(int docId) { - bytes = values.get(docId); - assert bytes.length % 8 == 0; - valueCount = bytes.length / 8; - } - - @Override - public int count() { - return valueCount; - } - - @Override - public double valueAt(int index) { - return ByteUtils.readDoubleLE(bytes.bytes, bytes.offset + index * 8); - } - - } - - private static class BinaryAsSortedNumericFloatValues extends SortedNumericDoubleValues { - - private final BinaryDocValues values; - private BytesRef bytes; - private int valueCount = 0; - - BinaryAsSortedNumericFloatValues(BinaryDocValues values) { - this.values = values; - } - - @Override - public void setDocument(int docId) { - bytes = values.get(docId); - assert bytes.length % 4 == 0; - valueCount = bytes.length / 4; - } - - @Override - public int count() { - return valueCount; - } - - @Override - public double valueAt(int index) { - return ByteUtils.readFloatLE(bytes.bytes, bytes.offset + index * 4); - } - - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java index b80aad90fb1..efe8bc97a30 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java @@ -22,13 +22,12 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Names; import org.elasticsearch.index.mapper.MapperService; @@ -65,11 +64,11 @@ public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData impleme public static class Builder implements IndexFieldData.Builder { @Override - public IndexFieldData build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { // Ignore breaker final Names fieldNames = fieldType.names(); - return new BytesBinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType()); + return new BytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldNames, fieldType.fieldDataType()); } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java index 03ac099489e..58a195057bb 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java @@ -20,14 +20,12 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Names; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -39,15 +37,15 @@ public final class DisabledIndexFieldData extends AbstractIndexFieldData build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { // Ignore Circuit Breaker - return new DisabledIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache); + return new DisabledIndexFieldData(indexSettings, fieldType.names(), fieldType.fieldDataType(), cache); } } - public DisabledIndexFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) { - super(index, indexSettings, fieldNames, fieldDataType, cache); + public DisabledIndexFieldData(IndexSettings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) { + super(indexSettings, fieldNames, fieldDataType, cache); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java index d7329be5d25..9cea1b00a94 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java @@ -20,11 +20,11 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.IndexReader; -import org.elasticsearch.Version; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -33,7 +33,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Names; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.IdFieldMapper; -import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -90,7 +89,7 @@ public abstract class DocValuesIndexFieldData { } @Override - public IndexFieldData build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { // Ignore Circuit Breaker final Names fieldNames = fieldType.names(); @@ -102,17 +101,11 @@ public abstract class DocValuesIndexFieldData { if (BINARY_INDEX_FIELD_NAMES.contains(fieldNames.indexName())) { assert numericType == null; - return new BinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType()); + return new BinaryDVIndexFieldData(indexSettings.getIndex(), fieldNames, fieldType.fieldDataType()); } else if (numericType != null) { - if (TimestampFieldMapper.NAME.equals(fieldNames.indexName()) - || Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1)) { - return new SortedNumericDVIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType()); - } else { - // prior to ES 1.4: multi-valued numerics were boxed inside a byte[] as BINARY - return new BinaryDVNumericIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType()); - } + return new SortedNumericDVIndexFieldData(indexSettings.getIndex(), fieldNames, numericType, fieldType.fieldDataType()); } else { - return new SortedSetDVOrdinalsIndexFieldData(index, cache, indexSettings, fieldNames, breakerService, fieldType.fieldDataType()); + return new SortedSetDVOrdinalsIndexFieldData(indexSettings, cache, fieldNames, breakerService, fieldType.fieldDataType()); } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java deleted file mode 100644 index 6b6131ebbaf..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.*; -import org.apache.lucene.util.*; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.DoubleArray; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.*; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; -import org.elasticsearch.index.fielddata.ordinals.Ordinals; -import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -/** - */ -public class DoubleArrayIndexFieldData extends AbstractIndexFieldData implements IndexNumericFieldData { - - private final CircuitBreakerService breakerService; - - public static class Builder implements IndexFieldData.Builder { - - @Override - public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, - CircuitBreakerService breakerService, MapperService mapperService) { - return new DoubleArrayIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, breakerService); - } - } - - public DoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, MappedFieldType.Names fieldNames, - FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) { - super(index, indexSettings, fieldNames, fieldDataType, cache); - this.breakerService = breakerService; - } - - @Override - public NumericType getNumericType() { - return NumericType.DOUBLE; - } - - @Override - public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Exception { - - final LeafReader reader = context.reader(); - Terms terms = reader.terms(getFieldNames().indexName()); - AtomicNumericFieldData data = null; - // TODO: Use an actual estimator to estimate before loading. - NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA)); - if (terms == null) { - data = AtomicDoubleFieldData.empty(reader.maxDoc()); - estimator.afterLoad(null, data.ramBytesUsed()); - return data; - } - // TODO: how can we guess the number of terms? numerics end up creating more terms per value... - DoubleArray values = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(128); - - final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); - boolean success = false; - try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) { - final BytesRefIterator iter = builder.buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator())); - BytesRef term; - long numTerms = 0; - while ((term = iter.next()) != null) { - values = BigArrays.NON_RECYCLING_INSTANCE.grow(values, numTerms + 1); - values.set(numTerms++, NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term))); - } - values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms); - final DoubleArray finalValues = values; - final Ordinals build = builder.build(fieldDataType.getSettings()); - RandomAccessOrds ordinals = build.ordinals(); - if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) { - final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed(); - data = new AtomicDoubleFieldData(ramBytesUsed) { - - @Override - public SortedNumericDoubleValues getDoubleValues() { - return withOrdinals(build, finalValues, reader.maxDoc()); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("ordinals", build)); - resources.add(Accountables.namedAccountable("values", finalValues)); - return Collections.unmodifiableList(resources); - } - - }; - } else { - final BitSet set = builder.buildDocsWithValuesSet(); - - // there's sweet spot where due to low unique value count, using ordinals will consume less memory - long singleValuesArraySize = reader.maxDoc() * RamUsageEstimator.NUM_BYTES_DOUBLE + (set == null ? 0 : set.ramBytesUsed()); - long uniqueValuesArraySize = values.ramBytesUsed(); - long ordinalsSize = build.ramBytesUsed(); - if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) { - final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed(); - success = true; - return data = new AtomicDoubleFieldData(ramBytesUsed) { - - @Override - public SortedNumericDoubleValues getDoubleValues() { - return withOrdinals(build, finalValues, reader.maxDoc()); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("ordinals", build)); - resources.add(Accountables.namedAccountable("values", finalValues)); - return Collections.unmodifiableList(resources); - } - - }; - } - - int maxDoc = reader.maxDoc(); - final DoubleArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(maxDoc); - for (int i = 0; i < maxDoc; i++) { - ordinals.setDocument(i); - final long ordinal = ordinals.nextOrd(); - if (ordinal != SortedSetDocValues.NO_MORE_ORDS) { - sValues.set(i, values.get(ordinal)); - } - } - assert sValues.size() == maxDoc; - final long ramBytesUsed = sValues.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed()); - data = new AtomicDoubleFieldData(ramBytesUsed) { - - @Override - public SortedNumericDoubleValues getDoubleValues() { - return singles(sValues, set); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("values", sValues)); - resources.add(Accountables.namedAccountable("missing bitset", set)); - return Collections.unmodifiableList(resources); - } - - }; - success = true; - } - success = true; - return data; - } finally { - if (success) { - estimator.afterLoad(null, data.ramBytesUsed()); - } - - } - - } - - @Override - protected AtomicNumericFieldData empty(int maxDoc) { - return AtomicDoubleFieldData.empty(maxDoc); - } - - @Override - public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { - return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); - } - - private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final DoubleArray values, int maxDoc) { - final RandomAccessOrds ords = ordinals.ordinals(); - final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords); - if (singleOrds != null) { - final NumericDoubleValues singleValues = new NumericDoubleValues() { - @Override - public double get(int docID) { - final int ord = singleOrds.getOrd(docID); - if (ord >= 0) { - return values.get(singleOrds.getOrd(docID)); - } else { - return 0; - } - } - }; - return FieldData.singleton(singleValues, DocValues.docsWithValue(ords, maxDoc)); - } else { - return new SortedNumericDoubleValues() { - @Override - public double valueAt(int index) { - return values.get(ords.ordAt(index)); - } - - @Override - public void setDocument(int doc) { - ords.setDocument(doc); - } - - @Override - public int count() { - return ords.cardinality(); - } - }; - } - } - - private static SortedNumericDoubleValues singles(final DoubleArray values, Bits set) { - final NumericDoubleValues numValues = new NumericDoubleValues() { - @Override - public double get(int docID) { - return values.get(docID); - } - }; - return FieldData.singleton(numValues, set); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java deleted file mode 100644 index 84b6dfe5c14..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.*; -import org.apache.lucene.util.*; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.FloatArray; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.*; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; -import org.elasticsearch.index.fielddata.ordinals.Ordinals; -import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -/** - */ -public class FloatArrayIndexFieldData extends AbstractIndexFieldData implements IndexNumericFieldData { - - private final CircuitBreakerService breakerService; - - public static class Builder implements IndexFieldData.Builder { - - @Override - public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, - CircuitBreakerService breakerService, MapperService mapperService) { - return new FloatArrayIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, breakerService); - } - } - - public FloatArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, MappedFieldType.Names fieldNames, - FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) { - super(index, indexSettings, fieldNames, fieldDataType, cache); - this.breakerService = breakerService; - } - - @Override - public NumericType getNumericType() { - return NumericType.FLOAT; - } - - @Override - public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Exception { - final LeafReader reader = context.reader(); - Terms terms = reader.terms(getFieldNames().indexName()); - AtomicNumericFieldData data = null; - // TODO: Use an actual estimator to estimate before loading. - NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA)); - if (terms == null) { - data = AtomicDoubleFieldData.empty(reader.maxDoc()); - estimator.afterLoad(null, data.ramBytesUsed()); - return data; - } - // TODO: how can we guess the number of terms? numerics end up creating more terms per value... - FloatArray values = BigArrays.NON_RECYCLING_INSTANCE.newFloatArray(128); - - final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); - boolean success = false; - try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) { - BytesRefIterator iter = builder.buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator())); - BytesRef term; - long numTerms = 0; - while ((term = iter.next()) != null) { - values = BigArrays.NON_RECYCLING_INSTANCE.grow(values, numTerms + 1); - values.set(numTerms++, NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term))); - } - values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms); - final FloatArray finalValues = values; - final Ordinals build = builder.build(fieldDataType.getSettings()); - RandomAccessOrds ordinals = build.ordinals(); - if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) { - final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed(); - data = new AtomicDoubleFieldData(ramBytesUsed) { - - @Override - public SortedNumericDoubleValues getDoubleValues() { - return withOrdinals(build, finalValues, reader.maxDoc()); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("ordinals", build)); - resources.add(Accountables.namedAccountable("values", finalValues)); - return Collections.unmodifiableList(resources); - } - - }; - } else { - final BitSet set = builder.buildDocsWithValuesSet(); - - // there's sweet spot where due to low unique value count, using ordinals will consume less memory - long singleValuesArraySize = reader.maxDoc() * RamUsageEstimator.NUM_BYTES_FLOAT + (set == null ? 0 : set.ramBytesUsed()); - long uniqueValuesArraySize = values.ramBytesUsed(); - long ordinalsSize = build.ramBytesUsed(); - if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) { - final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed(); - success = true; - return data = new AtomicDoubleFieldData(ramBytesUsed) { - - @Override - public SortedNumericDoubleValues getDoubleValues() { - return withOrdinals(build, finalValues, reader.maxDoc()); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("ordinals", build)); - resources.add(Accountables.namedAccountable("values", finalValues)); - return Collections.unmodifiableList(resources); - } - - }; - } - - int maxDoc = reader.maxDoc(); - final FloatArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newFloatArray(maxDoc); - for (int i = 0; i < maxDoc; i++) { - ordinals.setDocument(i); - final long ordinal = ordinals.nextOrd(); - if (ordinal != SortedSetDocValues.NO_MORE_ORDS) { - sValues.set(i, values.get(ordinal)); - } - } - assert sValues.size() == maxDoc; - final long ramBytesUsed = sValues.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed()); - data = new AtomicDoubleFieldData(ramBytesUsed) { - - @Override - public SortedNumericDoubleValues getDoubleValues() { - return singles(sValues, set); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("values", sValues)); - resources.add(Accountables.namedAccountable("missing bitset", set)); - return Collections.unmodifiableList(resources); - } - - }; - success = true; - } - success = true; - return data; - } finally { - if (success) { - estimator.afterLoad(null, data.ramBytesUsed()); - } - - } - - } - - @Override - protected AtomicNumericFieldData empty(int maxDoc) { - return AtomicDoubleFieldData.empty(maxDoc); - } - - @Override - public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { - return new FloatValuesComparatorSource(this, missingValue, sortMode, nested); - } - - private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final FloatArray values, int maxDoc) { - final RandomAccessOrds ords = ordinals.ordinals(); - final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords); - if (singleOrds != null) { - final NumericDoubleValues singleValues = new NumericDoubleValues() { - @Override - public double get(int docID) { - final int ord = singleOrds.getOrd(docID); - if (ord >= 0) { - return values.get(singleOrds.getOrd(docID)); - } else { - return 0; - } - } - }; - return FieldData.singleton(singleValues, DocValues.docsWithValue(ords, maxDoc)); - } else { - return new SortedNumericDoubleValues() { - @Override - public double valueAt(int index) { - return values.get(ords.ordAt(index)); - } - - @Override - public void setDocument(int doc) { - ords.setDocument(doc); - } - - @Override - public int count() { - return ords.cardinality(); - } - }; - } - } - - private static SortedNumericDoubleValues singles(final FloatArray values, Bits set) { - final NumericDoubleValues numValues = new NumericDoubleValues() { - @Override - public double get(int docID) { - return values.get(docID); - } - }; - return FieldData.singleton(numValues, set); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayAtomicFieldData.java new file mode 100644 index 00000000000..ce4dc2559e4 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayAtomicFieldData.java @@ -0,0 +1,150 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.Accountables; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.GeoUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.GeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.ordinals.Ordinals; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * + */ +public abstract class GeoPointArrayAtomicFieldData extends AbstractAtomicGeoPointFieldData { + @Override + public void close() { + } + + static class WithOrdinals extends GeoPointArrayAtomicFieldData { + private final LongArray indexedPoints; + private final Ordinals ordinals; + private final int maxDoc; + + public WithOrdinals(LongArray indexedPoints, Ordinals ordinals, int maxDoc) { + super(); + this.indexedPoints = indexedPoints; + this.ordinals = ordinals; + this.maxDoc = maxDoc; + } + + @Override + public long ramBytesUsed() { + return RamUsageEstimator.NUM_BYTES_INT + indexedPoints.ramBytesUsed(); + } + + @Override + public Collection getChildResources() { + List resources = new ArrayList<>(); + resources.add(Accountables.namedAccountable("indexedPoints", indexedPoints)); + return Collections.unmodifiableList(resources); + } + + @Override + public MultiGeoPointValues getGeoPointValues() { + final RandomAccessOrds ords = ordinals.ordinals(); + final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords); + final GeoPoint point = new GeoPoint(Double.NaN, Double.NaN); + if (singleOrds != null) { + final GeoPointValues values = new GeoPointValues() { + @Override + public GeoPoint get(int docID) { + final int ord = singleOrds.getOrd(docID); + if (ord >= 0) { + return point.resetFromIndexHash(indexedPoints.get(ord)); + } + return point.reset(Double.NaN, Double.NaN); + } + }; + return FieldData.singleton(values, DocValues.docsWithValue(singleOrds, maxDoc)); + } + return new MultiGeoPointValues() { + @Override + public GeoPoint valueAt(int index) { + return point.resetFromIndexHash(indexedPoints.get(ords.ordAt(index))); + } + + @Override + public void setDocument(int docId) { + ords.setDocument(docId); + } + + @Override + public int count() { + return ords.cardinality(); + } + }; + } + } + + public static class Single extends GeoPointArrayAtomicFieldData { + private final LongArray indexedPoint; + private final BitSet set; + + public Single(LongArray indexedPoint, BitSet set) { + this.indexedPoint = indexedPoint; + this.set = set; + } + + @Override + public long ramBytesUsed() { + return RamUsageEstimator.NUM_BYTES_INT + indexedPoint.ramBytesUsed() + + (set == null ? 0 : set.ramBytesUsed()); + } + + @Override + public Collection getChildResources() { + List resources = new ArrayList<>(); + resources.add(Accountables.namedAccountable("indexedPoints", indexedPoint)); + if (set != null) { + resources.add(Accountables.namedAccountable("missing bitset", set)); + } + return Collections.unmodifiableList(resources); + } + + @Override + public MultiGeoPointValues getGeoPointValues() { + final GeoPoint point = new GeoPoint(); + final GeoPointValues values = new GeoPointValues() { + @Override + public GeoPoint get(int docID) { + if (set == null || set.get(docID)) { + return point.resetFromIndexHash(indexedPoint.get(docID)); + } + return point.reset(Double.NaN, Double.NaN); + } + }; + return FieldData.singleton(values, set); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java new file mode 100644 index 00000000000..a0c0a55be71 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java @@ -0,0 +1,184 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.Terms; +import org.apache.lucene.util.BitSet; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.DoubleArray; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.ordinals.Ordinals; +import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; + +/** + * Loads FieldData for an array of GeoPoints supporting both long encoded points and backward compatible double arrays + */ +public class GeoPointArrayIndexFieldData extends AbstractIndexGeoPointFieldData { + private final CircuitBreakerService breakerService; + private final boolean indexCreatedBefore22; + + public static class Builder implements IndexFieldData.Builder { + @Override + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + CircuitBreakerService breakerService, MapperService mapperService) { + return new GeoPointArrayIndexFieldData(indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, + breakerService, fieldType.fieldDataType().getSettings() + .getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).before(Version.V_2_2_0) || + indexSettings.getIndexVersionCreated().before(Version.V_2_2_0)); + } + } + + public GeoPointArrayIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, + FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService, + final boolean indexCreatedBefore22) { + super(indexSettings, fieldNames, fieldDataType, cache); + this.breakerService = breakerService; + this.indexCreatedBefore22 = indexCreatedBefore22; + } + + @Override + public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { + LeafReader reader = context.reader(); + + Terms terms = reader.terms(getFieldNames().indexName()); + AtomicGeoPointFieldData data = null; + // TODO: Use an actual estimator to estimate before loading. + NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA)); + if (terms == null) { + data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc()); + estimator.afterLoad(null, data.ramBytesUsed()); + return data; + } + return (indexCreatedBefore22 == true) ? loadLegacyFieldData(reader, estimator, terms, data) : loadFieldData22(reader, estimator, terms, data); + } + + /** + * long encoded geopoint field data + */ + private AtomicGeoPointFieldData loadFieldData22(LeafReader reader, NonEstimatingEstimator estimator, Terms terms, + AtomicGeoPointFieldData data) throws Exception { + LongArray indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(128); + final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", + OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); + boolean success = false; + try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) { + final GeoPointTermsEnum iter = new GeoPointTermsEnum(builder.buildFromTerms(OrdinalsBuilder.wrapNumeric64Bit(terms.iterator()))); + Long hashedPoint; + long numTerms = 0; + while ((hashedPoint = iter.next()) != null) { + indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.resize(indexedPoints, numTerms + 1); + indexedPoints.set(numTerms++, hashedPoint); + } + indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.resize(indexedPoints, numTerms); + + Ordinals build = builder.build(fieldDataType.getSettings()); + RandomAccessOrds ordinals = build.ordinals(); + if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings + .MemoryStorageFormat.ORDINALS)) { + int maxDoc = reader.maxDoc(); + LongArray sIndexedPoint = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(reader.maxDoc()); + for (int i=0; i getChildResources() { List resources = new ArrayList<>(); @@ -83,9 +83,9 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicG public GeoPoint get(int docID) { final int ord = singleOrds.getOrd(docID); if (ord >= 0) { - point.reset(lat.get(ord), lon.get(ord)); + return point.reset(lat.get(ord), lon.get(ord)); } - return point; + return point.reset(Double.NaN, Double.NaN); } }; return FieldData.singleton(values, DocValues.docsWithValue(singleOrds, maxDoc)); @@ -96,8 +96,10 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicG @Override public GeoPoint valueAt(int index) { final long ord = ords.ordAt(index); - point.reset(lat.get(ord), lon.get(ord)); - return point; + if (ord >= 0) { + return point.reset(lat.get(ord), lon.get(ord)); + } + return point.reset(Double.NaN, Double.NaN); } @Override @@ -117,7 +119,7 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicG /** * Assumes unset values are marked in bitset, and docId is used as the index to the value array. */ - public static class Single extends GeoPointDoubleArrayAtomicFieldData { + public static class Single extends GeoPointArrayLegacyAtomicFieldData { private final DoubleArray lon, lat; private final BitSet set; @@ -130,9 +132,9 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicG @Override public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed()); + return RamUsageEstimator.NUM_BYTES_INT + lon.ramBytesUsed() + lat.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed()); } - + @Override public Collection getChildResources() { List resources = new ArrayList<>(); @@ -150,8 +152,10 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicG final GeoPointValues values = new GeoPointValues() { @Override public GeoPoint get(int docID) { - point.reset(lat.get(docID), lon.get(docID)); - return point; + if (set == null || set.get(docID)) { + return point.reset(lat.get(docID), lon.get(docID)); + } + return point.reset(Double.NaN, Double.NaN); } }; return FieldData.singleton(values, set); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java deleted file mode 100644 index 466ae0fcb65..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DocValues; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.*; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MappedFieldType.Names; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { - - public GeoPointBinaryDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType) { - super(index, fieldNames, fieldDataType); - } - - @Override - public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { - throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); - } - - @Override - public AtomicGeoPointFieldData load(LeafReaderContext context) { - try { - return new GeoPointBinaryDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName())); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - } - - @Override - public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { - return load(context); - } - - public static class Builder implements IndexFieldData.Builder { - - @Override - public IndexFieldData build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, - CircuitBreakerService breakerService, MapperService mapperService) { - // Ignore breaker - final Names fieldNames = fieldType.names(); - return new GeoPointBinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType()); - } - - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java new file mode 100644 index 00000000000..a71ea3a552e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +final class GeoPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { + + private final SortedNumericDocValues values; + + GeoPointDVAtomicFieldData(SortedNumericDocValues values) { + super(); + this.values = values; + } + + @Override + public long ramBytesUsed() { + return 0; // not exposed by Lucene + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() { + // no-op + } + + @Override + public MultiGeoPointValues getGeoPointValues() { + return new MultiGeoPointValues() { + GeoPoint[] points = new GeoPoint[0]; + private int count = 0; + + @Override + public void setDocument(int docId) { + values.setDocument(docId); + count = values.count(); + if (count > points.length) { + final int previousLength = points.length; + points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); + for (int i = previousLength; i < points.length; ++i) { + points[i] = new GeoPoint(Double.NaN, Double.NaN); + } + } + for (int i=0; i build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, - CircuitBreakerService breakerService, MapperService mapperService) { - return new GeoPointDoubleArrayIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, breakerService); - } - } - - public GeoPointDoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, MappedFieldType.Names fieldNames, - FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) { - super(index, indexSettings, fieldNames, fieldDataType, cache); - this.breakerService = breakerService; - } - - @Override - public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { - LeafReader reader = context.reader(); - - Terms terms = reader.terms(getFieldNames().indexName()); - AtomicGeoPointFieldData data = null; - // TODO: Use an actual estimator to estimate before loading. - NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA)); - if (terms == null) { - data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc()); - estimator.afterLoad(null, data.ramBytesUsed()); - return data; - } - DoubleArray lat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(128); - DoubleArray lon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(128); - final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); - boolean success = false; - try (OrdinalsBuilder builder = new OrdinalsBuilder(terms.size(), reader.maxDoc(), acceptableTransientOverheadRatio)) { - final GeoPointEnum iter = new GeoPointEnum(builder.buildFromTerms(terms.iterator())); - GeoPoint point; - long numTerms = 0; - while ((point = iter.next()) != null) { - lat = BigArrays.NON_RECYCLING_INSTANCE.resize(lat, numTerms + 1); - lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms + 1); - lat.set(numTerms, point.getLat()); - lon.set(numTerms, point.getLon()); - ++numTerms; - } - lat = BigArrays.NON_RECYCLING_INSTANCE.resize(lat, numTerms); - lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms); - - Ordinals build = builder.build(fieldDataType.getSettings()); - RandomAccessOrds ordinals = build.ordinals(); - if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) { - int maxDoc = reader.maxDoc(); - DoubleArray sLat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc()); - DoubleArray sLon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc()); - for (int i = 0; i < maxDoc; i++) { - ordinals.setDocument(i); - long nativeOrdinal = ordinals.nextOrd(); - if (nativeOrdinal != RandomAccessOrds.NO_MORE_ORDS) { - sLat.set(i, lat.get(nativeOrdinal)); - sLon.set(i, lon.get(nativeOrdinal)); - } - } - BitSet set = builder.buildDocsWithValuesSet(); - data = new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat, set); - } else { - data = new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(lon, lat, build, reader.maxDoc()); - } - success = true; - return data; - } finally { - if (success) { - estimator.afterLoad(null, data.ramBytesUsed()); - } - - } - - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointLegacyDVAtomicFieldData.java similarity index 94% rename from core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVAtomicFieldData.java rename to core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointLegacyDVAtomicFieldData.java index b1dd0a14e88..c51f2b96982 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointLegacyDVAtomicFieldData.java @@ -32,14 +32,14 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -final class GeoPointBinaryDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { +final class GeoPointLegacyDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { private static final int COORDINATE_SIZE = 8; // number of bytes per coordinate private static final int GEOPOINT_SIZE = COORDINATE_SIZE * 2; // lat + lon private final BinaryDocValues values; - GeoPointBinaryDVAtomicFieldData(BinaryDocValues values) { + GeoPointLegacyDVAtomicFieldData(BinaryDocValues values) { super(); this.values = values; } @@ -48,7 +48,7 @@ final class GeoPointBinaryDVAtomicFieldData extends AbstractAtomicGeoPointFieldD public long ramBytesUsed() { return 0; // not exposed by Lucene } - + @Override public Collection getChildResources() { return Collections.emptyList(); @@ -75,7 +75,7 @@ final class GeoPointBinaryDVAtomicFieldData extends AbstractAtomicGeoPointFieldD final int previousLength = points.length; points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); for (int i = previousLength; i < points.length; ++i) { - points[i] = new GeoPoint(); + points[i] = new GeoPoint(Double.NaN, Double.NaN); } } for (int i = 0; i < count; ++i) { @@ -97,5 +97,4 @@ final class GeoPointBinaryDVAtomicFieldData extends AbstractAtomicGeoPointFieldD }; } - } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java index 36bc96d5f6d..e836f13609b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java @@ -22,14 +22,12 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.*; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -42,9 +40,9 @@ public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData { public static class Builder implements IndexFieldData.Builder { @Override - public IndexFieldData build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new IndexIndexFieldData(index, fieldType.names()); + return new IndexIndexFieldData(indexSettings, fieldType.names()); } } @@ -98,8 +96,8 @@ public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData { private final AtomicOrdinalsFieldData atomicFieldData; - private IndexIndexFieldData(Index index, MappedFieldType.Names names) { - super(index, Settings.EMPTY, names, new FieldDataType("string"), null, null); + private IndexIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names names) { + super(indexSettings, names, new FieldDataType("string"), null, null); atomicFieldData = new IndexAtomicFieldData(index().name()); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java deleted file mode 100644 index 5f95d99f85d..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java +++ /dev/null @@ -1,540 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.*; -import org.apache.lucene.util.*; -import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.packed.PackedInts; -import org.apache.lucene.util.packed.PackedLongValues; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.*; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; -import org.elasticsearch.index.fielddata.ordinals.Ordinals; -import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.*; - -/** - * Stores numeric data into bit-packed arrays for better memory efficiency. - */ -public class PackedArrayIndexFieldData extends AbstractIndexFieldData implements IndexNumericFieldData { - - public static class Builder implements IndexFieldData.Builder { - - private NumericType numericType; - - public Builder setNumericType(NumericType numericType) { - this.numericType = numericType; - return this; - } - - @Override - public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, - IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new PackedArrayIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, numericType, breakerService); - } - } - - private final NumericType numericType; - private final CircuitBreakerService breakerService; - - public PackedArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, MappedFieldType.Names fieldNames, - FieldDataType fieldDataType, IndexFieldDataCache cache, NumericType numericType, - CircuitBreakerService breakerService) { - super(index, indexSettings, fieldNames, fieldDataType, cache); - Objects.requireNonNull(numericType); - if (!EnumSet.of(NumericType.BOOLEAN, NumericType.BYTE, NumericType.SHORT, NumericType.INT, NumericType.LONG).contains(numericType)) { - throw new IllegalArgumentException(getClass().getSimpleName() + " only supports integer types, not " + numericType); - } - this.numericType = numericType; - this.breakerService = breakerService; - } - - @Override - public NumericType getNumericType() { - return numericType; - } - - @Override - public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Exception { - final LeafReader reader = context.reader(); - Terms terms = reader.terms(getFieldNames().indexName()); - AtomicNumericFieldData data = null; - PackedArrayEstimator estimator = new PackedArrayEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA), getNumericType(), getFieldNames().fullName()); - if (terms == null) { - data = AtomicLongFieldData.empty(reader.maxDoc()); - estimator.adjustForNoTerms(data.ramBytesUsed()); - return data; - } - // TODO: how can we guess the number of terms? numerics end up creating more terms per value... - // Lucene encodes numeric data so that the lexicographical (encoded) order matches the integer order so we know the sequence of - // longs is going to be monotonically increasing - final PackedLongValues.Builder valuesBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); - - final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); - TermsEnum termsEnum = estimator.beforeLoad(terms); - assert !getNumericType().isFloatingPoint(); - boolean success = false; - try (OrdinalsBuilder builder = new OrdinalsBuilder(-1, reader.maxDoc(), acceptableTransientOverheadRatio)) { - BytesRefIterator iter = builder.buildFromTerms(termsEnum); - BytesRef term; - while ((term = iter.next()) != null) { - final long value = numericType.toLong(term); - valuesBuilder.add(value); - } - final PackedLongValues values = valuesBuilder.build(); - final Ordinals build = builder.build(fieldDataType.getSettings()); - CommonSettings.MemoryStorageFormat formatHint = CommonSettings.getMemoryStorageHint(fieldDataType); - - RandomAccessOrds ordinals = build.ordinals(); - if (FieldData.isMultiValued(ordinals) || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) { - final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed(); - data = new AtomicLongFieldData(ramBytesUsed) { - - @Override - public SortedNumericDocValues getLongValues() { - return withOrdinals(build, values, reader.maxDoc()); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("ordinals", build)); - resources.add(Accountables.namedAccountable("values", values)); - return Collections.unmodifiableList(resources); - } - }; - } else { - final BitSet docsWithValues = builder.buildDocsWithValuesSet(); - - long minV, maxV; - minV = maxV = 0; - if (values.size() > 0) { - minV = values.get(0); - maxV = values.get(values.size() - 1); - } - - - final float acceptableOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_overhead_ratio", PackedInts.DEFAULT); - final int pageSize = fieldDataType.getSettings().getAsInt("single_value_page_size", 1024); - - if (formatHint == null) { - formatHint = chooseStorageFormat(reader, values, build, ordinals, minV, maxV, acceptableOverheadRatio, pageSize); - } - - logger.trace("single value format for field [{}] set to [{}]", getFieldNames().fullName(), formatHint); - - switch (formatHint) { - case PACKED: - // Encode document without a value with a special value - long missingV = 0; - if (docsWithValues != null) { - if ((maxV - minV + 1) == values.size()) { - // values are dense - if (minV > Long.MIN_VALUE) { - missingV = --minV; - } else { - assert maxV != Long.MAX_VALUE; - missingV = ++maxV; - } - } else { - for (long i = 1; i < values.size(); ++i) { - if (values.get(i) > values.get(i - 1) + 1) { - missingV = values.get(i - 1) + 1; - break; - } - } - } - missingV -= minV; - } - final long missingValue = missingV; - final long minValue = minV; - final long maxValue = maxV; - - final long valuesDelta = maxValue - minValue; - int bitsRequired = valuesDelta < 0 ? 64 : PackedInts.bitsRequired(valuesDelta); - final PackedInts.Mutable sValues = PackedInts.getMutable(reader.maxDoc(), bitsRequired, acceptableOverheadRatio); - - if (docsWithValues != null) { - sValues.fill(0, sValues.size(), missingV); - } - - for (int i = 0; i < reader.maxDoc(); i++) { - ordinals.setDocument(i); - if (ordinals.cardinality() > 0) { - final long ord = ordinals.ordAt(0); - long value = values.get(ord); - sValues.set(i, value - minValue); - } - } - long ramBytesUsed = values.ramBytesUsed() + (docsWithValues == null ? 0 : docsWithValues.ramBytesUsed()); - data = new AtomicLongFieldData(ramBytesUsed) { - - @Override - public SortedNumericDocValues getLongValues() { - if (docsWithValues == null) { - return singles(sValues, minValue); - } else { - return sparseSingles(sValues, minValue, missingValue, reader.maxDoc()); - } - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("values", sValues)); - if (docsWithValues != null) { - resources.add(Accountables.namedAccountable("missing bitset", docsWithValues)); - } - return Collections.unmodifiableList(resources); - } - - }; - break; - case PAGED: - final PackedLongValues.Builder dpValues = PackedLongValues.deltaPackedBuilder(pageSize, acceptableOverheadRatio); - - long lastValue = 0; - for (int i = 0; i < reader.maxDoc(); i++) { - ordinals.setDocument(i); - if (ordinals.cardinality() > 0) { - final long ord = ordinals.ordAt(i); - lastValue = values.get(ord); - } - dpValues.add(lastValue); - } - final PackedLongValues pagedValues = dpValues.build(); - ramBytesUsed = pagedValues.ramBytesUsed(); - if (docsWithValues != null) { - ramBytesUsed += docsWithValues.ramBytesUsed(); - } - data = new AtomicLongFieldData(ramBytesUsed) { - - @Override - public SortedNumericDocValues getLongValues() { - return pagedSingles(pagedValues, docsWithValues); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("values", pagedValues)); - if (docsWithValues != null) { - resources.add(Accountables.namedAccountable("missing bitset", docsWithValues)); - } - return Collections.unmodifiableList(resources); - } - - }; - break; - case ORDINALS: - ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed(); - data = new AtomicLongFieldData(ramBytesUsed) { - - @Override - public SortedNumericDocValues getLongValues() { - return withOrdinals(build, values, reader.maxDoc()); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - resources.add(Accountables.namedAccountable("ordinals", build)); - resources.add(Accountables.namedAccountable("values", values)); - return Collections.unmodifiableList(resources); - } - - }; - break; - default: - throw new ElasticsearchException("unknown memory format: " + formatHint); - } - - } - - success = true; - return data; - } finally { - if (!success) { - // If something went wrong, unwind any current estimations we've made - estimator.afterLoad(termsEnum, 0); - } else { - // Adjust as usual, based on the actual size of the field data - estimator.afterLoad(termsEnum, data.ramBytesUsed()); - } - - } - - } - - protected CommonSettings.MemoryStorageFormat chooseStorageFormat(LeafReader reader, PackedLongValues values, Ordinals build, RandomAccessOrds ordinals, - long minValue, long maxValue, float acceptableOverheadRatio, int pageSize) { - - CommonSettings.MemoryStorageFormat format; - - // estimate memory usage for a single packed array - long packedDelta = maxValue - minValue + 1; // allow for a missing value - // valuesDelta can be negative if the difference between max and min values overflows the positive side of longs. - int bitsRequired = packedDelta < 0 ? 64 : PackedInts.bitsRequired(packedDelta); - PackedInts.FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits(reader.maxDoc(), bitsRequired, acceptableOverheadRatio); - final long singleValuesSize = formatAndBits.format.longCount(PackedInts.VERSION_CURRENT, reader.maxDoc(), formatAndBits.bitsPerValue) * 8L; - - // ordinal memory usage - final long ordinalsSize = build.ramBytesUsed() + values.ramBytesUsed(); - - // estimate the memory signature of paged packing - long pagedSingleValuesSize = (reader.maxDoc() / pageSize + 1) * RamUsageEstimator.NUM_BYTES_OBJECT_REF; // array of pages - int pageIndex = 0; - long pageMinOrdinal = Long.MAX_VALUE; - long pageMaxOrdinal = Long.MIN_VALUE; - for (int i = 1; i < reader.maxDoc(); ++i, pageIndex = (pageIndex + 1) % pageSize) { - ordinals.setDocument(i); - if (ordinals.cardinality() > 0) { - long ordinal = ordinals.ordAt(0); - pageMaxOrdinal = Math.max(ordinal, pageMaxOrdinal); - pageMinOrdinal = Math.min(ordinal, pageMinOrdinal); - } - if (pageIndex == pageSize - 1) { - // end of page, we now know enough to estimate memory usage - pagedSingleValuesSize += getPageMemoryUsage(values, acceptableOverheadRatio, pageSize, pageMinOrdinal, pageMaxOrdinal); - - pageMinOrdinal = Long.MAX_VALUE; - pageMaxOrdinal = Long.MIN_VALUE; - } - } - - if (pageIndex > 0) { - // last page estimation - pageIndex++; - pagedSingleValuesSize += getPageMemoryUsage(values, acceptableOverheadRatio, pageSize, pageMinOrdinal, pageMaxOrdinal); - } - - if (ordinalsSize < singleValuesSize) { - if (ordinalsSize < pagedSingleValuesSize) { - format = CommonSettings.MemoryStorageFormat.ORDINALS; - } else { - format = CommonSettings.MemoryStorageFormat.PAGED; - } - } else { - if (pagedSingleValuesSize < singleValuesSize) { - format = CommonSettings.MemoryStorageFormat.PAGED; - } else { - format = CommonSettings.MemoryStorageFormat.PACKED; - } - } - return format; - } - - private long getPageMemoryUsage(PackedLongValues values, float acceptableOverheadRatio, int pageSize, long pageMinOrdinal, long pageMaxOrdinal) { - int bitsRequired; - long pageMemorySize = 0; - PackedInts.FormatAndBits formatAndBits; - if (pageMaxOrdinal == Long.MIN_VALUE) { - // empty page - will use the null reader which just stores size - pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); - - } else { - long pageMinValue = values.get(pageMinOrdinal); - long pageMaxValue = values.get(pageMaxOrdinal); - long pageDelta = pageMaxValue - pageMinValue; - if (pageDelta != 0) { - bitsRequired = pageDelta < 0 ? 64 : PackedInts.bitsRequired(pageDelta); - formatAndBits = PackedInts.fastestFormatAndBits(pageSize, bitsRequired, acceptableOverheadRatio); - pageMemorySize += formatAndBits.format.longCount(PackedInts.VERSION_CURRENT, pageSize, formatAndBits.bitsPerValue) * RamUsageEstimator.NUM_BYTES_LONG; - pageMemorySize += RamUsageEstimator.NUM_BYTES_LONG; // min value per page storage - } else { - // empty page - pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); - } - } - return pageMemorySize; - } - - @Override - protected AtomicNumericFieldData empty(int maxDoc) { - return AtomicLongFieldData.empty(maxDoc); - } - - @Override - public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { - return new LongValuesComparatorSource(this, missingValue, sortMode, nested); - } - - /** - * Estimator that wraps numeric field data loading in a - * RamAccountingTermsEnum, adjusting the breaker after data has been - * loaded - */ - public class PackedArrayEstimator implements PerValueEstimator { - - private final CircuitBreaker breaker; - private final NumericType type; - private final String fieldName; - - public PackedArrayEstimator(CircuitBreaker breaker, NumericType type, String fieldName) { - this.breaker = breaker; - this.type = type; - this.fieldName = fieldName; - } - - /** - * @return number of bytes per term, based on the NumericValue.requiredBits() - */ - @Override - public long bytesPerValue(BytesRef term) { - // Estimate about about 0.8 (8 / 10) compression ratio for - // numbers, but at least 4 bytes - return Math.max(type.requiredBits() / 10, 4); - } - - /** - * @return A TermsEnum wrapped in a RamAccountingTermsEnum - */ - @Override - public TermsEnum beforeLoad(Terms terms) throws IOException { - return new RamAccountingTermsEnum(type.wrapTermsEnum(terms.iterator()), breaker, this, this.fieldName); - } - - /** - * Adjusts the breaker based on the aggregated value from the RamAccountingTermsEnum - * - * @param termsEnum terms that were wrapped and loaded - * @param actualUsed actual field data memory usage - */ - @Override - public void afterLoad(TermsEnum termsEnum, long actualUsed) { - assert termsEnum instanceof RamAccountingTermsEnum; - long estimatedBytes = ((RamAccountingTermsEnum) termsEnum).getTotalBytes(); - breaker.addWithoutBreaking(-(estimatedBytes - actualUsed)); - } - - /** - * Adjust the breaker when no terms were actually loaded, but the field - * data takes up space regardless. For instance, when ordinals are - * used. - * - * @param actualUsed bytes actually used - */ - public void adjustForNoTerms(long actualUsed) { - breaker.addWithoutBreaking(actualUsed); - } - } - - private static SortedNumericDocValues withOrdinals(Ordinals ordinals, final LongValues values, int maxDoc) { - final RandomAccessOrds ords = ordinals.ordinals(); - final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords); - if (singleOrds != null) { - final NumericDocValues singleValues = new NumericDocValues() { - @Override - public long get(int docID) { - final int ord = singleOrds.getOrd(docID); - if (ord >= 0) { - return values.get(singleOrds.getOrd(docID)); - } else { - return 0; - } - } - }; - return DocValues.singleton(singleValues, DocValues.docsWithValue(ords, maxDoc)); - } else { - return new SortedNumericDocValues() { - @Override - public long valueAt(int index) { - return values.get(ords.ordAt(index)); - } - - @Override - public void setDocument(int doc) { - ords.setDocument(doc); - } - - @Override - public int count() { - return ords.cardinality(); - } - }; - } - } - - private static SortedNumericDocValues singles(final NumericDocValues deltas, final long minValue) { - final NumericDocValues values; - if (minValue == 0) { - values = deltas; - } else { - values = new NumericDocValues() { - @Override - public long get(int docID) { - return minValue + deltas.get(docID); - } - }; - } - return DocValues.singleton(values, null); - } - - private static SortedNumericDocValues sparseSingles(final NumericDocValues deltas, final long minValue, final long missingValue, final int maxDoc) { - final NumericDocValues values = new NumericDocValues() { - @Override - public long get(int docID) { - final long delta = deltas.get(docID); - if (delta == missingValue) { - return 0; - } - return minValue + delta; - } - }; - final Bits docsWithFields = new Bits() { - @Override - public boolean get(int index) { - return deltas.get(index) != missingValue; - } - @Override - public int length() { - return maxDoc; - } - }; - return DocValues.singleton(values, docsWithFields); - } - - private static SortedNumericDocValues pagedSingles(final PackedLongValues values, final Bits docsWithValue) { - return DocValues.singleton(new NumericDocValues() { - // we need to wrap since NumericDocValues must return 0 when a doc has no value - @Override - public long get(int docID) { - if (docsWithValue == null || docsWithValue.get(docID)) { - return values.get(docID); - } else { - return 0; - } - } - }, docsWithValue); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java index 48c954394e5..2dcffbe12cc 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java @@ -27,15 +27,12 @@ import org.apache.lucene.util.PagedBytes; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.ordinals.Ordinals; import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; import java.io.IOException; @@ -48,15 +45,15 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData { public static class Builder implements IndexFieldData.Builder { @Override - public IndexOrdinalsFieldData build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, + public IndexOrdinalsFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new PagedBytesIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, breakerService); + return new PagedBytesIndexFieldData(indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, breakerService); } } - public PagedBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, MappedFieldType.Names fieldNames, + public PagedBytesIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) { - super(index, indexSettings, fieldNames, fieldDataType, cache, breakerService); + super(indexSettings, fieldNames, fieldDataType, cache, breakerService); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java index 187cd25d193..b1393542098 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java @@ -30,9 +30,9 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; @@ -41,7 +41,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Names; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; @@ -58,10 +57,10 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData parentTypes; private final CircuitBreakerService breakerService; - public ParentChildIndexFieldData(Index index, @IndexSettings Settings indexSettings, MappedFieldType.Names fieldNames, + public ParentChildIndexFieldData(IndexSettings indexSettings, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache, MapperService mapperService, CircuitBreakerService breakerService) { - super(index, indexSettings, fieldNames, fieldDataType, cache); + super(indexSettings, fieldNames, fieldDataType, cache); this.breakerService = breakerService; Set parentTypes = new HashSet<>(); for (DocumentMapper mapper : mapperService.docMappers(false)) { @@ -126,10 +125,11 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData build(Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, + public IndexFieldData build(IndexSettings indexSettings, + MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new ParentChildIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, + return new ParentChildIndexFieldData(indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, mapperService, breakerService); } } @@ -146,7 +146,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData fields; + private final List patterns; - public CustomFieldsVisitor(Set fields, boolean loadSource) { + public CustomFieldsVisitor(Set fields, List patterns, boolean loadSource) { super(loadSource); this.fields = fields; + this.patterns = patterns; + } + + public CustomFieldsVisitor(Set fields, boolean loadSource) { + this(fields, Collections.emptyList(), loadSource); } @Override @@ -42,7 +52,14 @@ public class CustomFieldsVisitor extends FieldsVisitor { if (super.needsField(fieldInfo) == Status.YES) { return Status.YES; } - - return fields.contains(fieldInfo.name) ? Status.YES : Status.NO; + if (fields.contains(fieldInfo.name)) { + return Status.YES; + } + for (String pattern : patterns) { + if (Regex.simpleMatch(pattern, fieldInfo.name)) { + return Status.YES; + } + } + return Status.NO; } } diff --git a/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java b/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java index 0566442c189..a1d18c13506 100644 --- a/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java +++ b/core/src/main/java/org/elasticsearch/index/flush/FlushStats.java @@ -50,6 +50,10 @@ public class FlushStats implements Streamable, ToXContent { } public void add(FlushStats flushStats) { + addTotals(flushStats); + } + + public void addTotals(FlushStats flushStats) { if (flushStats == null) { return; } diff --git a/core/src/main/java/org/elasticsearch/index/get/GetStats.java b/core/src/main/java/org/elasticsearch/index/get/GetStats.java index fa7730c21d7..876c9ac565a 100644 --- a/core/src/main/java/org/elasticsearch/index/get/GetStats.java +++ b/core/src/main/java/org/elasticsearch/index/get/GetStats.java @@ -51,6 +51,14 @@ public class GetStats implements Streamable, ToXContent { } public void add(GetStats stats) { + if (stats == null) { + return; + } + current += stats.current; + addTotals(stats); + } + + public void addTotals(GetStats stats) { if (stats == null) { return; } diff --git a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 1cf68ad383e..32e21be0326 100644 --- a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; @@ -72,9 +73,9 @@ public final class ShardGetService extends AbstractIndexShardComponent { private final CounterMetric currentMetric = new CounterMetric(); private final IndexShard indexShard; - public ShardGetService(IndexShard indexShard, + public ShardGetService(IndexSettings indexSettings, IndexShard indexShard, MapperService mapperService) { - super(indexShard.shardId(), indexShard.indexSettings()); + super(indexShard.shardId(), indexSettings); this.mapperService = mapperService; this.indexShard = indexShard; } @@ -281,14 +282,11 @@ public final class ShardGetService extends AbstractIndexShardComponent { boolean sourceFieldFiltering = sourceFieldMapper.includes().length > 0 || sourceFieldMapper.excludes().length > 0; boolean sourceFetchFiltering = fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0; - if (fetchSourceContext.transformSource() || sourceFieldFiltering || sourceFetchFiltering) { + if (sourceFieldFiltering || sourceFetchFiltering) { // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care? Tuple> typeMapTuple = XContentHelper.convertToMap(source.source, true); XContentType sourceContentType = typeMapTuple.v1(); Map sourceAsMap = typeMapTuple.v2(); - if (fetchSourceContext.transformSource()) { - sourceAsMap = docMapper.transformSourceAsMap(sourceAsMap); - } if (sourceFieldFiltering) { sourceAsMap = XContentMapValues.filter(sourceAsMap, sourceFieldMapper.includes(), sourceFieldMapper.excludes()); } @@ -396,16 +394,13 @@ public final class ShardGetService extends AbstractIndexShardComponent { if (!fetchSourceContext.fetchSource()) { source = null; - } else if (fetchSourceContext.transformSource() || fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) { + } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) { Map sourceAsMap; XContentType sourceContentType = null; // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care? Tuple> typeMapTuple = XContentHelper.convertToMap(source, true); sourceContentType = typeMapTuple.v1(); sourceAsMap = typeMapTuple.v2(); - if (fetchSourceContext.transformSource()) { - sourceAsMap = docMapper.transformSourceAsMap(sourceAsMap); - } sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes()); try { source = XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap).bytes(); diff --git a/core/src/main/java/org/elasticsearch/index/indexing/IndexingStats.java b/core/src/main/java/org/elasticsearch/index/indexing/IndexingStats.java index af4add91329..3df62994f96 100644 --- a/core/src/main/java/org/elasticsearch/index/indexing/IndexingStats.java +++ b/core/src/main/java/org/elasticsearch/index/indexing/IndexingStats.java @@ -232,7 +232,7 @@ public class IndexingStats implements Streamable, ToXContent { if (indexingStats == null) { return; } - totalStats.add(indexingStats.totalStats); + addTotals(indexingStats); if (includeTypes && indexingStats.typeStats != null && !indexingStats.typeStats.isEmpty()) { if (typeStats == null) { typeStats = new HashMap<>(indexingStats.typeStats.size()); @@ -248,6 +248,13 @@ public class IndexingStats implements Streamable, ToXContent { } } + public void addTotals(IndexingStats indexingStats) { + if (indexingStats == null) { + return; + } + totalStats.add(indexingStats.totalStats); + } + public Stats getTotal() { return this.totalStats; } diff --git a/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java b/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java index a3a1fa5b4a7..5cf180c3a2e 100644 --- a/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java +++ b/core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; @@ -48,9 +49,9 @@ public class ShardIndexingService extends AbstractIndexShardComponent { private volatile Map typesStats = emptyMap(); - public ShardIndexingService(ShardId shardId, Settings indexSettings) { + public ShardIndexingService(ShardId shardId, IndexSettings indexSettings) { super(shardId, indexSettings); - this.slowLog = new IndexingSlowLog(indexSettings); + this.slowLog = new IndexingSlowLog(this.indexSettings.getSettings()); } /** diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 0314f8f6aae..c4fec8cf095 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -20,10 +20,9 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Weight; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; @@ -36,9 +35,8 @@ import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.Mapping.SourceTransform; +import org.elasticsearch.index.mapper.MetadataFieldMapper.TypeParser; import org.elasticsearch.index.mapper.internal.AllFieldMapper; -import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.internal.IdFieldMapper; import org.elasticsearch.index.mapper.internal.IndexFieldMapper; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; @@ -48,14 +46,8 @@ import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.mapper.internal.VersionFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -78,9 +70,7 @@ public class DocumentMapper implements ToXContent { public static class Builder { - private Map, MetadataFieldMapper> rootMappers = new LinkedHashMap<>(); - - private List sourceTransforms = new ArrayList<>(1); + private Map, MetadataFieldMapper> metadataMappers = new LinkedHashMap<>(); private final Settings indexSettings; @@ -95,25 +85,12 @@ public class DocumentMapper implements ToXContent { this.builderContext = new Mapper.BuilderContext(indexSettings, new ContentPath(1)); this.rootObjectMapper = builder.build(builderContext); - // TODO: find a cleaner way to handle existing root mappings and using their field type as the default. - // the vast majority of these root mappers only need the existing type for backwards compatibility, since - // the pre 2.0 field type settings could be modified - - // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination - this.rootMappers.put(UidFieldMapper.class, new UidFieldMapper(indexSettings, mapperService.fullName(UidFieldMapper.NAME))); - this.rootMappers.put(IdFieldMapper.class, new IdFieldMapper(indexSettings, mapperService.fullName(IdFieldMapper.NAME))); - this.rootMappers.put(RoutingFieldMapper.class, new RoutingFieldMapper(indexSettings, mapperService.fullName(RoutingFieldMapper.NAME))); - // add default mappers, order is important (for example analyzer should come before the rest to set context.analyzer) - this.rootMappers.put(IndexFieldMapper.class, new IndexFieldMapper(indexSettings, mapperService.fullName(IndexFieldMapper.NAME))); - this.rootMappers.put(SourceFieldMapper.class, new SourceFieldMapper(indexSettings)); - this.rootMappers.put(TypeFieldMapper.class, new TypeFieldMapper(indexSettings, mapperService.fullName(TypeFieldMapper.NAME))); - this.rootMappers.put(AllFieldMapper.class, new AllFieldMapper(indexSettings, mapperService.fullName(AllFieldMapper.NAME))); - this.rootMappers.put(TimestampFieldMapper.class, new TimestampFieldMapper(indexSettings, mapperService.fullName(TimestampFieldMapper.NAME))); - this.rootMappers.put(TTLFieldMapper.class, new TTLFieldMapper(indexSettings)); - this.rootMappers.put(VersionFieldMapper.class, new VersionFieldMapper(indexSettings)); - this.rootMappers.put(ParentFieldMapper.class, new ParentFieldMapper(indexSettings, mapperService.fullName(ParentFieldMapper.NAME), /* parent type */builder.name())); - // _field_names last so that it can see all other fields - this.rootMappers.put(FieldNamesFieldMapper.class, new FieldNamesFieldMapper(indexSettings, mapperService.fullName(FieldNamesFieldMapper.NAME))); + for (Map.Entry entry : mapperService.mapperRegistry.getMetadataMapperParsers().entrySet()) { + final String name = entry.getKey(); + final TypeParser parser = entry.getValue(); + final MetadataFieldMapper metadataMapper = parser.getDefault(indexSettings, mapperService.fullName(name), builder.name()); + metadataMappers.put(metadataMapper.getClass(), metadataMapper); + } } public Builder meta(Map meta) { @@ -123,28 +100,13 @@ public class DocumentMapper implements ToXContent { public Builder put(MetadataFieldMapper.Builder mapper) { MetadataFieldMapper metadataMapper = mapper.build(builderContext); - rootMappers.put(metadataMapper.getClass(), metadataMapper); - return this; - } - - public Builder transform(ScriptService scriptService, Script script) { - sourceTransforms.add(new ScriptTransform(scriptService, script)); - return this; - } - - /** - * @deprecated Use {@link #transform(ScriptService, Script)} instead. - */ - @Deprecated - public Builder transform(ScriptService scriptService, String script, ScriptType scriptType, String language, - Map parameters) { - sourceTransforms.add(new ScriptTransform(scriptService, new Script(script, scriptType, language, parameters))); + metadataMappers.put(metadataMapper.getClass(), metadataMapper); return this; } public DocumentMapper build(MapperService mapperService, DocumentMapperParser docMapperParser) { Objects.requireNonNull(rootObjectMapper, "Mapper builder must have the root object mapper set"); - return new DocumentMapper(mapperService, indexSettings, docMapperParser, rootObjectMapper, meta, rootMappers, sourceTransforms, mapperService.mappingLock); + return new DocumentMapper(mapperService, indexSettings, docMapperParser, rootObjectMapper, meta, metadataMappers, mapperService.mappingLock); } } @@ -171,8 +133,7 @@ public class DocumentMapper implements ToXContent { public DocumentMapper(MapperService mapperService, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, RootObjectMapper rootObjectMapper, Map meta, - Map, MetadataFieldMapper> rootMappers, - List sourceTransforms, + Map, MetadataFieldMapper> metadataMappers, ReentrantReadWriteLock mappingLock) { this.mapperService = mapperService; this.type = rootObjectMapper.name(); @@ -180,17 +141,16 @@ public class DocumentMapper implements ToXContent { this.mapping = new Mapping( Version.indexCreated(indexSettings), rootObjectMapper, - rootMappers.values().toArray(new MetadataFieldMapper[rootMappers.values().size()]), - sourceTransforms.toArray(new SourceTransform[sourceTransforms.size()]), + metadataMappers.values().toArray(new MetadataFieldMapper[metadataMappers.values().size()]), meta); this.documentParser = new DocumentParser(indexSettings, docMapperParser, this, new ReleasableLock(mappingLock.readLock())); this.mappingWriteLock = new ReleasableLock(mappingLock.writeLock()); this.mappingLock = mappingLock; - if (rootMapper(ParentFieldMapper.class).active()) { + if (metadataMapper(ParentFieldMapper.class).active()) { // mark the routing field mapper as required - rootMapper(RoutingFieldMapper.class).markAsRequired(); + metadataMapper(RoutingFieldMapper.class).markAsRequired(); } // collect all the mappers for this type @@ -248,52 +208,52 @@ public class DocumentMapper implements ToXContent { } public UidFieldMapper uidMapper() { - return rootMapper(UidFieldMapper.class); + return metadataMapper(UidFieldMapper.class); } @SuppressWarnings({"unchecked"}) - public T rootMapper(Class type) { - return mapping.rootMapper(type); + public T metadataMapper(Class type) { + return mapping.metadataMapper(type); } public IndexFieldMapper indexMapper() { - return rootMapper(IndexFieldMapper.class); + return metadataMapper(IndexFieldMapper.class); } public TypeFieldMapper typeMapper() { - return rootMapper(TypeFieldMapper.class); + return metadataMapper(TypeFieldMapper.class); } public SourceFieldMapper sourceMapper() { - return rootMapper(SourceFieldMapper.class); + return metadataMapper(SourceFieldMapper.class); } public AllFieldMapper allFieldMapper() { - return rootMapper(AllFieldMapper.class); + return metadataMapper(AllFieldMapper.class); } public IdFieldMapper idFieldMapper() { - return rootMapper(IdFieldMapper.class); + return metadataMapper(IdFieldMapper.class); } public RoutingFieldMapper routingFieldMapper() { - return rootMapper(RoutingFieldMapper.class); + return metadataMapper(RoutingFieldMapper.class); } public ParentFieldMapper parentFieldMapper() { - return rootMapper(ParentFieldMapper.class); + return metadataMapper(ParentFieldMapper.class); } public TimestampFieldMapper timestampFieldMapper() { - return rootMapper(TimestampFieldMapper.class); + return metadataMapper(TimestampFieldMapper.class); } public TTLFieldMapper TTLFieldMapper() { - return rootMapper(TTLFieldMapper.class); + return metadataMapper(TTLFieldMapper.class); } public IndexFieldMapper IndexFieldMapper() { - return rootMapper(IndexFieldMapper.class); + return metadataMapper(IndexFieldMapper.class); } public Query typeFilter() { @@ -330,17 +290,14 @@ public class DocumentMapper implements ToXContent { continue; } - Filter filter = objectMapper.nestedTypeFilter(); + Query filter = objectMapper.nestedTypeFilter(); if (filter == null) { continue; } // We can pass down 'null' as acceptedDocs, because nestedDocId is a doc to be fetched and // therefor is guaranteed to be a live doc. - DocIdSet nestedTypeSet = filter.getDocIdSet(context, null); - if (nestedTypeSet == null) { - continue; - } - DocIdSetIterator iterator = nestedTypeSet.iterator(); + final Weight nestedWeight = filter.createWeight(sc.searcher(), false); + DocIdSetIterator iterator = nestedWeight.scorer(context); if (iterator == null) { continue; } @@ -373,23 +330,12 @@ public class DocumentMapper implements ToXContent { } } - /** - * Transform the source when it is expressed as a map. This is public so it can be transformed the source is loaded. - * @param sourceAsMap source to transform. This may be mutated by the script. - * @return transformed version of transformMe. This may actually be the same object as sourceAsMap - */ - public Map transformSourceAsMap(Map sourceAsMap) { - return DocumentParser.transformSourceAsMap(mapping, sourceAsMap); - } - public boolean isParent(String type) { return mapperService.getParentTypes().contains(type); } private void addMappers(Collection objectMappers, Collection fieldMappers, boolean updateAllTypes) { assert mappingLock.isWriteLockedByCurrentThread(); - // first ensure we don't have any incompatible new fields - mapperService.checkNewMappersCompatibility(objectMappers, fieldMappers, updateAllTypes); // update mappers for this document type Map builder = new HashMap<>(this.objectMappers); @@ -403,11 +349,12 @@ public class DocumentMapper implements ToXContent { this.fieldMappers = this.fieldMappers.copyAndAllAll(fieldMappers); // finally update for the entire index - mapperService.addMappers(objectMappers, fieldMappers); + mapperService.addMappers(type, objectMappers, fieldMappers); } public MergeResult merge(Mapping mapping, boolean simulate, boolean updateAllTypes) { try (ReleasableLock lock = mappingWriteLock.acquire()) { + mapperService.checkMappersCompatibility(type, mapping, updateAllTypes); final MergeResult mergeResult = new MergeResult(simulate, updateAllTypes); this.mapping.merge(mapping, mergeResult); if (simulate == false) { @@ -434,42 +381,4 @@ public class DocumentMapper implements ToXContent { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return mapping.toXContent(builder, params); } - - /** - * Script based source transformation. - */ - private static class ScriptTransform implements SourceTransform { - private final ScriptService scriptService; - /** - * The script to transform the source document before indexing. - */ - private final Script script; - - public ScriptTransform(ScriptService scriptService, Script script) { - this.scriptService = scriptService; - this.script = script; - } - - @Override - @SuppressWarnings("unchecked") - public Map transformSourceAsMap(Map sourceAsMap) { - try { - // We use the ctx variable and the _source name to be consistent with the update api. - ExecutableScript executable = scriptService.executable(script, ScriptContext.Standard.MAPPING, null); - Map ctx = new HashMap<>(1); - ctx.put("_source", sourceAsMap); - executable.setNextVar("ctx", ctx); - executable.run(); - ctx = (Map) executable.unwrap(ctx); - return (Map) ctx.get("_source"); - } catch (Exception e) { - throw new IllegalArgumentException("failed to execute script", e); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return script.toXContent(builder, params); - } - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 91371a2663a..8951ecf0f4e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -25,59 +25,21 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.mapper.core.BinaryFieldMapper; -import org.elasticsearch.index.mapper.core.BooleanFieldMapper; -import org.elasticsearch.index.mapper.core.ByteFieldMapper; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.index.mapper.core.DateFieldMapper; -import org.elasticsearch.index.mapper.core.DoubleFieldMapper; -import org.elasticsearch.index.mapper.core.FloatFieldMapper; -import org.elasticsearch.index.mapper.core.IntegerFieldMapper; -import org.elasticsearch.index.mapper.core.LongFieldMapper; -import org.elasticsearch.index.mapper.core.ShortFieldMapper; -import org.elasticsearch.index.mapper.core.StringFieldMapper; -import org.elasticsearch.index.mapper.core.TokenCountFieldMapper; -import org.elasticsearch.index.mapper.core.TypeParsers; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.internal.AllFieldMapper; -import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; -import org.elasticsearch.index.mapper.internal.IdFieldMapper; -import org.elasticsearch.index.mapper.internal.IndexFieldMapper; -import org.elasticsearch.index.mapper.internal.ParentFieldMapper; -import org.elasticsearch.index.mapper.internal.RoutingFieldMapper; -import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.index.mapper.internal.TTLFieldMapper; -import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; -import org.elasticsearch.index.mapper.internal.TypeFieldMapper; -import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.mapper.internal.VersionFieldMapper; -import org.elasticsearch.index.mapper.ip.IpFieldMapper; -import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; +import org.elasticsearch.indices.mapper.MapperRegistry; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.*; import static java.util.Collections.unmodifiableMap; -import static java.util.Collections.unmodifiableSortedMap; import static org.elasticsearch.index.mapper.MapperBuilders.doc; public class DocumentMapperParser { @@ -87,86 +49,25 @@ public class DocumentMapperParser { final AnalysisService analysisService; private static final ESLogger logger = Loggers.getLogger(DocumentMapperParser.class); private final SimilarityService similarityService; - private final ScriptService scriptService; private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser(); - private final Object typeParsersMutex = new Object(); private final Version indexVersionCreated; private final ParseFieldMatcher parseFieldMatcher; - private volatile Map typeParsers; - private volatile Map rootTypeParsers; - private volatile SortedMap additionalRootMappers; + private final Map typeParsers; + private final Map rootTypeParsers; - public DocumentMapperParser(@IndexSettings Settings indexSettings, MapperService mapperService, AnalysisService analysisService, - SimilarityService similarityService, ScriptService scriptService) { - this.indexSettings = indexSettings; - this.parseFieldMatcher = new ParseFieldMatcher(indexSettings); + public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperService, AnalysisService analysisService, + SimilarityService similarityService, MapperRegistry mapperRegistry) { + this.indexSettings = indexSettings.getSettings(); + this.parseFieldMatcher = new ParseFieldMatcher(this.indexSettings); this.mapperService = mapperService; this.analysisService = analysisService; this.similarityService = similarityService; - this.scriptService = scriptService; - Map typeParsers = new HashMap<>(); - typeParsers.put(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser()); - typeParsers.put(ShortFieldMapper.CONTENT_TYPE, new ShortFieldMapper.TypeParser()); - typeParsers.put(IntegerFieldMapper.CONTENT_TYPE, new IntegerFieldMapper.TypeParser()); - typeParsers.put(LongFieldMapper.CONTENT_TYPE, new LongFieldMapper.TypeParser()); - typeParsers.put(FloatFieldMapper.CONTENT_TYPE, new FloatFieldMapper.TypeParser()); - typeParsers.put(DoubleFieldMapper.CONTENT_TYPE, new DoubleFieldMapper.TypeParser()); - typeParsers.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()); - typeParsers.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser()); - typeParsers.put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser()); - typeParsers.put(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser()); - typeParsers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); - typeParsers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); - typeParsers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); - typeParsers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); - typeParsers.put(TypeParsers.MULTI_FIELD_CONTENT_TYPE, TypeParsers.multiFieldConverterTypeParser); - typeParsers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); - typeParsers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); - - if (ShapesAvailability.JTS_AVAILABLE) { - typeParsers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); - } - - this.typeParsers = unmodifiableMap(typeParsers); - - Map rootTypeParsers = new HashMap<>(); - rootTypeParsers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser()); - rootTypeParsers.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser()); - rootTypeParsers.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser()); - rootTypeParsers.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); - rootTypeParsers.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser()); - rootTypeParsers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); - rootTypeParsers.put(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser()); - rootTypeParsers.put(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser()); - rootTypeParsers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()); - rootTypeParsers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()); - rootTypeParsers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); - rootTypeParsers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()); - this.rootTypeParsers = unmodifiableMap(rootTypeParsers); - additionalRootMappers = Collections.emptySortedMap(); - indexVersionCreated = Version.indexCreated(indexSettings); - } - - public void putTypeParser(String type, Mapper.TypeParser typeParser) { - synchronized (typeParsersMutex) { - Map typeParsers = new HashMap<>(this.typeParsers); - typeParsers.put(type, typeParser); - this.typeParsers = unmodifiableMap(typeParsers); - } - } - - public void putRootTypeParser(String type, Mapper.TypeParser typeParser) { - synchronized (typeParsersMutex) { - Map rootTypeParsers = new HashMap<>(this.rootTypeParsers); - rootTypeParsers.put(type, typeParser); - this.rootTypeParsers = rootTypeParsers; - SortedMap additionalRootMappers = new TreeMap<>(this.additionalRootMappers); - additionalRootMappers.put(type, typeParser); - this.additionalRootMappers = unmodifiableSortedMap(additionalRootMappers); - } + this.typeParsers = mapperRegistry.getMapperParsers(); + this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(); + indexVersionCreated = indexSettings.getIndexVersionCreated(); } public Mapper.TypeParser.ParserContext parserContext(String type) { @@ -181,7 +82,6 @@ public class DocumentMapperParser { return parse(type, source, null); } - @SuppressWarnings({"unchecked"}) public DocumentMapper parse(@Nullable String type, String source, String defaultSource) throws MapperParsingException { Map mapping = null; if (source != null) { @@ -199,7 +99,6 @@ public class DocumentMapperParser { return parseCompressed(type, source, null); } - @SuppressWarnings({"unchecked"}) public DocumentMapper parseCompressed(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException { Map mapping = null; if (source != null) { @@ -231,10 +130,6 @@ public class DocumentMapperParser { Mapper.TypeParser.ParserContext parserContext = parserContext(type); // parse RootObjectMapper DocumentMapper.Builder docBuilder = doc(indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService); - // Add default mapping for the plugged-in meta mappers - for (Map.Entry entry : additionalRootMappers.entrySet()) { - docBuilder.put((MetadataFieldMapper.Builder) entry.getValue().parse(entry.getKey(), Collections.emptyMap(), parserContext)); - } Iterator> iterator = mapping.entrySet().iterator(); // parse DocumentMapper while(iterator.hasNext()) { @@ -242,29 +137,13 @@ public class DocumentMapperParser { String fieldName = Strings.toUnderscoreCase(entry.getKey()); Object fieldNode = entry.getValue(); - if ("transform".equals(fieldName)) { - if (fieldNode instanceof Map) { - parseTransform(docBuilder, (Map) fieldNode, parserContext.indexVersionCreated()); - } else if (fieldNode instanceof List) { - for (Object transformItem: (List)fieldNode) { - if (!(transformItem instanceof Map)) { - throw new MapperParsingException("Elements of transform list must be objects but one was: " + fieldNode); - } - parseTransform(docBuilder, (Map) transformItem, parserContext.indexVersionCreated()); - } - } else { - throw new MapperParsingException("Transform must be an object or an array but was: " + fieldNode); - } + MetadataFieldMapper.TypeParser typeParser = rootTypeParsers.get(fieldName); + if (typeParser != null) { iterator.remove(); - } else { - Mapper.TypeParser typeParser = rootTypeParsers.get(fieldName); - if (typeParser != null) { - iterator.remove(); - Map fieldNodeMap = (Map) fieldNode; - docBuilder.put((MetadataFieldMapper.Builder)typeParser.parse(fieldName, fieldNodeMap, parserContext)); - fieldNodeMap.remove("type"); - checkNoRemainingFields(fieldName, fieldNodeMap, parserContext.indexVersionCreated()); - } + Map fieldNodeMap = (Map) fieldNode; + docBuilder.put(typeParser.parse(fieldName, fieldNodeMap, parserContext)); + fieldNodeMap.remove("type"); + checkNoRemainingFields(fieldName, fieldNodeMap, parserContext.indexVersionCreated()); } } @@ -302,14 +181,6 @@ public class DocumentMapperParser { return remainingFields.toString(); } - private void parseTransform(DocumentMapper.Builder docBuilder, Map transformConfig, Version indexVersionCreated) { - Script script = Script.parse(transformConfig, true, parseFieldMatcher); - if (script != null) { - docBuilder.transform(scriptService, script); - } - checkNoRemainingFields(transformConfig, indexVersionCreated, "Transform config has unsupported parameters: "); - } - private Tuple> extractMapping(String type, String source) throws MapperParsingException { Map root; try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 97435e039e1..b0ad972d575 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -28,8 +28,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ReleasableLock; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.core.DateFieldMapper.DateFieldType; @@ -47,7 +45,6 @@ import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; /** A parser for documents, given mappings from a DocumentMapper */ @@ -79,6 +76,10 @@ class DocumentParser implements Closeable { } private ParsedDocument innerParseDocument(SourceToParse source) throws MapperParsingException { + if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) { + throw new IllegalArgumentException("It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]"); + } + ParseContext.InternalParseContext context = cache.get(); final Mapping mapping = docMapper.mapping(); @@ -92,9 +93,6 @@ class DocumentParser implements Closeable { if (parser == null) { parser = XContentHelper.createParser(source.source()); } - if (mapping.sourceTransforms.length > 0) { - parser = transform(mapping, parser); - } context.reset(parser, new ParseContext.Document(), source); // will result in START_OBJECT @@ -562,44 +560,19 @@ class DocumentParser implements Closeable { return builder; } else if (token == XContentParser.Token.VALUE_NUMBER) { XContentParser.NumberType numberType = context.parser().numberType(); - if (numberType == XContentParser.NumberType.INT) { - if (context.parser().estimatedNumberType()) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); - if (builder == null) { - builder = MapperBuilders.longField(currentFieldName); - } - return builder; - } else { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "integer"); - if (builder == null) { - builder = MapperBuilders.integerField(currentFieldName); - } - return builder; - } - } else if (numberType == XContentParser.NumberType.LONG) { + if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { builder = MapperBuilders.longField(currentFieldName); } return builder; - } else if (numberType == XContentParser.NumberType.FLOAT) { - if (context.parser().estimatedNumberType()) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); - if (builder == null) { - builder = MapperBuilders.doubleField(currentFieldName); - } - return builder; - } else { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "float"); - if (builder == null) { - builder = MapperBuilders.floatField(currentFieldName); - } - return builder; - } - } else if (numberType == XContentParser.NumberType.DOUBLE) { + } else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { - builder = MapperBuilders.doubleField(currentFieldName); + // no templates are defined, we use float by default instead of double + // since this is much more space-efficient and should be enough most of + // the time + builder = MapperBuilders.floatField(currentFieldName); } return builder; } @@ -715,37 +688,64 @@ class DocumentParser implements Closeable { // The path of the dest field might be completely different from the current one so we need to reset it context = context.overridePath(new ContentPath(0)); + String[] paths = Strings.splitStringToArray(field, '.'); + String fieldName = paths[paths.length-1]; ObjectMapper mapper = context.root(); - String objectPath = ""; - String fieldPath = field; - int posDot = field.lastIndexOf('.'); - if (posDot > 0) { - objectPath = field.substring(0, posDot); - context.path().add(objectPath); - mapper = context.docMapper().objectMappers().get(objectPath); - fieldPath = field.substring(posDot + 1); + ObjectMapper[] mappers = new ObjectMapper[paths.length-1]; + if (paths.length > 1) { + ObjectMapper parent = context.root(); + for (int i = 0; i < paths.length-1; i++) { + mapper = context.docMapper().objectMappers().get(context.path().fullPathAsText(paths[i])); + if (mapper == null) { + // One mapping is missing, check if we are allowed to create a dynamic one. + ObjectMapper.Dynamic dynamic = parent.dynamic(); + if (dynamic == null) { + dynamic = dynamicOrDefault(context.root().dynamic()); + } + + switch (dynamic) { + case STRICT: + throw new StrictDynamicMappingException(parent.fullPath(), paths[i]); + case TRUE: + Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], "object"); + if (builder == null) { + // if this is a non root object, then explicitly set the dynamic behavior if set + if (!(parent instanceof RootObjectMapper) && parent.dynamic() != ObjectMapper.Defaults.DYNAMIC) { + ((ObjectMapper.Builder) builder).dynamic(parent.dynamic()); + } + builder = MapperBuilders.object(paths[i]).enabled(true).pathType(parent.pathType()); + } + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path()); + mapper = (ObjectMapper) builder.build(builderContext); + if (mapper.nested() != ObjectMapper.Nested.NO) { + throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().fullPathAsText(paths[i]) + "]) through `copy_to`"); + } + break; + case FALSE: + // Maybe we should log something to tell the user that the copy_to is ignored in this case. + break; + default: + throw new AssertionError("Unexpected dynamic type " + dynamic); + + } + } + context.path().add(paths[i]); + mappers[i] = mapper; + parent = mapper; + } } - if (mapper == null) { - //TODO: Create an object dynamically? - throw new MapperParsingException("attempt to copy value to non-existing object [" + field + "]"); - } - ObjectMapper update = parseDynamicValue(context, mapper, fieldPath, context.parser().currentToken()); + ObjectMapper update = parseDynamicValue(context, mapper, fieldName, context.parser().currentToken()); assert update != null; // we are parsing a dynamic value so we necessarily created a new mapping - // propagate the update to the root - while (objectPath.length() > 0) { - String parentPath = ""; - ObjectMapper parent = context.root(); - posDot = objectPath.lastIndexOf('.'); - if (posDot > 0) { - parentPath = objectPath.substring(0, posDot); - parent = context.docMapper().objectMappers().get(parentPath); + if (paths.length > 1) { + for (int i = paths.length - 2; i >= 0; i--) { + ObjectMapper parent = context.root(); + if (i > 0) { + parent = mappers[i-1]; + } + assert parent != null; + update = parent.mappingUpdate(update); } - if (parent == null) { - throw new IllegalStateException("[" + objectPath + "] has no parent for path [" + parentPath + "]"); - } - update = parent.mappingUpdate(update); - objectPath = parentPath; } context.addDynamicMappingsUpdate(update); } @@ -764,29 +764,10 @@ class DocumentParser implements Closeable { return mapper; } - private static XContentParser transform(Mapping mapping, XContentParser parser) throws IOException { - Map transformed; - try (XContentParser autoCloses = parser) { - transformed = transformSourceAsMap(mapping, parser.mapOrdered()); - } - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()).value(transformed); - return parser.contentType().xContent().createParser(builder.bytes()); - } - private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper.Dynamic dynamic) { return dynamic == null ? ObjectMapper.Dynamic.TRUE : dynamic; } - static Map transformSourceAsMap(Mapping mapping, Map sourceAsMap) { - if (mapping.sourceTransforms.length == 0) { - return sourceAsMap; - } - for (Mapping.SourceTransform transform : mapping.sourceTransforms) { - sourceAsMap = transform.transformSourceAsMap(sourceAsMap); - } - return sourceAsMap; - } - @Override public void close() { cache.close(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 45bef68ee00..ced3f08b229 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -307,7 +307,6 @@ public abstract class FieldMapper extends Mapper { if (ref.get().equals(fieldType()) == false) { throw new IllegalStateException("Cannot overwrite field type reference to unequal reference"); } - ref.incrementAssociatedMappers(); this.fieldTypeRef = ref; } @@ -360,7 +359,7 @@ public abstract class FieldMapper extends Mapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { if (!this.getClass().equals(mergeWith.getClass())) { String mergedType = mergeWith.getClass().getSimpleName(); if (mergeWith instanceof FieldMapper) { @@ -371,20 +370,6 @@ public abstract class FieldMapper extends Mapper { return; } FieldMapper fieldMergeWith = (FieldMapper) mergeWith; - List subConflicts = new ArrayList<>(); // TODO: just expose list from MergeResult? - fieldType().checkTypeName(fieldMergeWith.fieldType(), subConflicts); - if (subConflicts.isEmpty() == false) { - // return early if field types don't match - assert subConflicts.size() == 1; - mergeResult.addConflict(subConflicts.get(0)); - return; - } - - boolean strict = this.fieldTypeRef.getNumAssociatedMappers() > 1 && mergeResult.updateAllTypes() == false; - fieldType().checkCompatibility(fieldMergeWith.fieldType(), subConflicts, strict); - for (String conflict : subConflicts) { - mergeResult.addConflict(conflict); - } multiFields.merge(mergeWith, mergeResult); if (mergeResult.simulate() == false && mergeResult.hasConflicts() == false) { @@ -614,7 +599,7 @@ public abstract class FieldMapper extends Mapper { } // No need for locking, because locking is taken care of in ObjectMapper#merge and DocumentMapper#merge - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { FieldMapper mergeWithMultiField = (FieldMapper) mergeWith; List newFieldMappers = null; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 1b0e827ac35..da21e599cc9 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -24,9 +24,11 @@ import org.elasticsearch.common.regex.Regex; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Set; /** @@ -37,18 +39,49 @@ class FieldTypeLookup implements Iterable { /** Full field name to field type */ private final CopyOnWriteHashMap fullNameToFieldType; + /** Full field name to types containing a mapping for this full name. */ + private final CopyOnWriteHashMap> fullNameToTypes; + /** Index field name to field type */ private final CopyOnWriteHashMap indexNameToFieldType; + /** Index field name to types containing a mapping for this index name. */ + private final CopyOnWriteHashMap> indexNameToTypes; + /** Create a new empty instance. */ public FieldTypeLookup() { fullNameToFieldType = new CopyOnWriteHashMap<>(); + fullNameToTypes = new CopyOnWriteHashMap<>(); indexNameToFieldType = new CopyOnWriteHashMap<>(); + indexNameToTypes = new CopyOnWriteHashMap<>(); } - private FieldTypeLookup(CopyOnWriteHashMap fullName, CopyOnWriteHashMap indexName) { - fullNameToFieldType = fullName; - indexNameToFieldType = indexName; + private FieldTypeLookup( + CopyOnWriteHashMap fullName, + CopyOnWriteHashMap> fullNameToTypes, + CopyOnWriteHashMap indexName, + CopyOnWriteHashMap> indexNameToTypes) { + this.fullNameToFieldType = fullName; + this.fullNameToTypes = fullNameToTypes; + this.indexNameToFieldType = indexName; + this.indexNameToTypes = indexNameToTypes; + } + + private static CopyOnWriteHashMap> addType(CopyOnWriteHashMap> map, String key, String type) { + Set types = map.get(key); + if (types == null) { + return map.copyAndPut(key, Collections.singleton(type)); + } else if (types.contains(type)) { + // noting to do + return map; + } else { + Set newTypes = new HashSet<>(types.size() + 1); + newTypes.addAll(types); + newTypes.add(type); + assert newTypes.size() == types.size() + 1; + newTypes = Collections.unmodifiableSet(newTypes); + return map.copyAndPut(key, newTypes); + } } /** @@ -56,9 +89,15 @@ class FieldTypeLookup implements Iterable { * from the provided fields. If a field already exists, the field type will be updated * to use the new mappers field type. */ - public FieldTypeLookup copyAndAddAll(Collection newFieldMappers) { + public FieldTypeLookup copyAndAddAll(String type, Collection newFieldMappers) { + Objects.requireNonNull(type, "type must not be null"); + if (MapperService.DEFAULT_MAPPING.equals(type)) { + throw new IllegalArgumentException("Default mappings should not be added to the lookup"); + } CopyOnWriteHashMap fullName = this.fullNameToFieldType; + CopyOnWriteHashMap> fullNameToTypes = this.fullNameToTypes; CopyOnWriteHashMap indexName = this.indexNameToFieldType; + CopyOnWriteHashMap> indexNameToTypes = this.indexNameToTypes; for (FieldMapper fieldMapper : newFieldMappers) { MappedFieldType fieldType = fieldMapper.fieldType(); @@ -86,8 +125,23 @@ class FieldTypeLookup implements Iterable { // this new field bridges between two existing field names (a full and index name), which we cannot support throw new IllegalStateException("insane mappings found. field " + fieldType.names().fullName() + " maps across types to field " + fieldType.names().indexName()); } + + fullNameToTypes = addType(fullNameToTypes, fieldType.names().fullName(), type); + indexNameToTypes = addType(indexNameToTypes, fieldType.names().indexName(), type); + } + return new FieldTypeLookup(fullName, fullNameToTypes, indexName, indexNameToTypes); + } + + private static boolean beStrict(String type, Set types, boolean updateAllTypes) { + assert types.size() >= 1; + if (updateAllTypes) { + return false; + } else if (types.size() == 1 && types.contains(type)) { + // we are implicitly updating all types + return false; + } else { + return true; } - return new FieldTypeLookup(fullName, indexName); } /** @@ -95,16 +149,14 @@ class FieldTypeLookup implements Iterable { * If any are not compatible, an IllegalArgumentException is thrown. * If updateAllTypes is true, only basic compatibility is checked. */ - public void checkCompatibility(Collection newFieldMappers, boolean updateAllTypes) { - for (FieldMapper fieldMapper : newFieldMappers) { + public void checkCompatibility(String type, Collection fieldMappers, boolean updateAllTypes) { + for (FieldMapper fieldMapper : fieldMappers) { MappedFieldTypeReference ref = fullNameToFieldType.get(fieldMapper.fieldType().names().fullName()); if (ref != null) { List conflicts = new ArrayList<>(); - ref.get().checkTypeName(fieldMapper.fieldType(), conflicts); - if (conflicts.isEmpty()) { // only check compat if they are the same type - boolean strict = updateAllTypes == false; - ref.get().checkCompatibility(fieldMapper.fieldType(), conflicts, strict); - } + final Set types = fullNameToTypes.get(fieldMapper.fieldType().names().fullName()); + boolean strict = beStrict(type, types, updateAllTypes); + ref.get().checkCompatibility(fieldMapper.fieldType(), conflicts, strict); if (conflicts.isEmpty() == false) { throw new IllegalArgumentException("Mapper for [" + fieldMapper.fieldType().names().fullName() + "] conflicts with existing mapping in other types:\n" + conflicts.toString()); } @@ -114,11 +166,9 @@ class FieldTypeLookup implements Iterable { MappedFieldTypeReference indexNameRef = indexNameToFieldType.get(fieldMapper.fieldType().names().indexName()); if (indexNameRef != null) { List conflicts = new ArrayList<>(); - indexNameRef.get().checkTypeName(fieldMapper.fieldType(), conflicts); - if (conflicts.isEmpty()) { // only check compat if they are the same type - boolean strict = updateAllTypes == false; - indexNameRef.get().checkCompatibility(fieldMapper.fieldType(), conflicts, strict); - } + final Set types = indexNameToTypes.get(fieldMapper.fieldType().names().indexName()); + boolean strict = beStrict(type, types, updateAllTypes); + indexNameRef.get().checkCompatibility(fieldMapper.fieldType(), conflicts, strict); if (conflicts.isEmpty() == false) { throw new IllegalArgumentException("Mapper for [" + fieldMapper.fieldType().names().fullName() + "] conflicts with mapping with the same index name in other types" + conflicts.toString()); } @@ -133,6 +183,15 @@ class FieldTypeLookup implements Iterable { return ref.get(); } + /** Get the set of types that have a mapping for the given field. */ + public Set getTypes(String field) { + Set types = fullNameToTypes.get(field); + if (types == null) { + types = Collections.emptySet(); + } + return types; + } + /** Returns the field type for the given index name */ public MappedFieldType getByIndexName(String field) { MappedFieldTypeReference ref = indexNameToFieldType.get(field); @@ -140,6 +199,15 @@ class FieldTypeLookup implements Iterable { return ref.get(); } + /** Get the set of types that have a mapping for the given field. */ + public Set getTypesByIndexName(String field) { + Set types = indexNameToTypes.get(field); + if (types == null) { + types = Collections.emptySet(); + } + return types; + } + /** * Returns a list of the index names of a simple match regex like pattern against full name and index name. */ diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 2e1f9df2d0d..32e749992e6 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.similarity.SimilarityProvider; import java.io.IOException; @@ -228,9 +229,9 @@ public abstract class MappedFieldType extends FieldType { public abstract String typeName(); /** Checks this type is the same type as other. Adds a conflict if they are different. */ - public final void checkTypeName(MappedFieldType other, List conflicts) { + private final void checkTypeName(MappedFieldType other) { if (typeName().equals(other.typeName()) == false) { - conflicts.add("mapper [" + names().fullName() + "] cannot be changed from type [" + typeName() + "] to [" + other.typeName() + "]"); + throw new IllegalArgumentException("mapper [" + names().fullName() + "] cannot be changed from type [" + typeName() + "] to [" + other.typeName() + "]"); } else if (getClass() != other.getClass()) { throw new IllegalStateException("Type names equal for class " + getClass().getSimpleName() + " and " + other.getClass().getSimpleName()); } @@ -242,6 +243,8 @@ public abstract class MappedFieldType extends FieldType { * Otherwise, only properties which must never change in an index are checked. */ public void checkCompatibility(MappedFieldType other, List conflicts, boolean strict) { + checkTypeName(other); + boolean indexed = indexOptions() != IndexOptions.NONE; boolean mergeWithIndexed = other.indexOptions() != IndexOptions.NONE; // TODO: should be validating if index options go "up" (but "down" is ok) @@ -481,6 +484,10 @@ public abstract class MappedFieldType extends FieldType { } public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) { + if (numericType() != null) { + throw new QueryShardException(context, "Cannot use regular expression to filter numeric field [" + names.fullName + "]"); + } + RegexpQuery query = new RegexpQuery(createTerm(value), flags, maxDeterminizedStates); if (method != null) { query.setRewriteMethod(method); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java index d3c6b83a6a3..1a9d0b70b37 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java @@ -23,12 +23,10 @@ package org.elasticsearch.index.mapper; */ public class MappedFieldTypeReference { private MappedFieldType fieldType; // the current field type this reference points to - private int numAssociatedMappers; public MappedFieldTypeReference(MappedFieldType fieldType) { fieldType.freeze(); // ensure frozen this.fieldType = fieldType; - this.numAssociatedMappers = 1; } public MappedFieldType get() { @@ -40,11 +38,4 @@ public class MappedFieldTypeReference { this.fieldType = fieldType; } - public int getNumAssociatedMappers() { - return numAssociatedMappers; - } - - public void incrementAssociatedMappers() { - ++numAssociatedMappers; - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 9ca34e1c573..33a4dabd3be 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -134,6 +134,26 @@ public abstract class Mapper implements ToXContent, Iterable { public ParseFieldMatcher parseFieldMatcher() { return parseFieldMatcher; } + + public boolean isWithinMultiField() { return false; } + + protected Function typeParsers() { return typeParsers; } + + protected Function similarityLookupService() { return similarityLookupService; } + + public ParserContext createMultiFieldContext(ParserContext in) { + return new MultiFieldParserContext(in) { + @Override + public boolean isWithinMultiField() { return true; } + }; + } + + static class MultiFieldParserContext extends ParserContext { + MultiFieldParserContext(ParserContext in) { + super(in.type(), in.analysisService, in.similarityLookupService(), in.mapperService(), in.typeParsers(), in.indexVersionCreated(), in.parseFieldMatcher()); + } + } + } Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException; @@ -154,5 +174,5 @@ public abstract class Mapper implements ToXContent, Iterable { /** Returns the canonical name which uniquely identifies the mapper against other mappers in a type. */ public abstract String name(); - public abstract void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException; + public abstract void merge(Mapper mergeWith, MergeResult mergeResult); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java index 41b657a73ca..0df3c06d042 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java @@ -21,9 +21,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.core.*; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.internal.*; import org.elasticsearch.index.mapper.ip.IpFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; @@ -92,10 +90,6 @@ public final class MapperBuilders { return new DoubleFieldMapper.Builder(name); } - public static GeoPointFieldMapper.Builder geoPointField(String name) { - return new GeoPointFieldMapper.Builder(name); - } - public static GeoShapeFieldMapper.Builder geoShapeField(String name) { return new GeoShapeFieldMapper.Builder(name); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperException.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperException.java index c4964a00a2b..0241f1c8e45 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperException.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperException.java @@ -39,4 +39,8 @@ public class MapperException extends ElasticsearchException { public MapperException(String message, Throwable cause) { super(message, cause); } + + public MapperException(String message, Throwable cause, Object... args) { + super(message, cause, args); + } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java index 8fb999e778a..df886c5ce9d 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java @@ -41,6 +41,10 @@ public class MapperParsingException extends MapperException { super(message, cause); } + public MapperParsingException(String message, Throwable cause, Object... args) { + super(message, cause, args); + } + @Override public RestStatus status() { return RestStatus.BAD_REQUEST; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index bbd96f7d930..938f610d6db 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -26,56 +26,40 @@ import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.queries.TermsQuery; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.InvalidTypeNameException; import org.elasticsearch.indices.TypeMissingException; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.percolator.PercolatorService; import org.elasticsearch.script.ScriptService; import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.stream.Collectors; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableMap; -import static java.util.Collections.unmodifiableSet; +import static java.util.Collections.*; import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; /** @@ -123,19 +107,20 @@ public class MapperService extends AbstractIndexComponent implements Closeable { private volatile Set parentTypes = emptySet(); - @Inject - public MapperService(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService, - SimilarityService similarityService, - ScriptService scriptService) { - super(index, indexSettings); + final MapperRegistry mapperRegistry; + + public MapperService(IndexSettings indexSettings, AnalysisService analysisService, + SimilarityService similarityService, MapperRegistry mapperRegistry) { + super(indexSettings); this.analysisService = analysisService; this.fieldTypes = new FieldTypeLookup(); - this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityService, scriptService); + this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityService, mapperRegistry); this.indexAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultIndexAnalyzer(), p -> p.indexAnalyzer()); this.searchAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchAnalyzer(), p -> p.searchAnalyzer()); this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); + this.mapperRegistry = mapperRegistry; - this.dynamic = indexSettings.getAsBoolean("index.mapper.dynamic", true); + this.dynamic = this.indexSettings.getSettings().getAsBoolean("index.mapper.dynamic", true); defaultPercolatorMappingSource = "{\n" + "\"_default_\":{\n" + "\"properties\" : {\n" + @@ -146,7 +131,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { "}\n" + "}\n" + "}"; - if (index.getName().equals(ScriptService.SCRIPT_INDEX)){ + if (index().getName().equals(ScriptService.SCRIPT_INDEX)){ defaultMappingSource = "{" + "\"_default_\": {" + "\"properties\": {" + @@ -238,7 +223,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { if (mapper.type().length() == 0) { throw new InvalidTypeNameException("mapping type name is empty"); } - if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0_beta1) && mapper.type().length() > 255) { + if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_2_0_0_beta1) && mapper.type().length() > 255) { throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] is too long; limit is length 255 but was [" + mapper.type().length() + "]"); } if (mapper.type().charAt(0) == '_') { @@ -254,7 +239,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { throw new IllegalArgumentException("The [_parent.type] option can't point to the same type"); } if (typeNameStartsWithIllegalDot(mapper)) { - if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0_beta1)) { + if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { throw new IllegalArgumentException("mapping type name [" + mapper.type() + "] must not start with a '.'"); } else { logger.warn("Type [{}] starts with a '.', it is recommended not to start a type name with a '.'", mapper.type()); @@ -266,23 +251,21 @@ public class MapperService extends AbstractIndexComponent implements Closeable { DocumentMapper oldMapper = mappers.get(mapper.type()); if (oldMapper != null) { - MergeResult result = oldMapper.merge(mapper.mapping(), false, updateAllTypes); + // simulate first + MergeResult result = oldMapper.merge(mapper.mapping(), true, updateAllTypes); if (result.hasConflicts()) { - // TODO: What should we do??? - if (logger.isDebugEnabled()) { - logger.debug("merging mapping for type [{}] resulted in conflicts: [{}]", mapper.type(), Arrays.toString(result.buildConflicts())); - } + throw new IllegalArgumentException("Merge failed with failures {" + Arrays.toString(result.buildConflicts()) + "}"); } + // then apply for real + result = oldMapper.merge(mapper.mapping(), false, updateAllTypes); + assert result.hasConflicts() == false; // we already simulated return oldMapper; } else { - List newObjectMappers = new ArrayList<>(); - List newFieldMappers = new ArrayList<>(); - for (MetadataFieldMapper metadataMapper : mapper.mapping().metadataMappers) { - newFieldMappers.add(metadataMapper); - } - MapperUtils.collect(mapper.mapping().root, newObjectMappers, newFieldMappers); - checkNewMappersCompatibility(newObjectMappers, newFieldMappers, updateAllTypes); - addMappers(newObjectMappers, newFieldMappers); + Tuple, Collection> newMappers = checkMappersCompatibility( + mapper.type(), mapper.mapping(), updateAllTypes); + Collection newObjectMappers = newMappers.v1(); + Collection newFieldMappers = newMappers.v2(); + addMappers(mapper.type(), newObjectMappers, newFieldMappers); for (DocumentTypeListener typeListener : typeListeners) { typeListener.beforeCreate(mapper); @@ -317,9 +300,9 @@ public class MapperService extends AbstractIndexComponent implements Closeable { return true; } - protected void checkNewMappersCompatibility(Collection newObjectMappers, Collection newFieldMappers, boolean updateAllTypes) { + protected void checkMappersCompatibility(String type, Collection objectMappers, Collection fieldMappers, boolean updateAllTypes) { assert mappingLock.isWriteLockedByCurrentThread(); - for (ObjectMapper newObjectMapper : newObjectMappers) { + for (ObjectMapper newObjectMapper : objectMappers) { ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath()); if (existingObjectMapper != null) { MergeResult result = new MergeResult(true, updateAllTypes); @@ -330,10 +313,22 @@ public class MapperService extends AbstractIndexComponent implements Closeable { } } } - fieldTypes.checkCompatibility(newFieldMappers, updateAllTypes); + fieldTypes.checkCompatibility(type, fieldMappers, updateAllTypes); } - protected void addMappers(Collection objectMappers, Collection fieldMappers) { + protected Tuple, Collection> checkMappersCompatibility( + String type, Mapping mapping, boolean updateAllTypes) { + List objectMappers = new ArrayList<>(); + List fieldMappers = new ArrayList<>(); + for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { + fieldMappers.add(metadataMapper); + } + MapperUtils.collect(mapping.root, objectMappers, fieldMappers); + checkMappersCompatibility(type, objectMappers, fieldMappers, updateAllTypes); + return new Tuple<>(objectMappers, fieldMappers); + } + + protected void addMappers(String type, Collection objectMappers, Collection fieldMappers) { assert mappingLock.isWriteLockedByCurrentThread(); ImmutableOpenMap.Builder fullPathObjectMappers = ImmutableOpenMap.builder(this.fullPathObjectMappers); for (ObjectMapper objectMapper : objectMappers) { @@ -343,7 +338,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { } } this.fullPathObjectMappers = fullPathObjectMappers.build(); - this.fieldTypes = this.fieldTypes.copyAndAddAll(fieldMappers); + this.fieldTypes = this.fieldTypes.copyAndAddAll(type, fieldMappers); } public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException { @@ -360,10 +355,21 @@ public class MapperService extends AbstractIndexComponent implements Closeable { return mappers.containsKey(mappingType); } + /** + * Return the set of concrete types that have a mapping. + * NOTE: this does not return the default mapping. + */ public Collection types() { - return mappers.keySet(); + final Set types = new HashSet<>(mappers.keySet()); + types.remove(DEFAULT_MAPPING); + return Collections.unmodifiableSet(types); } + /** + * Return the {@link DocumentMapper} for the given type. By using the special + * {@value #DEFAULT_MAPPING} type, you can get a {@link DocumentMapper} for + * the default mapping. + */ public DocumentMapper documentMapper(String type) { return mappers.get(type); } @@ -378,7 +384,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { return new DocumentMapperForType(mapper, null); } if (!dynamic) { - throw new TypeMissingException(index, type, "trying to auto create mapping, but dynamic mapping is disabled"); + throw new TypeMissingException(index(), type, "trying to auto create mapping, but dynamic mapping is disabled"); } mapper = parse(type, null, true); return new DocumentMapperForType(mapper, mapper.mapping()); @@ -547,7 +553,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } final Mapper.Builder builder = typeParser.parse("__anonymous_" + type, emptyMap(), parserContext); - final BuilderContext builderContext = new BuilderContext(indexSettings, new ContentPath(1)); + final BuilderContext builderContext = new BuilderContext(indexSettings.getSettings(), new ContentPath(1)); fieldType = ((FieldMapper)builder.build(builderContext)).fieldType(); // There is no need to synchronize writes here. In the case of concurrent access, we could just diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 6eeb520f86c..bac42162552 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -43,35 +43,22 @@ public final class Mapping implements ToXContent { public static final List LEGACY_INCLUDE_IN_OBJECT = Arrays.asList("_all", "_id", "_parent", "_routing", "_timestamp", "_ttl"); - /** - * Transformations to be applied to the source before indexing and/or after loading. - */ - public interface SourceTransform extends ToXContent { - /** - * Transform the source when it is expressed as a map. This is public so it can be transformed the source is loaded. - * @param sourceAsMap source to transform. This may be mutated by the script. - * @return transformed version of transformMe. This may actually be the same object as sourceAsMap - */ - Map transformSourceAsMap(Map sourceAsMap); - } - final Version indexCreated; final RootObjectMapper root; final MetadataFieldMapper[] metadataMappers; - final Map, MetadataFieldMapper> rootMappersMap; - final SourceTransform[] sourceTransforms; + final Map, MetadataFieldMapper> metadataMappersMap; volatile Map meta; - public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper, MetadataFieldMapper[] metadataMappers, SourceTransform[] sourceTransforms, Map meta) { + public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper, MetadataFieldMapper[] metadataMappers, Map meta) { this.indexCreated = indexCreated; this.root = rootObjectMapper; this.metadataMappers = metadataMappers; - Map, MetadataFieldMapper> rootMappersMap = new HashMap<>(); + Map, MetadataFieldMapper> metadataMappersMap = new HashMap<>(); for (MetadataFieldMapper metadataMapper : metadataMappers) { if (indexCreated.before(Version.V_2_0_0_beta1) && LEGACY_INCLUDE_IN_OBJECT.contains(metadataMapper.name())) { root.putMapper(metadataMapper); } - rootMappersMap.put(metadataMapper.getClass(), metadataMapper); + metadataMappersMap.put(metadataMapper.getClass(), metadataMapper); } // keep root mappers sorted for consistent serialization Arrays.sort(metadataMappers, new Comparator() { @@ -80,8 +67,7 @@ public final class Mapping implements ToXContent { return o1.name().compareTo(o2.name()); } }); - this.rootMappersMap = unmodifiableMap(rootMappersMap); - this.sourceTransforms = sourceTransforms; + this.metadataMappersMap = unmodifiableMap(metadataMappersMap); this.meta = meta; } @@ -94,13 +80,13 @@ public final class Mapping implements ToXContent { * Generate a mapping update for the given root object mapper. */ public Mapping mappingUpdate(Mapper rootObjectMapper) { - return new Mapping(indexCreated, (RootObjectMapper) rootObjectMapper, metadataMappers, sourceTransforms, meta); + return new Mapping(indexCreated, (RootObjectMapper) rootObjectMapper, metadataMappers, meta); } /** Get the root mapper with the given class. */ @SuppressWarnings("unchecked") - public T rootMapper(Class clazz) { - return (T) rootMappersMap.get(clazz); + public T metadataMapper(Class clazz) { + return (T) metadataMappersMap.get(clazz); } /** @see DocumentMapper#merge(Mapping, boolean, boolean) */ @@ -109,7 +95,7 @@ public final class Mapping implements ToXContent { root.merge(mergeWith.root, mergeResult); for (MetadataFieldMapper metadataMapper : metadataMappers) { - MetadataFieldMapper mergeWithMetadataMapper = mergeWith.rootMapper(metadataMapper.getClass()); + MetadataFieldMapper mergeWithMetadataMapper = mergeWith.metadataMapper(metadataMapper.getClass()); if (mergeWithMetadataMapper != null) { metadataMapper.merge(mergeWithMetadataMapper, mergeResult); } @@ -126,19 +112,6 @@ public final class Mapping implements ToXContent { root.toXContent(builder, params, new ToXContent() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (sourceTransforms.length > 0) { - if (sourceTransforms.length == 1) { - builder.field("transform"); - sourceTransforms[0].toXContent(builder, params); - } else { - builder.startArray("transform"); - for (SourceTransform transform: sourceTransforms) { - transform.toXContent(builder, params); - } - builder.endArray(); - } - } - if (meta != null && !meta.isEmpty()) { builder.field("_meta", meta); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MergeMappingException.java b/core/src/main/java/org/elasticsearch/index/mapper/MergeMappingException.java deleted file mode 100644 index 4a5e75fcb56..00000000000 --- a/core/src/main/java/org/elasticsearch/index/mapper/MergeMappingException.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.rest.RestStatus; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Objects; - -/** - * - */ -public final class MergeMappingException extends MapperException { - - private final String[] failures; - - public MergeMappingException(String[] failures) { - super("Merge failed with failures {" + Arrays.toString(failures) + "}"); - Objects.requireNonNull(failures, "failures must be non-null"); - this.failures = failures; - } - - public MergeMappingException(StreamInput in) throws IOException { - super(in); - failures = in.readStringArray(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeStringArray(failures); - } - - public String[] failures() { - return failures; - } - - @Override - public RestStatus status() { - return RestStatus.BAD_REQUEST; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 34e35131ba3..fc6d1fa9e1a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.object.RootObjectMapper; import java.io.IOException; +import java.util.Map; /** @@ -30,6 +31,25 @@ import java.io.IOException; */ public abstract class MetadataFieldMapper extends FieldMapper { + public static interface TypeParser extends Mapper.TypeParser { + + @Override + MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException; + + /** + * Get the default {@link MetadataFieldMapper} to use, if nothing had to be parsed. + * @param fieldType null if this is the first root mapper on this index, the existing + * fieldType for this index otherwise + * @param indexSettings the index-level settings + * @param fieldType the existing field type for this meta mapper on the current index + * or null if this is the first type being introduced + * @param typeName the name of the type that this mapper will be used on + */ + // TODO: remove the fieldType parameter which is only used for bw compat with pre-2.0 + // since settings could be modified + MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName); + } + public abstract static class Builder extends FieldMapper.Builder { public Builder(String name, MappedFieldType fieldType) { super(name, fieldType); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index e9d62c8e88d..f65072d489e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; /** @@ -159,16 +160,22 @@ public class SourceToParse { return this.ttl; } + public SourceToParse ttl(TimeValue ttl) { + if (ttl == null) { + this.ttl = -1; + return this; + } + this.ttl = ttl.millis(); + return this; + } + public SourceToParse ttl(long ttl) { this.ttl = ttl; return this; } - public static enum Origin { - + public enum Origin { PRIMARY, REPLICA - } - } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java index 3a3a8549151..5b4df635a34 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java @@ -18,144 +18,91 @@ */ package org.elasticsearch.index.mapper.core; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.Field; -import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; -import org.apache.lucene.util.BytesRef; +import org.apache.lucene.search.suggest.document.Completion50PostingsFormat; +import org.apache.lucene.search.suggest.document.CompletionAnalyzer; +import org.apache.lucene.search.suggest.document.CompletionQuery; +import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; +import org.apache.lucene.search.suggest.document.PrefixCompletionQuery; +import org.apache.lucene.search.suggest.document.RegexCompletionQuery; +import org.apache.lucene.search.suggest.document.SuggestField; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.NumberType; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperException; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; -import org.elasticsearch.index.mapper.MergeResult; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.search.suggest.completion.AnalyzingCompletionLookupProvider; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream; -import org.elasticsearch.search.suggest.context.ContextBuilder; -import org.elasticsearch.search.suggest.context.ContextMapping; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextConfig; +import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.object.ArrayValueMapperParser; +import org.elasticsearch.search.suggest.completion.CompletionSuggester; +import org.elasticsearch.search.suggest.completion.context.ContextMapping; +import org.elasticsearch.search.suggest.completion.context.ContextMappings; import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.*; import static org.elasticsearch.index.mapper.MapperBuilders.completionField; import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; /** + * Mapper for completion field. The field values are indexed as a weighted FST for + * fast auto-completion/search-as-you-type functionality.
    * + * Type properties:
    + *
      + *
    • "analyzer": "simple", (default)
      • + *
    • "search_analyzer": "simple", (default)
      • + *
    • "preserve_separators" : true, (default)
      • + *
    • "preserve_position_increments" : true (default)
      • + *
    • "min_input_length": 50 (default)
      • + *
    • "contexts" : CONTEXTS
      • + *
    + * see {@link ContextMappings#load(Object, Version)} for CONTEXTS
    + * see {@link #parse(ParseContext)} for acceptable inputs for indexing
    + *

    + * This field type constructs completion queries that are run + * against the weighted FST index by the {@link CompletionSuggester}. + * This field can also be extended to add search criteria to suggestions + * for query-time filtering and boosting (see {@link ContextMappings} */ -public class CompletionFieldMapper extends FieldMapper { +public class CompletionFieldMapper extends FieldMapper implements ArrayValueMapperParser { public static final String CONTENT_TYPE = "completion"; public static class Defaults { - public static final CompletionFieldType FIELD_TYPE = new CompletionFieldType(); - + public static final MappedFieldType FIELD_TYPE = new CompletionFieldType(); static { FIELD_TYPE.setOmitNorms(true); FIELD_TYPE.freeze(); } - public static final boolean DEFAULT_PRESERVE_SEPARATORS = true; public static final boolean DEFAULT_POSITION_INCREMENTS = true; - public static final boolean DEFAULT_HAS_PAYLOADS = false; public static final int DEFAULT_MAX_INPUT_LENGTH = 50; } public static class Fields { // Mapping field names - public static final String ANALYZER = "analyzer"; + public static final ParseField ANALYZER = new ParseField("analyzer"); public static final ParseField SEARCH_ANALYZER = new ParseField("search_analyzer"); public static final ParseField PRESERVE_SEPARATORS = new ParseField("preserve_separators"); public static final ParseField PRESERVE_POSITION_INCREMENTS = new ParseField("preserve_position_increments"); - public static final String PAYLOADS = "payloads"; - public static final String TYPE = "type"; + public static final ParseField TYPE = new ParseField("type"); + public static final ParseField CONTEXTS = new ParseField("contexts"); public static final ParseField MAX_INPUT_LENGTH = new ParseField("max_input_length", "max_input_len"); // Content field names public static final String CONTENT_FIELD_NAME_INPUT = "input"; - public static final String CONTENT_FIELD_NAME_OUTPUT = "output"; - public static final String CONTENT_FIELD_NAME_PAYLOAD = "payload"; public static final String CONTENT_FIELD_NAME_WEIGHT = "weight"; - public static final String CONTEXT = "context"; + public static final String CONTENT_FIELD_NAME_CONTEXTS = "contexts"; } public static final Set ALLOWED_CONTENT_FIELD_NAMES = Sets.newHashSet(Fields.CONTENT_FIELD_NAME_INPUT, - Fields.CONTENT_FIELD_NAME_OUTPUT, Fields.CONTENT_FIELD_NAME_PAYLOAD, Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTEXT); - - public static class Builder extends FieldMapper.Builder { - - private boolean preserveSeparators = Defaults.DEFAULT_PRESERVE_SEPARATORS; - private boolean payloads = Defaults.DEFAULT_HAS_PAYLOADS; - private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; - private int maxInputLength = Defaults.DEFAULT_MAX_INPUT_LENGTH; - private SortedMap contextMapping = ContextMapping.EMPTY_MAPPING; - - public Builder(String name) { - super(name, Defaults.FIELD_TYPE); - builder = this; - } - - public Builder payloads(boolean payloads) { - this.payloads = payloads; - return this; - } - - public Builder preserveSeparators(boolean preserveSeparators) { - this.preserveSeparators = preserveSeparators; - return this; - } - - public Builder preservePositionIncrements(boolean preservePositionIncrements) { - this.preservePositionIncrements = preservePositionIncrements; - return this; - } - - public Builder maxInputLength(int maxInputLength) { - if (maxInputLength <= 0) { - throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName() + " must be > 0 but was [" + maxInputLength + "]"); - } - this.maxInputLength = maxInputLength; - return this; - } - - public Builder contextMapping(SortedMap contextMapping) { - this.contextMapping = contextMapping; - return this; - } - - @Override - public CompletionFieldMapper build(Mapper.BuilderContext context) { - setupFieldType(context); - CompletionFieldType completionFieldType = (CompletionFieldType)fieldType; - completionFieldType.setProvider(new AnalyzingCompletionLookupProvider(preserveSeparators, false, preservePositionIncrements, payloads)); - completionFieldType.setContextMapping(contextMapping); - return new CompletionFieldMapper(name, fieldType, maxInputLength, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); - } - - } + Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTENT_FIELD_NAME_CONTEXTS); public static class TypeParser implements Mapper.TypeParser { @@ -171,17 +118,12 @@ public class CompletionFieldMapper extends FieldMapper { if (fieldName.equals("type")) { continue; } - if (Fields.ANALYZER.equals(fieldName) || // index_analyzer is for backcompat, remove for v3.0 - fieldName.equals("index_analyzer") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - + if (parserContext.parseFieldMatcher().match(fieldName, Fields.ANALYZER)) { indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); iterator.remove(); } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.SEARCH_ANALYZER)) { searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); iterator.remove(); - } else if (fieldName.equals(Fields.PAYLOADS)) { - builder.payloads(Boolean.parseBoolean(fieldNode.toString())); - iterator.remove(); } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_SEPARATORS)) { builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString())); iterator.remove(); @@ -191,14 +133,14 @@ public class CompletionFieldMapper extends FieldMapper { } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.MAX_INPUT_LENGTH)) { builder.maxInputLength(Integer.parseInt(fieldNode.toString())); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) { + } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.CONTEXTS)) { + builder.contextMappings(ContextMappings.load(fieldNode, parserContext.indexVersionCreated())); iterator.remove(); - } else if (fieldName.equals(Fields.CONTEXT)) { - builder.contextMapping(ContextBuilder.loadMappings(fieldNode, parserContext.indexVersionCreated())); + } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) { iterator.remove(); } } - + if (indexAnalyzer == null) { if (searchAnalyzer != null) { throw new MapperParsingException("analyzer on completion field [" + name + "] must be set when search_analyzer is set"); @@ -207,9 +149,9 @@ public class CompletionFieldMapper extends FieldMapper { } else if (searchAnalyzer == null) { searchAnalyzer = indexAnalyzer; } + builder.indexAnalyzer(indexAnalyzer); builder.searchAnalyzer(searchAnalyzer); - return builder; } @@ -223,40 +165,138 @@ public class CompletionFieldMapper extends FieldMapper { } public static final class CompletionFieldType extends MappedFieldType { - private PostingsFormat postingsFormat; - private AnalyzingCompletionLookupProvider analyzingSuggestLookupProvider; - private SortedMap contextMapping = ContextMapping.EMPTY_MAPPING; + + private static PostingsFormat postingsFormat; + + private boolean preserveSep = Defaults.DEFAULT_PRESERVE_SEPARATORS; + private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; + private ContextMappings contextMappings = null; public CompletionFieldType() { setFieldDataType(null); } - protected CompletionFieldType(CompletionFieldType ref) { + private CompletionFieldType(CompletionFieldType ref) { super(ref); - this.postingsFormat = ref.postingsFormat; - this.analyzingSuggestLookupProvider = ref.analyzingSuggestLookupProvider; - this.contextMapping = ref.contextMapping; + this.contextMappings = ref.contextMappings; + this.preserveSep = ref.preserveSep; + this.preservePositionIncrements = ref.preservePositionIncrements; + } + + public void setPreserveSep(boolean preserveSep) { + checkIfFrozen(); + this.preserveSep = preserveSep; + } + + public void setPreservePositionIncrements(boolean preservePositionIncrements) { + checkIfFrozen(); + this.preservePositionIncrements = preservePositionIncrements; + } + + public void setContextMappings(ContextMappings contextMappings) { + checkIfFrozen(); + this.contextMappings = contextMappings; + } + + @Override + public NamedAnalyzer indexAnalyzer() { + final NamedAnalyzer indexAnalyzer = super.indexAnalyzer(); + if (indexAnalyzer != null && !(indexAnalyzer.analyzer() instanceof CompletionAnalyzer)) { + return new NamedAnalyzer(indexAnalyzer.name(), + new CompletionAnalyzer(indexAnalyzer, preserveSep, preservePositionIncrements)); + + } + return indexAnalyzer; + } + + @Override + public NamedAnalyzer searchAnalyzer() { + final NamedAnalyzer searchAnalyzer = super.searchAnalyzer(); + if (searchAnalyzer != null && !(searchAnalyzer.analyzer() instanceof CompletionAnalyzer)) { + return new NamedAnalyzer(searchAnalyzer.name(), + new CompletionAnalyzer(searchAnalyzer, preserveSep, preservePositionIncrements)); + } + return searchAnalyzer; + } + + /** + * @return true if there are one or more context mappings defined + * for this field type + */ + public boolean hasContextMappings() { + return contextMappings != null; + } + + /** + * @return associated context mappings for this field type + */ + public ContextMappings getContextMappings() { + return contextMappings; + } + + public boolean preserveSep() { + return preserveSep; + } + + public boolean preservePositionIncrements() { + return preservePositionIncrements; + } + + /** + * @return postings format to use for this field-type + */ + public static synchronized PostingsFormat postingsFormat() { + if (postingsFormat == null) { + postingsFormat = new Completion50PostingsFormat(); + } + return postingsFormat; + } + + /** + * Completion prefix query + */ + public CompletionQuery prefixQuery(Object value) { + return new PrefixCompletionQuery(searchAnalyzer().analyzer(), createTerm(value)); + } + + /** + * Completion prefix regular expression query + */ + public CompletionQuery regexpQuery(Object value, int flags, int maxDeterminizedStates) { + return new RegexCompletionQuery(createTerm(value), flags, maxDeterminizedStates); + } + + /** + * Completion prefix fuzzy query + */ + public CompletionQuery fuzzyQuery(String value, Fuzziness fuzziness, int nonFuzzyPrefixLength, + int minFuzzyPrefixLength, int maxExpansions, boolean transpositions, + boolean unicodeAware) { + return new FuzzyCompletionQuery(searchAnalyzer().analyzer(), createTerm(value), null, + fuzziness.asDistance(), transpositions, nonFuzzyPrefixLength, minFuzzyPrefixLength, + unicodeAware, maxExpansions); } @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof CompletionFieldType)) return false; + if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; - CompletionFieldType fieldType = (CompletionFieldType) o; - return analyzingSuggestLookupProvider.getPreserveSep() == fieldType.analyzingSuggestLookupProvider.getPreserveSep() && - analyzingSuggestLookupProvider.getPreservePositionsIncrements() == fieldType.analyzingSuggestLookupProvider.getPreservePositionsIncrements() && - analyzingSuggestLookupProvider.hasPayloads() == fieldType.analyzingSuggestLookupProvider.hasPayloads() && - Objects.equals(getContextMapping(), fieldType.getContextMapping()); + + CompletionFieldType that = (CompletionFieldType) o; + + if (preserveSep != that.preserveSep) return false; + if (preservePositionIncrements != that.preservePositionIncrements) return false; + return !(contextMappings != null ? !contextMappings.equals(that.contextMappings) : that.contextMappings != null); + } @Override public int hashCode() { return Objects.hash(super.hashCode(), - analyzingSuggestLookupProvider.getPreserveSep(), - analyzingSuggestLookupProvider.getPreservePositionsIncrements(), - analyzingSuggestLookupProvider.hasPayloads(), - getContextMapping()); + preserveSep, + preservePositionIncrements, + contextMappings); } @Override @@ -273,50 +313,20 @@ public class CompletionFieldMapper extends FieldMapper { public void checkCompatibility(MappedFieldType fieldType, List conflicts, boolean strict) { super.checkCompatibility(fieldType, conflicts, strict); CompletionFieldType other = (CompletionFieldType)fieldType; - if (analyzingSuggestLookupProvider.hasPayloads() != other.analyzingSuggestLookupProvider.hasPayloads()) { - conflicts.add("mapper [" + names().fullName() + "] has different [payload] values"); - } - if (analyzingSuggestLookupProvider.getPreservePositionsIncrements() != other.analyzingSuggestLookupProvider.getPreservePositionsIncrements()) { + + if (preservePositionIncrements != other.preservePositionIncrements) { conflicts.add("mapper [" + names().fullName() + "] has different [preserve_position_increments] values"); } - if (analyzingSuggestLookupProvider.getPreserveSep() != other.analyzingSuggestLookupProvider.getPreserveSep()) { + if (preserveSep != other.preserveSep) { conflicts.add("mapper [" + names().fullName() + "] has different [preserve_separators] values"); } - if(!ContextMapping.mappingsAreEqual(getContextMapping(), other.getContextMapping())) { - conflicts.add("mapper [" + names().fullName() + "] has different [context_mapping] values"); + if (hasContextMappings() != other.hasContextMappings()) { + conflicts.add("mapper [" + names().fullName() + "] has different [context_mappings] values"); + } else if (hasContextMappings() && contextMappings.equals(other.contextMappings) == false) { + conflicts.add("mapper [" + names().fullName() + "] has different [context_mappings] values"); } } - public void setProvider(AnalyzingCompletionLookupProvider provider) { - checkIfFrozen(); - this.analyzingSuggestLookupProvider = provider; - } - - public synchronized PostingsFormat postingsFormat(PostingsFormat in) { - if (in instanceof Completion090PostingsFormat) { - throw new IllegalStateException("Double wrapping of " + Completion090PostingsFormat.class); - } - if (postingsFormat == null) { - postingsFormat = new Completion090PostingsFormat(in, analyzingSuggestLookupProvider); - } - return postingsFormat; - } - - public void setContextMapping(SortedMap contextMapping) { - checkIfFrozen(); - this.contextMapping = contextMapping; - } - - /** Get the context mapping associated with this completion field */ - public SortedMap getContextMapping() { - return contextMapping; - } - - /** @return true if a context mapping has been defined */ - public boolean requiresContext() { - return contextMapping.isEmpty() == false; - } - @Override public String value(Object value) { if (value == null) { @@ -329,13 +339,73 @@ public class CompletionFieldMapper extends FieldMapper { public boolean isSortable() { return false; } + } - private static final BytesRef EMPTY = new BytesRef(); + /** + * Builder for {@link CompletionFieldMapper} + */ + public static class Builder extends FieldMapper.Builder { + + private int maxInputLength = Defaults.DEFAULT_MAX_INPUT_LENGTH; + private ContextMappings contextMappings = null; + private boolean preserveSeparators = Defaults.DEFAULT_PRESERVE_SEPARATORS; + private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; + + /** + * @param name of the completion field to build + */ + public Builder(String name) { + super(name, new CompletionFieldType()); + builder = this; + } + + /** + * @param maxInputLength maximum expected prefix length + * NOTE: prefixes longer than this will + * be truncated + */ + public Builder maxInputLength(int maxInputLength) { + if (maxInputLength <= 0) { + throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName() + " must be > 0 but was [" + maxInputLength + "]"); + } + this.maxInputLength = maxInputLength; + return this; + } + + /** + * Add context mapping to this field + * @param contextMappings see {@link ContextMappings#load(Object, Version)} + */ + public Builder contextMappings(ContextMappings contextMappings) { + this.contextMappings = contextMappings; + return this; + } + + public Builder preserveSeparators(boolean preserveSeparators) { + this.preserveSeparators = preserveSeparators; + return this; + } + + public Builder preservePositionIncrements(boolean preservePositionIncrements) { + this.preservePositionIncrements = preservePositionIncrements; + return this; + } + + @Override + public CompletionFieldMapper build(BuilderContext context) { + setupFieldType(context); + CompletionFieldType completionFieldType = (CompletionFieldType) this.fieldType; + completionFieldType.setContextMappings(contextMappings); + completionFieldType.setPreservePositionIncrements(preservePositionIncrements); + completionFieldType.setPreserveSep(preserveSeparators); + return new CompletionFieldMapper(name, this.fieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, maxInputLength); + } + } private int maxInputLength; - public CompletionFieldMapper(String simpleName, MappedFieldType fieldType, int maxInputLength, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { + public CompletionFieldMapper(String simpleName, MappedFieldType fieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, int maxInputLength) { super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo); this.maxInputLength = maxInputLength; } @@ -345,216 +415,188 @@ public class CompletionFieldMapper extends FieldMapper { return (CompletionFieldType) super.fieldType(); } + /** + * Parses and indexes inputs + * + * Parsing: + * Acceptable format: + * "STRING" - interpreted as field value (input) + * "ARRAY" - each element can be one of "OBJECT" (see below) + * "OBJECT" - { "input": STRING|ARRAY, "weight": STRING|INT, "contexts": ARRAY|OBJECT } + * + * Indexing: + * if context mappings are defined, delegates to {@link ContextMappings#addField(ParseContext.Document, String, String, int, Map)} + * else adds inputs as a {@link org.apache.lucene.search.suggest.document.SuggestField} + */ @Override public Mapper parse(ParseContext context) throws IOException { + // parse XContentParser parser = context.parser(); - XContentParser.Token token = parser.currentToken(); - if (token == XContentParser.Token.VALUE_NULL) { + Token token = parser.currentToken(); + Map inputMap = new HashMap<>(1); + if (token == Token.VALUE_NULL) { throw new MapperParsingException("completion field [" + fieldType().names().fullName() + "] does not support null values"); - } - - String surfaceForm = null; - BytesRef payload = null; - long weight = -1; - List inputs = new ArrayList<>(4); - - SortedMap contextConfig = null; - - if (token == XContentParser.Token.VALUE_STRING) { - inputs.add(parser.text()); - multiFields.parse(this, context); - } else { - String currentFieldName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - if (!ALLOWED_CONTENT_FIELD_NAMES.contains(currentFieldName)) { - throw new IllegalArgumentException("Unknown field name[" + currentFieldName + "], must be one of " + ALLOWED_CONTENT_FIELD_NAMES); - } - } else if (Fields.CONTEXT.equals(currentFieldName)) { - SortedMap configs = new TreeMap<>(); - - if (token == Token.START_OBJECT) { - while ((token = parser.nextToken()) != Token.END_OBJECT) { - String name = parser.text(); - ContextMapping mapping = fieldType().getContextMapping().get(name); - if (mapping == null) { - throw new ElasticsearchParseException("context [{}] is not defined", name); - } else { - token = parser.nextToken(); - configs.put(name, mapping.parseContext(context, parser)); - } - } - contextConfig = new TreeMap<>(); - for (ContextMapping mapping : fieldType().getContextMapping().values()) { - ContextConfig config = configs.get(mapping.name()); - contextConfig.put(mapping.name(), config==null ? mapping.defaultConfig() : config); - } - } else { - throw new ElasticsearchParseException("context must be an object"); - } - } else if (Fields.CONTENT_FIELD_NAME_PAYLOAD.equals(currentFieldName)) { - if (!isStoringPayloads()) { - throw new MapperException("Payloads disabled in mapping"); - } - if (token == XContentParser.Token.START_OBJECT) { - XContentBuilder payloadBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); - payload = payloadBuilder.bytes().toBytesRef(); - payloadBuilder.close(); - } else if (token.isValue()) { - payload = parser.utf8BytesOrNull(); - } else { - throw new MapperException("payload doesn't support type " + token); - } - } else if (token == XContentParser.Token.VALUE_STRING) { - if (Fields.CONTENT_FIELD_NAME_OUTPUT.equals(currentFieldName)) { - surfaceForm = parser.text(); - } - if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) { - inputs.add(parser.text()); - } - if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { - Number weightValue; - try { - weightValue = Long.parseLong(parser.text()); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Weight must be a string representing a numeric value, but was [" + parser.text() + "]"); - } - weight = weightValue.longValue(); // always parse a long to make sure we don't get overflow - checkWeight(weight); - } - } else if (token == XContentParser.Token.VALUE_NUMBER) { - if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { - NumberType numberType = parser.numberType(); - if (NumberType.LONG != numberType && NumberType.INT != numberType) { - throw new IllegalArgumentException("Weight must be an integer, but was [" + parser.numberValue() + "]"); - } - weight = parser.longValue(); // always parse a long to make sure we don't get overflow - checkWeight(weight); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - inputs.add(parser.text()); - } - } - } - } - } - - if(contextConfig == null) { - contextConfig = new TreeMap<>(); - for (ContextMapping mapping : fieldType().getContextMapping().values()) { - contextConfig.put(mapping.name(), mapping.defaultConfig()); - } - } - - final ContextMapping.Context ctx = new ContextMapping.Context(contextConfig, context.doc()); - - payload = payload == null ? EMPTY : payload; - if (surfaceForm == null) { // no surface form use the input - for (String input : inputs) { - if (input.length() == 0) { - continue; - } - BytesRef suggestPayload = fieldType().analyzingSuggestLookupProvider.buildPayload(new BytesRef( - input), weight, payload); - context.doc().add(getCompletionField(ctx, input, suggestPayload)); + } else if (token == Token.START_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { + parse(context, token, parser, inputMap); } } else { - BytesRef suggestPayload = fieldType().analyzingSuggestLookupProvider.buildPayload(new BytesRef( - surfaceForm), weight, payload); - for (String input : inputs) { - if (input.length() == 0) { - continue; + parse(context, token, parser, inputMap); + } + + // index + for (Map.Entry completionInput : inputMap.entrySet()) { + String input = completionInput.getKey(); + // truncate input + if (input.length() > maxInputLength) { + int len = Math.min(maxInputLength, input.length()); + if (Character.isHighSurrogate(input.charAt(len - 1))) { + assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len)); + len += 1; } - context.doc().add(getCompletionField(ctx, input, suggestPayload)); + input = input.substring(0, len); + } + CompletionInputMetaData metaData = completionInput.getValue(); + if (fieldType().hasContextMappings()) { + fieldType().getContextMappings().addField(context.doc(), fieldType().names().indexName(), + input, metaData.weight, metaData.contexts); + } else { + context.doc().add(new SuggestField(fieldType().names().indexName(), input, metaData.weight)); } } + multiFields.parse(this, context); return null; } - private void checkWeight(long weight) { - if (weight < 0 || weight > Integer.MAX_VALUE) { - throw new IllegalArgumentException("Weight must be in the interval [0..2147483647], but was [" + weight + "]"); - } - } - - public Field getCompletionField(ContextMapping.Context ctx, String input, BytesRef payload) { - final String originalInput = input; - if (input.length() > maxInputLength) { - final int len = correctSubStringLen(input, Math.min(maxInputLength, input.length())); - input = input.substring(0, len); - } - for (int i = 0; i < input.length(); i++) { - if (isReservedChar(input.charAt(i))) { - throw new IllegalArgumentException("Illegal input [" + originalInput + "] UTF-16 codepoint [0x" - + Integer.toHexString((int) input.charAt(i)).toUpperCase(Locale.ROOT) - + "] at position " + i + " is a reserved character"); + /** + * Acceptable inputs: + * "STRING" - interpreted as the field value (input) + * "OBJECT" - { "input": STRING|ARRAY, "weight": STRING|INT, "contexts": ARRAY|OBJECT } + */ + private void parse(ParseContext parseContext, Token token, XContentParser parser, Map inputMap) throws IOException { + String currentFieldName = null; + if (token == Token.VALUE_STRING) { + inputMap.put(parser.text(), new CompletionInputMetaData(Collections.>emptyMap(), 1)); + } else if (token == Token.START_OBJECT) { + Set inputs = new HashSet<>(); + int weight = 1; + Map> contextsMap = new HashMap<>(); + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if (!ALLOWED_CONTENT_FIELD_NAMES.contains(currentFieldName)) { + throw new IllegalArgumentException("unknown field name [" + currentFieldName + "], must be one of " + ALLOWED_CONTENT_FIELD_NAMES); + } + } else if (currentFieldName != null) { + if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) { + if (token == Token.VALUE_STRING) { + inputs.add(parser.text()); + } else if (token == Token.START_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { + if (token == Token.VALUE_STRING) { + inputs.add(parser.text()); + } else { + throw new IllegalArgumentException("input array must have string values, but was [" + token.name() + "]"); + } + } + } else { + throw new IllegalArgumentException("input must be a string or array, but was [" + token.name() + "]"); + } + } else if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { + final Number weightValue; + if (token == Token.VALUE_STRING) { + try { + weightValue = Long.parseLong(parser.text()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("weight must be an integer, but was [" + parser.text() + "]"); + } + } else if (token == Token.VALUE_NUMBER) { + NumberType numberType = parser.numberType(); + if (NumberType.LONG != numberType && NumberType.INT != numberType) { + throw new IllegalArgumentException("weight must be an integer, but was [" + parser.numberValue() + "]"); + } + weightValue = parser.numberValue(); + } else { + throw new IllegalArgumentException("weight must be a number or string, but was [" + token.name() + "]"); + } + if (weightValue.longValue() < 0 || weightValue.longValue() > Integer.MAX_VALUE) { // always parse a long to make sure we don't get overflow + throw new IllegalArgumentException("weight must be in the interval [0..2147483647], but was [" + weightValue.longValue() + "]"); + } + weight = weightValue.intValue(); + } else if (Fields.CONTENT_FIELD_NAME_CONTEXTS.equals(currentFieldName)) { + if (fieldType().hasContextMappings() == false) { + throw new IllegalArgumentException("contexts field is not supported for field: [" + fieldType().names().fullName() + "]"); + } + ContextMappings contextMappings = fieldType().getContextMappings(); + XContentParser.Token currentToken = parser.currentToken(); + if (currentToken == XContentParser.Token.START_OBJECT) { + ContextMapping contextMapping = null; + String fieldName = null; + while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (currentToken == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + contextMapping = contextMappings.get(fieldName); + } else if (currentToken == XContentParser.Token.VALUE_STRING + || currentToken == XContentParser.Token.START_ARRAY + || currentToken == XContentParser.Token.START_OBJECT) { + assert fieldName != null; + assert !contextsMap.containsKey(fieldName); + contextsMap.put(fieldName, contextMapping.parseContext(parseContext, parser)); + } else { + throw new IllegalArgumentException("contexts must be an object or an array , but was [" + currentToken + "]"); + } + } + } else { + throw new IllegalArgumentException("contexts must be an object or an array , but was [" + currentToken + "]"); + } + } + } } - } - return new SuggestField(fieldType().names().indexName(), ctx, input, fieldType(), payload, fieldType().analyzingSuggestLookupProvider); - } - - public static int correctSubStringLen(String input, int len) { - if (Character.isHighSurrogate(input.charAt(len - 1))) { - assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len)); - return len + 1; - } - return len; - } - - public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { - return fieldType().analyzingSuggestLookupProvider.buildPayload(surfaceForm, weight, payload); - } - - private static final class SuggestField extends Field { - private final BytesRef payload; - private final CompletionTokenStream.ToFiniteStrings toFiniteStrings; - private final ContextMapping.Context ctx; - - public SuggestField(String name, ContextMapping.Context ctx, String value, MappedFieldType type, BytesRef payload, CompletionTokenStream.ToFiniteStrings toFiniteStrings) { - super(name, value, type); - this.payload = payload; - this.toFiniteStrings = toFiniteStrings; - this.ctx = ctx; - } - - @Override - public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) throws IOException { - TokenStream ts = ctx.wrapTokenStream(super.tokenStream(analyzer, previous)); - return new CompletionTokenStream(ts, payload, toFiniteStrings); + for (String input : inputs) { + if (inputMap.containsKey(input) == false || inputMap.get(input).weight < weight) { + inputMap.put(input, new CompletionInputMetaData(contextsMap, weight)); + } + } + } else { + throw new ElasticsearchParseException("failed to parse expected text or object got" + token.name()); } } - + + static class CompletionInputMetaData { + public final Map> contexts; + public final int weight; + + CompletionInputMetaData(Map> contexts, int weight) { + this.contexts = contexts; + this.weight = weight; + } + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(simpleName()) - .field(Fields.TYPE, CONTENT_TYPE); - - builder.field(Fields.ANALYZER, fieldType().indexAnalyzer().name()); + .field(Fields.TYPE.getPreferredName(), CONTENT_TYPE); + builder.field(Fields.ANALYZER.getPreferredName(), fieldType().indexAnalyzer().name()); if (fieldType().indexAnalyzer().name().equals(fieldType().searchAnalyzer().name()) == false) { builder.field(Fields.SEARCH_ANALYZER.getPreferredName(), fieldType().searchAnalyzer().name()); } - builder.field(Fields.PAYLOADS, fieldType().analyzingSuggestLookupProvider.hasPayloads()); - builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), fieldType().analyzingSuggestLookupProvider.getPreserveSep()); - builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), fieldType().analyzingSuggestLookupProvider.getPreservePositionsIncrements()); + builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), fieldType().preserveSep()); + builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), fieldType().preservePositionIncrements()); builder.field(Fields.MAX_INPUT_LENGTH.getPreferredName(), this.maxInputLength); - multiFields.toXContent(builder, params); - if(fieldType().requiresContext()) { - builder.startObject(Fields.CONTEXT); - for (ContextMapping mapping : fieldType().getContextMapping().values()) { - builder.value(mapping); - } - builder.endObject(); + if (fieldType().hasContextMappings()) { + builder.startArray(Fields.CONTEXTS.getPreferredName()); + fieldType().getContextMappings().toXContent(builder, params); + builder.endArray(); } + multiFields.toXContent(builder, params); return builder.endObject(); } @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { + // no-op } @Override @@ -562,34 +604,12 @@ public class CompletionFieldMapper extends FieldMapper { return CONTENT_TYPE; } - public boolean isStoringPayloads() { - return fieldType().analyzingSuggestLookupProvider.hasPayloads(); - } - @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { super.merge(mergeWith, mergeResult); CompletionFieldMapper fieldMergeWith = (CompletionFieldMapper) mergeWith; if (!mergeResult.simulate()) { this.maxInputLength = fieldMergeWith.maxInputLength; } } - - // this should be package private but our tests don't allow it. - public static boolean isReservedChar(char character) { - /* we use 0x001F as a SEP_LABEL in the suggester but we can use the UTF-16 representation since they - * are equivalent. We also don't need to convert the input character to UTF-8 here to check for - * the 0x00 end label since all multi-byte UTF-8 chars start with 0x10 binary so if the UTF-16 CP is == 0x00 - * it's the single byte UTF-8 CP */ - assert XAnalyzingSuggester.PAYLOAD_SEP == XAnalyzingSuggester.SEP_LABEL; // ensure they are the same! - switch(character) { - case XAnalyzingSuggester.END_BYTE: - case XAnalyzingSuggester.SEP_LABEL: - case XAnalyzingSuggester.HOLE_CHARACTER: - case ContextMapping.SEPARATOR: - return true; - default: - return false; - } - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 686cfcfe6e2..27b96b27a44 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -219,8 +219,9 @@ public class DateFieldMapper extends NumberFieldMapper { @Override public Query rewrite(IndexReader reader) throws IOException { - if (getBoost() != 1.0F) { - return super.rewrite(reader); + Query rewritten = super.rewrite(reader); + if (rewritten != this) { + return rewritten; } return innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser); } @@ -229,11 +230,9 @@ public class DateFieldMapper extends NumberFieldMapper { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; LateParsingQuery that = (LateParsingQuery) o; - if (includeLower != that.includeLower) return false; if (includeUpper != that.includeUpper) return false; if (lowerTerm != null ? !lowerTerm.equals(that.lowerTerm) : that.lowerTerm != null) return false; @@ -245,13 +244,7 @@ public class DateFieldMapper extends NumberFieldMapper { @Override public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (lowerTerm != null ? lowerTerm.hashCode() : 0); - result = 31 * result + (upperTerm != null ? upperTerm.hashCode() : 0); - result = 31 * result + (includeLower ? 1 : 0); - result = 31 * result + (includeUpper ? 1 : 0); - result = 31 * result + (timeZone != null ? timeZone.hashCode() : 0); - return result; + return Objects.hash(super.hashCode(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index 3fba511fb52..87a63de99ec 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -135,6 +135,15 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM super(ref); } + @Override + public void checkCompatibility(MappedFieldType other, + List conflicts, boolean strict) { + super.checkCompatibility(other, conflicts, strict); + if (numericPrecisionStep() != other.numericPrecisionStep()) { + conflicts.add("mapper [" + names().fullName() + "] has different [precision_step] values"); + } + } + public abstract NumberFieldType clone(); @Override @@ -245,17 +254,12 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { super.merge(mergeWith, mergeResult); if (!this.getClass().equals(mergeWith.getClass())) { return; } NumberFieldMapper nfmMergeWith = (NumberFieldMapper) mergeWith; - if (this.fieldTypeRef.getNumAssociatedMappers() > 1 && mergeResult.updateAllTypes() == false) { - if (fieldType().numericPrecisionStep() != nfmMergeWith.fieldType().numericPrecisionStep()) { - mergeResult.addConflict("mapper [" + fieldType().names().fullName() + "] is used by multiple types. Set update_all_types to true to update precision_step across all types."); - } - } if (mergeResult.simulate() == false && mergeResult.hasConflicts() == false) { this.includeInAll = nfmMergeWith.includeInAll; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java index cb89cb4973b..0a921ad85eb 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.internal.AllFieldMapper; @@ -47,7 +46,7 @@ import java.util.Map; import static org.apache.lucene.index.IndexOptions.NONE; import static org.elasticsearch.index.mapper.MapperBuilders.stringField; -import static org.elasticsearch.index.mapper.core.TypeParsers.parseField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseTextField; import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; public class StringFieldMapper extends FieldMapper implements AllFieldMapper.IncludeInAll { @@ -160,7 +159,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { StringFieldMapper.Builder builder = stringField(name); - parseField(builder, name, node, parserContext); + parseTextField(builder, name, node, parserContext); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String propName = Strings.toUnderscoreCase(entry.getKey()); @@ -360,7 +359,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { super.merge(mergeWith, mergeResult); if (!this.getClass().equals(mergeWith.getClass())) { return; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java index faa2b7e66a0..8348892e44a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapper.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.StringFieldMapper.ValueAndBoost; @@ -191,7 +190,7 @@ public class TokenCountFieldMapper extends IntegerFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { super.merge(mergeWith, mergeResult); if (!this.getClass().equals(mergeWith.getClass())) { return; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index 3f142cc2f9c..e530243657c 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -25,6 +25,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -181,34 +182,17 @@ public class TypeParsers { } } - public static void parseField(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { + private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { NamedAnalyzer indexAnalyzer = builder.fieldType().indexAnalyzer(); NamedAnalyzer searchAnalyzer = builder.fieldType().searchAnalyzer(); + for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); final String propName = Strings.toUnderscoreCase(entry.getKey()); final Object propNode = entry.getValue(); - if (propName.equals("index_name") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - builder.indexName(propNode.toString()); - iterator.remove(); - } else if (propName.equals("store")) { - builder.store(parseStore(name, propNode.toString())); - iterator.remove(); - } else if (propName.equals("index")) { - parseIndex(name, propNode.toString(), builder); - iterator.remove(); - } else if (propName.equals("tokenized")) { - builder.tokenized(nodeBooleanValue(propNode)); - iterator.remove(); - } else if (propName.equals(DOC_VALUES)) { - builder.docValues(nodeBooleanValue(propNode)); - iterator.remove(); - } else if (propName.equals("term_vector")) { + if (propName.equals("term_vector")) { parseTermVector(name, propNode.toString(), builder); iterator.remove(); - } else if (propName.equals("boost")) { - builder.boost(nodeFloatValue(propNode)); - iterator.remove(); } else if (propName.equals("store_term_vectors")) { builder.storeTermVectors(nodeBooleanValue(propNode)); iterator.remove(); @@ -221,6 +205,69 @@ public class TypeParsers { } else if (propName.equals("store_term_vector_payloads")) { builder.storeTermVectorPayloads(nodeBooleanValue(propNode)); iterator.remove(); + } else if (propName.equals("analyzer") || // for backcompat, reading old indexes, remove for v3.0 + propName.equals("index_analyzer") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + + NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); + if (analyzer == null) { + throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); + } + indexAnalyzer = analyzer; + iterator.remove(); + } else if (propName.equals("search_analyzer")) { + NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); + if (analyzer == null) { + throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); + } + searchAnalyzer = analyzer; + iterator.remove(); + } + } + + if (indexAnalyzer == null) { + if (searchAnalyzer != null) { + throw new MapperParsingException("analyzer on field [" + name + "] must be set when search_analyzer is set"); + } + } else if (searchAnalyzer == null) { + searchAnalyzer = indexAnalyzer; + } + builder.indexAnalyzer(indexAnalyzer); + builder.searchAnalyzer(searchAnalyzer); + } + + /** + * Parse text field attributes. In addition to {@link #parseField common attributes} + * this will parse analysis and term-vectors related settings. + */ + public static void parseTextField(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { + parseField(builder, name, fieldNode, parserContext); + parseAnalyzersAndTermVectors(builder, name, fieldNode, parserContext); + } + + /** + * Parse common field attributes such as {@code doc_values} or {@code store}. + */ + public static void parseField(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { + Version indexVersionCreated = parserContext.indexVersionCreated(); + for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + final String propName = Strings.toUnderscoreCase(entry.getKey()); + final Object propNode = entry.getValue(); + if (propName.equals("index_name") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { + builder.indexName(propNode.toString()); + iterator.remove(); + } else if (propName.equals("store")) { + builder.store(parseStore(name, propNode.toString())); + iterator.remove(); + } else if (propName.equals("index")) { + parseIndex(name, propNode.toString(), builder); + iterator.remove(); + } else if (propName.equals(DOC_VALUES)) { + builder.docValues(nodeBooleanValue(propNode)); + iterator.remove(); + } else if (propName.equals("boost")) { + builder.boost(nodeFloatValue(propNode)); + iterator.remove(); } else if (propName.equals("omit_norms")) { builder.omitNorms(nodeBooleanValue(propNode)); iterator.remove(); @@ -242,7 +289,7 @@ public class TypeParsers { iterator.remove(); } else if (propName.equals("omit_term_freq_and_positions")) { final IndexOptions op = nodeBooleanValue(propNode) ? IndexOptions.DOCS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_0_0_RC2)) { + if (indexVersionCreated.onOrAfter(Version.V_1_0_0_RC2)) { throw new ElasticsearchParseException("'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : 'docs'] instead"); } // deprecated option for BW compat @@ -251,29 +298,13 @@ public class TypeParsers { } else if (propName.equals("index_options")) { builder.indexOptions(nodeIndexOptionValue(propNode)); iterator.remove(); - } else if (propName.equals("analyzer") || // for backcompat, reading old indexes, remove for v3.0 - propName.equals("index_analyzer") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - - NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); - if (analyzer == null) { - throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); - } - indexAnalyzer = analyzer; - iterator.remove(); - } else if (propName.equals("search_analyzer")) { - NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); - if (analyzer == null) { - throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); - } - searchAnalyzer = analyzer; - iterator.remove(); } else if (propName.equals("include_in_all")) { builder.includeInAll(nodeBooleanValue(propNode)); iterator.remove(); - } else if (propName.equals("postings_format") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + } else if (propName.equals("postings_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { // ignore for old indexes iterator.remove(); - } else if (propName.equals("doc_values_format") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + } else if (propName.equals("doc_values_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { // ignore for old indexes iterator.remove(); } else if (propName.equals("similarity")) { @@ -284,23 +315,28 @@ public class TypeParsers { builder.fieldDataSettings(settings); iterator.remove(); } else if (propName.equals("copy_to")) { - parseCopyFields(propNode, builder); + if (parserContext.isWithinMultiField()) { + if (indexVersionCreated.after(Version.V_2_1_0) || + (indexVersionCreated.after(Version.V_2_0_1) && indexVersionCreated.before(Version.V_2_1_0))) { + throw new MapperParsingException("copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] which is within a multi field."); + } else { + ESLoggerFactory.getLogger("mapping [" + parserContext.type() + "]").warn("Found a copy_to in field [" + name + "] which is within a multi field. This feature has been removed and the copy_to will be removed from the mapping."); + } + } else { + parseCopyFields(propNode, builder); + } iterator.remove(); } } - - if (indexAnalyzer == null) { - if (searchAnalyzer != null) { - throw new MapperParsingException("analyzer on field [" + name + "] must be set when search_analyzer is set"); - } - } else if (searchAnalyzer == null) { - searchAnalyzer = indexAnalyzer; + if (indexVersionCreated.before(Version.V_2_2_0)) { + // analyzer, search_analyzer, term_vectors were accepted on all fields + // before 2.2, even though it made little sense + parseAnalyzersAndTermVectors(builder, name, fieldNode, parserContext); } - builder.indexAnalyzer(indexAnalyzer); - builder.searchAnalyzer(searchAnalyzer); } public static boolean parseMultiField(FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) { + parserContext = parserContext.createMultiFieldContext(parserContext); if (propName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { builder.multiFieldPathType(parsePathType(name, propNode.toString())); return true; @@ -319,6 +355,9 @@ public class TypeParsers { for (Map.Entry multiFieldEntry : multiFieldsPropNodes.entrySet()) { String multiFieldName = multiFieldEntry.getKey(); + if (multiFieldName.contains(".")) { + throw new MapperParsingException("Field name [" + multiFieldName + "] which is a multi field of [" + name + "] cannot contain '.'"); + } if (!(multiFieldEntry.getValue() instanceof Map)) { throw new MapperParsingException("illegal field [" + multiFieldName + "], only fields can be specified inside fields"); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java new file mode 100644 index 00000000000..0b57d866ddd --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java @@ -0,0 +1,537 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.geo; + +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.util.GeoHashUtils; +import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MergeResult; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.DoubleFieldMapper; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.index.mapper.object.ArrayValueMapperParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.index.mapper.MapperBuilders.doubleField; +import static org.elasticsearch.index.mapper.MapperBuilders.stringField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; + +/** + * GeoPointFieldMapper base class to maintain backward compatibility + */ +public abstract class BaseGeoPointFieldMapper extends FieldMapper implements ArrayValueMapperParser { + public static final String CONTENT_TYPE = "geo_point"; + + public static class Names { + public static final String LAT = "lat"; + public static final String LAT_SUFFIX = "." + LAT; + public static final String LON = "lon"; + public static final String LON_SUFFIX = "." + LON; + public static final String GEOHASH = "geohash"; + public static final String GEOHASH_SUFFIX = "." + GEOHASH; + public static final String IGNORE_MALFORMED = "ignore_malformed"; + } + + public static class Defaults { + public static final ContentPath.Type PATH_TYPE = ContentPath.Type.FULL; + public static final boolean ENABLE_LATLON = false; + public static final boolean ENABLE_GEOHASH = false; + public static final boolean ENABLE_GEOHASH_PREFIX = false; + public static final int GEO_HASH_PRECISION = GeoHashUtils.PRECISION; + public static final Explicit IGNORE_MALFORMED = new Explicit(false, false); + } + + public abstract static class Builder extends FieldMapper.Builder { + protected ContentPath.Type pathType = Defaults.PATH_TYPE; + + protected boolean enableLatLon = Defaults.ENABLE_LATLON; + + protected Integer precisionStep; + + protected boolean enableGeoHash = Defaults.ENABLE_GEOHASH; + + protected boolean enableGeoHashPrefix = Defaults.ENABLE_GEOHASH_PREFIX; + + protected int geoHashPrecision = Defaults.GEO_HASH_PRECISION; + + protected Boolean ignoreMalformed; + + public Builder(String name, GeoPointFieldType fieldType) { + super(name, fieldType); + } + + @Override + public GeoPointFieldType fieldType() { + return (GeoPointFieldType)fieldType; + } + + @Override + public T multiFieldPathType(ContentPath.Type pathType) { + this.pathType = pathType; + return builder; + } + + @Override + public T fieldDataSettings(Settings settings) { + this.fieldDataSettings = settings; + return builder; + } + + public T enableLatLon(boolean enableLatLon) { + this.enableLatLon = enableLatLon; + return builder; + } + + public T precisionStep(int precisionStep) { + this.precisionStep = precisionStep; + return builder; + } + + public T enableGeoHash(boolean enableGeoHash) { + this.enableGeoHash = enableGeoHash; + return builder; + } + + public T geoHashPrefix(boolean enableGeoHashPrefix) { + this.enableGeoHashPrefix = enableGeoHashPrefix; + return builder; + } + + public T geoHashPrecision(int precision) { + this.geoHashPrecision = precision; + return builder; + } + + public T ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return builder; + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(context.indexSettings().getAsBoolean("index.mapping.ignore_malformed", Defaults.IGNORE_MALFORMED.value()), false); + } + return Defaults.IGNORE_MALFORMED; + } + + public abstract Y build(BuilderContext context, String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Settings indexSettings, ContentPath.Type pathType, DoubleFieldMapper latMapper, DoubleFieldMapper lonMapper, + StringFieldMapper geoHashMapper, MultiFields multiFields, Explicit ignoreMalformed, CopyTo copyTo); + + public Y build(Mapper.BuilderContext context) { + ContentPath.Type origPathType = context.path().pathType(); + context.path().pathType(pathType); + + GeoPointFieldType geoPointFieldType = (GeoPointFieldType)fieldType; + + DoubleFieldMapper latMapper = null; + DoubleFieldMapper lonMapper = null; + + context.path().add(name); + if (enableLatLon) { + NumberFieldMapper.Builder latMapperBuilder = doubleField(Names.LAT).includeInAll(false); + NumberFieldMapper.Builder lonMapperBuilder = doubleField(Names.LON).includeInAll(false); + if (precisionStep != null) { + latMapperBuilder.precisionStep(precisionStep); + lonMapperBuilder.precisionStep(precisionStep); + } + latMapper = (DoubleFieldMapper) latMapperBuilder.includeInAll(false).store(fieldType.stored()).docValues(false).build(context); + lonMapper = (DoubleFieldMapper) lonMapperBuilder.includeInAll(false).store(fieldType.stored()).docValues(false).build(context); + geoPointFieldType.setLatLonEnabled(latMapper.fieldType(), lonMapper.fieldType()); + } + StringFieldMapper geoHashMapper = null; + if (enableGeoHash || enableGeoHashPrefix) { + // TODO: possible also implicitly enable geohash if geohash precision is set + geoHashMapper = stringField(Names.GEOHASH).index(true).tokenized(false).includeInAll(false).store(fieldType.stored()) + .omitNorms(true).indexOptions(IndexOptions.DOCS).build(context); + geoPointFieldType.setGeoHashEnabled(geoHashMapper.fieldType(), geoHashPrecision, enableGeoHashPrefix); + } + context.path().remove(); + context.path().pathType(origPathType); + + return build(context, name, fieldType, defaultFieldType, context.indexSettings(), origPathType, + latMapper, lonMapper, geoHashMapper, multiFieldsBuilder.build(this, context), ignoreMalformed(context), copyTo); + } + } + + public abstract static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder; + if (parserContext.indexVersionCreated().before(Version.V_2_2_0)) { + builder = new GeoPointFieldMapperLegacy.Builder(name); + } else { + builder = new GeoPointFieldMapper.Builder(name); + } + parseField(builder, name, node, parserContext); + + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = Strings.toUnderscoreCase(entry.getKey()); + Object propNode = entry.getValue(); + if (propName.equals("lat_lon")) { + builder.enableLatLon(XContentMapValues.nodeBooleanValue(propNode)); + iterator.remove(); + } else if (propName.equals("precision_step")) { + builder.precisionStep(XContentMapValues.nodeIntegerValue(propNode)); + iterator.remove(); + } else if (propName.equals("geohash")) { + builder.enableGeoHash(XContentMapValues.nodeBooleanValue(propNode)); + iterator.remove(); + } else if (propName.equals("geohash_prefix")) { + builder.geoHashPrefix(XContentMapValues.nodeBooleanValue(propNode)); + if (XContentMapValues.nodeBooleanValue(propNode)) { + builder.enableGeoHash(true); + } + iterator.remove(); + } else if (propName.equals("geohash_precision")) { + if (propNode instanceof Integer) { + builder.geoHashPrecision(XContentMapValues.nodeIntegerValue(propNode)); + } else { + builder.geoHashPrecision(GeoUtils.geoHashLevelsForPrecision(propNode.toString())); + } + iterator.remove(); + } else if (propName.equals(Names.IGNORE_MALFORMED)) { + builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode)); + iterator.remove(); + } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { + iterator.remove(); + } + } + + if (builder instanceof GeoPointFieldMapperLegacy.Builder) { + return GeoPointFieldMapperLegacy.parse((GeoPointFieldMapperLegacy.Builder) builder, node, parserContext); + } + + return (GeoPointFieldMapper.Builder) builder; + } + } + + public static class GeoPointFieldType extends MappedFieldType { + protected MappedFieldType geoHashFieldType; + protected int geoHashPrecision; + protected boolean geoHashPrefixEnabled; + + protected MappedFieldType latFieldType; + protected MappedFieldType lonFieldType; + + GeoPointFieldType() {} + + GeoPointFieldType(GeoPointFieldType ref) { + super(ref); + this.geoHashFieldType = ref.geoHashFieldType; // copying ref is ok, this can never be modified + this.geoHashPrecision = ref.geoHashPrecision; + this.geoHashPrefixEnabled = ref.geoHashPrefixEnabled; + this.latFieldType = ref.latFieldType; // copying ref is ok, this can never be modified + this.lonFieldType = ref.lonFieldType; // copying ref is ok, this can never be modified + } + + @Override + public MappedFieldType clone() { + return new GeoPointFieldType(this); + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) return false; + GeoPointFieldType that = (GeoPointFieldType) o; + return geoHashPrecision == that.geoHashPrecision && + geoHashPrefixEnabled == that.geoHashPrefixEnabled && + java.util.Objects.equals(geoHashFieldType, that.geoHashFieldType) && + java.util.Objects.equals(latFieldType, that.latFieldType) && + java.util.Objects.equals(lonFieldType, that.lonFieldType); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(super.hashCode(), geoHashFieldType, geoHashPrecision, geoHashPrefixEnabled, latFieldType, + lonFieldType); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public void checkCompatibility(MappedFieldType fieldType, List conflicts, boolean strict) { + super.checkCompatibility(fieldType, conflicts, strict); + GeoPointFieldType other = (GeoPointFieldType)fieldType; + if (isLatLonEnabled() != other.isLatLonEnabled()) { + conflicts.add("mapper [" + names().fullName() + "] has different [lat_lon]"); + } + if (isLatLonEnabled() && other.isLatLonEnabled() && + latFieldType().numericPrecisionStep() != other.latFieldType().numericPrecisionStep()) { + conflicts.add("mapper [" + names().fullName() + "] has different [precision_step]"); + } + if (isGeoHashEnabled() != other.isGeoHashEnabled()) { + conflicts.add("mapper [" + names().fullName() + "] has different [geohash]"); + } + if (geoHashPrecision() != other.geoHashPrecision()) { + conflicts.add("mapper [" + names().fullName() + "] has different [geohash_precision]"); + } + if (isGeoHashPrefixEnabled() != other.isGeoHashPrefixEnabled()) { + conflicts.add("mapper [" + names().fullName() + "] has different [geohash_prefix]"); + } + } + + public boolean isGeoHashEnabled() { + return geoHashFieldType != null; + } + + public MappedFieldType geoHashFieldType() { + return geoHashFieldType; + } + + public int geoHashPrecision() { + return geoHashPrecision; + } + + public boolean isGeoHashPrefixEnabled() { + return geoHashPrefixEnabled; + } + + public void setGeoHashEnabled(MappedFieldType geoHashFieldType, int geoHashPrecision, boolean geoHashPrefixEnabled) { + checkIfFrozen(); + this.geoHashFieldType = geoHashFieldType; + this.geoHashPrecision = geoHashPrecision; + this.geoHashPrefixEnabled = geoHashPrefixEnabled; + } + + public boolean isLatLonEnabled() { + return latFieldType != null; + } + + public MappedFieldType latFieldType() { + return latFieldType; + } + + public MappedFieldType lonFieldType() { + return lonFieldType; + } + + public void setLatLonEnabled(MappedFieldType latFieldType, MappedFieldType lonFieldType) { + checkIfFrozen(); + this.latFieldType = latFieldType; + this.lonFieldType = lonFieldType; + } + } + + protected final DoubleFieldMapper latMapper; + + protected final DoubleFieldMapper lonMapper; + + protected final ContentPath.Type pathType; + + protected final StringFieldMapper geoHashMapper; + + protected Explicit ignoreMalformed; + + protected BaseGeoPointFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, + ContentPath.Type pathType, DoubleFieldMapper latMapper, DoubleFieldMapper lonMapper, StringFieldMapper geoHashMapper, + MultiFields multiFields, Explicit ignoreMalformed, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); + this.pathType = pathType; + this.latMapper = latMapper; + this.lonMapper = lonMapper; + this.geoHashMapper = geoHashMapper; + this.ignoreMalformed = ignoreMalformed; + } + + @Override + public GeoPointFieldType fieldType() { + return (GeoPointFieldType) super.fieldType(); + } + + @Override + public void merge(Mapper mergeWith, MergeResult mergeResult) { + super.merge(mergeWith, mergeResult); + if (!this.getClass().equals(mergeWith.getClass())) { + return; + } + + BaseGeoPointFieldMapper gpfmMergeWith = (BaseGeoPointFieldMapper) mergeWith; + if (mergeResult.simulate() == false && mergeResult.hasConflicts() == false) { + if (gpfmMergeWith.ignoreMalformed.explicit()) { + this.ignoreMalformed = gpfmMergeWith.ignoreMalformed; + } + } + } + + @Override + public Iterator iterator() { + List extras = new ArrayList<>(); + if (fieldType().isGeoHashEnabled()) { + extras.add(geoHashMapper); + } + if (fieldType().isLatLonEnabled()) { + extras.add(latMapper); + extras.add(lonMapper); + } + return Iterators.concat(super.iterator(), extras.iterator()); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); + } + + protected void parse(ParseContext context, GeoPoint point, String geoHash) throws IOException { + if (fieldType().isGeoHashEnabled()) { + if (geoHash == null) { + geoHash = GeoHashUtils.stringEncode(point.lon(), point.lat()); + } + addGeoHashField(context, geoHash); + } + if (fieldType().isLatLonEnabled()) { + latMapper.parse(context.createExternalValueContext(point.lat())); + lonMapper.parse(context.createExternalValueContext(point.lon())); + } + multiFields.parse(this, context); + } + + @Override + public Mapper parse(ParseContext context) throws IOException { + ContentPath.Type origPathType = context.path().pathType(); + context.path().pathType(pathType); + context.path().add(simpleName()); + + GeoPoint sparse = context.parseExternalValue(GeoPoint.class); + + if (sparse != null) { + parse(context, sparse, null); + } else { + sparse = new GeoPoint(); + XContentParser.Token token = context.parser().currentToken(); + if (token == XContentParser.Token.START_ARRAY) { + token = context.parser().nextToken(); + if (token == XContentParser.Token.START_ARRAY) { + // its an array of array of lon/lat [ [1.2, 1.3], [1.4, 1.5] ] + while (token != XContentParser.Token.END_ARRAY) { + parse(context, GeoUtils.parseGeoPoint(context.parser(), sparse), null); + token = context.parser().nextToken(); + } + } else { + // its an array of other possible values + if (token == XContentParser.Token.VALUE_NUMBER) { + double lon = context.parser().doubleValue(); + token = context.parser().nextToken(); + double lat = context.parser().doubleValue(); + while ((token = context.parser().nextToken()) != XContentParser.Token.END_ARRAY); + parse(context, sparse.reset(lat, lon), null); + } else { + while (token != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + parsePointFromString(context, sparse, context.parser().text()); + } else { + parse(context, GeoUtils.parseGeoPoint(context.parser(), sparse), null); + } + token = context.parser().nextToken(); + } + } + } + } else if (token == XContentParser.Token.VALUE_STRING) { + parsePointFromString(context, sparse, context.parser().text()); + } else if (token != XContentParser.Token.VALUE_NULL) { + parse(context, GeoUtils.parseGeoPoint(context.parser(), sparse), null); + } + } + + context.path().remove(); + context.path().pathType(origPathType); + return null; + } + + private void addGeoHashField(ParseContext context, String geoHash) throws IOException { + int len = Math.min(fieldType().geoHashPrecision(), geoHash.length()); + int min = fieldType().isGeoHashPrefixEnabled() ? 1 : len; + + for (int i = len; i >= min; i--) { + // side effect of this call is adding the field + geoHashMapper.parse(context.createExternalValueContext(geoHash.substring(0, i))); + } + } + + private void parsePointFromString(ParseContext context, GeoPoint sparse, String point) throws IOException { + if (point.indexOf(',') < 0) { + parse(context, sparse.resetFromGeoHash(point), point); + } else { + parse(context, sparse.resetFromString(point), null); + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + if (includeDefaults || pathType != Defaults.PATH_TYPE) { + builder.field("path", pathType.name().toLowerCase(Locale.ROOT)); + } + if (includeDefaults || fieldType().isLatLonEnabled() != GeoPointFieldMapper.Defaults.ENABLE_LATLON) { + builder.field("lat_lon", fieldType().isLatLonEnabled()); + } + if (fieldType().isLatLonEnabled() && (includeDefaults || fieldType().latFieldType().numericPrecisionStep() != NumericUtils.PRECISION_STEP_DEFAULT)) { + builder.field("precision_step", fieldType().latFieldType().numericPrecisionStep()); + } + if (includeDefaults || fieldType().isGeoHashEnabled() != Defaults.ENABLE_GEOHASH) { + builder.field("geohash", fieldType().isGeoHashEnabled()); + } + if (includeDefaults || fieldType().isGeoHashPrefixEnabled() != Defaults.ENABLE_GEOHASH_PREFIX) { + builder.field("geohash_prefix", fieldType().isGeoHashPrefixEnabled()); + } + if (fieldType().isGeoHashEnabled() && (includeDefaults || fieldType().geoHashPrecision() != Defaults.GEO_HASH_PRECISION)) { + builder.field("geohash_precision", fieldType().geoHashPrecision()); + } + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field(Names.IGNORE_MALFORMED, ignoreMalformed.value()); + } + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java index b264bfa4bc3..286aca29727 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java @@ -19,38 +19,24 @@ package org.elasticsearch.index.mapper.geo; -import com.carrotsearch.hppc.ObjectHashSet; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.GeoPointField; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.XGeoHashUtils; -import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.DoubleFieldMapper; -import org.elasticsearch.index.mapper.core.NumberFieldMapper; -import org.elasticsearch.index.mapper.core.NumberFieldMapper.CustomNumericDocValuesField; import org.elasticsearch.index.mapper.core.StringFieldMapper; -import org.elasticsearch.index.mapper.object.ArrayValueMapperParser; import java.io.IOException; -import java.util.*; - -import static org.elasticsearch.index.mapper.MapperBuilders.*; -import static org.elasticsearch.index.mapper.core.TypeParsers.*; +import java.util.Map; /** * Parsing: We handle: @@ -62,736 +48,84 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.*; * "lon" : 2.1 * } */ -public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapperParser { +public class GeoPointFieldMapper extends BaseGeoPointFieldMapper { public static final String CONTENT_TYPE = "geo_point"; - public static class Names { - public static final String LAT = "lat"; - public static final String LAT_SUFFIX = "." + LAT; - public static final String LON = "lon"; - public static final String LON_SUFFIX = "." + LON; - public static final String GEOHASH = "geohash"; - public static final String GEOHASH_SUFFIX = "." + GEOHASH; - public static final String IGNORE_MALFORMED = "ignore_malformed"; - public static final String COERCE = "coerce"; - } + public static class Defaults extends BaseGeoPointFieldMapper.Defaults { - public static class Defaults { - public static final ContentPath.Type PATH_TYPE = ContentPath.Type.FULL; - public static final boolean ENABLE_LATLON = false; - public static final boolean ENABLE_GEOHASH = false; - public static final boolean ENABLE_GEOHASH_PREFIX = false; - public static final int GEO_HASH_PRECISION = XGeoHashUtils.PRECISION; - - public static final Explicit IGNORE_MALFORMED = new Explicit(false, false); - public static final Explicit COERCE = new Explicit(false, false); - - public static final MappedFieldType FIELD_TYPE = new GeoPointFieldType(); + public static final GeoPointFieldType FIELD_TYPE = new GeoPointFieldType(); static { FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); FIELD_TYPE.setTokenized(false); FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setNumericType(FieldType.NumericType.LONG); + FIELD_TYPE.setNumericPrecisionStep(GeoPointField.PRECISION_STEP); + FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC); + FIELD_TYPE.setHasDocValues(true); + FIELD_TYPE.setStored(true); FIELD_TYPE.freeze(); } } - public static class Builder extends FieldMapper.Builder { - - private ContentPath.Type pathType = Defaults.PATH_TYPE; - - private boolean enableGeoHash = Defaults.ENABLE_GEOHASH; - - private boolean enableGeohashPrefix = Defaults.ENABLE_GEOHASH_PREFIX; - - private boolean enableLatLon = Defaults.ENABLE_LATLON; - - private Integer precisionStep; - - private int geoHashPrecision = Defaults.GEO_HASH_PRECISION; - - private Boolean ignoreMalformed; - - private Boolean coerce; + /** + * Concrete builder for indexed GeoPointField type + */ + public static class Builder extends BaseGeoPointFieldMapper.Builder { public Builder(String name) { super(name, Defaults.FIELD_TYPE); this.builder = this; } - public Builder ignoreMalformed(boolean ignoreMalformed) { - this.ignoreMalformed = ignoreMalformed; - return builder; - } - - protected Explicit ignoreMalformed(BuilderContext context) { - if (ignoreMalformed != null) { - return new Explicit<>(ignoreMalformed, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(context.indexSettings().getAsBoolean("index.mapping.ignore_malformed", Defaults.IGNORE_MALFORMED.value()), false); - } - return Defaults.IGNORE_MALFORMED; - } - - public Builder coerce(boolean coerce) { - this.coerce = coerce; - return builder; - } - - protected Explicit coerce(BuilderContext context) { - if (coerce != null) { - return new Explicit<>(coerce, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(context.indexSettings().getAsBoolean("index.mapping.coerce", Defaults.COERCE.value()), false); - } - return Defaults.COERCE; - } - @Override - public GeoPointFieldType fieldType() { - return (GeoPointFieldType)fieldType; - } - - @Override - public Builder multiFieldPathType(ContentPath.Type pathType) { - this.pathType = pathType; - return this; - } - - public Builder enableGeoHash(boolean enableGeoHash) { - this.enableGeoHash = enableGeoHash; - return this; - } - - public Builder geohashPrefix(boolean enableGeohashPrefix) { - this.enableGeohashPrefix = enableGeohashPrefix; - return this; - } - - public Builder enableLatLon(boolean enableLatLon) { - this.enableLatLon = enableLatLon; - return this; - } - - public Builder precisionStep(int precisionStep) { - this.precisionStep = precisionStep; - return this; - } - - public Builder geoHashPrecision(int precision) { - this.geoHashPrecision = precision; - return this; - } - - @Override - public Builder fieldDataSettings(Settings settings) { - this.fieldDataSettings = settings; - return builder; + public GeoPointFieldMapper build(BuilderContext context, String simpleName, MappedFieldType fieldType, + MappedFieldType defaultFieldType, Settings indexSettings, ContentPath.Type pathType, DoubleFieldMapper latMapper, + DoubleFieldMapper lonMapper, StringFieldMapper geoHashMapper, MultiFields multiFields, Explicit ignoreMalformed, + CopyTo copyTo) { + fieldType.setTokenized(false); + setupFieldType(context); + return new GeoPointFieldMapper(simpleName, fieldType, defaultFieldType, indexSettings, pathType, latMapper, lonMapper, + geoHashMapper, multiFields, ignoreMalformed, copyTo); } @Override public GeoPointFieldMapper build(BuilderContext context) { - ContentPath.Type origPathType = context.path().pathType(); - context.path().pathType(pathType); - - DoubleFieldMapper latMapper = null; - DoubleFieldMapper lonMapper = null; - GeoPointFieldType geoPointFieldType = (GeoPointFieldType)fieldType; - - context.path().add(name); - if (enableLatLon) { - NumberFieldMapper.Builder latMapperBuilder = doubleField(Names.LAT).includeInAll(false); - NumberFieldMapper.Builder lonMapperBuilder = doubleField(Names.LON).includeInAll(false); - if (precisionStep != null) { - latMapperBuilder.precisionStep(precisionStep); - lonMapperBuilder.precisionStep(precisionStep); - } - latMapper = (DoubleFieldMapper) latMapperBuilder.includeInAll(false).store(fieldType.stored()).docValues(false).build(context); - lonMapper = (DoubleFieldMapper) lonMapperBuilder.includeInAll(false).store(fieldType.stored()).docValues(false).build(context); - geoPointFieldType.setLatLonEnabled(latMapper.fieldType(), lonMapper.fieldType()); - } - StringFieldMapper geohashMapper = null; - if (enableGeoHash || enableGeohashPrefix) { - // TODO: possible also implicitly enable geohash if geohash precision is set - geohashMapper = stringField(Names.GEOHASH).index(true).tokenized(false).includeInAll(false).store(fieldType.stored()) - .omitNorms(true).indexOptions(IndexOptions.DOCS).build(context); - geoPointFieldType.setGeohashEnabled(geohashMapper.fieldType(), geoHashPrecision, enableGeohashPrefix); - } - context.path().remove(); - - context.path().pathType(origPathType); - - // this is important: even if geo points feel like they need to be tokenized to distinguish lat from lon, we actually want to - // store them as a single token. - fieldType.setTokenized(false); - setupFieldType(context); - fieldType.setHasDocValues(false); - defaultFieldType.setHasDocValues(false); - return new GeoPointFieldMapper(name, fieldType, defaultFieldType, context.indexSettings(), origPathType, - latMapper, lonMapper, geohashMapper, multiFieldsBuilder.build(this, context), ignoreMalformed(context), coerce(context)); + return super.build(context); } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser extends BaseGeoPointFieldMapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = geoPointField(name); - final boolean indexCreatedBeforeV2_0 = parserContext.indexVersionCreated().before(Version.V_2_0_0); - parseField(builder, name, node, parserContext); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = Strings.toUnderscoreCase(entry.getKey()); - Object propNode = entry.getValue(); - if (propName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - builder.multiFieldPathType(parsePathType(name, propNode.toString())); - iterator.remove(); - } else if (propName.equals("lat_lon")) { - builder.enableLatLon(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (propName.equals("geohash")) { - builder.enableGeoHash(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (propName.equals("geohash_prefix")) { - builder.geohashPrefix(XContentMapValues.nodeBooleanValue(propNode)); - if (XContentMapValues.nodeBooleanValue(propNode)) { - builder.enableGeoHash(true); - } - iterator.remove(); - } else if (propName.equals("precision_step")) { - builder.precisionStep(XContentMapValues.nodeIntegerValue(propNode)); - iterator.remove(); - } else if (propName.equals("geohash_precision")) { - if (propNode instanceof Integer) { - builder.geoHashPrecision(XContentMapValues.nodeIntegerValue(propNode)); - } else { - builder.geoHashPrecision(GeoUtils.geoHashLevelsForPrecision(propNode.toString())); - } - iterator.remove(); - } else if (propName.equals(Names.IGNORE_MALFORMED)) { - builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (indexCreatedBeforeV2_0 && propName.equals("validate")) { - builder.ignoreMalformed(!XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (indexCreatedBeforeV2_0 && propName.equals("validate_lon")) { - builder.ignoreMalformed(!XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (indexCreatedBeforeV2_0 && propName.equals("validate_lat")) { - builder.ignoreMalformed(!XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (propName.equals(Names.COERCE)) { - builder.coerce(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (indexCreatedBeforeV2_0 && propName.equals("normalize")) { - builder.coerce(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (indexCreatedBeforeV2_0 && propName.equals("normalize_lat")) { - builder.coerce(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (indexCreatedBeforeV2_0 && propName.equals("normalize_lon")) { - builder.coerce(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); - } - } - return builder; + return super.parse(name, node, parserContext); } } - public static final class GeoPointFieldType extends MappedFieldType { - - private MappedFieldType geohashFieldType; - private int geohashPrecision; - private boolean geohashPrefixEnabled; - - private MappedFieldType latFieldType; - private MappedFieldType lonFieldType; - - public GeoPointFieldType() {} - - protected GeoPointFieldType(GeoPointFieldType ref) { - super(ref); - this.geohashFieldType = ref.geohashFieldType; // copying ref is ok, this can never be modified - this.geohashPrecision = ref.geohashPrecision; - this.geohashPrefixEnabled = ref.geohashPrefixEnabled; - this.latFieldType = ref.latFieldType; // copying ref is ok, this can never be modified - this.lonFieldType = ref.lonFieldType; // copying ref is ok, this can never be modified - } - - @Override - public MappedFieldType clone() { - return new GeoPointFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - GeoPointFieldType that = (GeoPointFieldType) o; - return geohashPrecision == that.geohashPrecision && - geohashPrefixEnabled == that.geohashPrefixEnabled && - java.util.Objects.equals(geohashFieldType, that.geohashFieldType) && - java.util.Objects.equals(latFieldType, that.latFieldType) && - java.util.Objects.equals(lonFieldType, that.lonFieldType); - } - - @Override - public int hashCode() { - return java.util.Objects.hash(super.hashCode(), geohashFieldType, geohashPrecision, geohashPrefixEnabled, latFieldType, - lonFieldType); - } - - @Override - public String typeName() { - return CONTENT_TYPE; - } - - @Override - public void checkCompatibility(MappedFieldType fieldType, List conflicts, boolean strict) { - super.checkCompatibility(fieldType, conflicts, strict); - GeoPointFieldType other = (GeoPointFieldType)fieldType; - if (isLatLonEnabled() != other.isLatLonEnabled()) { - conflicts.add("mapper [" + names().fullName() + "] has different [lat_lon]"); - } - if (isGeohashEnabled() != other.isGeohashEnabled()) { - conflicts.add("mapper [" + names().fullName() + "] has different [geohash]"); - } - if (geohashPrecision() != other.geohashPrecision()) { - conflicts.add("mapper [" + names().fullName() + "] has different [geohash_precision]"); - } - if (isGeohashPrefixEnabled() != other.isGeohashPrefixEnabled()) { - conflicts.add("mapper [" + names().fullName() + "] has different [geohash_prefix]"); - } - if (isLatLonEnabled() && other.isLatLonEnabled() && - latFieldType().numericPrecisionStep() != other.latFieldType().numericPrecisionStep()) { - conflicts.add("mapper [" + names().fullName() + "] has different [precision_step]"); - } - } - - public boolean isGeohashEnabled() { - return geohashFieldType != null; - } - - public MappedFieldType geohashFieldType() { - return geohashFieldType; - } - - public int geohashPrecision() { - return geohashPrecision; - } - - public boolean isGeohashPrefixEnabled() { - return geohashPrefixEnabled; - } - - public void setGeohashEnabled(MappedFieldType geohashFieldType, int geohashPrecision, boolean geohashPrefixEnabled) { - checkIfFrozen(); - this.geohashFieldType = geohashFieldType; - this.geohashPrecision = geohashPrecision; - this.geohashPrefixEnabled = geohashPrefixEnabled; - } - - public boolean isLatLonEnabled() { - return latFieldType != null; - } - - public MappedFieldType latFieldType() { - return latFieldType; - } - - public MappedFieldType lonFieldType() { - return lonFieldType; - } - - public void setLatLonEnabled(MappedFieldType latFieldType, MappedFieldType lonFieldType) { - checkIfFrozen(); - this.latFieldType = latFieldType; - this.lonFieldType = lonFieldType; - } - - @Override - public GeoPoint value(Object value) { - if (value instanceof GeoPoint) { - return (GeoPoint) value; - } else { - return GeoPoint.parseFromLatLon(value.toString()); - } - } - } - - /** - * A byte-aligned fixed-length encoding for latitudes and longitudes. - */ - public static final class Encoding { - - // With 14 bytes we already have better precision than a double since a double has 11 bits of exponent - private static final int MAX_NUM_BYTES = 14; - - private static final Encoding[] INSTANCES; - static { - INSTANCES = new Encoding[MAX_NUM_BYTES + 1]; - for (int numBytes = 2; numBytes <= MAX_NUM_BYTES; numBytes += 2) { - INSTANCES[numBytes] = new Encoding(numBytes); - } - } - - /** Get an instance based on the number of bytes that has been used to encode values. */ - public static final Encoding of(int numBytesPerValue) { - final Encoding instance = INSTANCES[numBytesPerValue]; - if (instance == null) { - throw new IllegalStateException("No encoding for " + numBytesPerValue + " bytes per value"); - } - return instance; - } - - /** Get an instance based on the expected precision. Here are examples of the number of required bytes per value depending on the - * expected precision:

      - *
    • 1km: 4 bytes
      • - *
    • 3m: 6 bytes
      • - *
    • 1m: 8 bytes
      • - *
    • 1cm: 8 bytes
      • - *
    • 1mm: 10 bytes
    */ - public static final Encoding of(DistanceUnit.Distance precision) { - for (Encoding encoding : INSTANCES) { - if (encoding != null && encoding.precision().compareTo(precision) <= 0) { - return encoding; - } - } - return INSTANCES[MAX_NUM_BYTES]; - } - - private final DistanceUnit.Distance precision; - private final int numBytes; - private final int numBytesPerCoordinate; - private final double factor; - - private Encoding(int numBytes) { - assert numBytes >= 1 && numBytes <= MAX_NUM_BYTES; - assert (numBytes & 1) == 0; // we don't support odd numBytes for the moment - this.numBytes = numBytes; - this.numBytesPerCoordinate = numBytes / 2; - this.factor = Math.pow(2, - numBytesPerCoordinate * 8 + 9); - assert (1L << (numBytesPerCoordinate * 8 - 1)) * factor > 180 && (1L << (numBytesPerCoordinate * 8 - 2)) * factor < 180 : numBytesPerCoordinate + " " + factor; - if (numBytes == MAX_NUM_BYTES) { - // no precision loss compared to a double - precision = new DistanceUnit.Distance(0, DistanceUnit.DEFAULT); - } else { - precision = new DistanceUnit.Distance( - GeoDistance.PLANE.calculate(0, 0, factor / 2, factor / 2, DistanceUnit.DEFAULT), // factor/2 because we use Math.round instead of a cast to convert the double to a long - DistanceUnit.DEFAULT); - } - } - - public DistanceUnit.Distance precision() { - return precision; - } - - /** The number of bytes required to encode a single geo point. */ - public final int numBytes() { - return numBytes; - } - - /** The number of bits required to encode a single coordinate of a geo point. */ - public int numBitsPerCoordinate() { - return numBytesPerCoordinate << 3; - } - - /** Return the bits that encode a latitude/longitude. */ - public long encodeCoordinate(double lat) { - return Math.round((lat + 180) / factor); - } - - /** Decode a sequence of bits into the original coordinate. */ - public double decodeCoordinate(long bits) { - return bits * factor - 180; - } - - private void encodeBits(long bits, byte[] out, int offset) { - for (int i = 0; i < numBytesPerCoordinate; ++i) { - out[offset++] = (byte) bits; - bits >>>= 8; - } - assert bits == 0; - } - - private long decodeBits(byte [] in, int offset) { - long r = in[offset++] & 0xFFL; - for (int i = 1; i < numBytesPerCoordinate; ++i) { - r = (in[offset++] & 0xFFL) << (i * 8); - } - return r; - } - - /** Encode a geo point into a byte-array, over {@link #numBytes()} bytes. */ - public void encode(double lat, double lon, byte[] out, int offset) { - encodeBits(encodeCoordinate(lat), out, offset); - encodeBits(encodeCoordinate(lon), out, offset + numBytesPerCoordinate); - } - - /** Decode a geo point from a byte-array, reading {@link #numBytes()} bytes. */ - public GeoPoint decode(byte[] in, int offset, GeoPoint out) { - final long latBits = decodeBits(in, offset); - final long lonBits = decodeBits(in, offset + numBytesPerCoordinate); - return decode(latBits, lonBits, out); - } - - /** Decode a geo point from the bits of the encoded latitude and longitudes. */ - public GeoPoint decode(long latBits, long lonBits, GeoPoint out) { - final double lat = decodeCoordinate(latBits); - final double lon = decodeCoordinate(lonBits); - return out.reset(lat, lon); - } - - } - - private final ContentPath.Type pathType; - - private final DoubleFieldMapper latMapper; - - private final DoubleFieldMapper lonMapper; - - private final StringFieldMapper geohashMapper; - - protected Explicit ignoreMalformed; - - protected Explicit coerce; - public GeoPointFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, - ContentPath.Type pathType, DoubleFieldMapper latMapper, DoubleFieldMapper lonMapper, StringFieldMapper geohashMapper, - MultiFields multiFields, Explicit ignoreMalformed, Explicit coerce) { - super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, null); - this.pathType = pathType; - this.latMapper = latMapper; - this.lonMapper = lonMapper; - this.geohashMapper = geohashMapper; - this.ignoreMalformed = ignoreMalformed; - this.coerce = coerce; + ContentPath.Type pathType, DoubleFieldMapper latMapper, DoubleFieldMapper lonMapper, + StringFieldMapper geoHashMapper, MultiFields multiFields, Explicit ignoreMalformed, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, indexSettings, pathType, latMapper, lonMapper, geoHashMapper, multiFields, + ignoreMalformed, copyTo); } @Override - protected String contentType() { - return CONTENT_TYPE; - } - - @Override - public GeoPointFieldType fieldType() { - return (GeoPointFieldType) super.fieldType(); - } - - @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { - super.merge(mergeWith, mergeResult); - if (!this.getClass().equals(mergeWith.getClass())) { - return; - } - - GeoPointFieldMapper gpfmMergeWith = (GeoPointFieldMapper) mergeWith; - if (gpfmMergeWith.coerce.explicit()) { - if (coerce.explicit() && coerce.value() != gpfmMergeWith.coerce.value()) { - mergeResult.addConflict("mapper [" + fieldType().names().fullName() + "] has different [coerce]"); - } - } - - if (mergeResult.simulate() == false && mergeResult.hasConflicts() == false) { - if (gpfmMergeWith.ignoreMalformed.explicit()) { - this.ignoreMalformed = gpfmMergeWith.ignoreMalformed; - } - if (gpfmMergeWith.coerce.explicit()) { - this.coerce = gpfmMergeWith.coerce; - } - } - } - - @Override - protected void parseCreateField(ParseContext context, List fields) throws IOException { - throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); - } - - @Override - public Mapper parse(ParseContext context) throws IOException { - ContentPath.Type origPathType = context.path().pathType(); - context.path().pathType(pathType); - context.path().add(simpleName()); - - GeoPoint sparse = context.parseExternalValue(GeoPoint.class); - - if (sparse != null) { - parse(context, sparse, null); - } else { - sparse = new GeoPoint(); - XContentParser.Token token = context.parser().currentToken(); - if (token == XContentParser.Token.START_ARRAY) { - token = context.parser().nextToken(); - if (token == XContentParser.Token.START_ARRAY) { - // its an array of array of lon/lat [ [1.2, 1.3], [1.4, 1.5] ] - while (token != XContentParser.Token.END_ARRAY) { - parse(context, GeoUtils.parseGeoPoint(context.parser(), sparse), null); - token = context.parser().nextToken(); - } - } else { - // its an array of other possible values - if (token == XContentParser.Token.VALUE_NUMBER) { - double lon = context.parser().doubleValue(); - token = context.parser().nextToken(); - double lat = context.parser().doubleValue(); - while ((token = context.parser().nextToken()) != XContentParser.Token.END_ARRAY); - parse(context, sparse.reset(lat, lon), null); - } else { - while (token != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - parsePointFromString(context, sparse, context.parser().text()); - } else { - parse(context, GeoUtils.parseGeoPoint(context.parser(), sparse), null); - } - token = context.parser().nextToken(); - } - } - } - } else if (token == XContentParser.Token.VALUE_STRING) { - parsePointFromString(context, sparse, context.parser().text()); - } else if (token != XContentParser.Token.VALUE_NULL) { - parse(context, GeoUtils.parseGeoPoint(context.parser(), sparse), null); - } - } - - context.path().remove(); - context.path().pathType(origPathType); - return null; - } - - private void addGeohashField(ParseContext context, String geohash) throws IOException { - int len = Math.min(fieldType().geohashPrecision(), geohash.length()); - int min = fieldType().isGeohashPrefixEnabled() ? 1 : len; - - for (int i = len; i >= min; i--) { - // side effect of this call is adding the field - geohashMapper.parse(context.createExternalValueContext(geohash.substring(0, i))); - } - } - - private void parsePointFromString(ParseContext context, GeoPoint sparse, String point) throws IOException { - if (point.indexOf(',') < 0) { - parse(context, sparse.resetFromGeoHash(point), point); - } else { - parse(context, sparse.resetFromString(point), null); - } - } - - private void parse(ParseContext context, GeoPoint point, String geohash) throws IOException { - boolean validPoint = false; - if (coerce.value() == false && ignoreMalformed.value() == false) { + protected void parse(ParseContext context, GeoPoint point, String geoHash) throws IOException { + if (ignoreMalformed.value() == false) { if (point.lat() > 90.0 || point.lat() < -90.0) { throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "] for " + name()); } if (point.lon() > 180.0 || point.lon() < -180) { throw new IllegalArgumentException("illegal longitude value [" + point.lon() + "] for " + name()); } - validPoint = true; - } - - if (coerce.value() == true && validPoint == false) { - // by setting coerce to false we are assuming all geopoints are already in a valid coordinate system - // thus this extra step can be skipped + } else { // LUCENE WATCH: This will be folded back into Lucene's GeoPointField - GeoUtils.normalizePoint(point, true, true); + GeoUtils.normalizePoint(point); } - if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { - Field field = new Field(fieldType().names().indexName(), Double.toString(point.lat()) + ',' + Double.toString(point.lon()), fieldType()); - context.doc().add(field); + context.doc().add(new GeoPointField(fieldType().names().indexName(), point.lon(), point.lat(), fieldType() )); } - if (fieldType().isGeohashEnabled()) { - if (geohash == null) { - geohash = XGeoHashUtils.stringEncode(point.lon(), point.lat()); - } - addGeohashField(context, geohash); - } - if (fieldType().isLatLonEnabled()) { - latMapper.parse(context.createExternalValueContext(point.lat())); - lonMapper.parse(context.createExternalValueContext(point.lon())); - } - if (fieldType().hasDocValues()) { - CustomGeoPointDocValuesField field = (CustomGeoPointDocValuesField) context.doc().getByKey(fieldType().names().indexName()); - if (field == null) { - field = new CustomGeoPointDocValuesField(fieldType().names().indexName(), point.lat(), point.lon()); - context.doc().addWithKey(fieldType().names().indexName(), field); - } else { - field.add(point.lat(), point.lon()); - } - } - multiFields.parse(this, context); + super.parse(context, point, geoHash); } - - @Override - public Iterator iterator() { - List extras = new ArrayList<>(); - if (fieldType().isGeohashEnabled()) { - extras.add(geohashMapper); - } - if (fieldType().isLatLonEnabled()) { - extras.add(latMapper); - extras.add(lonMapper); - } - return Iterators.concat(super.iterator(), extras.iterator()); - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - if (includeDefaults || pathType != Defaults.PATH_TYPE) { - builder.field("path", pathType.name().toLowerCase(Locale.ROOT)); - } - if (includeDefaults || fieldType().isLatLonEnabled() != Defaults.ENABLE_LATLON) { - builder.field("lat_lon", fieldType().isLatLonEnabled()); - } - if (includeDefaults || fieldType().isGeohashEnabled() != Defaults.ENABLE_GEOHASH) { - builder.field("geohash", fieldType().isGeohashEnabled()); - } - if (includeDefaults || fieldType().isGeohashPrefixEnabled() != Defaults.ENABLE_GEOHASH_PREFIX) { - builder.field("geohash_prefix", fieldType().isGeohashPrefixEnabled()); - } - if (fieldType().isGeohashEnabled() && (includeDefaults || fieldType().geohashPrecision() != Defaults.GEO_HASH_PRECISION)) { - builder.field("geohash_precision", fieldType().geohashPrecision()); - } - if (fieldType().isLatLonEnabled() && (includeDefaults || fieldType().latFieldType().numericPrecisionStep() != NumericUtils.PRECISION_STEP_DEFAULT)) { - builder.field("precision_step", fieldType().latFieldType().numericPrecisionStep()); - } - if (includeDefaults || coerce.explicit()) { - builder.field(Names.COERCE, coerce.value()); - } - if (includeDefaults || ignoreMalformed.explicit()) { - builder.field(Names.IGNORE_MALFORMED, ignoreMalformed.value()); - } - } - - public static class CustomGeoPointDocValuesField extends CustomNumericDocValuesField { - - private final ObjectHashSet points; - - public CustomGeoPointDocValuesField(String name, double lat, double lon) { - super(name); - points = new ObjectHashSet<>(2); - points.add(new GeoPoint(lat, lon)); - } - - public void add(double lat, double lon) { - points.add(new GeoPoint(lat, lon)); - } - - @Override - public BytesRef binaryValue() { - final byte[] bytes = new byte[points.size() * 16]; - int off = 0; - for (Iterator> it = points.iterator(); it.hasNext(); ) { - final GeoPoint point = it.next().value; - ByteUtils.writeDoubleLE(point.getLat(), bytes, off); - ByteUtils.writeDoubleLE(point.getLon(), bytes, off + 8); - off += 16; - } - return new BytesRef(bytes); - } - } -} +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperLegacy.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperLegacy.java new file mode 100644 index 00000000000..84e6bde07ac --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperLegacy.java @@ -0,0 +1,393 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.geo; + +import com.carrotsearch.hppc.ObjectHashSet; +import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.geo.GeoDistance; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MergeResult; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.DoubleFieldMapper; +import org.elasticsearch.index.mapper.core.NumberFieldMapper.CustomNumericDocValuesField; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.index.mapper.object.ArrayValueMapperParser; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; + + +/** + * Parsing: We handle: + *

    + * - "field" : "geo_hash" + * - "field" : "lat,lon" + * - "field" : { + * "lat" : 1.1, + * "lon" : 2.1 + * } + */ +public class GeoPointFieldMapperLegacy extends BaseGeoPointFieldMapper implements ArrayValueMapperParser { + + public static final String CONTENT_TYPE = "geo_point"; + + public static class Names extends BaseGeoPointFieldMapper.Names { + public static final String COERCE = "coerce"; + } + + public static class Defaults extends BaseGeoPointFieldMapper.Defaults{ + public static final Explicit COERCE = new Explicit(false, false); + + public static final GeoPointFieldType FIELD_TYPE = new GeoPointFieldType(); + + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + } + } + + /** + * Concrete builder for legacy GeoPointField + */ + public static class Builder extends BaseGeoPointFieldMapper.Builder { + + private Boolean coerce; + + public Builder(String name) { + super(name, Defaults.FIELD_TYPE); + this.builder = this; + } + + public Builder coerce(boolean coerce) { + this.coerce = coerce; + return builder; + } + + protected Explicit coerce(BuilderContext context) { + if (coerce != null) { + return new Explicit<>(coerce, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(context.indexSettings().getAsBoolean("index.mapping.coerce", Defaults.COERCE.value()), false); + } + return Defaults.COERCE; + } + + @Override + public GeoPointFieldMapperLegacy build(BuilderContext context, String simpleName, MappedFieldType fieldType, + MappedFieldType defaultFieldType, Settings indexSettings, ContentPath.Type pathType, DoubleFieldMapper latMapper, + DoubleFieldMapper lonMapper, StringFieldMapper geoHashMapper, MultiFields multiFields, Explicit ignoreMalformed, + CopyTo copyTo) { + fieldType.setTokenized(false); + setupFieldType(context); + fieldType.setHasDocValues(false); + defaultFieldType.setHasDocValues(false); + return new GeoPointFieldMapperLegacy(simpleName, fieldType, defaultFieldType, indexSettings, pathType, latMapper, lonMapper, + geoHashMapper, multiFields, ignoreMalformed, coerce(context), copyTo); + } + + @Override + public GeoPointFieldMapperLegacy build(BuilderContext context) { + return super.build(context); + } + } + + public static Builder parse(Builder builder, Map node, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException { + final boolean indexCreatedBeforeV2_0 = parserContext.indexVersionCreated().before(Version.V_2_0_0); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = Strings.toUnderscoreCase(entry.getKey()); + Object propNode = entry.getValue(); + if (indexCreatedBeforeV2_0 && propName.equals("validate")) { + builder.ignoreMalformed = !XContentMapValues.nodeBooleanValue(propNode); + iterator.remove(); + } else if (indexCreatedBeforeV2_0 && propName.equals("validate_lon")) { + builder.ignoreMalformed = !XContentMapValues.nodeBooleanValue(propNode); + iterator.remove(); + } else if (indexCreatedBeforeV2_0 && propName.equals("validate_lat")) { + builder.ignoreMalformed = !XContentMapValues.nodeBooleanValue(propNode); + iterator.remove(); + } else if (propName.equals(Names.COERCE)) { + builder.coerce = XContentMapValues.nodeBooleanValue(propNode); + iterator.remove(); + } else if (indexCreatedBeforeV2_0 && propName.equals("normalize")) { + builder.coerce = XContentMapValues.nodeBooleanValue(propNode); + iterator.remove(); + } else if (indexCreatedBeforeV2_0 && propName.equals("normalize_lat")) { + builder.coerce = XContentMapValues.nodeBooleanValue(propNode); + iterator.remove(); + } else if (indexCreatedBeforeV2_0 && propName.equals("normalize_lon")) { + builder.coerce = XContentMapValues.nodeBooleanValue(propNode); + iterator.remove(); + } + } + return builder; + } + + /** + * A byte-aligned fixed-length encoding for latitudes and longitudes. + */ + public static final class Encoding { + + // With 14 bytes we already have better precision than a double since a double has 11 bits of exponent + private static final int MAX_NUM_BYTES = 14; + + private static final Encoding[] INSTANCES; + static { + INSTANCES = new Encoding[MAX_NUM_BYTES + 1]; + for (int numBytes = 2; numBytes <= MAX_NUM_BYTES; numBytes += 2) { + INSTANCES[numBytes] = new Encoding(numBytes); + } + } + + /** Get an instance based on the number of bytes that has been used to encode values. */ + public static final Encoding of(int numBytesPerValue) { + final Encoding instance = INSTANCES[numBytesPerValue]; + if (instance == null) { + throw new IllegalStateException("No encoding for " + numBytesPerValue + " bytes per value"); + } + return instance; + } + + /** Get an instance based on the expected precision. Here are examples of the number of required bytes per value depending on the + * expected precision:

      + *
    • 1km: 4 bytes
      • + *
    • 3m: 6 bytes
      • + *
    • 1m: 8 bytes
      • + *
    • 1cm: 8 bytes
      • + *
    • 1mm: 10 bytes
    */ + public static final Encoding of(DistanceUnit.Distance precision) { + for (Encoding encoding : INSTANCES) { + if (encoding != null && encoding.precision().compareTo(precision) <= 0) { + return encoding; + } + } + return INSTANCES[MAX_NUM_BYTES]; + } + + private final DistanceUnit.Distance precision; + private final int numBytes; + private final int numBytesPerCoordinate; + private final double factor; + + private Encoding(int numBytes) { + assert numBytes >= 1 && numBytes <= MAX_NUM_BYTES; + assert (numBytes & 1) == 0; // we don't support odd numBytes for the moment + this.numBytes = numBytes; + this.numBytesPerCoordinate = numBytes / 2; + this.factor = Math.pow(2, - numBytesPerCoordinate * 8 + 9); + assert (1L << (numBytesPerCoordinate * 8 - 1)) * factor > 180 && (1L << (numBytesPerCoordinate * 8 - 2)) * factor < 180 : numBytesPerCoordinate + " " + factor; + if (numBytes == MAX_NUM_BYTES) { + // no precision loss compared to a double + precision = new DistanceUnit.Distance(0, DistanceUnit.DEFAULT); + } else { + precision = new DistanceUnit.Distance( + GeoDistance.PLANE.calculate(0, 0, factor / 2, factor / 2, DistanceUnit.DEFAULT), // factor/2 because we use Math.round instead of a cast to convert the double to a long + DistanceUnit.DEFAULT); + } + } + + public DistanceUnit.Distance precision() { + return precision; + } + + /** The number of bytes required to encode a single geo point. */ + public final int numBytes() { + return numBytes; + } + + /** The number of bits required to encode a single coordinate of a geo point. */ + public int numBitsPerCoordinate() { + return numBytesPerCoordinate << 3; + } + + /** Return the bits that encode a latitude/longitude. */ + public long encodeCoordinate(double lat) { + return Math.round((lat + 180) / factor); + } + + /** Decode a sequence of bits into the original coordinate. */ + public double decodeCoordinate(long bits) { + return bits * factor - 180; + } + + private void encodeBits(long bits, byte[] out, int offset) { + for (int i = 0; i < numBytesPerCoordinate; ++i) { + out[offset++] = (byte) bits; + bits >>>= 8; + } + assert bits == 0; + } + + private long decodeBits(byte [] in, int offset) { + long r = in[offset++] & 0xFFL; + for (int i = 1; i < numBytesPerCoordinate; ++i) { + r = (in[offset++] & 0xFFL) << (i * 8); + } + return r; + } + + /** Encode a geo point into a byte-array, over {@link #numBytes()} bytes. */ + public void encode(double lat, double lon, byte[] out, int offset) { + encodeBits(encodeCoordinate(lat), out, offset); + encodeBits(encodeCoordinate(lon), out, offset + numBytesPerCoordinate); + } + + /** Decode a geo point from a byte-array, reading {@link #numBytes()} bytes. */ + public GeoPoint decode(byte[] in, int offset, GeoPoint out) { + final long latBits = decodeBits(in, offset); + final long lonBits = decodeBits(in, offset + numBytesPerCoordinate); + return decode(latBits, lonBits, out); + } + + /** Decode a geo point from the bits of the encoded latitude and longitudes. */ + public GeoPoint decode(long latBits, long lonBits, GeoPoint out) { + final double lat = decodeCoordinate(latBits); + final double lon = decodeCoordinate(lonBits); + return out.reset(lat, lon); + } + + } + + protected Explicit coerce; + + public GeoPointFieldMapperLegacy(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, + ContentPath.Type pathType, DoubleFieldMapper latMapper, DoubleFieldMapper lonMapper, + StringFieldMapper geoHashMapper, MultiFields multiFields, Explicit ignoreMalformed, + Explicit coerce, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, indexSettings, pathType, latMapper, lonMapper, geoHashMapper, multiFields, + ignoreMalformed, copyTo); + this.coerce = coerce; + } + + @Override + public void merge(Mapper mergeWith, MergeResult mergeResult) { + super.merge(mergeWith, mergeResult); + if (!this.getClass().equals(mergeWith.getClass())) { + return; + } + + GeoPointFieldMapperLegacy gpfmMergeWith = (GeoPointFieldMapperLegacy) mergeWith; + if (gpfmMergeWith.coerce.explicit()) { + if (coerce.explicit() && coerce.value() != gpfmMergeWith.coerce.value()) { + mergeResult.addConflict("mapper [" + fieldType().names().fullName() + "] has different [coerce]"); + } + } + + if (mergeResult.simulate() == false && mergeResult.hasConflicts() == false) { + if (gpfmMergeWith.coerce.explicit()) { + this.coerce = gpfmMergeWith.coerce; + } + } + } + + @Override + protected void parse(ParseContext context, GeoPoint point, String geoHash) throws IOException { + boolean validPoint = false; + if (coerce.value() == false && ignoreMalformed.value() == false) { + if (point.lat() > 90.0 || point.lat() < -90.0) { + throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "] for " + name()); + } + if (point.lon() > 180.0 || point.lon() < -180) { + throw new IllegalArgumentException("illegal longitude value [" + point.lon() + "] for " + name()); + } + validPoint = true; + } + + if (coerce.value() == true && validPoint == false) { + // by setting coerce to false we are assuming all geopoints are already in a valid coordinate system + // thus this extra step can be skipped + GeoUtils.normalizePoint(point, true, true); + } + + if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { + Field field = new Field(fieldType().names().indexName(), Double.toString(point.lat()) + ',' + Double.toString(point.lon()), fieldType()); + context.doc().add(field); + } + + super.parse(context, point, geoHash); + + if (fieldType().hasDocValues()) { + CustomGeoPointDocValuesField field = (CustomGeoPointDocValuesField) context.doc().getByKey(fieldType().names().indexName()); + if (field == null) { + field = new CustomGeoPointDocValuesField(fieldType().names().indexName(), point.lat(), point.lon()); + context.doc().addWithKey(fieldType().names().indexName(), field); + } else { + field.add(point.lat(), point.lon()); + } + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + if (includeDefaults || coerce.explicit()) { + builder.field(Names.COERCE, coerce.value()); + } + } + + public static class CustomGeoPointDocValuesField extends CustomNumericDocValuesField { + + private final ObjectHashSet points; + + public CustomGeoPointDocValuesField(String name, double lat, double lon) { + super(name); + points = new ObjectHashSet<>(2); + points.add(new GeoPoint(lat, lon)); + } + + public void add(double lat, double lon) { + points.add(new GeoPoint(lat, lon)); + } + + @Override + public BytesRef binaryValue() { + final byte[] bytes = new byte[points.size() * 16]; + int off = 0; + for (Iterator> it = points.iterator(); it.hasNext(); ) { + final GeoPoint point = it.next().value; + ByteUtils.writeDoubleLE(point.getLat(), bytes, off); + ByteUtils.writeDoubleLE(point.getLon(), bytes, off + 8); + off += 16; + } + return new BytesRef(bytes); + } + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java index 6d385875b18..71b6d89610f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java @@ -45,7 +45,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParseContext; @@ -194,7 +193,8 @@ public class GeoShapeFieldMapper extends FieldMapper { } else if (Names.COERCE.equals(fieldName)) { builder.coerce(nodeBooleanValue(fieldNode)); iterator.remove(); - } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)) { + } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName) + && builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) == false) { builder.fieldType().setPointsOnly(XContentMapValues.nodeBooleanValue(fieldNode)); iterator.remove(); } @@ -285,6 +285,7 @@ public class GeoShapeFieldMapper extends FieldMapper { termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, names().indexName()); termStrategy.setDistErrPct(distanceErrorPct()); defaultStrategy = resolveStrategy(strategyName); + defaultStrategy.setPointsOnly(pointsOnly); } @Override @@ -348,6 +349,9 @@ public class GeoShapeFieldMapper extends FieldMapper { public void setStrategyName(String strategyName) { checkIfFrozen(); this.strategyName = strategyName; + if (this.strategyName.equals(SpatialStrategy.TERM)) { + this.pointsOnly = true; + } } public boolean pointsOnly() { @@ -407,7 +411,6 @@ public class GeoShapeFieldMapper extends FieldMapper { public PrefixTreeStrategy resolveStrategy(String strategyName) { if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { - recursiveStrategy.setPointsOnly(pointsOnly()); return recursiveStrategy; } if (SpatialStrategy.TERM.getStrategyName().equals(strategyName)) { @@ -447,7 +450,7 @@ public class GeoShapeFieldMapper extends FieldMapper { } shape = shapeBuilder.build(); } - if (fieldType().defaultStrategy() instanceof RecursivePrefixTreeStrategy && fieldType().pointsOnly() && !(shape instanceof Point)) { + if (fieldType().pointsOnly() && !(shape instanceof Point)) { throw new MapperParsingException("[{" + fieldType().names().fullName() + "}] is configured for points only but a " + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) + " was found"); } @@ -472,7 +475,7 @@ public class GeoShapeFieldMapper extends FieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { super.merge(mergeWith, mergeResult); if (!this.getClass().equals(mergeWith.getClass())) { return; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java index 59b664dbd65..645c36a4855 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -50,7 +49,7 @@ import java.util.Map; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue; -import static org.elasticsearch.index.mapper.core.TypeParsers.parseField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseTextField; /** * @@ -113,9 +112,9 @@ public class AllFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); // parseField below will happily parse the doc_values setting, but it is then never passed to @@ -135,7 +134,7 @@ public class AllFieldMapper extends MetadataFieldMapper { } } - parseField(builder, builder.name, node, parserContext); + parseTextField(builder, builder.name, node, parserContext); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String fieldName = Strings.toUnderscoreCase(entry.getKey()); @@ -150,6 +149,11 @@ public class AllFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new AllFieldMapper(indexSettings, fieldType); + } } static final class AllFieldType extends MappedFieldType { @@ -193,11 +197,11 @@ public class AllFieldMapper extends MetadataFieldMapper { private EnabledAttributeMapper enabledState; - public AllFieldMapper(Settings indexSettings, MappedFieldType existing) { + private AllFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), Defaults.ENABLED, indexSettings); } - protected AllFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, Settings indexSettings) { + private AllFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.enabledState = enabled; @@ -305,7 +309,7 @@ public class AllFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { if (((AllFieldMapper)mergeWith).enabled() != this.enabled() && ((AllFieldMapper)mergeWith).enabledState != Defaults.ENABLED) { mergeResult.addConflict("mapper [" + fieldType().names().fullName() + "] enabled is " + this.enabled() + " now encountering "+ ((AllFieldMapper)mergeWith).enabled()); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java index 500f973e0ea..7883415e59a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java @@ -104,9 +104,9 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().before(Version.V_1_3_0)) { throw new IllegalArgumentException("type="+CONTENT_TYPE+" is not supported on indices created before version 1.3.0. Is your cluster running multiple datanode versions?"); } @@ -127,6 +127,11 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new FieldNamesFieldMapper(indexSettings, fieldType); + } } public static final class FieldNamesFieldType extends MappedFieldType { @@ -200,11 +205,11 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { private final boolean pre13Index; // if the index was created before 1.3, _field_names is always disabled - public FieldNamesFieldMapper(Settings indexSettings, MappedFieldType existing) { + private FieldNamesFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), indexSettings); } - public FieldNamesFieldMapper(MappedFieldType fieldType, Settings indexSettings) { + private FieldNamesFieldMapper(MappedFieldType fieldType, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.pre13Index = Version.indexCreated(indexSettings).before(Version.V_1_3_0); if (this.pre13Index) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 70948b154ed..16b6c4c56da 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -112,9 +111,9 @@ public class IdFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { throw new MapperParsingException(NAME + " is not configurable"); } @@ -131,6 +130,11 @@ public class IdFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new IdFieldMapper(indexSettings, fieldType); + } } static final class IdFieldType extends MappedFieldType { @@ -228,11 +232,11 @@ public class IdFieldMapper extends MetadataFieldMapper { private final String path; - public IdFieldMapper(Settings indexSettings, MappedFieldType existing) { + private IdFieldMapper(Settings indexSettings, MappedFieldType existing) { this(idFieldType(indexSettings, existing), Defaults.PATH, indexSettings); } - protected IdFieldMapper(MappedFieldType fieldType, String path, Settings indexSettings) { + private IdFieldMapper(MappedFieldType fieldType, String path, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.path = path; } @@ -327,7 +331,7 @@ public class IdFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { // do nothing here, no merging, but also no exception } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java index 1b7168a2d1c..962332b5c4b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -98,9 +97,9 @@ public class IndexFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { return builder; @@ -119,6 +118,11 @@ public class IndexFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new IndexFieldMapper(indexSettings, fieldType); + } } static final class IndexFieldType extends MappedFieldType { @@ -206,11 +210,11 @@ public class IndexFieldMapper extends MetadataFieldMapper { private EnabledAttributeMapper enabledState; - public IndexFieldMapper(Settings indexSettings, MappedFieldType existing) { + private IndexFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing, Defaults.ENABLED_STATE, indexSettings); } - public IndexFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabledState, Settings indexSettings) { + private IndexFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabledState, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.enabledState = enabledState; } @@ -275,7 +279,7 @@ public class IndexFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { IndexFieldMapper indexFieldMapperMergeWith = (IndexFieldMapper) mergeWith; if (!mergeResult.simulate()) { if (indexFieldMapperMergeWith.enabledState != enabledState && !indexFieldMapperMergeWith.enabledState.unset()) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java index ca792b8705b..760259a1802 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -129,9 +128,9 @@ public class ParentFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.type()); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -155,6 +154,11 @@ public class ParentFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String parentType) { + return new ParentFieldMapper(indexSettings, fieldType, parentType); + } } static final class ParentFieldType extends MappedFieldType { @@ -220,8 +224,8 @@ public class ParentFieldMapper extends MetadataFieldMapper { return super.termsQuery(values, context); } - List types = new ArrayList<>(context.mapperService().types().size()); - for (DocumentMapper documentMapper : context.mapperService().docMappers(false)) { + List types = new ArrayList<>(context.getMapperService().types().size()); + for (DocumentMapper documentMapper : context.getMapperService().docMappers(false)) { if (!documentMapper.parentFieldMapper().active()) { types.add(documentMapper.type()); } @@ -249,7 +253,7 @@ public class ParentFieldMapper extends MetadataFieldMapper { // has no impact of field data settings, is just here for creating a join field, the parent field mapper in the child type pointing to this type determines the field data settings for this join field private final MappedFieldType parentJoinFieldType; - protected ParentFieldMapper(MappedFieldType fieldType, MappedFieldType parentJoinFieldType, MappedFieldType childJoinFieldType, String parentType, Settings indexSettings) { + private ParentFieldMapper(MappedFieldType fieldType, MappedFieldType parentJoinFieldType, MappedFieldType childJoinFieldType, String parentType, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.parentType = parentType; this.parentJoinFieldType = parentJoinFieldType; @@ -260,7 +264,7 @@ public class ParentFieldMapper extends MetadataFieldMapper { } } - public ParentFieldMapper(Settings indexSettings, MappedFieldType existing, String parentType) { + private ParentFieldMapper(Settings indexSettings, MappedFieldType existing, String parentType) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), joinFieldTypeForParentType(parentType, indexSettings), null, null, indexSettings); } @@ -367,7 +371,7 @@ public class ParentFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { super.merge(mergeWith, mergeResult); ParentFieldMapper fieldMergeWith = (ParentFieldMapper) mergeWith; if (Objects.equals(parentType, fieldMergeWith.parentType) == false) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java index a95329251cd..18d0645d2d5 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java @@ -31,7 +31,6 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -98,9 +97,9 @@ public class RoutingFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { parseField(builder, builder.name, node, parserContext); @@ -119,6 +118,11 @@ public class RoutingFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new RoutingFieldMapper(indexSettings, fieldType); + } } static final class RoutingFieldType extends MappedFieldType { @@ -153,11 +157,11 @@ public class RoutingFieldMapper extends MetadataFieldMapper { private boolean required; private final String path; - public RoutingFieldMapper(Settings indexSettings, MappedFieldType existing) { + private RoutingFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), Defaults.REQUIRED, Defaults.PATH, indexSettings); } - protected RoutingFieldMapper(MappedFieldType fieldType, boolean required, String path, Settings indexSettings) { + private RoutingFieldMapper(MappedFieldType fieldType, boolean required, String path, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.required = required; this.path = path; @@ -245,7 +249,7 @@ public class RoutingFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { // do nothing here, no merging, but also no exception } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index 177764a5184..f9bcb31b406 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -29,13 +29,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; @@ -44,22 +41,17 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; -import java.io.BufferedInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue; /** * @@ -73,8 +65,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; public static final boolean ENABLED = true; - public static final long COMPRESS_THRESHOLD = -1; - public static final String FORMAT = null; // default format is to use the one provided public static final MappedFieldType FIELD_TYPE = new SourceFieldType(); @@ -94,12 +84,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { private boolean enabled = Defaults.ENABLED; - private long compressThreshold = Defaults.COMPRESS_THRESHOLD; - - private Boolean compress = null; - - private String format = Defaults.FORMAT; - private String[] includes = null; private String[] excludes = null; @@ -112,21 +96,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { return this; } - public Builder compress(boolean compress) { - this.compress = compress; - return this; - } - - public Builder compressThreshold(long compressThreshold) { - this.compressThreshold = compressThreshold; - return this; - } - - public Builder format(String format) { - this.format = format; - return this; - } - public Builder includes(String[] includes) { this.includes = includes; return this; @@ -139,13 +108,13 @@ public class SourceFieldMapper extends MetadataFieldMapper { @Override public SourceFieldMapper build(BuilderContext context) { - return new SourceFieldMapper(enabled, format, compress, compressThreshold, includes, excludes, context.indexSettings()); + return new SourceFieldMapper(enabled, includes, excludes, context.indexSettings()); } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { @@ -155,24 +124,8 @@ public class SourceFieldMapper extends MetadataFieldMapper { if (fieldName.equals("enabled")) { builder.enabled(nodeBooleanValue(fieldNode)); iterator.remove(); - } else if (fieldName.equals("compress") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - if (fieldNode != null) { - builder.compress(nodeBooleanValue(fieldNode)); - } - iterator.remove(); - } else if (fieldName.equals("compress_threshold") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - if (fieldNode != null) { - if (fieldNode instanceof Number) { - builder.compressThreshold(((Number) fieldNode).longValue()); - builder.compress(true); - } else { - builder.compressThreshold(ByteSizeValue.parseBytesSizeValue(fieldNode.toString(), "compress_threshold").bytes()); - builder.compress(true); - } - } - iterator.remove(); - } else if ("format".equals(fieldName)) { - builder.format(nodeStringValue(fieldNode, null)); + } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_3_0_0)) { + // ignore on old indices, reject on and after 3.0 iterator.remove(); } else if (fieldName.equals("includes")) { List values = (List) fieldNode; @@ -194,6 +147,11 @@ public class SourceFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new SourceFieldMapper(indexSettings); + } } static final class SourceFieldType extends MappedFieldType { @@ -238,30 +196,18 @@ public class SourceFieldMapper extends MetadataFieldMapper { /** indicates whether the source will always exist and be complete, for use by features like the update API */ private final boolean complete; - private Boolean compress; - private long compressThreshold; - private final String[] includes; private final String[] excludes; - private String format; - - private XContentType formatContentType; - - public SourceFieldMapper(Settings indexSettings) { - this(Defaults.ENABLED, Defaults.FORMAT, null, -1, null, null, indexSettings); + private SourceFieldMapper(Settings indexSettings) { + this(Defaults.ENABLED, null, null, indexSettings); } - protected SourceFieldMapper(boolean enabled, String format, Boolean compress, long compressThreshold, - String[] includes, String[] excludes, Settings indexSettings) { + private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes, Settings indexSettings) { super(NAME, Defaults.FIELD_TYPE.clone(), Defaults.FIELD_TYPE, indexSettings); // Only stored. this.enabled = enabled; - this.compress = compress; - this.compressThreshold = compressThreshold; this.includes = includes; this.excludes = excludes; - this.format = format; - this.formatContentType = format == null ? null : XContentType.fromRestContentType(format); this.complete = enabled && includes == null && excludes == null; } @@ -317,71 +263,11 @@ public class SourceFieldMapper extends MetadataFieldMapper { Tuple> mapTuple = XContentHelper.convertToMap(source, true); Map filteredSource = XContentMapValues.filter(mapTuple.v2(), includes, excludes); BytesStreamOutput bStream = new BytesStreamOutput(); - StreamOutput streamOutput = bStream; - if (compress != null && compress && (compressThreshold == -1 || source.length() > compressThreshold)) { - streamOutput = CompressorFactory.defaultCompressor().streamOutput(bStream); - } - XContentType contentType = formatContentType; - if (contentType == null) { - contentType = mapTuple.v1(); - } - XContentBuilder builder = XContentFactory.contentBuilder(contentType, streamOutput).map(filteredSource); + XContentType contentType = mapTuple.v1(); + XContentBuilder builder = XContentFactory.contentBuilder(contentType, bStream).map(filteredSource); builder.close(); source = bStream.bytes(); - } else if (compress != null && compress && !CompressorFactory.isCompressed(source)) { - if (compressThreshold == -1 || source.length() > compressThreshold) { - BytesStreamOutput bStream = new BytesStreamOutput(); - XContentType contentType = XContentFactory.xContentType(source); - if (formatContentType != null && formatContentType != contentType) { - XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, CompressorFactory.defaultCompressor().streamOutput(bStream)); - builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(source)); - builder.close(); - } else { - StreamOutput streamOutput = CompressorFactory.defaultCompressor().streamOutput(bStream); - source.writeTo(streamOutput); - streamOutput.close(); - } - source = bStream.bytes(); - // update the data in the context, so it can be compressed and stored compressed outside... - context.source(source); - } - } else if (formatContentType != null) { - // see if we need to convert the content type - Compressor compressor = CompressorFactory.compressor(source); - if (compressor != null) { - InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); - if (compressedStreamInput.markSupported() == false) { - compressedStreamInput = new BufferedInputStream(compressedStreamInput); - } - XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - if (contentType != formatContentType) { - // we need to reread and store back, compressed.... - BytesStreamOutput bStream = new BytesStreamOutput(); - StreamOutput streamOutput = CompressorFactory.defaultCompressor().streamOutput(bStream); - XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, streamOutput); - builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(compressedStreamInput)); - builder.close(); - source = bStream.bytes(); - // update the data in the context, so we store it in the translog in this format - context.source(source); - } else { - compressedStreamInput.close(); - } - } else { - XContentType contentType = XContentFactory.xContentType(source); - if (contentType != formatContentType) { - // we need to reread and store back - // we need to reread and store back, compressed.... - BytesStreamOutput bStream = new BytesStreamOutput(); - XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, bStream); - builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(source)); - builder.close(); - source = bStream.bytes(); - // update the data in the context, so we store it in the translog in this format - context.source(source); - } - } } if (!source.hasArray()) { source = source.toBytesArray(); @@ -399,26 +285,13 @@ public class SourceFieldMapper extends MetadataFieldMapper { boolean includeDefaults = params.paramAsBoolean("include_defaults", false); // all are defaults, no need to write it at all - if (!includeDefaults && enabled == Defaults.ENABLED && compress == null && compressThreshold == -1 && includes == null && excludes == null) { + if (!includeDefaults && enabled == Defaults.ENABLED && includes == null && excludes == null) { return builder; } builder.startObject(contentType()); if (includeDefaults || enabled != Defaults.ENABLED) { builder.field("enabled", enabled); } - if (includeDefaults || !Objects.equals(format, Defaults.FORMAT)) { - builder.field("format", format); - } - if (compress != null) { - builder.field("compress", compress); - } else if (includeDefaults) { - builder.field("compress", false); - } - if (compressThreshold != -1) { - builder.field("compress_threshold", new ByteSizeValue(compressThreshold).toString()); - } else if (includeDefaults) { - builder.field("compress_threshold", -1); - } if (includes != null) { builder.field("includes", includes); @@ -437,7 +310,7 @@ public class SourceFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { SourceFieldMapper sourceMergeWith = (SourceFieldMapper) mergeWith; if (mergeResult.simulate()) { if (this.enabled != sourceMergeWith.enabled) { @@ -449,13 +322,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { if (Arrays.equals(excludes(), sourceMergeWith.excludes()) == false) { mergeResult.addConflict("Cannot update excludes setting for [_source]"); } - } else { - if (sourceMergeWith.compress != null) { - this.compress = sourceMergeWith.compress; - } - if (sourceMergeWith.compressThreshold != -1) { - this.compressThreshold = sourceMergeWith.compressThreshold; - } } } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index 96ed142029c..9a18befe622 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.analysis.NumericLongAnalyzer; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -101,9 +100,9 @@ public class TTLFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -123,6 +122,11 @@ public class TTLFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new TTLFieldMapper(indexSettings); + } } public static final class TTLFieldType extends LongFieldMapper.LongFieldType { @@ -157,11 +161,11 @@ public class TTLFieldMapper extends MetadataFieldMapper { private EnabledAttributeMapper enabledState; private long defaultTTL; - public TTLFieldMapper(Settings indexSettings) { + private TTLFieldMapper(Settings indexSettings) { this(Defaults.TTL_FIELD_TYPE.clone(), Defaults.ENABLED_STATE, Defaults.DEFAULT, null, indexSettings); } - protected TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, + private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings); this.enabledState = enabled; @@ -254,7 +258,7 @@ public class TTLFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { TTLFieldMapper ttlMergeWith = (TTLFieldMapper) mergeWith; if (((TTLFieldMapper) mergeWith).enabledState != Defaults.ENABLED_STATE) {//only do something if actually something was set for the document mapper that we merge with if (this.enabledState == EnabledAttributeMapper.ENABLED && ((TTLFieldMapper) mergeWith).enabledState == EnabledAttributeMapper.DISABLED) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 253cc6dfcbf..468243d63cf 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.analysis.NumericDateAnalyzer; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.MetadataFieldMapper; @@ -165,9 +164,9 @@ public class TimestampFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { parseField(builder, builder.name, node, parserContext); @@ -218,6 +217,11 @@ public class TimestampFieldMapper extends MetadataFieldMapper { return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new TimestampFieldMapper(indexSettings, fieldType); + } } public static final class TimestampFieldType extends DateFieldMapper.DateFieldType { @@ -255,11 +259,11 @@ public class TimestampFieldMapper extends MetadataFieldMapper { private final String defaultTimestamp; private final Boolean ignoreMissing; - public TimestampFieldMapper(Settings indexSettings, MappedFieldType existing) { + private TimestampFieldMapper(Settings indexSettings, MappedFieldType existing) { this(chooseFieldType(indexSettings, existing).clone(), chooseFieldType(indexSettings, null), Defaults.ENABLED, Defaults.PATH, Defaults.DEFAULT_TIMESTAMP, null, indexSettings); } - protected TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState, String path, + private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState, String path, String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) { super(NAME, fieldType, defaultFieldType, indexSettings); this.enabledState = enabledState; @@ -375,7 +379,7 @@ public class TimestampFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { TimestampFieldMapper timestampFieldMapperMergeWith = (TimestampFieldMapper) mergeWith; super.merge(mergeWith, mergeResult); if (!mergeResult.simulate()) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java index 12e40dec47a..d4acc3c5975 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java @@ -30,6 +30,8 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -38,7 +40,6 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -86,14 +87,14 @@ public class TypeFieldMapper extends MetadataFieldMapper { @Override public TypeFieldMapper build(BuilderContext context) { - fieldType.setNames(new MappedFieldType.Names(indexName, indexName, name)); + fieldType.setNames(buildNames(context)); return new TypeFieldMapper(fieldType, context.indexSettings()); } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { throw new MapperParsingException(NAME + " is not configurable"); } @@ -101,6 +102,11 @@ public class TypeFieldMapper extends MetadataFieldMapper { parseField(builder, builder.name, node, parserContext); return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new TypeFieldMapper(indexSettings, fieldType); + } } static final class TypeFieldType extends MappedFieldType { @@ -145,13 +151,22 @@ public class TypeFieldMapper extends MetadataFieldMapper { } } - public TypeFieldMapper(Settings indexSettings, MappedFieldType existing) { - this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), + private TypeFieldMapper(Settings indexSettings, MappedFieldType existing) { + this(existing == null ? defaultFieldType(indexSettings) : existing.clone(), indexSettings); } - public TypeFieldMapper(MappedFieldType fieldType, Settings indexSettings) { - super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); + private TypeFieldMapper(MappedFieldType fieldType, Settings indexSettings) { + super(NAME, fieldType, defaultFieldType(indexSettings), indexSettings); + } + + private static MappedFieldType defaultFieldType(Settings indexSettings) { + MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone(); + Version indexCreated = Version.indexCreated(indexSettings); + if (indexCreated.onOrAfter(Version.V_2_1_0)) { + defaultFieldType.setHasDocValues(true); + } + return defaultFieldType; } @Override @@ -210,7 +225,7 @@ public class TypeFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { // do nothing here, no merging, but also no exception } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java index 92688c213b9..ef4c48e62e3 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -92,9 +91,9 @@ public class UidFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { throw new MapperParsingException(NAME + " is not configurable"); } @@ -102,6 +101,11 @@ public class UidFieldMapper extends MetadataFieldMapper { parseField(builder, builder.name, node, parserContext); return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new UidFieldMapper(indexSettings, fieldType); + } } static final class UidFieldType extends MappedFieldType { @@ -133,11 +137,11 @@ public class UidFieldMapper extends MetadataFieldMapper { } } - public UidFieldMapper(Settings indexSettings, MappedFieldType existing) { + private UidFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing, Defaults.FIELD_TYPE, indexSettings); } - protected UidFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) { + private UidFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) { super(NAME, fieldType, defaultFieldType, indexSettings); } @@ -221,7 +225,7 @@ public class UidFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { // do nothing here, no merging, but also no exception } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java index 0fbf2a3a83c..292a622ab73 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -72,9 +71,9 @@ public class VersionFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -86,6 +85,11 @@ public class VersionFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new VersionFieldMapper(indexSettings); + } } static final class VersionFieldType extends MappedFieldType { @@ -118,7 +122,7 @@ public class VersionFieldMapper extends MetadataFieldMapper { } } - public VersionFieldMapper(Settings indexSettings) { + private VersionFieldMapper(Settings indexSettings) { super(NAME, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE, indexSettings); } @@ -162,7 +166,7 @@ public class VersionFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { // nothing to do } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index 1d73139fb25..e57ceaf8ca8 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -28,9 +28,11 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.Cidrs; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -46,10 +48,14 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.LongFieldMapper; import org.elasticsearch.index.mapper.core.LongFieldMapper.CustomLongNumericField; import org.elasticsearch.index.mapper.core.NumberFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.bucket.range.ipv4.InternalIPv4Range; + import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.elasticsearch.index.mapper.MapperBuilders.ipField; @@ -61,6 +67,7 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.parseNumberField; public class IpFieldMapper extends NumberFieldMapper { public static final String CONTENT_TYPE = "ip"; + public static final long MAX_IP = 4294967296l; public static String longToIp(long longIp) { int octet3 = (int) ((longIp >> 24) % 256); @@ -205,6 +212,23 @@ public class IpFieldMapper extends NumberFieldMapper { return bytesRef.get(); } + @Override + public Query termQuery(Object value, @Nullable QueryShardContext context) { + if (value != null) { + long[] fromTo; + if (value instanceof BytesRef) { + fromTo = Cidrs.cidrMaskToMinMax(((BytesRef) value).utf8ToString()); + } else { + fromTo = Cidrs.cidrMaskToMinMax(value.toString()); + } + if (fromTo != null) { + return rangeQuery(fromTo[0] == 0 ? null : fromTo[0], + fromTo[1] == InternalIPv4Range.MAX_IP ? null : fromTo[1], true, false); + } + } + return super.termQuery(value, context); + } + @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeQuery.newLongRange(names().indexName(), numericPrecisionStep(), diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index 24be42c6094..88f89719050 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -20,8 +20,7 @@ package org.elasticsearch.index.mapper.object; import org.apache.lucene.index.Term; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; @@ -35,7 +34,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; -import org.elasticsearch.index.settings.IndexSettings; import java.io.IOException; import java.util.*; @@ -167,7 +165,7 @@ public class ObjectMapper extends Mapper implements AllFieldMapper.IncludeInAll, return (Y) objectMapper; } - protected ObjectMapper createMapper(String name, String fullPath, boolean enabled, Nested nested, Dynamic dynamic, ContentPath.Type pathType, Map mappers, @Nullable @IndexSettings Settings settings) { + protected ObjectMapper createMapper(String name, String fullPath, boolean enabled, Nested nested, Dynamic dynamic, ContentPath.Type pathType, Map mappers, @Nullable Settings settings) { return new ObjectMapper(name, fullPath, enabled, nested, dynamic, pathType, mappers); } } @@ -324,7 +322,7 @@ public class ObjectMapper extends Mapper implements AllFieldMapper.IncludeInAll, private final String nestedTypePathAsString; private final BytesRef nestedTypePathAsBytes; - private final Filter nestedTypeFilter; + private final Query nestedTypeFilter; private volatile Dynamic dynamic; @@ -348,7 +346,7 @@ public class ObjectMapper extends Mapper implements AllFieldMapper.IncludeInAll, } this.nestedTypePathAsString = "__" + fullPath; this.nestedTypePathAsBytes = new BytesRef(nestedTypePathAsString); - this.nestedTypeFilter = new QueryWrapperFilter(new TermQuery(new Term(TypeFieldMapper.NAME, nestedTypePathAsBytes))); + this.nestedTypeFilter = new TermQuery(new Term(TypeFieldMapper.NAME, nestedTypePathAsBytes)); } @Override @@ -357,7 +355,7 @@ public class ObjectMapper extends Mapper implements AllFieldMapper.IncludeInAll, try { clone = (ObjectMapper) super.clone(); } catch (CloneNotSupportedException e) { - throw new RuntimeException(); + throw new RuntimeException(e); } return clone; } @@ -432,7 +430,7 @@ public class ObjectMapper extends Mapper implements AllFieldMapper.IncludeInAll, return this.nested; } - public Filter nestedTypeFilter() { + public Query nestedTypeFilter() { return this.nestedTypeFilter; } @@ -466,7 +464,7 @@ public class ObjectMapper extends Mapper implements AllFieldMapper.IncludeInAll, } @Override - public void merge(final Mapper mergeWith, final MergeResult mergeResult) throws MergeMappingException { + public void merge(final Mapper mergeWith, final MergeResult mergeResult) { if (!(mergeWith instanceof ObjectMapper)) { mergeResult.addConflict("Can't merge a non object mapping [" + mergeWith.name() + "] with an object mapping [" + name() + "]"); return; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java index b2e572f5291..a0c989abd7d 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java @@ -26,22 +26,11 @@ import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeResult; -import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.core.DateFieldMapper; -import org.elasticsearch.index.settings.IndexSettings; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.index.mapper.core.TypeParsers.parseDateTimeFormatter; @@ -106,7 +95,7 @@ public class RootObjectMapper extends ObjectMapper { @Override - protected ObjectMapper createMapper(String name, String fullPath, boolean enabled, Nested nested, Dynamic dynamic, ContentPath.Type pathType, Map mappers, @Nullable @IndexSettings Settings settings) { + protected ObjectMapper createMapper(String name, String fullPath, boolean enabled, Nested nested, Dynamic dynamic, ContentPath.Type pathType, Map mappers, @Nullable Settings settings) { assert !nested.isNested(); FormatDateTimeFormatter[] dates = null; if (dynamicDateTimeFormatters == null) { diff --git a/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java b/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java index 055558cc563..bcacb3516da 100644 --- a/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java +++ b/core/src/main/java/org/elasticsearch/index/merge/MergeStats.java @@ -76,6 +76,17 @@ public class MergeStats implements Streamable, ToXContent { } public void add(MergeStats mergeStats) { + if (mergeStats == null) { + return; + } + this.current += mergeStats.current; + this.currentNumDocs += mergeStats.currentNumDocs; + this.currentSizeInBytes += mergeStats.currentSizeInBytes; + + addTotals(mergeStats); + } + + public void addTotals(MergeStats mergeStats) { if (mergeStats == null) { return; } @@ -83,9 +94,6 @@ public class MergeStats implements Streamable, ToXContent { this.totalTimeInMillis += mergeStats.totalTimeInMillis; this.totalNumDocs += mergeStats.totalNumDocs; this.totalSizeInBytes += mergeStats.totalSizeInBytes; - this.current += mergeStats.current; - this.currentNumDocs += mergeStats.currentNumDocs; - this.currentSizeInBytes += mergeStats.currentSizeInBytes; this.totalStoppedTimeInMillis += mergeStats.totalStoppedTimeInMillis; this.totalThrottledTimeInMillis += mergeStats.totalThrottledTimeInMillis; if (this.totalBytesPerSecAutoThrottle == Long.MAX_VALUE || mergeStats.totalBytesPerSecAutoThrottle == Long.MAX_VALUE) { diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java index 1f8a4c61f9a..eaf562e2127 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java @@ -30,12 +30,12 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.indexing.IndexingOperationListener; @@ -44,9 +44,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentTypeListener; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.percolator.PercolatorService; @@ -70,7 +68,6 @@ public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent public final String MAP_UNMAPPED_FIELDS_AS_STRING = "index.percolator.map_unmapped_fields_as_string"; // This is a shard level service, but these below are index level service: - private final IndexQueryParserService queryParserService; private final MapperService mapperService; private final IndexFieldDataService indexFieldDataService; @@ -80,20 +77,22 @@ public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent private final RealTimePercolatorOperationListener realTimePercolatorOperationListener = new RealTimePercolatorOperationListener(); private final PercolateTypeListener percolateTypeListener = new PercolateTypeListener(); private final AtomicBoolean realTimePercolatorEnabled = new AtomicBoolean(false); + private final QueryShardContext queryShardContext; private boolean mapUnmappedFieldsAsString; private final MeanMetric percolateMetric = new MeanMetric(); private final CounterMetric currentMetric = new CounterMetric(); private final CounterMetric numberOfQueries = new CounterMetric(); - public PercolatorQueriesRegistry(ShardId shardId, @IndexSettings Settings indexSettings, IndexQueryParserService queryParserService, + public PercolatorQueriesRegistry(ShardId shardId, IndexSettings indexSettings, ShardIndexingService indexingService, MapperService mapperService, + QueryShardContext queryShardContext, IndexFieldDataService indexFieldDataService) { super(shardId, indexSettings); - this.queryParserService = queryParserService; this.mapperService = mapperService; this.indexingService = indexingService; + this.queryShardContext = queryShardContext; this.indexFieldDataService = indexFieldDataService; - this.mapUnmappedFieldsAsString = indexSettings.getAsBoolean(MAP_UNMAPPED_FIELDS_AS_STRING, false); + this.mapUnmappedFieldsAsString = this.indexSettings.getSettings().getAsBoolean(MAP_UNMAPPED_FIELDS_AS_STRING, false); mapperService.addTypeListener(percolateTypeListener); } @@ -180,7 +179,7 @@ public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent if (type != null) { previousTypes = QueryShardContext.setTypesWithPrevious(type); } - QueryShardContext context = queryParserService.getShardContext(); + QueryShardContext context = new QueryShardContext(queryShardContext); try { context.reset(parser); // This means that fields in the query need to exist in the mapping prior to registering this query @@ -197,7 +196,7 @@ public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent // as an analyzed string. context.setAllowUnmappedFields(false); context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString); - return queryParserService.parseInnerQuery(context); + return context.parseInnerQuery(); } catch (IOException e) { throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e); } finally { diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java index 560476a69d8..79ba3804ecd 100644 --- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java @@ -19,7 +19,10 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.spans.SpanBoostQuery; +import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; @@ -64,9 +67,9 @@ public abstract class AbstractQueryBuilder exte protected abstract void doXContent(XContentBuilder builder, Params params) throws IOException; protected void printBoostAndQueryName(XContentBuilder builder) throws IOException { - builder.field("boost", boost); + builder.field(BOOST_FIELD.getPreferredName(), boost); if (queryName != null) { - builder.field("_name", queryName); + builder.field(NAME_FIELD.getPreferredName(), queryName); } } @@ -74,7 +77,13 @@ public abstract class AbstractQueryBuilder exte public final Query toQuery(QueryShardContext context) throws IOException { Query query = doToQuery(context); if (query != null) { - setFinalBoost(query); + if (boost != DEFAULT_BOOST) { + if (query instanceof SpanQuery) { + query = new SpanBoostQuery((SpanQuery) query, boost); + } else { + query = new BoostQuery(query, boost); + } + } if (queryName != null) { context.addNamedQuery(queryName, query); } @@ -82,20 +91,6 @@ public abstract class AbstractQueryBuilder exte return query; } - /** - * Sets the main boost to the query obtained by converting the current query into a lucene query. - * The default behaviour is to set the main boost, after verifying that we are not overriding any non default boost - * value that was previously set to the lucene query. That case would require some manual decision on how to combine - * the main boost with the boost coming from lucene by overriding this method. - * @throws IllegalStateException if the lucene query boost has already been set - */ - protected void setFinalBoost(Query query) { - if (query.getBoost() != AbstractQueryBuilder.DEFAULT_BOOST) { - throw new IllegalStateException("lucene query boost is already set, override setFinalBoost to define how to combine lucene boost with main boost"); - } - query.setBoost(boost); - } - @Override public final Query toFilter(QueryShardContext context) throws IOException { Query result = null; @@ -112,7 +107,7 @@ public abstract class AbstractQueryBuilder exte protected abstract Query doToQuery(QueryShardContext context) throws IOException; /** - * Returns the query name for the query. + * Sets the query name for the query. */ @SuppressWarnings("unchecked") @Override @@ -122,7 +117,7 @@ public abstract class AbstractQueryBuilder exte } /** - * Sets the query name for the query. + * Returns the query name for the query. */ @Override public final String queryName() { diff --git a/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java index 06666b74120..fd5f93f6f7f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -30,6 +31,8 @@ import java.util.Objects; public abstract class BaseTermQueryBuilder> extends AbstractQueryBuilder { + public static final ParseField VALUE_FIELD = new ParseField("value"); + /** Name of field to match against. */ protected final String fieldName; @@ -133,7 +136,7 @@ public abstract class BaseTermQueryBuilder> protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(getName()); builder.startObject(fieldName); - builder.field("value", convertToStringIfBytesRef(this.value)); + builder.field(VALUE_FIELD.getPreferredName(), convertToStringIfBytesRef(this.value)); printBoostAndQueryName(builder); builder.endObject(); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 25821e1dd1d..b8170a3195a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -231,14 +231,14 @@ public class BoolQueryBuilder extends AbstractQueryBuilder { @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - doXArrayContent("must", mustClauses, builder, params); - doXArrayContent("filter", filterClauses, builder, params); - doXArrayContent("must_not", mustNotClauses, builder, params); - doXArrayContent("should", shouldClauses, builder, params); - builder.field("disable_coord", disableCoord); - builder.field("adjust_pure_negative", adjustPureNegative); + doXArrayContent(BoolQueryParser.MUST, mustClauses, builder, params); + doXArrayContent(BoolQueryParser.FILTER, filterClauses, builder, params); + doXArrayContent(BoolQueryParser.MUST_NOT, mustNotClauses, builder, params); + doXArrayContent(BoolQueryParser.SHOULD, shouldClauses, builder, params); + builder.field(BoolQueryParser.DISABLE_COORD_FIELD.getPreferredName(), disableCoord); + builder.field(BoolQueryParser.ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative); if (minimumShouldMatch != null) { - builder.field("minimum_should_match", minimumShouldMatch); + builder.field(BoolQueryParser.MINIMUM_SHOULD_MATCH.getPreferredName(), minimumShouldMatch); } printBoostAndQueryName(builder); builder.endObject(); @@ -272,26 +272,29 @@ public class BoolQueryBuilder extends AbstractQueryBuilder { if (booleanQuery.clauses().isEmpty()) { return new MatchAllDocsQuery(); } - booleanQuery = Queries.applyMinimumShouldMatch(booleanQuery, minimumShouldMatch); - return adjustPureNegative ? fixNegativeQueryIfNeeded(booleanQuery) : booleanQuery; + final String minimumShouldMatch; + if (context.isFilter() && this.minimumShouldMatch == null) { + //will be applied for real only if there are should clauses + minimumShouldMatch = "1"; + } else { + minimumShouldMatch = this.minimumShouldMatch; + } + Query query = Queries.applyMinimumShouldMatch(booleanQuery, minimumShouldMatch); + return adjustPureNegative ? fixNegativeQueryIfNeeded(query) : query; } - private void addBooleanClauses(QueryShardContext context, BooleanQuery.Builder booleanQueryBuilder, List clauses, Occur occurs) throws IOException { + private static void addBooleanClauses(QueryShardContext context, BooleanQuery.Builder booleanQueryBuilder, List clauses, Occur occurs) throws IOException { for (QueryBuilder query : clauses) { Query luceneQuery = null; switch (occurs) { - case SHOULD: - if (context.isFilter() && minimumShouldMatch == null) { - minimumShouldMatch = "1"; - } - luceneQuery = query.toQuery(context); - break; - case FILTER: - case MUST_NOT: - luceneQuery = query.toFilter(context); - break; - case MUST: - luceneQuery = query.toQuery(context); + case MUST: + case SHOULD: + luceneQuery = query.toQuery(context); + break; + case FILTER: + case MUST_NOT: + luceneQuery = query.toFilter(context); + break; } if (luceneQuery != null) { booleanQueryBuilder.add(new BooleanClause(luceneQuery, occurs)); diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java index 13b5f509084..d0d130f9774 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.BooleanQuery; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -34,6 +35,16 @@ import java.util.List; */ public class BoolQueryParser implements QueryParser { + public static final String MUSTNOT = "mustNot"; + public static final String MUST_NOT = "must_not"; + public static final String FILTER = "filter"; + public static final String SHOULD = "should"; + public static final String MUST = "must"; + public static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord"); + public static final ParseField MINIMUM_SHOULD_MATCH = new ParseField("minimum_should_match"); + public static final ParseField MINIMUM_NUMBER_SHOULD_MATCH = new ParseField("minimum_number_should_match"); + public static final ParseField ADJUST_PURE_NEGATIVE = new ParseField("adjust_pure_negative"); + @Inject public BoolQueryParser(Settings settings) { BooleanQuery.setMaxClauseCount(settings.getAsInt("index.query.bool.max_clause_count", settings.getAsInt("indices.query.bool.max_clause_count", BooleanQuery.getMaxClauseCount()))); @@ -69,20 +80,20 @@ public class BoolQueryParser implements QueryParser { // skip } else if (token == XContentParser.Token.START_OBJECT) { switch (currentFieldName) { - case "must": + case MUST: query = parseContext.parseInnerQueryBuilder(); mustClauses.add(query); break; - case "should": + case SHOULD: query = parseContext.parseInnerQueryBuilder(); shouldClauses.add(query); break; - case "filter": + case FILTER: query = parseContext.parseInnerQueryBuilder(); filterClauses.add(query); break; - case "must_not": - case "mustNot": + case MUST_NOT: + case MUSTNOT: query = parseContext.parseInnerQueryBuilder(); mustNotClauses.add(query); break; @@ -92,20 +103,20 @@ public class BoolQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_ARRAY) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { switch (currentFieldName) { - case "must": + case MUST: query = parseContext.parseInnerQueryBuilder(); mustClauses.add(query); break; - case "should": + case SHOULD: query = parseContext.parseInnerQueryBuilder(); shouldClauses.add(query); break; - case "filter": + case FILTER: query = parseContext.parseInnerQueryBuilder(); filterClauses.add(query); break; - case "must_not": - case "mustNot": + case MUST_NOT: + case MUSTNOT: query = parseContext.parseInnerQueryBuilder(); mustNotClauses.add(query); break; @@ -114,17 +125,17 @@ public class BoolQueryParser implements QueryParser { } } } else if (token.isValue()) { - if ("disable_coord".equals(currentFieldName) || "disableCoord".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, DISABLE_COORD_FIELD)) { disableCoord = parser.booleanValue(); - } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH)) { minimumShouldMatch = parser.textOrNull(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("minimum_number_should_match".equals(currentFieldName) || "minimumNumberShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_NUMBER_SHOULD_MATCH)) { minimumShouldMatch = parser.textOrNull(); - } else if ("adjust_pure_negative".equals(currentFieldName) || "adjustPureNegative".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ADJUST_PURE_NEGATIVE)) { adjustPureNegative = parser.booleanValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[bool] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java index c1994a6033e..c7349cca3e6 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java @@ -104,11 +104,11 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder { + public static final ParseField POSITIVE_FIELD = new ParseField("positive"); + public static final ParseField NEGATIVE_FIELD = new ParseField("negative"); + public static final ParseField NEGATIVE_BOOST_FIELD = new ParseField("negative_boost"); + @Override public String[] names() { return new String[]{BoostingQueryBuilder.NAME}; @@ -52,21 +57,21 @@ public class BoostingQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("positive".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, POSITIVE_FIELD)) { positiveQuery = parseContext.parseInnerQueryBuilder(); positiveQueryFound = true; - } else if ("negative".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, NEGATIVE_FIELD)) { negativeQuery = parseContext.parseInnerQueryBuilder(); negativeQueryFound = true; } else { throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("negative_boost".equals(currentFieldName) || "negativeBoost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, NEGATIVE_BOOST_FIELD)) { negativeBoost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java index c6b82535408..20d0b62b725 100644 --- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java @@ -202,21 +202,21 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder { + public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField LOW_FREQ_OPERATOR_FIELD = new ParseField("low_freq_operator"); + public static final ParseField HIGH_FREQ_OPERATOR_FIELD = new ParseField("high_freq_operator"); + public static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField HIGH_FREQ_FIELD = new ParseField("high_freq"); + public static final ParseField LOW_FREQ_FIELD = new ParseField("low_freq"); + @Override public String[] names() { return new String[] { CommonTermsQueryBuilder.NAME }; @@ -39,7 +50,7 @@ public class CommonTermsQueryParser implements QueryParser { - private static final ParseField INNER_QUERY_FIELD = new ParseField("filter", "query"); + public static final ParseField INNER_QUERY_FIELD = new ParseField("filter", "query"); @Override public String[] names() { @@ -62,9 +62,9 @@ public class ConstantScoreQueryParser implements QueryParser @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("tie_breaker", tieBreaker); - builder.startArray("queries"); + builder.field(DisMaxQueryParser.TIE_BREAKER_FIELD.getPreferredName(), tieBreaker); + builder.startArray(DisMaxQueryParser.QUERIES_FIELD.getPreferredName()); for (QueryBuilder queryBuilder : queries) { queryBuilder.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java index a280cdfd837..660790e21ae 100644 --- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; @@ -32,6 +33,9 @@ import java.util.List; */ public class DisMaxQueryParser implements QueryParser { + public static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker"); + public static final ParseField QUERIES_FIELD = new ParseField("queries"); + @Override public String[] names() { return new String[]{DisMaxQueryBuilder.NAME, Strings.toCamelCase(DisMaxQueryBuilder.NAME)}; @@ -54,7 +58,7 @@ public class DisMaxQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("queries".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) { queriesFound = true; QueryBuilder query = parseContext.parseInnerQueryBuilder(); queries.add(query); @@ -62,7 +66,7 @@ public class DisMaxQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { - if ("queries".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) { queriesFound = true; while (token != XContentParser.Token.END_ARRAY) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); @@ -73,11 +77,11 @@ public class DisMaxQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]"); } } else { - if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("tie_breaker".equals(currentFieldName) || "tieBreaker".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) { tieBreaker = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java index 89a738ebac0..02af9b7675a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java @@ -61,7 +61,7 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("field", fieldName); + builder.field(ExistsQueryParser.FIELD_FIELD.getPreferredName(), fieldName); printBoostAndQueryName(builder); builder.endObject(); } @@ -72,7 +72,7 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder } public static Query newFilter(QueryShardContext context, String fieldPattern) { - final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType)context.mapperService().fullName(FieldNamesFieldMapper.NAME); + final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType)context.getMapperService().fullName(FieldNamesFieldMapper.NAME); if (fieldNamesFieldType == null) { // can only happen when no types exist, so no docs exist either return Queries.newMatchNoDocsQuery(); diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java index f0df84f9aa8..86a5311ff40 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; @@ -29,6 +30,8 @@ import java.io.IOException; */ public class ExistsQueryParser implements QueryParser { + public static final ParseField FIELD_FIELD = new ParseField("field"); + @Override public String[] names() { return new String[]{ExistsQueryBuilder.NAME}; @@ -48,20 +51,22 @@ public class ExistsQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("field".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, FIELD_FIELD)) { fieldPattern = parser.text(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { - throw new ParsingException(parser.getTokenLocation(), "[exists] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + ExistsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + ExistsQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } if (fieldPattern == null) { - throw new ParsingException(parser.getTokenLocation(), "exists must be provided with a [field]"); + throw new ParsingException(parser.getTokenLocation(), "[" + ExistsQueryBuilder.NAME + "] must be provided with a [field]"); } ExistsQueryBuilder builder = new ExistsQueryBuilder(fieldPattern); diff --git a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java index 671cfda3e7a..e9258d7cfc1 100644 --- a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java @@ -75,9 +75,9 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder { + public static final ParseField FIELD_FIELD = new ParseField("field"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + @Override public String[] names() { return new String[]{FieldMaskingSpanQueryBuilder.NAME, Strings.toCamelCase(FieldMaskingSpanQueryBuilder.NAME)}; @@ -50,7 +54,7 @@ public class FieldMaskingSpanQueryParser implements QueryParser i } @Override - public void doXContent(XContentBuilder builder, Params params) throws IOException { + protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("value", convertToStringIfBytesRef(this.value)); + builder.field(FuzzyQueryParser.VALUE_FIELD.getPreferredName(), convertToStringIfBytesRef(this.value)); fuzziness.toXContent(builder, params); - builder.field("prefix_length", prefixLength); - builder.field("max_expansions", maxExpansions); - builder.field("transpositions", transpositions); + builder.field(FuzzyQueryParser.PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); + builder.field(FuzzyQueryParser.MAX_EXPANSIONS_FIELD.getPreferredName(), maxExpansions); + builder.field(FuzzyQueryParser.TRANSPOSITIONS_FIELD.getPreferredName(), transpositions); if (rewrite != null) { - builder.field("rewrite", rewrite); + builder.field(FuzzyQueryParser.REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); builder.endObject(); @@ -231,7 +231,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public Query doToQuery(QueryShardContext context) throws IOException { + protected Query doToQuery(QueryShardContext context) throws IOException { Query query = null; String rewrite = this.rewrite; if (rewrite == null && context.isFilter()) { @@ -253,7 +253,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public FuzzyQueryBuilder doReadFrom(StreamInput in) throws IOException { + protected FuzzyQueryBuilder doReadFrom(StreamInput in) throws IOException { FuzzyQueryBuilder fuzzyQueryBuilder = new FuzzyQueryBuilder(in.readString(), in.readGenericValue()); fuzzyQueryBuilder.fuzziness = Fuzziness.readFuzzinessFrom(in); fuzzyQueryBuilder.prefixLength = in.readVInt(); @@ -264,7 +264,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public void doWriteTo(StreamOutput out) throws IOException { + protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(this.fieldName); out.writeGenericValue(this.value); this.fuzziness.writeTo(out); @@ -275,12 +275,12 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public int doHashCode() { + protected int doHashCode() { return Objects.hash(fieldName, value, fuzziness, prefixLength, maxExpansions, transpositions, rewrite); } @Override - public boolean doEquals(FuzzyQueryBuilder other) { + protected boolean doEquals(FuzzyQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(value, other.value) && Objects.equals(fuzziness, other.fuzziness) && diff --git a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java index 340094af7ce..85365f84fb9 100644 --- a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java @@ -23,11 +23,17 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; + import java.io.IOException; public class FuzzyQueryParser implements QueryParser { - private static final ParseField FUZZINESS = Fuzziness.FIELD.withDeprecation("min_similarity"); + public static final ParseField TERM_FIELD = new ParseField("term"); + public static final ParseField VALUE_FIELD = new ParseField("value"); + public static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length"); + public static final ParseField MAX_EXPANSIONS_FIELD = new ParseField("max_expansions"); + public static final ParseField TRANSPOSITIONS_FIELD = new ParseField("transpositions"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); @Override public String[] names() { @@ -62,23 +68,23 @@ public class FuzzyQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("term".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TERM_FIELD)) { value = parser.objectBytes(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.objectBytes(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZINESS)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) { fuzziness = Fuzziness.parse(parser); - } else if ("prefix_length".equals(currentFieldName) || "prefixLength".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) { prefixLength = parser.intValue(); - } else if ("max_expansions".equals(currentFieldName) || "maxExpansions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) { maxExpansions = parser.intValue(); - } else if ("transpositions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TRANSPOSITIONS_FIELD)) { transpositions = parser.booleanValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[fuzzy] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java index 59d20cef611..dec14f59bf8 100644 --- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.GeoPointInBBoxQuery; import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.common.Numbers; @@ -29,7 +30,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapperLegacy; import org.elasticsearch.index.search.geo.InMemoryGeoBoundingBoxQuery; import org.elasticsearch.index.search.geo.IndexedGeoBoundingBoxQuery; @@ -232,6 +234,14 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder { + public static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method"); + public static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed"); + public static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize"); + public static final ParseField OPTIMIZE_BBOX_FIELD = new ParseField("optimize_bbox"); + public static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type"); + public static final ParseField UNIT_FIELD = new ParseField("unit"); + public static final ParseField DISTANCE_FIELD = new ParseField("distance"); + @Override public String[] names() { return new String[]{GeoDistanceQueryBuilder.NAME, "geoDistance"}; @@ -95,15 +104,15 @@ public class GeoDistanceQueryParser implements QueryParser { public static final String NAME = "geo_distance_range"; @@ -177,7 +181,7 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder field : { lat : 30, lon : 12 } - fieldName = currentFieldName; - if (point == null) { - point = new GeoPoint(); + if (fieldName == null) { + fieldName = currentFieldName; + if (point == null) { + point = new GeoPoint(); + } + GeoUtils.parseGeoPoint(parser, point); + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + GeoDistanceRangeQueryBuilder.NAME + + "] field name already set to [" + fieldName + "] but found [" + currentFieldName + "]"); } - GeoUtils.parseGeoPoint(parser, point); } else if (token.isValue()) { if (parseContext.parseFieldMatcher().match(currentFieldName, FROM_FIELD)) { if (token == XContentParser.Token.VALUE_NULL) { @@ -162,20 +173,38 @@ public class GeoDistanceRangeQueryParser implements QueryParser shell = new ArrayList(); for (GeoPoint geoPoint : this.shell) { shell.add(new GeoPoint(geoPoint)); } + final int shellSize = shell.size(); final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0); // validation was not available prior to 2.x, so to support bwc @@ -127,16 +136,20 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { + if (fieldName != null) { + throw new ParsingException(parser.getTokenLocation(), "[" + GeoShapeQueryBuilder.NAME + "] point specified twice. [" + currentFieldName + "]"); + } fieldName = currentFieldName; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); token = parser.nextToken(); if (parseContext.parseFieldMatcher().match(currentFieldName, SHAPE_FIELD)) { - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + XContentBuilder builder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); shape = builder.bytes(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, STRATEGY_FIELD)) { String strategyName = parser.text(); @@ -104,10 +106,12 @@ public class GeoShapeQueryParser implements QueryParser { } else if (parseContext.parseFieldMatcher().match(currentFieldName, SHAPE_PATH_FIELD)) { shapePath = parser.text(); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + GeoShapeQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } } else { - throw new ParsingException(parser.getTokenLocation(), "[geo_shape] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + GeoShapeQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } } } @@ -117,7 +121,7 @@ public class GeoShapeQueryParser implements QueryParser { } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { - throw new ParsingException(parser.getTokenLocation(), "[geo_shape] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + GeoShapeQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } } } diff --git a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java index a385e978487..1649d12f186 100644 --- a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java +++ b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java @@ -20,10 +20,11 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.Query; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; @@ -35,7 +36,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper; import java.io.IOException; import java.util.ArrayList; @@ -74,8 +75,8 @@ public class GeohashCellQuery { * @param geohashes optional array of additional geohashes * @return a new GeoBoundinboxfilter */ - public static Query create(QueryShardContext context, GeoPointFieldMapper.GeoPointFieldType fieldType, String geohash, @Nullable List geohashes) { - MappedFieldType geoHashMapper = fieldType.geohashFieldType(); + public static Query create(QueryShardContext context, BaseGeoPointFieldMapper.GeoPointFieldType fieldType, String geohash, @Nullable List geohashes) { + MappedFieldType geoHashMapper = fieldType.geoHashFieldType(); if (geoHashMapper == null) { throw new IllegalArgumentException("geohash filter needs geohash_prefix to be enabled"); } @@ -131,7 +132,7 @@ public class GeohashCellQuery { } public Builder point(double lat, double lon) { - this.geohash = XGeoHashUtils.stringEncode(lon, lat); + this.geohash = GeoHashUtils.stringEncode(lon, lat); return this; } @@ -184,15 +185,15 @@ public class GeohashCellQuery { MappedFieldType fieldType = context.fieldMapper(fieldName); if (fieldType == null) { throw new QueryShardException(context, "failed to parse [{}] query. missing [{}] field [{}]", NAME, - GeoPointFieldMapper.CONTENT_TYPE, fieldName); + BaseGeoPointFieldMapper.CONTENT_TYPE, fieldName); } - if (!(fieldType instanceof GeoPointFieldMapper.GeoPointFieldType)) { + if (!(fieldType instanceof BaseGeoPointFieldMapper.GeoPointFieldType)) { throw new QueryShardException(context, "failed to parse [{}] query. field [{}] is not a geo_point field", NAME, fieldName); } - GeoPointFieldMapper.GeoPointFieldType geoFieldType = ((GeoPointFieldMapper.GeoPointFieldType) fieldType); - if (!geoFieldType.isGeohashPrefixEnabled()) { + BaseGeoPointFieldMapper.GeoPointFieldType geoFieldType = ((BaseGeoPointFieldMapper.GeoPointFieldType) fieldType); + if (!geoFieldType.isGeoHashPrefixEnabled()) { throw new QueryShardException(context, "failed to parse [{}] query. [geohash_prefix] is not enabled for field [{}]", NAME, fieldName); } @@ -205,7 +206,7 @@ public class GeohashCellQuery { Query query; if (neighbors) { - query = create(context, geoFieldType, geohash, XGeoHashUtils.addNeighbors(geohash, new ArrayList(8))); + query = create(context, geoFieldType, geohash, GeoHashUtils.addNeighbors(geohash, new ArrayList(8))); } else { query = create(context, geoFieldType, geohash, null); } @@ -318,19 +319,25 @@ public class GeohashCellQuery { parser.nextToken(); boost = parser.floatValue(); } else { - fieldName = field; - token = parser.nextToken(); - if (token == Token.VALUE_STRING) { - // A string indicates either a geohash or a lat/lon - // string - String location = parser.text(); - if (location.indexOf(",") > 0) { - geohash = GeoUtils.parseGeoPoint(parser).geohash(); + if (fieldName == null) { + fieldName = field; + token = parser.nextToken(); + if (token == Token.VALUE_STRING) { + // A string indicates either a geohash or a + // lat/lon + // string + String location = parser.text(); + if (location.indexOf(",") > 0) { + geohash = GeoUtils.parseGeoPoint(parser).geohash(); + } else { + geohash = location; + } } else { - geohash = location; + geohash = GeoUtils.parseGeoPoint(parser).geohash(); } } else { - geohash = GeoUtils.parseGeoPoint(parser).geohash(); + throw new ParsingException(parser.getTokenLocation(), "[" + NAME + + "] field name already set to [" + fieldName + "] but found [" + field + "]"); } } } else { diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java index a7b7506ebae..3589215d5f8 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java @@ -185,12 +185,12 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder { - private static final ParseField QUERY_FIELD = new ParseField("query", "filter"); + public static final ParseField QUERY_FIELD = new ParseField("query", "filter"); + public static final ParseField TYPE_FIELD = new ParseField("type", "child_type"); + public static final ParseField MAX_CHILDREN_FIELD = new ParseField("max_children"); + public static final ParseField MIN_CHILDREN_FIELD = new ParseField("min_children"); + public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode"); + public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); @Override public String[] names() { @@ -61,23 +66,23 @@ public class HasChildQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_OBJECT) { if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { iqb = parseContext.parseInnerQueryBuilder(); - } else if ("inner_hits".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) { queryInnerHits = new QueryInnerHits(parser); } else { throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("type".equals(currentFieldName) || "child_type".equals(currentFieldName) || "childType".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { childType = parser.text(); - } else if ("score_mode".equals(currentFieldName) || "scoreMode".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) { scoreMode = parseScoreMode(parser.text()); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("min_children".equals(currentFieldName) || "minChildren".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MIN_CHILDREN_FIELD)) { minChildren = parser.intValue(true); - } else if ("max_children".equals(currentFieldName) || "maxChildren".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_CHILDREN_FIELD)) { maxChildren = parser.intValue(true); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java index c2a2c33d518..0ad734ec552 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java @@ -129,8 +129,7 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder parentTypes = new HashSet<>(5); parentTypes.add(parentDocMapper.type()); ParentChildIndexFieldData parentChildIndexFieldData = null; - for (DocumentMapper documentMapper : context.mapperService().docMappers(false)) { + for (DocumentMapper documentMapper : context.getMapperService().docMappers(false)) { ParentFieldMapper parentFieldMapper = documentMapper.parentFieldMapper(); if (parentFieldMapper.active()) { - DocumentMapper parentTypeDocumentMapper = context.mapperService().documentMapper(parentFieldMapper.type()); + DocumentMapper parentTypeDocumentMapper = context.getMapperService().documentMapper(parentFieldMapper.type()); parentChildIndexFieldData = context.getForField(parentFieldMapper.fieldType()); if (parentTypeDocumentMapper == null) { // Only add this, if this parentFieldMapper (also a parent) isn't a child of another parent. @@ -172,14 +171,14 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder { private static final HasParentQueryBuilder PROTOTYPE = new HasParentQueryBuilder("", EmptyQueryBuilder.PROTOTYPE); - private static final ParseField QUERY_FIELD = new ParseField("query", "filter"); - private static final ParseField SCORE_FIELD = new ParseField("score_mode").withAllDeprecated("score"); - private static final ParseField TYPE_FIELD = new ParseField("parent_type", "type"); + public static final ParseField QUERY_FIELD = new ParseField("query", "filter"); + //public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode").withAllDeprecated("score"); + public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode").withAllDeprecated("score"); + public static final ParseField TYPE_FIELD = new ParseField("parent_type", "type"); + public static final ParseField SCORE_FIELD = new ParseField("score"); @Override public String[] names() { @@ -42,7 +44,6 @@ public class HasParentQueryParser implements QueryParser @Override public HasParentQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException { XContentParser parser = parseContext.parser(); - float boost = AbstractQueryBuilder.DEFAULT_BOOST; String parentType = null; boolean score = HasParentQueryBuilder.DEFAULT_SCORE; @@ -66,7 +67,7 @@ public class HasParentQueryParser implements QueryParser } else if (token.isValue()) { if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { parentType = parser.text(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_FIELD)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) { String scoreModeValue = parser.text(); if ("score".equals(scoreModeValue)) { score = true; @@ -75,11 +76,11 @@ public class HasParentQueryParser implements QueryParser } else { throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + scoreModeValue + "] as an option for score_mode"); } - } else if ("score".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_FIELD)) { score = parser.booleanValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java index 1de8db2e801..4f9574f2981 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java @@ -90,8 +90,8 @@ public class IdsQueryBuilder extends AbstractQueryBuilder { @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.array("types", types); - builder.startArray("values"); + builder.array(IdsQueryParser.TYPE_FIELD.getPreferredName(), types); + builder.startArray(IdsQueryParser.VALUES_FIELD.getPreferredName()); for (String value : ids) { builder.value(value); } @@ -115,7 +115,7 @@ public class IdsQueryBuilder extends AbstractQueryBuilder { if (types.length == 0) { typesForQuery = context.queryTypes(); } else if (types.length == 1 && MetaData.ALL.equals(types[0])) { - typesForQuery = context.mapperService().types(); + typesForQuery = context.getMapperService().types(); } else { typesForQuery = new HashSet<>(); Collections.addAll(typesForQuery, types); diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java index 0ffd31644e5..46058d98eb5 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; @@ -32,6 +33,10 @@ import java.util.List; */ public class IdsQueryParser implements QueryParser { + public static final ParseField TYPE_FIELD = new ParseField("type", "types", "_type"); + + public static final ParseField VALUES_FIELD = new ParseField("values"); + @Override public String[] names() { return new String[]{IdsQueryBuilder.NAME}; @@ -55,7 +60,7 @@ public class IdsQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("values".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, VALUES_FIELD)) { idsProvided = true; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if ((token == XContentParser.Token.VALUE_STRING) || @@ -70,7 +75,7 @@ public class IdsQueryParser implements QueryParser { + token); } } - } else if ("types".equals(currentFieldName) || "type".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { String value = parser.textOrNull(); if (value == null) { @@ -79,22 +84,24 @@ public class IdsQueryParser implements QueryParser { types.add(value); } } else { - throw new ParsingException(parser.getTokenLocation(), "[ids] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("type".equals(currentFieldName) || "_type".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { types = Collections.singletonList(parser.text()); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { - throw new ParsingException(parser.getTokenLocation(), "[ids] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } if (!idsProvided) { - throw new ParsingException(parser.getTokenLocation(), "[ids] query, no ids values provided"); + throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME + "] query, no ids values provided"); } IdsQueryBuilder query = new IdsQueryBuilder(types.toArray(new String[types.size()])); diff --git a/core/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java b/core/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java deleted file mode 100644 index bbd9f84e81e..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.apache.lucene.search.Query; -import org.apache.lucene.util.CloseableThreadLocal; -import org.elasticsearch.Version; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.cache.IndexCache; -import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.internal.AllFieldMapper; -import org.elasticsearch.index.query.support.InnerHitsQueryParserHelper; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; -import org.elasticsearch.script.ScriptService; - -import java.io.IOException; - -public class IndexQueryParserService extends AbstractIndexComponent { - - public static final String DEFAULT_FIELD = "index.query.default_field"; - public static final String QUERY_STRING_LENIENT = "index.query_string.lenient"; - public static final String QUERY_STRING_ANALYZE_WILDCARD = "indices.query.query_string.analyze_wildcard"; - public static final String QUERY_STRING_ALLOW_LEADING_WILDCARD = "indices.query.query_string.allowLeadingWildcard"; - public static final String PARSE_STRICT = "index.query.parse.strict"; - public static final String ALLOW_UNMAPPED = "index.query.parse.allow_unmapped_fields"; - private final InnerHitsQueryParserHelper innerHitsQueryParserHelper; - - private CloseableThreadLocal cache = new CloseableThreadLocal() { - @Override - protected QueryShardContext initialValue() { - return new QueryShardContext(index, IndexQueryParserService.this); - } - }; - - final AnalysisService analysisService; - - final ScriptService scriptService; - - final MapperService mapperService; - - final SimilarityService similarityService; - - final IndexCache indexCache; - - protected IndexFieldDataService fieldDataService; - - final ClusterService clusterService; - - final IndexNameExpressionResolver indexNameExpressionResolver; - - final BitsetFilterCache bitsetFilterCache; - - private final IndicesQueriesRegistry indicesQueriesRegistry; - - private final String defaultField; - private final boolean queryStringLenient; - private final boolean queryStringAnalyzeWildcard; - private final boolean queryStringAllowLeadingWildcard; - private final ParseFieldMatcher parseFieldMatcher; - private final boolean defaultAllowUnmappedFields; - private final Client client; - - @Inject - public IndexQueryParserService(Index index, @IndexSettings Settings indexSettings, Settings settings, - IndicesQueriesRegistry indicesQueriesRegistry, - ScriptService scriptService, AnalysisService analysisService, - MapperService mapperService, IndexCache indexCache, IndexFieldDataService fieldDataService, - BitsetFilterCache bitsetFilterCache, - @Nullable SimilarityService similarityService, ClusterService clusterService, - IndexNameExpressionResolver indexNameExpressionResolver, - InnerHitsQueryParserHelper innerHitsQueryParserHelper, Client client) { - super(index, indexSettings); - this.scriptService = scriptService; - this.analysisService = analysisService; - this.mapperService = mapperService; - this.similarityService = similarityService; - this.indexCache = indexCache; - this.fieldDataService = fieldDataService; - this.bitsetFilterCache = bitsetFilterCache; - this.clusterService = clusterService; - this.indexNameExpressionResolver = indexNameExpressionResolver; - - this.defaultField = indexSettings.get(DEFAULT_FIELD, AllFieldMapper.NAME); - this.queryStringLenient = indexSettings.getAsBoolean(QUERY_STRING_LENIENT, false); - this.queryStringAnalyzeWildcard = settings.getAsBoolean(QUERY_STRING_ANALYZE_WILDCARD, false); - this.queryStringAllowLeadingWildcard = settings.getAsBoolean(QUERY_STRING_ALLOW_LEADING_WILDCARD, true); - this.parseFieldMatcher = new ParseFieldMatcher(indexSettings); - this.defaultAllowUnmappedFields = indexSettings.getAsBoolean(ALLOW_UNMAPPED, true); - this.indicesQueriesRegistry = indicesQueriesRegistry; - this.innerHitsQueryParserHelper = innerHitsQueryParserHelper; - this.client = client; - } - - public void close() { - cache.close(); - } - - public String defaultField() { - return this.defaultField; - } - - public boolean queryStringAnalyzeWildcard() { - return this.queryStringAnalyzeWildcard; - } - - public boolean queryStringAllowLeadingWildcard() { - return this.queryStringAllowLeadingWildcard; - } - - public boolean queryStringLenient() { - return this.queryStringLenient; - } - - IndicesQueriesRegistry indicesQueriesRegistry() { - return indicesQueriesRegistry; - } - - public ParsedQuery parse(BytesReference source) { - QueryShardContext context = cache.get(); - XContentParser parser = null; - try { - parser = XContentFactory.xContent(source).createParser(source); - return innerParse(context, parser); - } catch (ParsingException e) { - throw e; - } catch (Exception e) { - throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e); - } finally { - if (parser != null) { - parser.close(); - } - } - } - - public ParsedQuery parse(XContentParser parser) { - try { - return innerParse(cache.get(), parser); - } catch(IOException e) { - throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e); - } - } - - /** - * Parses an inner filter, returning null if the filter should be ignored. - */ - @Nullable - public ParsedQuery parseInnerFilter(XContentParser parser) throws IOException { - QueryShardContext context = cache.get(); - context.reset(parser); - try { - context.parseFieldMatcher(parseFieldMatcher); - Query filter = context.parseContext().parseInnerQueryBuilder().toFilter(context); - if (filter == null) { - return null; - } - return new ParsedQuery(filter, context.copyNamedQueries()); - } finally { - context.reset(null); - } - } - - public QueryShardContext getShardContext() { - return cache.get(); - } - - public boolean defaultAllowUnmappedFields() { - return defaultAllowUnmappedFields; - } - - /** - * @return The lowest node version in the cluster when the index was created or null if that was unknown - */ - public Version getIndexCreatedVersion() { - return Version.indexCreated(indexSettings); - } - - /** - * Selectively parses a query from a top level query or query_binary json field from the specified source. - */ - public ParsedQuery parseQuery(BytesReference source) { - XContentParser parser = null; - try { - parser = XContentHelper.createParser(source); - ParsedQuery parsedQuery = null; - for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { - if (token == XContentParser.Token.FIELD_NAME) { - String fieldName = parser.currentName(); - if ("query".equals(fieldName)) { - parsedQuery = parse(parser); - } else if ("query_binary".equals(fieldName) || "queryBinary".equals(fieldName)) { - byte[] querySource = parser.binaryValue(); - XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource); - parsedQuery = parse(qSourceParser); - } else { - throw new ParsingException(parser.getTokenLocation(), "request does not support [" + fieldName + "]"); - } - } - } - if (parsedQuery == null) { - throw new ParsingException(parser.getTokenLocation(), "Required query is missing"); - } - return parsedQuery; - } catch (ParsingException | QueryShardException e) { - throw e; - } catch (Throwable e) { - throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e); - } - } - - private ParsedQuery innerParse(QueryShardContext context, XContentParser parser) throws IOException, QueryShardException { - context.reset(parser); - try { - context.parseFieldMatcher(parseFieldMatcher); - Query query = parseInnerQuery(context); - return new ParsedQuery(query, context.copyNamedQueries()); - } finally { - context.reset(null); - } - } - - public Query parseInnerQuery(QueryShardContext context) throws IOException { - return toQuery(context.parseContext().parseInnerQueryBuilder(), context); - } - - public ParsedQuery toQuery(QueryBuilder queryBuilder) { - QueryShardContext context = cache.get(); - context.reset(); - context.parseFieldMatcher(parseFieldMatcher); - try { - Query query = toQuery(queryBuilder, context); - return new ParsedQuery(query, context.copyNamedQueries()); - } catch(QueryShardException | ParsingException e ) { - throw e; - } catch(Exception e) { - throw new QueryShardException(context, "failed to create query: {}", e, queryBuilder); - } finally { - context.reset(); - } - } - - private static Query toQuery(QueryBuilder queryBuilder, QueryShardContext context) throws IOException { - Query query = queryBuilder.toQuery(context); - if (query == null) { - query = Queries.newMatchNoDocsQuery(); - } - return query; - } - - public ParseFieldMatcher parseFieldMatcher() { - return parseFieldMatcher; - } - - public boolean matchesIndices(String... indices) { - final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterService.state(), IndicesOptions.lenientExpandOpen(), indices); - for (String index : concreteIndices) { - if (Regex.simpleMatch(index, this.index.name())) { - return true; - } - } - return false; - } - - public InnerHitsQueryParserHelper getInnerHitsQueryParserHelper() { - return innerHitsQueryParserHelper; - } - - public Client getClient() { - return client; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java index b4c7b53a99e..5185dfda3b0 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java @@ -93,10 +93,10 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { - throw new ParsingException(parser.getTokenLocation(), "[match_all] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + MatchAllQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + MatchAllQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } MatchAllQueryBuilder queryBuilder = new MatchAllQueryBuilder(); diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java index 0c466a43d10..4fc96f37f7a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java @@ -39,6 +39,7 @@ public class MatchNoneQueryBuilder extends AbstractQueryBuilder public MatchNoneQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException { XContentParser parser = parseContext.parser(); - XContentParser.Token token = parser.nextToken(); - if (token != XContentParser.Token.END_OBJECT) { - throw new ParsingException(parser.getTokenLocation(), "[match_none] query malformed"); + String currentFieldName = null; + XContentParser.Token token; + String queryName = null; + float boost = AbstractQueryBuilder.DEFAULT_BOOST; + while (((token = parser.nextToken()) != XContentParser.Token.END_OBJECT && token != XContentParser.Token.END_ARRAY)) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { + queryName = parser.text(); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { + boost = parser.floatValue(); + } else { + throw new ParsingException(parser.getTokenLocation(), "["+MatchNoneQueryBuilder.NAME+"] query does not support [" + currentFieldName + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + MatchNoneQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); + } } - return new MatchNoneQueryBuilder(); + MatchNoneQueryBuilder matchNoneQueryBuilder = new MatchNoneQueryBuilder(); + matchNoneQueryBuilder.boost(boost); + matchNoneQueryBuilder.queryName(queryName); + return matchNoneQueryBuilder; } @Override diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java index e959dbee88c..d9a99cc50cb 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java @@ -129,6 +129,11 @@ public class MatchQueryBuilder extends AbstractQueryBuilder { return this; } + /** Returns the operator to use in a boolean query.*/ + public Operator operator() { + return this.operator; + } + /** * Explicitly set the analyzer to use. Defaults to use explicit mapping config for the field, or, if not * set, the default search analyzer. @@ -301,8 +306,7 @@ public class MatchQueryBuilder extends AbstractQueryBuilder { } /** - * Get the setting for handling zero terms queries. - * @see #zeroTermsQuery(ZeroTermsQuery) + * Returns the setting for handling zero terms queries. */ public MatchQuery.ZeroTermsQuery zeroTermsQuery() { return this.zeroTermsQuery; @@ -313,30 +317,30 @@ public class MatchQueryBuilder extends AbstractQueryBuilder { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("query", value); - builder.field("type", type.toString().toLowerCase(Locale.ENGLISH)); - builder.field("operator", operator.toString()); + builder.field(MatchQueryParser.QUERY_FIELD.getPreferredName(), value); + builder.field(MatchQueryParser.TYPE_FIELD.getPreferredName(), type.toString().toLowerCase(Locale.ENGLISH)); + builder.field(MatchQueryParser.OPERATOR_FIELD.getPreferredName(), operator.toString()); if (analyzer != null) { - builder.field("analyzer", analyzer); + builder.field(MatchQueryParser.ANALYZER_FIELD.getPreferredName(), analyzer); } - builder.field("slop", slop); + builder.field(MatchQueryParser.SLOP_FIELD.getPreferredName(), slop); if (fuzziness != null) { fuzziness.toXContent(builder, params); } - builder.field("prefix_length", prefixLength); - builder.field("max_expansions", maxExpansions); + builder.field(MatchQueryParser.PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); + builder.field(MatchQueryParser.MAX_EXPANSIONS_FIELD.getPreferredName(), maxExpansions); if (minimumShouldMatch != null) { - builder.field("minimum_should_match", minimumShouldMatch); + builder.field(MatchQueryParser.MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), minimumShouldMatch); } if (fuzzyRewrite != null) { - builder.field("fuzzy_rewrite", fuzzyRewrite); + builder.field(MatchQueryParser.FUZZY_REWRITE_FIELD.getPreferredName(), fuzzyRewrite); } // LUCENE 4 UPGRADE we need to document this & test this - builder.field("fuzzy_transpositions", fuzzyTranspositions); - builder.field("lenient", lenient); - builder.field("zero_terms_query", zeroTermsQuery.toString()); + builder.field(MatchQueryParser.FUZZY_TRANSPOSITIONS_FIELD.getPreferredName(), fuzzyTranspositions); + builder.field(MatchQueryParser.LENIENT_FIELD.getPreferredName(), lenient); + builder.field(MatchQueryParser.ZERO_TERMS_QUERY_FIELD.getPreferredName(), zeroTermsQuery.toString()); if (cutoffFrequency != null) { - builder.field("cutoff_frequency", cutoffFrequency); + builder.field(MatchQueryParser.CUTOFF_FREQUENCY_FIELD.getPreferredName(), cutoffFrequency); } printBoostAndQueryName(builder); builder.endObject(); @@ -346,7 +350,7 @@ public class MatchQueryBuilder extends AbstractQueryBuilder { @Override protected Query doToQuery(QueryShardContext context) throws IOException { // validate context specific fields - if (analyzer != null && context.analysisService().analyzer(analyzer) == null) { + if (analyzer != null && context.getAnalysisService().analyzer(analyzer) == null) { throw new QueryShardException(context, "[match] analyzer [" + analyzer + "] not found"); } diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java index afcf25ca2a7..4b149dd6be3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.FuzzyQuery; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; @@ -33,6 +34,22 @@ import java.io.IOException; */ public class MatchQueryParser implements QueryParser { + public static final ParseField MATCH_PHRASE_FIELD = new ParseField("match_phrase", "text_phrase"); + public static final ParseField MATCH_PHRASE_PREFIX_FIELD = new ParseField("match_phrase_prefix", "text_phrase_prefix"); + public static final ParseField SLOP_FIELD = new ParseField("slop", "phrase_slop"); + public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query"); + public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField FUZZY_TRANSPOSITIONS_FIELD = new ParseField("fuzzy_transpositions"); + public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField OPERATOR_FIELD = new ParseField("operator"); + public static final ParseField MAX_EXPANSIONS_FIELD = new ParseField("max_expansions"); + public static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + @Override public String[] names() { return new String[]{ @@ -45,17 +62,15 @@ public class MatchQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); MatchQuery.Type type = MatchQuery.Type.BOOLEAN; - if ("match_phrase".equals(parser.currentName()) || "matchPhrase".equals(parser.currentName()) || - "text_phrase".equals(parser.currentName()) || "textPhrase".equals(parser.currentName())) { + if (parseContext.parseFieldMatcher().match(parser.currentName(), MATCH_PHRASE_FIELD)) { type = MatchQuery.Type.PHRASE; - } else if ("match_phrase_prefix".equals(parser.currentName()) || "matchPhrasePrefix".equals(parser.currentName()) || - "text_phrase_prefix".equals(parser.currentName()) || "textPhrasePrefix".equals(parser.currentName())) { + } else if (parseContext.parseFieldMatcher().match(parser.currentName(), MATCH_PHRASE_PREFIX_FIELD)) { type = MatchQuery.Type.PHRASE_PREFIX; } XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { - throw new ParsingException(parser.getTokenLocation(), "[match] query malformed, no field"); + throw new ParsingException(parser.getTokenLocation(), "[" + MatchQueryBuilder.NAME + "] query malformed, no field"); } String fieldName = parser.currentName(); @@ -82,44 +97,44 @@ public class MatchQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("query".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { value = parser.objectText(); - } else if ("type".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { String tStr = parser.text(); if ("boolean".equals(tStr)) { type = MatchQuery.Type.BOOLEAN; } else if ("phrase".equals(tStr)) { type = MatchQuery.Type.PHRASE; - } else if ("phrase_prefix".equals(tStr) || "phrasePrefix".equals(currentFieldName)) { + } else if ("phrase_prefix".equals(tStr) || ("phrasePrefix".equals(tStr))) { type = MatchQuery.Type.PHRASE_PREFIX; } else { - throw new ParsingException(parser.getTokenLocation(), "[match] query does not support type " + tStr); + throw new ParsingException(parser.getTokenLocation(), "[" + MatchQueryBuilder.NAME + "] query does not support type " + tStr); } - } else if ("analyzer".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) { analyzer = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("slop".equals(currentFieldName) || "phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SLOP_FIELD)) { slop = parser.intValue(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) { fuzziness = Fuzziness.parse(parser); - } else if ("prefix_length".equals(currentFieldName) || "prefixLength".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) { prefixLength = parser.intValue(); - } else if ("max_expansions".equals(currentFieldName) || "maxExpansions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) { maxExpansion = parser.intValue(); - } else if ("operator".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, OPERATOR_FIELD)) { operator = Operator.fromString(parser.text()); - } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) { minimumShouldMatch = parser.textOrNull(); - } else if ("fuzzy_rewrite".equals(currentFieldName) || "fuzzyRewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) { fuzzyRewrite = parser.textOrNull(); - } else if ("fuzzy_transpositions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_TRANSPOSITIONS_FIELD)) { fuzzyTranspositions = parser.booleanValue(); - } else if ("lenient".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) { lenient = parser.booleanValue(); - } else if ("cutoff_frequency".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) { cutOffFrequency = parser.floatValue(); - } else if ("zero_terms_query".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ZERO_TERMS_QUERY_FIELD)) { String zeroTermsDocs = parser.text(); if ("none".equalsIgnoreCase(zeroTermsDocs)) { zeroTermsQuery = MatchQuery.ZeroTermsQuery.NONE; @@ -128,11 +143,13 @@ public class MatchQueryParser implements QueryParser { } else { throw new ParsingException(parser.getTokenLocation(), "Unsupported zero_terms_docs value [" + zeroTermsDocs + "]"); } - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { - throw new ParsingException(parser.getTokenLocation(), "[match] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + MatchQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + MatchQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } parser.nextToken(); diff --git a/core/src/main/java/org/elasticsearch/index/query/MissingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MissingQueryBuilder.java deleted file mode 100644 index a3374bf1c7f..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/MissingQueryBuilder.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermRangeQuery; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; -import org.elasticsearch.index.mapper.object.ObjectMapper; - -import java.io.IOException; -import java.util.Collection; -import java.util.Objects; - -/** - * Constructs a filter that have only null values or no value in the original field. - */ -public class MissingQueryBuilder extends AbstractQueryBuilder { - - public static final String NAME = "missing"; - - public static final boolean DEFAULT_NULL_VALUE = false; - - public static final boolean DEFAULT_EXISTENCE_VALUE = true; - - private final String fieldPattern; - - private final boolean nullValue; - - private final boolean existence; - - static final MissingQueryBuilder PROTOTYPE = new MissingQueryBuilder("field", DEFAULT_NULL_VALUE, DEFAULT_EXISTENCE_VALUE); - - /** - * Constructs a filter that returns documents with only null values or no value in the original field. - * @param fieldPattern the field to query - * @param nullValue should the missing filter automatically include fields with null value configured in the - * mappings. Defaults to false. - * @param existence should the missing filter include documents where the field doesn't exist in the docs. - * Defaults to true. - * @throws IllegalArgumentException when both existence and nullValue are set to false - */ - public MissingQueryBuilder(String fieldPattern, boolean nullValue, boolean existence) { - if (Strings.isEmpty(fieldPattern)) { - throw new IllegalArgumentException("missing query must be provided with a [field]"); - } - if (nullValue == false && existence == false) { - throw new IllegalArgumentException("missing query must have either 'existence', or 'null_value', or both set to true"); - } - this.fieldPattern = fieldPattern; - this.nullValue = nullValue; - this.existence = existence; - } - - public MissingQueryBuilder(String fieldPattern) { - this(fieldPattern, DEFAULT_NULL_VALUE, DEFAULT_EXISTENCE_VALUE); - } - - public String fieldPattern() { - return this.fieldPattern; - } - - /** - * Returns true if the missing filter will include documents where the field contains a null value, otherwise - * these documents will not be included. - */ - public boolean nullValue() { - return this.nullValue; - } - - /** - * Returns true if the missing filter will include documents where the field has no values, otherwise - * these documents will not be included. - */ - public boolean existence() { - return this.existence; - } - - @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(NAME); - builder.field("field", fieldPattern); - builder.field("null_value", nullValue); - builder.field("existence", existence); - printBoostAndQueryName(builder); - builder.endObject(); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - protected Query doToQuery(QueryShardContext context) throws IOException { - return newFilter(context, fieldPattern, existence, nullValue); - } - - public static Query newFilter(QueryShardContext context, String fieldPattern, boolean existence, boolean nullValue) { - if (!existence && !nullValue) { - throw new QueryShardException(context, "missing must have either existence, or null_value, or both set to true"); - } - - final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType) context.mapperService().fullName(FieldNamesFieldMapper.NAME); - if (fieldNamesFieldType == null) { - // can only happen when no types exist, so no docs exist either - return Queries.newMatchNoDocsQuery(); - } - - ObjectMapper objectMapper = context.getObjectMapper(fieldPattern); - if (objectMapper != null) { - // automatic make the object mapper pattern - fieldPattern = fieldPattern + ".*"; - } - - Collection fields = context.simpleMatchToIndexNames(fieldPattern); - if (fields.isEmpty()) { - if (existence) { - // if we ask for existence of fields, and we found none, then we should match on all - return Queries.newMatchAllQuery(); - } - return null; - } - - Query existenceFilter = null; - Query nullFilter = null; - - if (existence) { - BooleanQuery.Builder boolFilter = new BooleanQuery.Builder(); - for (String field : fields) { - MappedFieldType fieldType = context.fieldMapper(field); - Query filter = null; - if (fieldNamesFieldType.isEnabled()) { - final String f; - if (fieldType != null) { - f = fieldType.names().indexName(); - } else { - f = field; - } - filter = fieldNamesFieldType.termQuery(f, context); - } - // if _field_names are not indexed, we need to go the slow way - if (filter == null && fieldType != null) { - filter = fieldType.rangeQuery(null, null, true, true); - } - if (filter == null) { - filter = new TermRangeQuery(field, null, null, true, true); - } - boolFilter.add(filter, BooleanClause.Occur.SHOULD); - } - - existenceFilter = boolFilter.build(); - existenceFilter = Queries.not(existenceFilter);; - } - - if (nullValue) { - for (String field : fields) { - MappedFieldType fieldType = context.fieldMapper(field); - if (fieldType != null) { - nullFilter = fieldType.nullValueQuery(); - } - } - } - - Query filter; - if (nullFilter != null) { - if (existenceFilter != null) { - filter = new BooleanQuery.Builder() - .add(existenceFilter, BooleanClause.Occur.SHOULD) - .add(nullFilter, BooleanClause.Occur.SHOULD) - .build(); - } else { - filter = nullFilter; - } - } else { - filter = existenceFilter; - } - - if (filter == null) { - return null; - } - - return new ConstantScoreQuery(filter); - } - - @Override - protected MissingQueryBuilder doReadFrom(StreamInput in) throws IOException { - return new MissingQueryBuilder(in.readString(), in.readBoolean(), in.readBoolean()); - } - - @Override - protected void doWriteTo(StreamOutput out) throws IOException { - out.writeString(fieldPattern); - out.writeBoolean(nullValue); - out.writeBoolean(existence); - } - - @Override - protected int doHashCode() { - return Objects.hash(fieldPattern, nullValue, existence); - } - - @Override - protected boolean doEquals(MissingQueryBuilder other) { - return Objects.equals(fieldPattern, other.fieldPattern) && - Objects.equals(nullValue, other.nullValue) && - Objects.equals(existence, other.existence); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/query/MissingQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/MissingQueryParser.java deleted file mode 100644 index 8d8c5aec01f..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/MissingQueryParser.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; - -/** - * Parser for missing query - */ -public class MissingQueryParser implements QueryParser { - - @Override - public String[] names() { - return new String[]{MissingQueryBuilder.NAME}; - } - - @Override - public MissingQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - - String fieldPattern = null; - String queryName = null; - float boost = AbstractQueryBuilder.DEFAULT_BOOST; - boolean nullValue = MissingQueryBuilder.DEFAULT_NULL_VALUE; - boolean existence = MissingQueryBuilder.DEFAULT_EXISTENCE_VALUE; - - XContentParser.Token token; - String currentFieldName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - if ("field".equals(currentFieldName)) { - fieldPattern = parser.text(); - } else if ("null_value".equals(currentFieldName)) { - nullValue = parser.booleanValue(); - } else if ("existence".equals(currentFieldName)) { - existence = parser.booleanValue(); - } else if ("_name".equals(currentFieldName)) { - queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { - boost = parser.floatValue(); - } else { - throw new ParsingException(parser.getTokenLocation(), "[missing] query does not support [" + currentFieldName + "]"); - } - } - } - - if (fieldPattern == null) { - throw new ParsingException(parser.getTokenLocation(), "missing must be provided with a [field]"); - } - return new MissingQueryBuilder(fieldPattern, nullValue, existence) - .boost(boost) - .queryName(queryName); - } - - @Override - public MissingQueryBuilder getBuilderPrototype() { - return MissingQueryBuilder.PROTOTYPE; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index e4fd6aaf572..39612b7a856 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -775,7 +775,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder fieldEntry : this.fieldsBoosts.entrySet()) { builder.value(fieldEntry.getKey() + "^" + fieldEntry.getValue()); } builder.endArray(); - builder.field("type", type.toString().toLowerCase(Locale.ENGLISH)); - builder.field("operator", operator.toString()); + builder.field(MultiMatchQueryParser.TYPE_FIELD.getPreferredName(), type.toString().toLowerCase(Locale.ENGLISH)); + builder.field(MultiMatchQueryParser.OPERATOR_FIELD.getPreferredName(), operator.toString()); if (analyzer != null) { - builder.field("analyzer", analyzer); + builder.field(MultiMatchQueryParser.ANALYZER_FIELD.getPreferredName(), analyzer); } - builder.field("slop", slop); + builder.field(MultiMatchQueryParser.SLOP_FIELD.getPreferredName(), slop); if (fuzziness != null) { fuzziness.toXContent(builder, params); } - builder.field("prefix_length", prefixLength); - builder.field("max_expansions", maxExpansions); + builder.field(MultiMatchQueryParser.PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); + builder.field(MultiMatchQueryParser.MAX_EXPANSIONS_FIELD.getPreferredName(), maxExpansions); if (minimumShouldMatch != null) { - builder.field("minimum_should_match", minimumShouldMatch); + builder.field(MultiMatchQueryParser.MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), minimumShouldMatch); } if (fuzzyRewrite != null) { - builder.field("fuzzy_rewrite", fuzzyRewrite); + builder.field(MultiMatchQueryParser.FUZZY_REWRITE_FIELD.getPreferredName(), fuzzyRewrite); } if (useDisMax != null) { - builder.field("use_dis_max", useDisMax); + builder.field(MultiMatchQueryParser.USE_DIS_MAX_FIELD.getPreferredName(), useDisMax); } if (tieBreaker != null) { - builder.field("tie_breaker", tieBreaker); + builder.field(MultiMatchQueryParser.TIE_BREAKER_FIELD.getPreferredName(), tieBreaker); } - builder.field("lenient", lenient); + builder.field(MultiMatchQueryParser.LENIENT_FIELD.getPreferredName(), lenient); if (cutoffFrequency != null) { - builder.field("cutoff_frequency", cutoffFrequency); + builder.field(MultiMatchQueryParser.CUTOFF_FREQUENCY_FIELD.getPreferredName(), cutoffFrequency); } - builder.field("zero_terms_query", zeroTermsQuery.toString()); + builder.field(MultiMatchQueryParser.ZERO_TERMS_QUERY_FIELD.getPreferredName(), zeroTermsQuery.toString()); printBoostAndQueryName(builder); builder.endObject(); } @@ -504,7 +504,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder newFieldsBoosts = handleFieldsMatchPattern(context.mapperService(), fieldsBoosts); + Map newFieldsBoosts = handleFieldsMatchPattern(context.getMapperService(), fieldsBoosts); Query query = multiMatchQuery.parse(type, newFieldsBoosts, value, minimumShouldMatch); if (query == null) { @@ -548,12 +548,6 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder handleFieldsMatchPattern(MapperService mapperService, Map fieldsBoosts) { Map newFieldsBoosts = new TreeMap<>(); for (Map.Entry fieldBoost : fieldsBoosts.entrySet()) { diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java index c86d0295577..af212e7aedf 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.MoreLikeThisQueryParser.Field; import org.elasticsearch.index.search.MatchQuery; import java.io.IOException; @@ -33,6 +35,22 @@ import java.util.Map; */ public class MultiMatchQueryParser implements QueryParser { + public static final ParseField SLOP_FIELD = new ParseField("slop", "phrase_slop"); + public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency"); + public static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker"); + public static final ParseField USE_DIS_MAX_FIELD = new ParseField("use_dis_max"); + public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField OPERATOR_FIELD = new ParseField("operator"); + public static final ParseField MAX_EXPANSIONS_FIELD = new ParseField("max_expansions"); + public static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + @Override public String[] names() { return new String[]{ @@ -69,7 +87,7 @@ public class MultiMatchQueryParser implements QueryParser @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("query"); + builder.field(NestedQueryParser.QUERY_FIELD.getPreferredName()); query.toXContent(builder, params); - builder.field("path", path); + builder.field(NestedQueryParser.PATH_FIELD.getPreferredName(), path); if (scoreMode != null) { - builder.field("score_mode", scoreMode.name().toLowerCase(Locale.ROOT)); + builder.field(NestedQueryParser.SCORE_MODE_FIELD.getPreferredName(), scoreMode.name().toLowerCase(Locale.ROOT)); } printBoostAndQueryName(builder); if (queryInnerHits != null) { @@ -187,7 +186,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder throw new IllegalStateException("[" + NAME + "] nested object under path [" + path + "] is not of nested type"); } final BitSetProducer parentFilter; - final Filter childFilter; + final Query childFilter; final ObjectMapper parentObjectMapper; final Query innerQuery; ObjectMapper objectMapper = context.nestedScope().getObjectMapper(); @@ -213,7 +212,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder if (token != XContentParser.Token.START_OBJECT) { throw new IllegalStateException("start object expected but was: [" + token + "]"); } - InnerHitsSubSearchContext innerHits = context.indexQueryParserService().getInnerHitsQueryParserHelper().parse(parser); + InnerHitsSubSearchContext innerHits = context.getInnerHitsContext(parser); if (innerHits != null) { ParsedQuery parsedQuery = new ParsedQuery(innerQuery, context.copyNamedQueries()); diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java index 1fabfede29d..7cdb66bd126 100644 --- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java @@ -25,12 +25,16 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.support.QueryInnerHits; + import java.io.IOException; public class NestedQueryParser implements QueryParser { - private static final ParseField FILTER_FIELD = new ParseField("filter").withAllDeprecated("query"); private static final NestedQueryBuilder PROTOTYPE = new NestedQueryBuilder("", EmptyQueryBuilder.PROTOTYPE); + public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode"); + public static final ParseField PATH_FIELD = new ParseField("path"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); @Override public String[] names() { @@ -52,21 +56,19 @@ public class NestedQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("query".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { query = parseContext.parseInnerQueryBuilder(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, FILTER_FIELD)) { - query = parseContext.parseInnerQueryBuilder(); - } else if ("inner_hits".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) { queryInnerHits = new QueryInnerHits(parser); } else { throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("path".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, PATH_FIELD)) { path = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("score_mode".equals(currentFieldName) || "scoreMode".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) { String sScoreMode = parser.text(); if ("avg".equals(sScoreMode)) { scoreMode = ScoreMode.Avg; @@ -81,7 +83,7 @@ public class NestedQueryParser implements QueryParser { } else { throw new ParsingException(parser.getTokenLocation(), "illegal score_mode for nested query [" + sScoreMode + "]"); } - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/NotQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NotQueryBuilder.java deleted file mode 100644 index 72b70a7d126..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/NotQueryBuilder.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.apache.lucene.search.Query; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Objects; - -/** - * A filter that matches documents matching boolean combinations of other filters. - */ -public class NotQueryBuilder extends AbstractQueryBuilder { - - public static final String NAME = "not"; - - private final QueryBuilder filter; - - static final NotQueryBuilder PROTOTYPE = new NotQueryBuilder(EmptyQueryBuilder.PROTOTYPE); - - public NotQueryBuilder(QueryBuilder filter) { - if (filter == null) { - throw new IllegalArgumentException("inner filter cannot be null"); - } - this.filter = filter; - } - - /** - * @return the query added to "not". - */ - public QueryBuilder innerQuery() { - return this.filter; - } - - @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(NAME); - builder.field("query"); - filter.toXContent(builder, params); - printBoostAndQueryName(builder); - builder.endObject(); - } - - @Override - protected Query doToQuery(QueryShardContext context) throws IOException { - Query luceneQuery = filter.toFilter(context); - if (luceneQuery == null) { - return null; - } - return Queries.not(luceneQuery); - } - - @Override - protected int doHashCode() { - return Objects.hash(filter); - } - - @Override - protected boolean doEquals(NotQueryBuilder other) { - return Objects.equals(filter, other.filter); - } - - @Override - protected NotQueryBuilder doReadFrom(StreamInput in) throws IOException { - QueryBuilder queryBuilder = in.readQuery(); - return new NotQueryBuilder(queryBuilder); - } - - @Override - protected void doWriteTo(StreamOutput out) throws IOException { - out.writeQuery(filter); - } - - @Override - public String getWriteableName() { - return NAME; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/query/NotQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/NotQueryParser.java deleted file mode 100644 index de458209b07..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/NotQueryParser.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; - -/** - * Parser for not query - */ -public class NotQueryParser implements QueryParser { - - private static final ParseField QUERY_FIELD = new ParseField("query", "filter"); - - @Override - public String[] names() { - return new String[]{NotQueryBuilder.NAME}; - } - - @Override - public NotQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException { - XContentParser parser = parseContext.parser(); - - QueryBuilder query = null; - boolean queryFound = false; - - String queryName = null; - String currentFieldName = null; - float boost = AbstractQueryBuilder.DEFAULT_BOOST; - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (parseContext.isDeprecatedSetting(currentFieldName)) { - // skip - } else if (token == XContentParser.Token.START_OBJECT) { - if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { - query = parseContext.parseInnerQueryBuilder(); - queryFound = true; - } else { - queryFound = true; - // its the filter, and the name is the field - query = parseContext.parseInnerQueryBuilderByName(currentFieldName); - } - } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { - queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { - boost = parser.floatValue(); - } else { - throw new ParsingException(parser.getTokenLocation(), "[not] query does not support [" + currentFieldName + "]"); - } - } - } - - if (!queryFound) { - throw new ParsingException(parser.getTokenLocation(), "query is required when using `not` query"); - } - - NotQueryBuilder notQueryBuilder = new NotQueryBuilder(query); - notQueryBuilder.queryName(queryName); - notQueryBuilder.boost(boost); - return notQueryBuilder; - } - - @Override - public NotQueryBuilder getBuilderPrototype() { - return NotQueryBuilder.PROTOTYPE; - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java index f5ca1360268..fbd1bbd05a6 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java @@ -87,9 +87,9 @@ public class PrefixQueryBuilder extends AbstractQueryBuilder public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("prefix", this.value); + builder.field(PrefixQueryParser.PREFIX_FIELD.getPreferredName(), this.value); if (rewrite != null) { - builder.field("rewrite", rewrite); + builder.field(PrefixQueryParser.REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java index a8dca4c7816..e13d937d847 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java @@ -30,7 +30,8 @@ import java.io.IOException; */ public class PrefixQueryParser implements QueryParser { - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of prefix query"); + public static final ParseField PREFIX_FIELD = new ParseField("value", "prefix"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); @Override public String[] names() { @@ -60,13 +61,13 @@ public class PrefixQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("value".equals(currentFieldName) || "prefix".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PREFIX_FIELD)) { value = parser.textOrNull(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); } else { throw new ParsingException(parser.getTokenLocation(), "[regexp] query does not support [" + currentFieldName + "]"); @@ -74,12 +75,8 @@ public class PrefixQueryParser implements QueryParser { } } } else { - if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { - queryName = parser.text(); - } else { fieldName = currentFieldName; value = parser.textOrNull(); - } } } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java index 67e654cb0d8..3fb09679204 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java @@ -810,31 +810,6 @@ public abstract class QueryBuilders { return new ExistsQueryBuilder(name); } - /** - * A filter to filter only documents where a field does not exists in them. - * @param name the field to query - */ - public static MissingQueryBuilder missingQuery(String name) { - return missingQuery(name, MissingQueryBuilder.DEFAULT_NULL_VALUE, MissingQueryBuilder.DEFAULT_EXISTENCE_VALUE); - } - - /** - * A filter to filter only documents where a field does not exists in them. - * @param name the field to query - * @param nullValue should the missing filter automatically include fields with null value configured in the - * mappings. Defaults to false. - * @param existence should the missing filter include documents where the field doesn't exist in the docs. - * Defaults to true. - * @throws IllegalArgumentException when both existence and nullValue are set to false - */ - public static MissingQueryBuilder missingQuery(String name, boolean nullValue, boolean existence) { - return new MissingQueryBuilder(name, nullValue, existence); - } - - public static NotQueryBuilder notQuery(QueryBuilder filter) { - return new NotQueryBuilder(filter); - } - private QueryBuilders() { } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryFilterBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryFilterBuilder.java deleted file mode 100644 index 4ca9e1598e2..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/QueryFilterBuilder.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Query; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Objects; - -/** - * A filter that simply wraps a query. - * @deprecated Useless now that queries and filters are merged: pass the - * query as a filter directly. - */ -//TODO: remove when https://github.com/elastic/elasticsearch/issues/13326 is fixed -@Deprecated -public class QueryFilterBuilder extends AbstractQueryBuilder { - - public static final String NAME = "query"; - - private final QueryBuilder queryBuilder; - - static final QueryFilterBuilder PROTOTYPE = new QueryFilterBuilder(EmptyQueryBuilder.PROTOTYPE); - - /** - * A filter that simply wraps a query. - * - * @param queryBuilder The query to wrap as a filter - */ - public QueryFilterBuilder(QueryBuilder queryBuilder) { - if (queryBuilder == null) { - throw new IllegalArgumentException("inner query cannot be null"); - } - this.queryBuilder = queryBuilder; - } - - /** - * @return the query builder that is wrapped by this {@link QueryFilterBuilder} - */ - public QueryBuilder innerQuery() { - return this.queryBuilder; - } - - @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(NAME); - queryBuilder.toXContent(builder, params); - } - - @Override - protected Query doToQuery(QueryShardContext context) throws IOException { - // inner query builder can potentially be `null`, in that case we ignore it - Query innerQuery = this.queryBuilder.toQuery(context); - if (innerQuery == null) { - return null; - } - return new ConstantScoreQuery(innerQuery); - } - - @Override - protected void setFinalBoost(Query query) { - //no-op this query doesn't support boost - } - - @Override - protected int doHashCode() { - return Objects.hash(queryBuilder); - } - - @Override - protected boolean doEquals(QueryFilterBuilder other) { - return Objects.equals(queryBuilder, other.queryBuilder); - } - - @Override - protected QueryFilterBuilder doReadFrom(StreamInput in) throws IOException { - QueryBuilder innerQueryBuilder = in.readQuery(); - return new QueryFilterBuilder(innerQueryBuilder); - } - - @Override - protected void doWriteTo(StreamOutput out) throws IOException { - out.writeQuery(queryBuilder); - } - - @Override - public String getWriteableName() { - return NAME; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryFilterParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryFilterParser.java deleted file mode 100644 index e13661c814c..00000000000 --- a/core/src/main/java/org/elasticsearch/index/query/QueryFilterParser.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import java.io.IOException; - -/** - * Parser for query filter - * @deprecated use any query instead directly, possible since queries and filters are merged. - */ -// TODO: remove when https://github.com/elastic/elasticsearch/issues/13326 is fixed -@Deprecated -public class QueryFilterParser implements QueryParser { - - @Override - public String[] names() { - return new String[]{QueryFilterBuilder.NAME}; - } - - @Override - public QueryFilterBuilder fromXContent(QueryParseContext parseContext) throws IOException { - return new QueryFilterBuilder(parseContext.parseInnerQueryBuilder()); - } - - @Override - public QueryFilterBuilder getBuilderPrototype() { - return QueryFilterBuilder.PROTOTYPE; - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java index de85abd347a..78d76d8292e 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java @@ -65,9 +65,36 @@ public class QueryParseContext { } /** - * @return a new QueryBuilder based on the current state of the parser + * Parses a top level query including the query element that wraps it */ - public QueryBuilder parseInnerQueryBuilder() throws IOException { + public QueryBuilder parseTopLevelQueryBuilder() { + try { + QueryBuilder queryBuilder = null; + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + String fieldName = parser.currentName(); + if ("query".equals(fieldName)) { + queryBuilder = parseInnerQueryBuilder(); + } else { + throw new ParsingException(parser.getTokenLocation(), "request does not support [" + parser.currentName() + "]"); + } + } + } + if (queryBuilder == null) { + throw new ParsingException(parser.getTokenLocation(), "Required query is missing"); + } + return queryBuilder; + } catch (ParsingException e) { + throw e; + } catch (Throwable e) { + throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e); + } + } + + /** + * Parses a query excluding the query element that wraps it + */ + public QueryBuilder parseInnerQueryBuilder() throws IOException { // move to START object XContentParser.Token token; if (parser.currentToken() != XContentParser.Token.START_OBJECT) { @@ -91,7 +118,12 @@ public class QueryParseContext { throw new ParsingException(parser.getTokenLocation(), "[_na] query malformed, no field after start_object"); } - QueryBuilder result = parseInnerQueryBuilderByName(queryName); + QueryParser queryParser = queryParser(queryName); + if (queryParser == null) { + throw new ParsingException(parser.getTokenLocation(), "No query registered for [" + queryName + "]"); + } + + QueryBuilder result = queryParser.fromXContent(this); if (parser.currentToken() == XContentParser.Token.END_OBJECT || parser.currentToken() == XContentParser.Token.END_ARRAY) { // if we are at END_OBJECT, move to the next one... parser.nextToken(); @@ -99,14 +131,6 @@ public class QueryParseContext { return result; } - public QueryBuilder parseInnerQueryBuilderByName(String queryName) throws IOException { - QueryParser queryParser = queryParser(queryName); - if (queryParser == null) { - throw new ParsingException(parser.getTokenLocation(), "No query registered for [" + queryName + "]"); - } - return queryParser.fromXContent(this); - } - public ParseFieldMatcher parseFieldMatcher() { return parseFieldMatcher; } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index e2a16dfddfa..65dfb559e3f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -22,20 +22,24 @@ package org.elasticsearch.index.query; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.queryparser.classic.MapperQueryParser; import org.apache.lucene.queryparser.classic.QueryParserSettings; -import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; import org.elasticsearch.Version; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -43,15 +47,20 @@ import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; +import org.elasticsearch.index.query.support.InnerHitsQueryParserHelper; import org.elasticsearch.index.query.support.NestedScope; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.Template; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; +import org.elasticsearch.search.fetch.innerhits.InnerHitsSubSearchContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SearchLookup; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -64,7 +73,14 @@ import static java.util.Collections.unmodifiableMap; */ public class QueryShardContext { - private static ThreadLocal typesContext = new ThreadLocal<>(); + private static final ThreadLocal typesContext = new ThreadLocal<>(); + private final MapperService mapperService; + private final ScriptService scriptService; + private final SimilarityService similarityService; + private final BitsetFilterCache bitsetFilterCache; + private final IndexFieldDataService indexFieldDataService; + private final IndexSettings indexSettings; + private final Client client; public static void setTypes(String[] types) { typesContext.set(types); @@ -84,31 +100,36 @@ public class QueryShardContext { typesContext.remove(); } - private final Index index; - - private final Version indexVersionCreated; - - private final IndexQueryParserService indexQueryParser; - private final Map namedQueries = new HashMap<>(); - private final MapperQueryParser queryParser = new MapperQueryParser(this); - + private final IndicesQueriesRegistry indicesQueriesRegistry; private boolean allowUnmappedFields; - private boolean mapUnmappedFieldAsString; - private NestedScope nestedScope; - private QueryParseContext parseContext; + boolean isFilter; // pkg private for testing - boolean isFilter; + public QueryShardContext(IndexSettings indexSettings, Client client, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService, + final IndicesQueriesRegistry indicesQueriesRegistry) { + this.indexSettings = indexSettings; + this.scriptService = scriptService; + this.client = client; + this.similarityService = similarityService; + this.mapperService = mapperService; + this.bitsetFilterCache = bitsetFilterCache; + this.indexFieldDataService = indexFieldDataService; + this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); + this.indicesQueriesRegistry = indicesQueriesRegistry; + this.parseContext = new QueryParseContext(indicesQueriesRegistry); + } - public QueryShardContext(Index index, IndexQueryParserService indexQueryParser) { - this.index = index; - this.indexVersionCreated = Version.indexCreated(indexQueryParser.indexSettings()); - this.indexQueryParser = indexQueryParser; - this.parseContext = new QueryParseContext(indexQueryParser.indicesQueriesRegistry()); + public QueryShardContext(QueryShardContext source) { + this(source.indexSettings, source.client, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry); + } + + + public QueryShardContext clone() { + return new QueryShardContext(indexSettings, client, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry); } public void parseFieldMatcher(ParseFieldMatcher parseFieldMatcher) { @@ -120,11 +141,12 @@ public class QueryShardContext { } public void reset() { - allowUnmappedFields = indexQueryParser.defaultAllowUnmappedFields(); + allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); this.parseFieldMatcher(ParseFieldMatcher.EMPTY); this.lookup = null; this.namedQueries.clear(); this.nestedScope = new NestedScope(); + this.isFilter = false; } public void reset(XContentParser jp) { @@ -133,43 +155,43 @@ public class QueryShardContext { } public Index index() { - return this.index; + return this.mapperService.getIndexSettings().getIndex(); } - public IndexQueryParserService indexQueryParserService() { - return indexQueryParser; + public InnerHitsSubSearchContext getInnerHitsContext(XContentParser parser) throws IOException { + return InnerHitsQueryParserHelper.parse(parser); } - public AnalysisService analysisService() { - return indexQueryParser.analysisService; + public AnalysisService getAnalysisService() { + return mapperService.analysisService(); } - public ScriptService scriptService() { - return indexQueryParser.scriptService; + public ScriptService getScriptService() { + return scriptService; } - public MapperService mapperService() { - return indexQueryParser.mapperService; + public MapperService getMapperService() { + return mapperService; } - public Similarity searchSimilarity() { - return indexQueryParser.similarityService != null ? indexQueryParser.similarityService.similarity(indexQueryParser.mapperService) : null; + public Similarity getSearchSimilarity() { + return similarityService != null ? similarityService.similarity(mapperService) : null; } public String defaultField() { - return indexQueryParser.defaultField(); + return indexSettings.getDefaultField(); } public boolean queryStringLenient() { - return indexQueryParser.queryStringLenient(); + return indexSettings.isQueryStringLenient(); } public boolean queryStringAnalyzeWildcard() { - return indexQueryParser.queryStringAnalyzeWildcard(); + return indexSettings.isQueryStringAnalyzeWildcard(); } public boolean queryStringAllowLeadingWildcard() { - return indexQueryParser.queryStringAllowLeadingWildcard(); + return indexSettings.isQueryStringAllowLeadingWildcard(); } public MapperQueryParser queryParser(QueryParserSettings settings) { @@ -177,12 +199,12 @@ public class QueryShardContext { return queryParser; } - public BitSetProducer bitsetFilter(Filter filter) { - return indexQueryParser.bitsetFilterCache.getBitSetProducer(filter); + public BitSetProducer bitsetFilter(Query filter) { + return bitsetFilterCache.getBitSetProducer(filter); } public > IFD getForField(MappedFieldType mapper) { - return indexQueryParser.fieldDataService.getForField(mapper); + return indexFieldDataService.getForField(mapper); } public void addNamedQuery(String name, Query query) { @@ -215,7 +237,7 @@ public class QueryShardContext { InnerHitsContext innerHitsContext; if (sc.innerHits() == null) { - innerHitsContext = new InnerHitsContext(new HashMap()); + innerHitsContext = new InnerHitsContext(new HashMap<>()); sc.innerHits(innerHitsContext); } else { innerHitsContext = sc.innerHits(); @@ -224,15 +246,15 @@ public class QueryShardContext { } public Collection simpleMatchToIndexNames(String pattern) { - return indexQueryParser.mapperService.simpleMatchToIndexNames(pattern); + return mapperService.simpleMatchToIndexNames(pattern); } public MappedFieldType fieldMapper(String name) { - return failIfFieldMappingNotFound(name, indexQueryParser.mapperService.smartNameFieldType(name, getTypes())); + return failIfFieldMappingNotFound(name, mapperService.smartNameFieldType(name, getTypes())); } public ObjectMapper getObjectMapper(String name) { - return indexQueryParser.mapperService.getObjectMapper(name, getTypes()); + return mapperService.getObjectMapper(name, getTypes()); } /** @@ -243,7 +265,7 @@ public class QueryShardContext { if (fieldType.searchAnalyzer() != null) { return fieldType.searchAnalyzer(); } - return mapperService().searchAnalyzer(); + return getMapperService().searchAnalyzer(); } /** @@ -254,7 +276,7 @@ public class QueryShardContext { if (fieldType.searchQuoteAnalyzer() != null) { return fieldType.searchQuoteAnalyzer(); } - return mapperService().searchQuoteAnalyzer(); + return getMapperService().searchQuoteAnalyzer(); } public void setAllowUnmappedFields(boolean allowUnmappedFields) { @@ -270,11 +292,9 @@ public class QueryShardContext { return fieldMapping; } else if (mapUnmappedFieldAsString) { StringFieldMapper.Builder builder = MapperBuilders.stringField(name); - // it would be better to pass the real index settings, but they are not easily accessible from here... - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexQueryParser.getIndexCreatedVersion()).build(); - return builder.build(new Mapper.BuilderContext(settings, new ContentPath(1))).fieldType(); + return builder.build(new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1))).fieldType(); } else { - Version indexCreatedVersion = indexQueryParser.getIndexCreatedVersion(); + Version indexCreatedVersion = indexSettings.getIndexVersionCreated(); if (fieldMapping == null && indexCreatedVersion.onOrAfter(Version.V_1_4_0_Beta1)) { throw new QueryShardException(this, "Strict field resolution and no field mapping can be found for the field with name [" + name + "]"); @@ -290,10 +310,10 @@ public class QueryShardContext { public Collection queryTypes() { String[] types = getTypes(); if (types == null || types.length == 0) { - return mapperService().types(); + return getMapperService().types(); } if (types.length == 1 && types[0].equals("_all")) { - return mapperService().types(); + return getMapperService().types(); } return Arrays.asList(types); } @@ -306,7 +326,7 @@ public class QueryShardContext { return current.lookup(); } if (lookup == null) { - lookup = new SearchLookup(mapperService(), indexQueryParser.fieldDataService, null); + lookup = new SearchLookup(getMapperService(), indexFieldDataService, null); } return lookup; } @@ -324,7 +344,7 @@ public class QueryShardContext { } public Version indexVersionCreated() { - return indexVersionCreated; + return indexSettings.getIndexVersionCreated(); } public QueryParseContext parseContext() { @@ -332,18 +352,105 @@ public class QueryShardContext { } public boolean matchesIndices(String... indices) { - return this.indexQueryParser.matchesIndices(indices); + for (String index : indices) { + if (indexSettings.matchesIndexName(index)) { + return true; + } + } + return false; } /* * Executes the given template, and returns the response. */ public BytesReference executeQueryTemplate(Template template, SearchContext searchContext) { - ExecutableScript executable = scriptService().executable(template, ScriptContext.Standard.SEARCH, searchContext); + ExecutableScript executable = getScriptService().executable(template, ScriptContext.Standard.SEARCH, searchContext); return (BytesReference) executable.run(); } public Client getClient() { - return indexQueryParser.getClient(); + return client; } + + public ParsedQuery parse(BytesReference source) { + XContentParser parser = null; + try { + parser = XContentFactory.xContent(source).createParser(source); + return innerParse(parser); + } catch (ParsingException e) { + throw e; + } catch (Exception e) { + throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e); + } finally { + if (parser != null) { + parser.close(); + } + } + } + + public ParsedQuery parse(XContentParser parser) { + try { + return innerParse(parser); + } catch(IOException e) { + throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e); + } + } + + /** + * Parses an inner filter, returning null if the filter should be ignored. + */ + @Nullable + public ParsedQuery parseInnerFilter(XContentParser parser) throws IOException { + reset(parser); + try { + parseFieldMatcher(indexSettings.getParseFieldMatcher()); + Query filter = parseContext().parseInnerQueryBuilder().toFilter(this); + if (filter == null) { + return null; + } + return new ParsedQuery(filter, copyNamedQueries()); + } finally { + reset(null); + } + } + + + private ParsedQuery innerParse(XContentParser parser) throws IOException, QueryShardException { + reset(parser); + try { + parseFieldMatcher(indexSettings.getParseFieldMatcher()); + Query query = parseInnerQuery(); + return new ParsedQuery(query, copyNamedQueries()); + } finally { + reset(null); + } + } + + public Query parseInnerQuery() throws IOException { + return toQuery(this.parseContext().parseInnerQueryBuilder(), this); + } + + public ParsedQuery toQuery(QueryBuilder queryBuilder) { + reset(); + parseFieldMatcher(indexSettings.getParseFieldMatcher()); + try { + Query query = toQuery(queryBuilder, this); + return new ParsedQuery(query, copyNamedQueries()); + } catch(QueryShardException | ParsingException e ) { + throw e; + } catch(Exception e) { + throw new QueryShardException(this, "failed to create query: {}", e, queryBuilder); + } finally { + this.reset(); + } + } + + private static Query toQuery(QueryBuilder queryBuilder, QueryShardContext context) throws IOException { + Query query = queryBuilder.toQuery(context); + if (query == null) { + query = Queries.newMatchNoDocsQuery(); + } + return query; + } + } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java index 16d65162cef..16107d4ec97 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.queryparser.classic.MapperQueryParser; import org.apache.lucene.queryparser.classic.QueryParserSettings; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.automaton.Operations; @@ -36,10 +37,7 @@ import org.elasticsearch.index.query.support.QueryParsers; import org.joda.time.DateTimeZone; import java.io.IOException; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.TreeMap; +import java.util.*; /** * A query that parses a query string and runs it. There are two modes that this operates. The first, @@ -470,58 +468,58 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder fieldEntry : this.fieldsAndWeights.entrySet()) { builder.value(fieldEntry.getKey() + "^" + fieldEntry.getValue()); } builder.endArray(); - builder.field("use_dis_max", this.useDisMax); - builder.field("tie_breaker", this.tieBreaker); - builder.field("default_operator", this.defaultOperator.name().toLowerCase(Locale.ROOT)); + builder.field(QueryStringQueryParser.USE_DIS_MAX_FIELD.getPreferredName(), this.useDisMax); + builder.field(QueryStringQueryParser.TIE_BREAKER_FIELD.getPreferredName(), this.tieBreaker); + builder.field(QueryStringQueryParser.DEFAULT_OPERATOR_FIELD.getPreferredName(), this.defaultOperator.name().toLowerCase(Locale.ROOT)); if (this.analyzer != null) { - builder.field("analyzer", this.analyzer); + builder.field(QueryStringQueryParser.ANALYZER_FIELD.getPreferredName(), this.analyzer); } if (this.quoteAnalyzer != null) { - builder.field("quote_analyzer", this.quoteAnalyzer); + builder.field(QueryStringQueryParser.QUOTE_ANALYZER_FIELD.getPreferredName(), this.quoteAnalyzer); } - builder.field("auto_generate_phrase_queries", this.autoGeneratePhraseQueries); - builder.field("max_determinized_states", this.maxDeterminizedStates); + builder.field(QueryStringQueryParser.AUTO_GENERATED_PHRASE_QUERIES_FIELD.getPreferredName(), this.autoGeneratePhraseQueries); + builder.field(QueryStringQueryParser.MAX_DETERMINED_STATES_FIELD.getPreferredName(), this.maxDeterminizedStates); if (this.allowLeadingWildcard != null) { - builder.field("allow_leading_wildcard", this.allowLeadingWildcard); + builder.field(QueryStringQueryParser.ALLOW_LEADING_WILDCARD_FIELD.getPreferredName(), this.allowLeadingWildcard); } - builder.field("lowercase_expanded_terms", this.lowercaseExpandedTerms); - builder.field("enable_position_increments", this.enablePositionIncrements); + builder.field(QueryStringQueryParser.LOWERCASE_EXPANDED_TERMS_FIELD.getPreferredName(), this.lowercaseExpandedTerms); + builder.field(QueryStringQueryParser.ENABLE_POSITION_INCREMENTS_FIELD.getPreferredName(), this.enablePositionIncrements); this.fuzziness.toXContent(builder, params); - builder.field("fuzzy_prefix_length", this.fuzzyPrefixLength); - builder.field("fuzzy_max_expansions", this.fuzzyMaxExpansions); + builder.field(QueryStringQueryParser.FUZZY_PREFIX_LENGTH_FIELD.getPreferredName(), this.fuzzyPrefixLength); + builder.field(QueryStringQueryParser.FUZZY_MAX_EXPANSIONS_FIELD.getPreferredName(), this.fuzzyMaxExpansions); if (this.fuzzyRewrite != null) { - builder.field("fuzzy_rewrite", this.fuzzyRewrite); + builder.field(QueryStringQueryParser.FUZZY_REWRITE_FIELD.getPreferredName(), this.fuzzyRewrite); } - builder.field("phrase_slop", this.phraseSlop); + builder.field(QueryStringQueryParser.PHRASE_SLOP_FIELD.getPreferredName(), this.phraseSlop); if (this.analyzeWildcard != null) { - builder.field("analyze_wildcard", this.analyzeWildcard); + builder.field(QueryStringQueryParser.ANALYZE_WILDCARD_FIELD.getPreferredName(), this.analyzeWildcard); } if (this.rewrite != null) { - builder.field("rewrite", this.rewrite); + builder.field(QueryStringQueryParser.REWRITE_FIELD.getPreferredName(), this.rewrite); } if (this.minimumShouldMatch != null) { - builder.field("minimum_should_match", this.minimumShouldMatch); + builder.field(QueryStringQueryParser.MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), this.minimumShouldMatch); } if (this.quoteFieldSuffix != null) { - builder.field("quote_field_suffix", this.quoteFieldSuffix); + builder.field(QueryStringQueryParser.QUOTE_FIELD_SUFFIX_FIELD.getPreferredName(), this.quoteFieldSuffix); } if (this.lenient != null) { - builder.field("lenient", this.lenient); + builder.field(QueryStringQueryParser.LENIENT_FIELD.getPreferredName(), this.lenient); } - builder.field("locale", this.locale.toLanguageTag()); + builder.field(QueryStringQueryParser.LOCALE_FIELD.getPreferredName(), this.locale.toLanguageTag()); if (this.timeZone != null) { - builder.field("time_zone", this.timeZone.getID()); + builder.field(QueryStringQueryParser.TIME_ZONE_FIELD.getPreferredName(), this.timeZone.getID()); } - builder.field("escape", this.escape); + builder.field(QueryStringQueryParser.ESCAPE_FIELD.getPreferredName(), this.escape); printBoostAndQueryName(builder); builder.endObject(); } @@ -661,7 +659,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder boosts = new ArrayList<>(); + while(query instanceof BoostQuery) { + BoostQuery boostQuery = (BoostQuery) query; + boosts.add(boostQuery.getBoost()); + query = boostQuery.getQuery(); + } + query = Queries.fixNegativeQueryIfNeeded(query); if (query instanceof BooleanQuery) { query = Queries.applyMinimumShouldMatch((BooleanQuery) query, this.minimumShouldMatch()); } + + //restore the previous BoostQuery wrapping + for (int i = boosts.size() - 1; i >= 0; i--) { + query = new BoostQuery(query, boosts.get(i)); + } + return query; } - - @Override - protected void setFinalBoost(Query query) { - //we need to preserve the boost that came out of the parsing phase - query.setBoost(query.getBoost() * boost); - } } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java index f5dbb250805..f7d9d2989dd 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java @@ -35,8 +35,32 @@ import java.util.Map; */ public class QueryStringQueryParser implements QueryParser { - private static final ParseField FUZZINESS = Fuzziness.FIELD.withDeprecation("fuzzy_min_sim"); - + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + public static final ParseField DEFAULT_FIELD_FIELD = new ParseField("default_field"); + public static final ParseField DEFAULT_OPERATOR_FIELD = new ParseField("default_operator"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField QUOTE_ANALYZER_FIELD = new ParseField("quote_analyzer"); + public static final ParseField ALLOW_LEADING_WILDCARD_FIELD = new ParseField("allow_leading_wildcard"); + public static final ParseField AUTO_GENERATED_PHRASE_QUERIES_FIELD = new ParseField("auto_generated_phrase_queries"); + public static final ParseField MAX_DETERMINED_STATES_FIELD = new ParseField("max_determined_states"); + public static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms"); + public static final ParseField ENABLE_POSITION_INCREMENTS_FIELD = new ParseField("enable_position_increment"); + public static final ParseField ESCAPE_FIELD = new ParseField("escape"); + public static final ParseField USE_DIS_MAX_FIELD = new ParseField("use_dis_max"); + public static final ParseField FUZZY_PREFIX_LENGTH_FIELD = new ParseField("fuzzy_prefix_length"); + public static final ParseField FUZZY_MAX_EXPANSIONS_FIELD = new ParseField("fuzzy_max_expansions"); + public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite"); + public static final ParseField PHRASE_SLOP_FIELD = new ParseField("phrase_slop"); + public static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker"); + public static final ParseField ANALYZE_WILDCARD_FIELD = new ParseField("analyze_wildcard"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField QUOTE_FIELD_SUFFIX_FIELD = new ParseField("quote_field_suffix"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField LOCALE_FIELD = new ParseField("locale"); + public static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); + @Override public String[] names() { return new String[]{QueryStringQueryBuilder.NAME, Strings.toCamelCase(QueryStringQueryBuilder.NAME)}; @@ -79,7 +103,7 @@ public class QueryStringQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("fields".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { String fField = null; float fBoost = AbstractQueryBuilder.DEFAULT_BOOST; @@ -99,75 +123,77 @@ public class QueryStringQueryParser implements QueryParser { fieldsAndWeights.put(fField, fBoost); } } else { - throw new ParsingException(parser.getTokenLocation(), "[query_string] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("query".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { queryString = parser.text(); - } else if ("default_field".equals(currentFieldName) || "defaultField".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, DEFAULT_FIELD_FIELD)) { defaultField = parser.text(); - } else if ("default_operator".equals(currentFieldName) || "defaultOperator".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, DEFAULT_OPERATOR_FIELD)) { defaultOperator = Operator.fromString(parser.text()); - } else if ("analyzer".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) { analyzer = parser.text(); - } else if ("quote_analyzer".equals(currentFieldName) || "quoteAnalyzer".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, QUOTE_ANALYZER_FIELD)) { quoteAnalyzer = parser.text(); - } else if ("allow_leading_wildcard".equals(currentFieldName) || "allowLeadingWildcard".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ALLOW_LEADING_WILDCARD_FIELD)) { allowLeadingWildcard = parser.booleanValue(); - } else if ("auto_generate_phrase_queries".equals(currentFieldName) || "autoGeneratePhraseQueries".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AUTO_GENERATED_PHRASE_QUERIES_FIELD)) { autoGeneratePhraseQueries = parser.booleanValue(); - } else if ("max_determinized_states".equals(currentFieldName) || "maxDeterminizedStates".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_DETERMINED_STATES_FIELD)) { maxDeterminizedStates = parser.intValue(); - } else if ("lowercase_expanded_terms".equals(currentFieldName) || "lowercaseExpandedTerms".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) { lowercaseExpandedTerms = parser.booleanValue(); - } else if ("enable_position_increments".equals(currentFieldName) || "enablePositionIncrements".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ENABLE_POSITION_INCREMENTS_FIELD)) { enablePositionIncrements = parser.booleanValue(); - } else if ("escape".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ESCAPE_FIELD)) { escape = parser.booleanValue(); - } else if ("use_dis_max".equals(currentFieldName) || "useDisMax".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, USE_DIS_MAX_FIELD)) { useDisMax = parser.booleanValue(); - } else if ("fuzzy_prefix_length".equals(currentFieldName) || "fuzzyPrefixLength".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_PREFIX_LENGTH_FIELD)) { fuzzyPrefixLength = parser.intValue(); - } else if ("fuzzy_max_expansions".equals(currentFieldName) || "fuzzyMaxExpansions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_MAX_EXPANSIONS_FIELD)) { fuzzyMaxExpansions = parser.intValue(); - } else if ("fuzzy_rewrite".equals(currentFieldName) || "fuzzyRewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) { fuzzyRewrite = parser.textOrNull(); - } else if ("phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PHRASE_SLOP_FIELD)) { phraseSlop = parser.intValue(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZINESS)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) { fuzziness = Fuzziness.parse(parser); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("tie_breaker".equals(currentFieldName) || "tieBreaker".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) { tieBreaker = parser.floatValue(); - } else if ("analyze_wildcard".equals(currentFieldName) || "analyzeWildcard".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ANALYZE_WILDCARD_FIELD)) { analyzeWildcard = parser.booleanValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) { minimumShouldMatch = parser.textOrNull(); - } else if ("quote_field_suffix".equals(currentFieldName) || "quoteFieldSuffix".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, QUOTE_FIELD_SUFFIX_FIELD)) { quoteFieldSuffix = parser.textOrNull(); - } else if ("lenient".equalsIgnoreCase(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) { lenient = parser.booleanValue(); - } else if ("locale".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) { String localeStr = parser.text(); locale = Locale.forLanguageTag(localeStr); - } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) { try { timeZone = parser.text(); } catch (IllegalArgumentException e) { - throw new ParsingException(parser.getTokenLocation(), "[query_string] time_zone [" + parser.text() + "] is unknown"); + throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] time_zone [" + parser.text() + "] is unknown"); } - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { - throw new ParsingException(parser.getTokenLocation(), "[query_string] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } if (queryString == null) { - throw new ParsingException(parser.getTokenLocation(), "query_string must be provided with a [query]"); + throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] must be provided with a [query]"); } QueryStringQueryBuilder queryStringQuery = new QueryStringQueryBuilder(queryString); diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index 1c8b57c3879..cd99bec0f74 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -233,15 +233,15 @@ public class RangeQueryBuilder extends AbstractQueryBuilder i protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("from", convertToStringIfBytesRef(this.from)); - builder.field("to", convertToStringIfBytesRef(this.to)); - builder.field("include_lower", includeLower); - builder.field("include_upper", includeUpper); + builder.field(RangeQueryParser.FROM_FIELD.getPreferredName(), convertToStringIfBytesRef(this.from)); + builder.field(RangeQueryParser.TO_FIELD.getPreferredName(), convertToStringIfBytesRef(this.to)); + builder.field(RangeQueryParser.INCLUDE_LOWER_FIELD.getPreferredName(), includeLower); + builder.field(RangeQueryParser.INCLUDE_UPPER_FIELD.getPreferredName(), includeUpper); if (timeZone != null) { - builder.field("time_zone", timeZone.getID()); + builder.field(RangeQueryParser.TIME_ZONE_FIELD.getPreferredName(), timeZone.getID()); } if (format != null) { - builder.field("format", format.format()); + builder.field(RangeQueryParser.FORMAT_FIELD.getPreferredName(), format.format()); } printBoostAndQueryName(builder); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java index dcd07b3e4eb..10a13dd52f5 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java @@ -30,8 +30,18 @@ import java.io.IOException; */ public class RangeQueryParser implements QueryParser { - private static final ParseField FIELDDATA_FIELD = new ParseField("fielddata").withAllDeprecated("[no replacement]"); - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of range query"); + public static final ParseField FIELDDATA_FIELD = new ParseField("fielddata").withAllDeprecated("[no replacement]"); + public static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of range query"); + public static final ParseField LTE_FIELD = new ParseField("lte", "le"); + public static final ParseField GTE_FIELD = new ParseField("gte", "ge"); + public static final ParseField FROM_FIELD = new ParseField("from"); + public static final ParseField TO_FIELD = new ParseField("to"); + public static final ParseField INCLUDE_LOWER_FIELD = new ParseField("include_lower"); + public static final ParseField INCLUDE_UPPER_FIELD = new ParseField("include_upper"); + public static final ParseField GT_FIELD = new ParseField("gt"); + public static final ParseField LT_FIELD = new ParseField("lt"); + public static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); + public static final ParseField FORMAT_FIELD = new ParseField("format"); @Override public String[] names() { @@ -65,33 +75,33 @@ public class RangeQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("from".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, FROM_FIELD)) { from = parser.objectBytes(); - } else if ("to".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TO_FIELD)) { to = parser.objectBytes(); - } else if ("include_lower".equals(currentFieldName) || "includeLower".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INCLUDE_LOWER_FIELD)) { includeLower = parser.booleanValue(); - } else if ("include_upper".equals(currentFieldName) || "includeUpper".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INCLUDE_UPPER_FIELD)) { includeUpper = parser.booleanValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("gt".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, GT_FIELD)) { from = parser.objectBytes(); includeLower = false; - } else if ("gte".equals(currentFieldName) || "ge".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, GTE_FIELD)) { from = parser.objectBytes(); includeLower = true; - } else if ("lt".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LT_FIELD)) { to = parser.objectBytes(); includeUpper = false; - } else if ("lte".equals(currentFieldName) || "le".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LTE_FIELD)) { to = parser.objectBytes(); includeUpper = true; - } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) { timeZone = parser.text(); - } else if ("format".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FORMAT_FIELD)) { format = parser.text(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[range] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java index 45f58c47da6..1ebf44e23f2 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java +++ b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java @@ -108,7 +108,7 @@ public enum RegexpFlag { * @param flags A string representing a list of regular expression flags * @return The combined OR'ed value for all the flags */ - static int resolveValue(String flags) { + public static int resolveValue(String flags) { if (flags == null || flags.isEmpty()) { return RegExp.ALL; } diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java index f596bf84d5b..6f78a91a02a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java @@ -135,14 +135,14 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public void doXContent(XContentBuilder builder, Params params) throws IOException { + protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("value", this.value); - builder.field("flags_value", flagsValue); - builder.field("max_determinized_states", maxDeterminizedStates); + builder.field(RegexpQueryParser.VALUE_FIELD.getPreferredName(), this.value); + builder.field(RegexpQueryParser.FLAGS_VALUE_FIELD.getPreferredName(), flagsValue); + builder.field(RegexpQueryParser.MAX_DETERMINIZED_STATES_FIELD.getPreferredName(), maxDeterminizedStates); if (rewrite != null) { - builder.field("rewrite", rewrite); + builder.field(RegexpQueryParser.REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); builder.endObject(); @@ -155,7 +155,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public Query doToQuery(QueryShardContext context) throws QueryShardException, IOException { + protected Query doToQuery(QueryShardContext context) throws QueryShardException, IOException { MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(context.parseFieldMatcher(), rewrite, null); Query query = null; @@ -174,7 +174,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public RegexpQueryBuilder doReadFrom(StreamInput in) throws IOException { + protected RegexpQueryBuilder doReadFrom(StreamInput in) throws IOException { RegexpQueryBuilder regexpQueryBuilder = new RegexpQueryBuilder(in.readString(), in.readString()); regexpQueryBuilder.flagsValue = in.readVInt(); regexpQueryBuilder.maxDeterminizedStates = in.readVInt(); @@ -183,7 +183,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public void doWriteTo(StreamOutput out) throws IOException { + protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); out.writeString(value); out.writeVInt(flagsValue); @@ -192,12 +192,12 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public int doHashCode() { + protected int doHashCode() { return Objects.hash(fieldName, value, flagsValue, maxDeterminizedStates, rewrite); } @Override - public boolean doEquals(RegexpQueryBuilder other) { + protected boolean doEquals(RegexpQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(value, other.value) && Objects.equals(flagsValue, other.flagsValue) && diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java index d07c23da171..92305abc1df 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java @@ -30,7 +30,12 @@ import java.io.IOException; */ public class RegexpQueryParser implements QueryParser { - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of regexp query"); + public static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of regexp query"); + public static final ParseField FLAGS_VALUE_FIELD = new ParseField("flags_value"); + public static final ParseField MAX_DETERMINIZED_STATES_FIELD = new ParseField("max_determinized_states"); + public static final ParseField FLAGS_FIELD = new ParseField("flags"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + public static final ParseField VALUE_FIELD = new ParseField("value"); @Override public String[] names() { @@ -62,20 +67,20 @@ public class RegexpQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("value".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.textOrNull(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("flags".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FLAGS_FIELD)) { String flags = parser.textOrNull(); flagsValue = RegexpFlag.resolveValue(flags); - } else if ("max_determinized_states".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_DETERMINIZED_STATES_FIELD)) { maxDeterminizedStates = parser.intValue(); - } else if ("flags_value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FLAGS_VALUE_FIELD)) { flagsValue = parser.intValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[regexp] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 45ab7454d6a..f69ac8c0548 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -69,7 +69,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder @Override protected Query doToQuery(QueryShardContext context) throws IOException { - return new ScriptQuery(script, context.scriptService(), context.lookup()); + return new ScriptQuery(script, context.getScriptService(), context.lookup()); } static class ScriptQuery extends Query { @@ -104,10 +104,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder @Override public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + Objects.hashCode(script); - return result; + return Objects.hash(super.hashCode(), script); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java index 97ad0a21873..51e299815bc 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.Script; @@ -35,6 +36,8 @@ import java.util.Map; */ public class ScriptQueryParser implements QueryParser { + public static final ParseField PARAMS_FIELD = new ParseField("params"); + @Override public String[] names() { return new String[]{ScriptQueryBuilder.NAME}; @@ -62,15 +65,15 @@ public class ScriptQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_OBJECT) { if (parseContext.parseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { script = Script.parse(parser, parseContext.parseFieldMatcher()); - } else if ("params".equals(currentFieldName)) { // TODO remove in 3.0 (here to support old script APIs) + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PARAMS_FIELD)) { // TODO remove in 3.0 (here to support old script APIs) params = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else if (!scriptParameterParser.token(currentFieldName, token, parser, parseContext.parseFieldMatcher())) { throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java index f8b0deaf9be..7627644e750 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java @@ -63,8 +63,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp try { Query q = createBooleanQuery(entry.getKey(), text, super.getDefaultOperator()); if (q != null) { - q.setBoost(entry.getValue()); - bq.add(q, BooleanClause.Occur.SHOULD); + bq.add(wrapWithBoost(q, entry.getValue()), BooleanClause.Occur.SHOULD); } } catch (RuntimeException e) { rethrowUnlessLenient(e); @@ -86,9 +85,8 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { try { - Query q = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness); - q.setBoost(entry.getValue()); - bq.add(q, BooleanClause.Occur.SHOULD); + Query query = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness); + bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD); } catch (RuntimeException e) { rethrowUnlessLenient(e); } @@ -104,8 +102,7 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp try { Query q = createPhraseQuery(entry.getKey(), text, slop); if (q != null) { - q.setBoost(entry.getValue()); - bq.add(q, BooleanClause.Occur.SHOULD); + bq.add(wrapWithBoost(q, entry.getValue()), BooleanClause.Occur.SHOULD); } } catch (RuntimeException e) { rethrowUnlessLenient(e); @@ -129,12 +126,12 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp try { if (settings.analyzeWildcard()) { Query analyzedQuery = newPossiblyAnalyzedQuery(entry.getKey(), text); - analyzedQuery.setBoost(entry.getValue()); - bq.add(analyzedQuery, BooleanClause.Occur.SHOULD); + if (analyzedQuery != null) { + bq.add(wrapWithBoost(analyzedQuery, entry.getValue()), BooleanClause.Occur.SHOULD); + } } else { - PrefixQuery prefix = new PrefixQuery(new Term(entry.getKey(), text)); - prefix.setBoost(entry.getValue()); - bq.add(prefix, BooleanClause.Occur.SHOULD); + Query query = new PrefixQuery(new Term(entry.getKey(), text)); + bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD); } } catch (RuntimeException e) { return rethrowUnlessLenient(e); @@ -143,6 +140,13 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp return super.simplify(bq.build()); } + private static Query wrapWithBoost(Query query, float boost) { + if (boost != AbstractQueryBuilder.DEFAULT_BOOST) { + return new BoostQuery(query, boost); + } + return query; + } + /** * Analyze the given string using its analyzer, constructing either a * {@code PrefixQuery} or a {@code BooleanQuery} made up diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java index 3f8cc5d7e23..092f966d8d7 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java @@ -121,6 +121,7 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder fields) { + Objects.requireNonNull(fields, "fields cannot be null"); this.fieldsAndWeights.putAll(fields); return this; } @@ -258,7 +259,7 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder fieldEntry : fieldsAndWeights.entrySet()) { if (Regex.isSimpleMatchPattern(fieldEntry.getKey())) { - for (String fieldName : context.mapperService().simpleMatchToIndexNames(fieldEntry.getKey())) { + for (String fieldName : context.getMapperService().simpleMatchToIndexNames(fieldEntry.getKey())) { resolvedFieldsAndWeights.put(fieldName, fieldEntry.getValue()); } } else { @@ -270,9 +271,9 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder 0) { - builder.startArray("fields"); + builder.startArray(SimpleQueryStringParser.FIELDS_FIELD.getPreferredName()); for (Map.Entry entry : fieldsAndWeights.entrySet()) { builder.value(entry.getKey() + "^" + entry.getValue()); } @@ -318,18 +314,18 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder { + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField ANALYZE_WILDCARD_FIELD = new ParseField("analyze_wildcard"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms"); + public static final ParseField LOCALE_FIELD = new ParseField("locale"); + public static final ParseField FLAGS_FIELD = new ParseField("flags"); + public static final ParseField DEFAULT_OPERATOR_FIELD = new ParseField("default_operator"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + @Override public String[] names() { return new String[]{SimpleQueryStringBuilder.NAME, Strings.toCamelCase(SimpleQueryStringBuilder.NAME)}; @@ -88,7 +100,7 @@ public class SimpleQueryStringParser implements QueryParser { + public static final ParseField BIG_FIELD = new ParseField("big"); + public static final ParseField LITTLE_FIELD = new ParseField("little"); + @Override public String[] names() { return new String[]{SpanContainingQueryBuilder.NAME, Strings.toCamelCase(SpanContainingQueryBuilder.NAME)}; @@ -49,13 +53,13 @@ public class SpanContainingQueryParser implements QueryParser)) { throw new ParsingException(parser.getTokenLocation(), "span_containing [big] must be of type span query"); } big = (SpanQueryBuilder) query; - } else if ("little".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "span_containing [little] must be of type span query"); @@ -64,9 +68,9 @@ public class SpanContainingQueryParser implements QueryParser { + public static final ParseField MATCH_FIELD = new ParseField("match"); + public static final ParseField END_FIELD = new ParseField("end"); + @Override public String[] names() { return new String[]{SpanFirstQueryBuilder.NAME, Strings.toCamelCase(SpanFirstQueryBuilder.NAME)}; @@ -51,7 +55,7 @@ public class SpanFirstQueryParser implements QueryParser if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("match".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, MATCH_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "spanFirst [match] must be of type span query"); @@ -61,11 +65,11 @@ public class SpanFirstQueryParser implements QueryParser throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]"); } } else { - if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("end".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, END_FIELD)) { end = parser.intValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java index eac2e6a0a7a..21c9c615551 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java @@ -18,9 +18,12 @@ */ package org.elasticsearch.index.query; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.spans.SpanBoostQuery; import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -53,7 +56,7 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder((MultiTermQuery) subQuery); + SpanQuery wrapper = new SpanMultiTermQueryWrapper<>((MultiTermQuery) subQuery); + if (boost != AbstractQueryBuilder.DEFAULT_BOOST) { + wrapper = new SpanBoostQuery(wrapper, boost); + } + return wrapper; } @Override diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryParser.java index e51b693187c..a3b1c09b14e 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryParser.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; @@ -29,7 +30,7 @@ import java.io.IOException; */ public class SpanMultiTermQueryParser implements QueryParser { - public static final String MATCH_NAME = "match"; + public static final ParseField MATCH_FIELD = new ParseField("match"); @Override public String[] names() { @@ -48,19 +49,19 @@ public class SpanMultiTermQueryParser implements QueryParser { + public static final ParseField SLOP_FIELD = new ParseField("slop"); + public static final ParseField COLLECT_PAYLOADS_FIELD = new ParseField("collect_payloads"); + public static final ParseField CLAUSES_FIELD = new ParseField("clauses"); + public static final ParseField IN_ORDER_FIELD = new ParseField("in_order"); + @Override public String[] names() { return new String[]{SpanNearQueryBuilder.NAME, Strings.toCamelCase(SpanNearQueryBuilder.NAME)}; @@ -55,7 +61,7 @@ public class SpanNearQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("clauses".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { @@ -67,15 +73,15 @@ public class SpanNearQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("in_order".equals(currentFieldName) || "inOrder".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, IN_ORDER_FIELD)) { inOrder = parser.booleanValue(); - } else if ("collect_payloads".equals(currentFieldName) || "collectPayloads".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, COLLECT_PAYLOADS_FIELD)) { collectPayloads = parser.booleanValue(); - } else if ("slop".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SLOP_FIELD)) { slop = parser.intValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java index ffe3cecf412..780344b70b2 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java @@ -125,12 +125,12 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder { + public static final ParseField POST_FIELD = new ParseField("post"); + public static final ParseField PRE_FIELD = new ParseField("pre"); + public static final ParseField DIST_FIELD = new ParseField("dist"); + public static final ParseField EXCLUDE_FIELD = new ParseField("exclude"); + public static final ParseField INCLUDE_FIELD = new ParseField("include"); + @Override public String[] names() { return new String[]{SpanNotQueryBuilder.NAME, Strings.toCamelCase(SpanNotQueryBuilder.NAME)}; @@ -56,13 +63,13 @@ public class SpanNotQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("include".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, INCLUDE_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "spanNot [include] must be of type span query"); } include = (SpanQueryBuilder) query; - } else if ("exclude".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, EXCLUDE_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "spanNot [exclude] must be of type span query"); @@ -72,15 +79,15 @@ public class SpanNotQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]"); } } else { - if ("dist".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, DIST_FIELD)) { dist = parser.intValue(); - } else if ("pre".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PRE_FIELD)) { pre = parser.intValue(); - } else if ("post".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, POST_FIELD)) { post = parser.intValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java index a46bef4e520..3b8681c685b 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java @@ -67,7 +67,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.startArray("clauses"); + builder.startArray(SpanOrQueryParser.CLAUSES_FIELD.getPreferredName()); for (SpanQueryBuilder clause : clauses) { clause.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java index a0dabbdad06..50500def865 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; @@ -32,6 +33,8 @@ import java.util.List; */ public class SpanOrQueryParser implements QueryParser { + public static final ParseField CLAUSES_FIELD = new ParseField("clauses"); + @Override public String[] names() { return new String[]{SpanOrQueryBuilder.NAME, Strings.toCamelCase(SpanOrQueryBuilder.NAME)}; @@ -52,7 +55,7 @@ public class SpanOrQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("clauses".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { @@ -64,9 +67,9 @@ public class SpanOrQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]"); } } else { - if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java index fc41dc4ba0d..7e234e551c3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java @@ -68,7 +68,7 @@ public class SpanTermQueryBuilder extends BaseTermQueryBuilder { + public static final ParseField TERM_FIELD = new ParseField("term"); + @Override public String[] names() { return new String[]{SpanTermQueryBuilder.NAME, Strings.toCamelCase(SpanTermQueryBuilder.NAME)}; @@ -58,13 +61,13 @@ public class SpanTermQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("term".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TERM_FIELD)) { value = parser.objectBytes(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, BaseTermQueryBuilder.VALUE_FIELD)) { value = parser.objectBytes(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_term] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java index c3a11c8f325..440e1797dfa 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java @@ -73,10 +73,10 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder { + public static final ParseField BIG_FIELD = new ParseField("big"); + public static final ParseField LITTLE_FIELD = new ParseField("little"); + @Override public String[] names() { return new String[]{SpanWithinQueryBuilder.NAME, Strings.toCamelCase(SpanWithinQueryBuilder.NAME)}; @@ -50,13 +54,13 @@ public class SpanWithinQueryParser implements QueryParser { public TemplateQueryBuilder getBuilderPrototype() { return TemplateQueryBuilder.PROTOTYPE; } + + } diff --git a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java index bed373b9f0c..7d226d97b72 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java @@ -71,7 +71,7 @@ public class TermQueryBuilder extends BaseTermQueryBuilder { } @Override - public Query doToQuery(QueryShardContext context) throws IOException { + protected Query doToQuery(QueryShardContext context) throws IOException { Query query = null; MappedFieldType mapper = context.fieldMapper(this.fieldName); if (mapper != null) { diff --git a/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java index 0591497a3c8..5ac083d6217 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java @@ -30,8 +30,8 @@ import java.io.IOException; */ public class TermQueryParser implements QueryParser { - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of term query"); - private static final ParseField BOOST_FIELD = new ParseField("boost").withAllDeprecated("boost is not supported in short version of term query"); + public static final ParseField TERM_FIELD = new ParseField("term"); + public static final ParseField VALUE_FIELD = new ParseField("value"); @Override public String[] names() { @@ -63,13 +63,13 @@ public class TermQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("term".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TERM_FIELD)) { value = parser.objectBytes(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.objectBytes(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[term] query does not support [" + currentFieldName + "]"); @@ -77,17 +77,11 @@ public class TermQueryParser implements QueryParser { } } } else if (token.isValue()) { - if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { - queryName = parser.text(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, BOOST_FIELD)) { - boost = parser.floatValue(); - } else { - if (fieldName != null) { - throw new ParsingException(parser.getTokenLocation(), "[term] query does not support different field names, use [bool] query instead"); - } - fieldName = currentFieldName; - value = parser.objectBytes(); + if (fieldName != null) { + throw new ParsingException(parser.getTokenLocation(), "[term] query does not support different field names, use [bool] query instead"); } + fieldName = currentFieldName; + value = parser.objectBytes(); } else if (token == XContentParser.Token.START_ARRAY) { throw new ParsingException(parser.getTokenLocation(), "[term] query does not support array of values"); } diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java index 97508a8a16f..b2bcce4dfaf 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java @@ -57,24 +57,18 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { static final TermsQueryBuilder PROTOTYPE = new TermsQueryBuilder("field", "value"); - public static final boolean DEFAULT_DISABLE_COORD = false; - private final String fieldName; private final List values; - @Deprecated - private String minimumShouldMatch; - @Deprecated - private boolean disableCoord = DEFAULT_DISABLE_COORD; private final TermsLookup termsLookup; public TermsQueryBuilder(String fieldName, TermsLookup termsLookup) { - this(fieldName, null, null, DEFAULT_DISABLE_COORD, termsLookup); + this(fieldName, null, termsLookup); } /** * constructor used internally for serialization of both value / termslookup variants */ - TermsQueryBuilder(String fieldName, List values, String minimumShouldMatch, boolean disableCoord, TermsLookup termsLookup) { + TermsQueryBuilder(String fieldName, List values, TermsLookup termsLookup) { if (Strings.isEmpty(fieldName)) { throw new IllegalArgumentException("field name cannot be null."); } @@ -86,8 +80,6 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { } this.fieldName = fieldName; this.values = values; - this.disableCoord = disableCoord; - this.minimumShouldMatch = minimumShouldMatch; this.termsLookup = termsLookup; } @@ -178,34 +170,6 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { return convertToStringListIfBytesRefList(this.values); } - /** - * Sets the minimum number of matches across the provided terms. Defaults to 1. - * @deprecated use [bool] query instead - */ - @Deprecated - public TermsQueryBuilder minimumShouldMatch(String minimumShouldMatch) { - this.minimumShouldMatch = minimumShouldMatch; - return this; - } - - public String minimumShouldMatch() { - return this.minimumShouldMatch; - } - - /** - * Disables Similarity#coord(int,int) in scoring. Defaults to false. - * @deprecated use [bool] query instead - */ - @Deprecated - public TermsQueryBuilder disableCoord(boolean disableCoord) { - this.disableCoord = disableCoord; - return this; - } - - boolean disableCoord() { - return this.disableCoord; - } - public TermsLookup termsLookup() { return this.termsLookup; } @@ -243,7 +207,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { } @Override - public void doXContent(XContentBuilder builder, Params params) throws IOException { + protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); if (this.termsLookup != null) { builder.startObject(fieldName); @@ -252,12 +216,6 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { } else { builder.field(fieldName, convertToStringListIfBytesRefList(values)); } - if (minimumShouldMatch != null) { - builder.field("minimum_should_match", minimumShouldMatch); - } - if (disableCoord != DEFAULT_DISABLE_COORD) { - builder.field("disable_coord", disableCoord); - } printBoostAndQueryName(builder); builder.endObject(); } @@ -284,7 +242,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { if (terms == null || terms.isEmpty()) { return Queries.newMatchNoDocsQuery(); } - return handleTermsQuery(terms, fieldName, context, minimumShouldMatch, disableCoord); + return handleTermsQuery(terms, fieldName, context); } private List fetch(TermsLookup termsLookup, Client client) { @@ -300,7 +258,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { return terms; } - private static Query handleTermsQuery(List terms, String fieldName, QueryShardContext context, String minimumShouldMatch, boolean disableCoord) { + private static Query handleTermsQuery(List terms, String fieldName, QueryShardContext context) { MappedFieldType fieldType = context.fieldMapper(fieldName); String indexFieldName; if (fieldType != null) { @@ -322,7 +280,6 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { } } else { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(disableCoord); for (Object term : terms) { if (fieldType != null) { bq.add(fieldType.termQuery(term, context), BooleanClause.Occur.SHOULD); @@ -330,7 +287,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { bq.add(new TermQuery(new Term(indexFieldName, BytesRefs.toBytesRef(term))), BooleanClause.Occur.SHOULD); } } - query = Queries.applyMinimumShouldMatch(bq.build(), minimumShouldMatch); + query = bq.build(); } return query; } @@ -344,9 +301,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { lookup = TermsLookup.readTermsLookupFrom(in); } List values = (List) in.readGenericValue(); - String minimumShouldMatch = in.readOptionalString(); - boolean disableCoord = in.readBoolean(); - return new TermsQueryBuilder(field, values, minimumShouldMatch, disableCoord, lookup); + return new TermsQueryBuilder(field, values, lookup); } @Override @@ -357,21 +312,17 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { termsLookup.writeTo(out); } out.writeGenericValue(values); - out.writeOptionalString(minimumShouldMatch); - out.writeBoolean(disableCoord); } @Override protected int doHashCode() { - return Objects.hash(fieldName, values, minimumShouldMatch, disableCoord, termsLookup); + return Objects.hash(fieldName, values, termsLookup); } @Override protected boolean doEquals(TermsQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(values, other.values) && - Objects.equals(minimumShouldMatch, other.minimumShouldMatch) && - Objects.equals(disableCoord, other.disableCoord) && Objects.equals(termsLookup, other.termsLookup); } } diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java index c76369195a3..310256556c8 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.query; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.indices.cache.query.terms.TermsLookup; @@ -38,11 +37,6 @@ import java.util.List; */ public class TermsQueryParser implements QueryParser { - private static final ParseField MIN_SHOULD_MATCH_FIELD = new ParseField("min_match", "min_should_match", "minimum_should_match") - .withAllDeprecated("Use [bool] query instead"); - private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord").withAllDeprecated("Use [bool] query instead"); - private static final ParseField EXECUTION_FIELD = new ParseField("execution").withAllDeprecated("execution is deprecated and has no effect"); - @Override public String[] names() { return new String[]{TermsQueryBuilder.NAME, "in"}; @@ -54,8 +48,6 @@ public class TermsQueryParser implements QueryParser { String fieldName = null; List values = null; - String minShouldMatch = null; - boolean disableCoord = TermsQueryBuilder.DEFAULT_DISABLE_COORD; TermsLookup termsLookup = null; String queryName = null; @@ -70,37 +62,34 @@ public class TermsQueryParser implements QueryParser { // skip } else if (token == XContentParser.Token.START_ARRAY) { if (fieldName != null) { - throw new ParsingException(parser.getTokenLocation(), "[terms] query does not support multiple fields"); + throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query does not support multiple fields"); } fieldName = currentFieldName; values = parseValues(parser); } else if (token == XContentParser.Token.START_OBJECT) { + if (fieldName != null) { + throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query does not support more than one field. " + + "Already got: [" + fieldName + "] but also found [" + currentFieldName +"]"); + } fieldName = currentFieldName; termsLookup = TermsLookup.parseTermsLookup(parser); } else if (token.isValue()) { - if (parseContext.parseFieldMatcher().match(currentFieldName, EXECUTION_FIELD)) { - // ignore - } else if (parseContext.parseFieldMatcher().match(currentFieldName, MIN_SHOULD_MATCH_FIELD)) { - if (minShouldMatch != null) { - throw new IllegalArgumentException("[" + currentFieldName + "] is not allowed in a filter context for the [" + TermsQueryBuilder.NAME + "] query"); - } - minShouldMatch = parser.textOrNull(); - } else if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, DISABLE_COORD_FIELD)) { - disableCoord = parser.booleanValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { - throw new ParsingException(parser.getTokenLocation(), "[terms] query does not support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } if (fieldName == null) { - throw new ParsingException(parser.getTokenLocation(), "terms query requires a field name, followed by array of terms or a document lookup specification"); + throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query requires a field name, followed by array of terms or a document lookup specification"); } - return new TermsQueryBuilder(fieldName, values, minShouldMatch, disableCoord, termsLookup) + return new TermsQueryBuilder(fieldName, values, termsLookup) .boost(boost) .queryName(queryName); } diff --git a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java index 94cdc243bf8..975736e842a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java @@ -62,7 +62,7 @@ public class TypeQueryBuilder extends AbstractQueryBuilder { @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("value", type.utf8ToString()); + builder.field(TypeQueryParser.VALUE_FIELD.getPreferredName(), type.utf8ToString()); printBoostAndQueryName(builder); builder.endObject(); } @@ -76,7 +76,7 @@ public class TypeQueryBuilder extends AbstractQueryBuilder { protected Query doToQuery(QueryShardContext context) throws IOException { Query filter; //LUCENE 4 UPGRADE document mapper should use bytesref as well? - DocumentMapper documentMapper = context.mapperService().documentMapper(type.utf8ToString()); + DocumentMapper documentMapper = context.getMapperService().documentMapper(type.utf8ToString()); if (documentMapper == null) { filter = new TermQuery(new Term(TypeFieldMapper.NAME, type)); } else { diff --git a/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java index e2b4e13c65e..d746b4656a0 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; @@ -30,6 +31,8 @@ import java.io.IOException; */ public class TypeQueryParser implements QueryParser { + public static final ParseField VALUE_FIELD = new ParseField("value"); + @Override public String[] names() { return new String[]{TypeQueryBuilder.NAME}; @@ -49,20 +52,22 @@ public class TypeQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { type = parser.utf8Bytes(); + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + TypeQueryBuilder.NAME + "] filter doesn't support [" + currentFieldName + "]"); } } else { - throw new ParsingException(parser.getTokenLocation(), "[type] filter doesn't support [" + currentFieldName + "]"); + throw new ParsingException(parser.getTokenLocation(), "[" + TypeQueryBuilder.NAME + "] filter doesn't support [" + currentFieldName + "]"); } } if (type == null) { - throw new ParsingException(parser.getTokenLocation(), "[type] filter needs to be provided with a value for the type"); + throw new ParsingException(parser.getTokenLocation(), "[" + TypeQueryBuilder.NAME + "] filter needs to be provided with a value for the type"); } return new TypeQueryBuilder(type) .boost(boost) diff --git a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index 44775926400..7c3cc1c30a3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -99,12 +99,12 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder { + public static final ParseField WILDCARD_FIELD = new ParseField("wildcard"); + public static final ParseField VALUE_FIELD = new ParseField("value"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + @Override public String[] names() { return new String[]{WildcardQueryBuilder.NAME}; @@ -55,15 +60,15 @@ public class WildcardQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("wildcard".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, WILDCARD_FIELD)) { value = parser.text(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[wildcard] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java index ef106a4354e..e908d763311 100644 --- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java @@ -20,9 +20,6 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.Query; - -import java.nio.charset.StandardCharsets; - import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -32,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; /** @@ -96,7 +94,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder result = contextCopy.parseContext().parseInnerQueryBuilder(); context.combineNamedQueries(contextCopy); return result.toQuery(context); } } - @Override - protected void setFinalBoost(Query query) { - //no-op this query doesn't support boost - } - @Override protected WrapperQueryBuilder doReadFrom(StreamInput in) throws IOException { return new WrapperQueryBuilder(in.readByteArray()); diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java index 59c570e97f9..a18ad52fcfe 100644 --- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; @@ -29,6 +30,8 @@ import java.io.IOException; */ public class WrapperQueryParser implements QueryParser { + public static final ParseField QUERY_FIELD = new ParseField("query"); + @Override public String[] names() { return new String[]{WrapperQueryBuilder.NAME}; @@ -43,8 +46,8 @@ public class WrapperQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed"); } String fieldName = parser.currentName(); - if (!fieldName.equals("query")) { - throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed"); + if (! parseContext.parseFieldMatcher().match(fieldName, QUERY_FIELD)) { + throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed, expected `query` but was" + fieldName); } parser.nextToken(); diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 8302b874533..5c1fe0ede88 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -41,7 +41,7 @@ import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.index.mapper.core.NumberFieldMapper; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.MultiValueMode; @@ -187,8 +187,8 @@ public abstract class DecayFunctionBuilder ext parser.nextToken(); if (fieldType instanceof DateFieldMapper.DateFieldType) { return parseDateVariable(parser, context, (DateFieldMapper.DateFieldType) fieldType, mode); - } else if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType) { - return parseGeoVariable(parser, context, (GeoPointFieldMapper.GeoPointFieldType) fieldType, mode); + } else if (fieldType instanceof BaseGeoPointFieldMapper.GeoPointFieldType) { + return parseGeoVariable(parser, context, (BaseGeoPointFieldMapper.GeoPointFieldType) fieldType, mode); } else if (fieldType instanceof NumberFieldMapper.NumberFieldType) { return parseNumberVariable(parser, context, (NumberFieldMapper.NumberFieldType) fieldType, mode); } else { @@ -231,7 +231,7 @@ public abstract class DecayFunctionBuilder ext } private AbstractDistanceScoreFunction parseGeoVariable(XContentParser parser, QueryShardContext context, - GeoPointFieldMapper.GeoPointFieldType fieldType, MultiValueMode mode) throws IOException { + BaseGeoPointFieldMapper.GeoPointFieldType fieldType, MultiValueMode mode) throws IOException { XContentParser.Token token; String parameterName = null; GeoPoint origin = new GeoPoint(); diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryParser.java index 7adde617009..d738f3a259d 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryParser.java @@ -46,7 +46,6 @@ public class FunctionScoreQueryParser implements QueryParser>> 32)); + return Long.hashCode(value); } } diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java index 266e75f6ce2..92308466312 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java @@ -89,7 +89,7 @@ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder()); + groups.put(actualAnalyzer, new ArrayList<>()); } Float boost = entry.getValue(); boost = boost == null ? Float.valueOf(1.0f) : boost; groups.get(actualAnalyzer).add(new FieldAndFieldType(name, fieldType, boost)); } else { - missing.add(new Tuple(name, entry.getValue())); + missing.add(new Tuple<>(name, entry.getValue())); } } diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java index c590ea08301..f68699ac2a2 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java @@ -29,14 +29,13 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapperLegacy; import java.io.IOException; @@ -58,8 +57,9 @@ public class GeoDistanceRangeQuery extends Query { private final IndexGeoPointFieldData indexFieldData; - public GeoDistanceRangeQuery(GeoPoint point, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper, GeoDistance geoDistance, GeoPointFieldMapper.GeoPointFieldType fieldType, IndexGeoPointFieldData indexFieldData, - String optimizeBbox) { + public GeoDistanceRangeQuery(GeoPoint point, Double lowerVal, Double upperVal, boolean includeLower, + boolean includeUpper, GeoDistance geoDistance, GeoPointFieldMapperLegacy.GeoPointFieldType fieldType, + IndexGeoPointFieldData indexFieldData, String optimizeBbox) { this.lat = point.lat(); this.lon = point.lon(); this.geoDistance = geoDistance; @@ -170,6 +170,16 @@ public class GeoDistanceRangeQuery extends Query { } return false; } + + @Override + public float matchCost() { + if (distanceBoundingCheck == GeoDistance.ALWAYS_INSTANCE) { + return 0.0f; + } else { + // TODO: is this right (up to 4 comparisons from GeoDistance.SimpleDistanceBoundingCheck)? + return 4.0f; + } + } }; return new ConstantScoreScorer(this, score(), twoPhaseIterator); } @@ -196,7 +206,7 @@ public class GeoDistanceRangeQuery extends Query { @Override public String toString(String field) { - return "GeoDistanceRangeFilter(" + indexFieldData.getFieldNames().indexName() + ", " + geoDistance + ", [" + inclusiveLowerPoint + " - " + inclusiveUpperPoint + "], " + lat + ", " + lon + ")"; + return "GeoDistanceRangeQuery(" + indexFieldData.getFieldNames().indexName() + ", " + geoDistance + ", [" + inclusiveLowerPoint + " - " + inclusiveUpperPoint + "], " + lat + ", " + lon + ")"; } @Override @@ -204,13 +214,13 @@ public class GeoDistanceRangeQuery extends Query { int result = super.hashCode(); long temp; temp = lat != +0.0d ? Double.doubleToLongBits(lat) : 0L; - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + Long.hashCode(temp); temp = lon != +0.0d ? Double.doubleToLongBits(lon) : 0L; - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + Long.hashCode(temp); temp = inclusiveLowerPoint != +0.0d ? Double.doubleToLongBits(inclusiveLowerPoint) : 0L; - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + Long.hashCode(temp); temp = inclusiveUpperPoint != +0.0d ? Double.doubleToLongBits(inclusiveUpperPoint) : 0L; - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + Long.hashCode(temp); result = 31 * result + (geoDistance != null ? geoDistance.hashCode() : 0); result = 31 * result + indexFieldData.getFieldNames().indexName().hashCode(); return result; diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java index 5fef3c906f3..71e369cce0b 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java @@ -103,7 +103,7 @@ public class GeoPolygonQuery extends Query { @Override public String toString(String field) { - StringBuilder sb = new StringBuilder("GeoPolygonFilter("); + StringBuilder sb = new StringBuilder("GeoPolygonQuery("); sb.append(indexFieldData.getFieldNames().indexName()); sb.append(", ").append(Arrays.toString(points)).append(')'); return sb.toString(); diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java index 8d7dba292f4..a2e9e1b689d 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import java.io.IOException; +import java.util.Objects; /** * @@ -94,11 +95,7 @@ public class InMemoryGeoBoundingBoxQuery extends Query { @Override public int hashCode() { - int h = super.hashCode(); - h = 31 * h + fieldName().hashCode(); - h = 31 * h + topLeft.hashCode(); - h = 31 * h + bottomRight.hashCode(); - return h; + return Objects.hash(super.hashCode(), fieldName(), topLeft, bottomRight); } private static class Meridian180GeoBoundingBoxBits implements Bits { diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxQuery.java index 2430ac3cf7e..13290f98920 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxQuery.java @@ -25,12 +25,13 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapperLegacy; /** */ public class IndexedGeoBoundingBoxQuery { - public static Query create(GeoPoint topLeft, GeoPoint bottomRight, GeoPointFieldMapper.GeoPointFieldType fieldType) { + public static Query create(GeoPoint topLeft, GeoPoint bottomRight, GeoPointFieldMapperLegacy.GeoPointFieldType fieldType) { if (!fieldType.isLatLonEnabled()) { throw new IllegalArgumentException("lat/lon is not enabled (indexed) for field [" + fieldType.names().fullName() + "], can't use indexed filter on it"); } @@ -42,7 +43,7 @@ public class IndexedGeoBoundingBoxQuery { } } - private static Query westGeoBoundingBoxFilter(GeoPoint topLeft, GeoPoint bottomRight, GeoPointFieldMapper.GeoPointFieldType fieldType) { + private static Query westGeoBoundingBoxFilter(GeoPoint topLeft, GeoPoint bottomRight, GeoPointFieldMapperLegacy.GeoPointFieldType fieldType) { BooleanQuery.Builder filter = new BooleanQuery.Builder(); filter.setMinimumNumberShouldMatch(1); filter.add(fieldType.lonFieldType().rangeQuery(null, bottomRight.lon(), true, true), Occur.SHOULD); @@ -51,7 +52,7 @@ public class IndexedGeoBoundingBoxQuery { return new ConstantScoreQuery(filter.build()); } - private static Query eastGeoBoundingBoxFilter(GeoPoint topLeft, GeoPoint bottomRight, GeoPointFieldMapper.GeoPointFieldType fieldType) { + private static Query eastGeoBoundingBoxFilter(GeoPoint topLeft, GeoPoint bottomRight, GeoPointFieldMapperLegacy.GeoPointFieldType fieldType) { BooleanQuery.Builder filter = new BooleanQuery.Builder(); filter.add(fieldType.lonFieldType().rangeQuery(topLeft.lon(), bottomRight.lon(), true, true), Occur.MUST); filter.add(fieldType.latFieldType().rangeQuery(bottomRight.lat(), topLeft.lat(), true, true), Occur.MUST); diff --git a/core/src/main/java/org/elasticsearch/index/search/nested/IncludeNestedDocsQuery.java b/core/src/main/java/org/elasticsearch/index/search/nested/IncludeNestedDocsQuery.java deleted file mode 100644 index e3631269fbe..00000000000 --- a/core/src/main/java/org/elasticsearch/index/search/nested/IncludeNestedDocsQuery.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.search.nested; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.util.BitSet; - -import java.io.IOException; -import java.util.Collection; -import java.util.Set; - -/** - * A special query that accepts a top level parent matching query, and returns the nested docs of the matching parent - * doc as well. This is handy when deleting by query, don't use it for other purposes. - * - * @elasticsearch.internal - */ -public class IncludeNestedDocsQuery extends Query { - - private final BitSetProducer parentFilter; - private final Query parentQuery; - - // If we are rewritten, this is the original childQuery we - // were passed; we use this for .equals() and - // .hashCode(). This makes rewritten query equal the - // original, so that user does not have to .rewrite() their - // query before searching: - private final Query origParentQuery; - - - public IncludeNestedDocsQuery(Query parentQuery, BitSetProducer parentFilter) { - this.origParentQuery = parentQuery; - this.parentQuery = parentQuery; - this.parentFilter = parentFilter; - } - - // For rewriting - IncludeNestedDocsQuery(Query rewrite, Query originalQuery, IncludeNestedDocsQuery previousInstance) { - this.origParentQuery = originalQuery; - this.parentQuery = rewrite; - this.parentFilter = previousInstance.parentFilter; - setBoost(previousInstance.getBoost()); - } - - // For cloning - IncludeNestedDocsQuery(Query originalQuery, IncludeNestedDocsQuery previousInstance) { - this.origParentQuery = originalQuery; - this.parentQuery = originalQuery; - this.parentFilter = previousInstance.parentFilter; - } - - @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new IncludeNestedDocsWeight(this, parentQuery, parentQuery.createWeight(searcher, needsScores), parentFilter); - } - - static class IncludeNestedDocsWeight extends Weight { - - private final Query parentQuery; - private final Weight parentWeight; - private final BitSetProducer parentsFilter; - - IncludeNestedDocsWeight(Query query, Query parentQuery, Weight parentWeight, BitSetProducer parentsFilter) { - super(query); - this.parentQuery = parentQuery; - this.parentWeight = parentWeight; - this.parentsFilter = parentsFilter; - } - - @Override - public void extractTerms(Set terms) { - parentWeight.extractTerms(terms); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - parentWeight.normalize(norm, topLevelBoost); - } - - @Override - public float getValueForNormalization() throws IOException { - return parentWeight.getValueForNormalization(); // this query is never boosted so just delegate... - } - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - final Scorer parentScorer = parentWeight.scorer(context); - - // no matches - if (parentScorer == null) { - return null; - } - - BitSet parents = parentsFilter.getBitSet(context); - if (parents == null) { - // No matches - return null; - } - - int firstParentDoc = parentScorer.nextDoc(); - if (firstParentDoc == DocIdSetIterator.NO_MORE_DOCS) { - // No matches - return null; - } - return new IncludeNestedDocsScorer(this, parentScorer, parents, firstParentDoc); - } - - @Override - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - return null; //Query is used internally and not by users, so explain can be empty - } - } - - static class IncludeNestedDocsScorer extends Scorer { - - final Scorer parentScorer; - final BitSet parentBits; - - int currentChildPointer = -1; - int currentParentPointer = -1; - int currentDoc = -1; - - IncludeNestedDocsScorer(Weight weight, Scorer parentScorer, BitSet parentBits, int currentParentPointer) { - super(weight); - this.parentScorer = parentScorer; - this.parentBits = parentBits; - this.currentParentPointer = currentParentPointer; - if (currentParentPointer == 0) { - currentChildPointer = 0; - } else { - this.currentChildPointer = this.parentBits.prevSetBit(currentParentPointer - 1); - if (currentChildPointer == -1) { - // no previous set parent, we delete from doc 0 - currentChildPointer = 0; - } else { - currentChildPointer++; // we only care about children - } - } - - currentDoc = currentChildPointer; - } - - @Override - public Collection getChildren() { - return parentScorer.getChildren(); - } - - @Override - public int nextDoc() throws IOException { - if (currentParentPointer == NO_MORE_DOCS) { - return (currentDoc = NO_MORE_DOCS); - } - - if (currentChildPointer == currentParentPointer) { - // we need to return the current parent as well, but prepare to return - // the next set of children - currentDoc = currentParentPointer; - currentParentPointer = parentScorer.nextDoc(); - if (currentParentPointer != NO_MORE_DOCS) { - currentChildPointer = parentBits.prevSetBit(currentParentPointer - 1); - if (currentChildPointer == -1) { - // no previous set parent, just set the child to the current parent - currentChildPointer = currentParentPointer; - } else { - currentChildPointer++; // we only care about children - } - } - } else { - currentDoc = currentChildPointer++; - } - - assert currentDoc != -1; - return currentDoc; - } - - @Override - public int advance(int target) throws IOException { - if (target == NO_MORE_DOCS) { - return (currentDoc = NO_MORE_DOCS); - } - - if (target == 0) { - return nextDoc(); - } - - if (target < currentParentPointer) { - currentDoc = currentParentPointer = parentScorer.advance(target); - if (currentParentPointer == NO_MORE_DOCS) { - return (currentDoc = NO_MORE_DOCS); - } - if (currentParentPointer == 0) { - currentChildPointer = 0; - } else { - currentChildPointer = parentBits.prevSetBit(currentParentPointer - 1); - if (currentChildPointer == -1) { - // no previous set parent, just set the child to 0 to delete all up to the parent - currentChildPointer = 0; - } else { - currentChildPointer++; // we only care about children - } - } - } else { - currentDoc = currentChildPointer++; - } - - return currentDoc; - } - - @Override - public float score() throws IOException { - return parentScorer.score(); - } - - @Override - public int freq() throws IOException { - return parentScorer.freq(); - } - - @Override - public int docID() { - return currentDoc; - } - - @Override - public long cost() { - return parentScorer.cost(); - } - } - - @Override - public Query rewrite(IndexReader reader) throws IOException { - final Query parentRewrite = parentQuery.rewrite(reader); - if (parentRewrite != parentQuery) { - return new IncludeNestedDocsQuery(parentRewrite, parentQuery, this); - } else { - return this; - } - } - - @Override - public String toString(String field) { - return "IncludeNestedDocsQuery (" + parentQuery.toString() + ")"; - } - - @Override - public boolean equals(Object _other) { - if (_other instanceof IncludeNestedDocsQuery) { - final IncludeNestedDocsQuery other = (IncludeNestedDocsQuery) _other; - return origParentQuery.equals(other.origParentQuery) && parentFilter.equals(other.parentFilter); - } else { - return false; - } - } - - @Override - public int hashCode() { - final int prime = 31; - int hash = 1; - hash = prime * hash + origParentQuery.hashCode(); - hash = prime * hash + parentFilter.hashCode(); - return hash; - } - - @Override - public Query clone() { - Query clonedQuery = origParentQuery.clone(); - return new IncludeNestedDocsQuery(clonedQuery, this); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java b/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java index cfb74027a79..108dab449a3 100644 --- a/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java +++ b/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java @@ -189,24 +189,11 @@ public final class SearchSlowLog{ sb.append("], "); } sb.append("search_type[").append(context.searchType()).append("], total_shards[").append(context.numberOfShards()).append("], "); - if (context.request().source() != null && context.request().source().length() > 0) { - try { - sb.append("source[").append(XContentHelper.convertToJson(context.request().source(), reformat)).append("], "); - } catch (IOException e) { - sb.append("source[_failed_to_convert_], "); - } + if (context.request().source() != null) { + sb.append("source[").append(context.request().source()).append("], "); } else { sb.append("source[], "); } - if (context.request().extraSource() != null && context.request().extraSource().length() > 0) { - try { - sb.append("extra_source[").append(XContentHelper.convertToJson(context.request().extraSource(), reformat)).append("], "); - } catch (IOException e) { - sb.append("extra_source[_failed_to_convert_], "); - } - } else { - sb.append("extra_source[], "); - } return sb.toString(); } } diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java index 2d3ee81d0ce..c35a4cdbadb 100644 --- a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java +++ b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java @@ -221,7 +221,7 @@ public class SearchStats implements Streamable, ToXContent { if (searchStats == null) { return; } - totalStats.add(searchStats.totalStats); + addTotals(searchStats); openContexts += searchStats.openContexts; if (includeTypes && searchStats.groupStats != null && !searchStats.groupStats.isEmpty()) { if (groupStats == null) { @@ -238,6 +238,13 @@ public class SearchStats implements Streamable, ToXContent { } } + public void addTotals(SearchStats searchStats) { + if (searchStats == null) { + return; + } + totalStats.add(searchStats.totalStats); + } + public Stats getTotal() { return this.totalStats; } diff --git a/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsProvider.java b/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsProvider.java deleted file mode 100644 index f1d58580a4c..00000000000 --- a/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsProvider.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.settings; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.Provider; -import org.elasticsearch.common.settings.Settings; - -/** - * A wrapper around the {@link IndexSettingsService} allowing to get the current - * settings associated with an index (thus, allowing to change those settings and - * new shards will use the new settings). - */ -public class IndexSettingsProvider implements Provider { - - private final IndexSettingsService indexSettingsService; - - @Inject - public IndexSettingsProvider(IndexSettingsService indexSettingsService) { - this.indexSettingsService = indexSettingsService; - } - - @Override - public Settings get() { - return indexSettingsService.getSettings(); - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsService.java b/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsService.java deleted file mode 100644 index d76540e1e85..00000000000 --- a/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsService.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.settings; - -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; - -import java.util.concurrent.CopyOnWriteArrayList; - -/** - * A holds to the latest, updated settings for an index. - */ -public class IndexSettingsService extends AbstractIndexComponent { - - private volatile Settings settings; - - private final CopyOnWriteArrayList listeners = new CopyOnWriteArrayList<>(); - - @Inject - public IndexSettingsService(Index index, Settings settings) { - super(index, settings); - this.settings = settings; - } - - public synchronized void refreshSettings(Settings settings) { - // this.settings include also the node settings - if (this.settings.getByPrefix(IndexMetaData.INDEX_SETTING_PREFIX).getAsMap().equals(settings.getByPrefix(IndexMetaData.INDEX_SETTING_PREFIX).getAsMap())) { - // nothing to update, same settings - return; - } - this.settings = Settings.settingsBuilder().put(this.settings).put(settings).build(); - for (Listener listener : listeners) { - try { - listener.onRefreshSettings(settings); - } catch (Exception e) { - logger.warn("failed to refresh settings for [{}]", e, listener); - } - } - } - - public Settings getSettings() { - return this.settings; - } - - /** - * Only settings registered in {@link org.elasticsearch.cluster.ClusterModule} can be changed dynamically. - */ - public void addListener(Listener listener) { - this.listeners.add(listener); - } - - public void removeListener(Listener listener) { - this.listeners.remove(listener); - } - - /** - * Returns true iff the given listener is already registered otherwise false - */ - public boolean isRegistered(Listener listener) { - return listeners.contains(listener); - } - public interface Listener { - void onRefreshSettings(Settings settings); - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/shard/AbstractIndexShardComponent.java b/core/src/main/java/org/elasticsearch/index/shard/AbstractIndexShardComponent.java index 76ae65522c4..c8719a610e2 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/AbstractIndexShardComponent.java +++ b/core/src/main/java/org/elasticsearch/index/shard/AbstractIndexShardComponent.java @@ -22,8 +22,7 @@ package org.elasticsearch.index.shard; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** * @@ -33,12 +32,12 @@ public abstract class AbstractIndexShardComponent implements IndexShardComponent protected final ESLogger logger; protected final DeprecationLogger deprecationLogger; protected final ShardId shardId; - protected final Settings indexSettings; + protected final IndexSettings indexSettings; - protected AbstractIndexShardComponent(ShardId shardId, @IndexSettings Settings indexSettings) { + protected AbstractIndexShardComponent(ShardId shardId, IndexSettings indexSettings) { this.shardId = shardId; this.indexSettings = indexSettings; - this.logger = Loggers.getLogger(getClass(), indexSettings, shardId); + this.logger = Loggers.getLogger(getClass(), this.indexSettings.getSettings(), shardId); this.deprecationLogger = new DeprecationLogger(logger); } @@ -48,12 +47,12 @@ public abstract class AbstractIndexShardComponent implements IndexShardComponent } @Override - public Settings indexSettings() { - return this.indexSettings; + public IndexSettings indexSettings() { + return indexSettings; } public String nodeName() { - return indexSettings.get("name", ""); + return indexSettings.getNodeName(); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java b/core/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java new file mode 100644 index 00000000000..9a55b9b6161 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java @@ -0,0 +1,188 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; + +/** + * An index event listener is the primary extension point for plugins and build-in services + * to react / listen to per-index and per-shard events. These listeners are registered per-index + * via {@link org.elasticsearch.index.IndexModule#addIndexEventListener(IndexEventListener)}. All listeners have the same + * lifecycle as the {@link IndexService} they are created for. + *

    + * An IndexEventListener can be used across multiple indices and shards since all callback methods receive sufficient + * local state via their arguments. Yet, if an instance is shared across indices they might be called concurrently and should not + * modify local state without sufficient synchronization. + *

    + */ +public interface IndexEventListener { + + /** + * Called when the shard routing has changed state. + * + * @param indexShard The index shard + * @param oldRouting The old routing state (can be null) + * @param newRouting The new routing state + */ + default void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting oldRouting, ShardRouting newRouting) {} + + /** + * Called after the index shard has been created. + */ + default void afterIndexShardCreated(IndexShard indexShard) {} + + /** + * Called after the index shard has been started. + */ + default void afterIndexShardStarted(IndexShard indexShard) {} + + /** + * Called before the index shard gets closed. + * + * @param indexShard The index shard + */ + default void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) {} + + /** + * Called after the index shard has been closed. + * + * @param shardId The shard id + */ + default void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) {} + + + /** + * Called after a shard's {@link org.elasticsearch.index.shard.IndexShardState} changes. + * The order of concurrent events is preserved. The execution must be lightweight. + * + * @param indexShard the shard the new state was applied to + * @param previousState the previous index shard state if there was one, null otherwise + * @param currentState the new shard state + * @param reason the reason for the state change if there is one, null otherwise + */ + default void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) {} + + /** + * Called when a shard is marked as inactive + * + * @param indexShard The shard that was marked inactive + */ + default void onShardInactive(IndexShard indexShard) {} + + /** + * Called when a shard is marked as active ie. was previously inactive and is now active again. + * + * @param indexShard The shard that was marked active + */ + default void onShardActive(IndexShard indexShard) {} + + /** + * Called before the index gets created. Note that this is also called + * when the index is created on data nodes + */ + default void beforeIndexCreated(Index index, Settings indexSettings) { + + } + + /** + * Called after the index has been created. + */ + default void afterIndexCreated(IndexService indexService) { + + } + + /** + * Called before the index shard gets created. + */ + default void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) { + } + + + /** + * Called before the index get closed. + * + * @param indexService The index service + */ + default void beforeIndexClosed(IndexService indexService) { + + } + + /** + * Called after the index has been closed. + * + * @param index The index + */ + default void afterIndexClosed(Index index, Settings indexSettings) { + + } + + /** + * Called before the index shard gets deleted from disk + * Note: this method is only executed on the first attempt of deleting the shard. Retries are will not invoke + * this method. + * @param shardId The shard id + * @param indexSettings the shards index settings + */ + default void beforeIndexShardDeleted(ShardId shardId, Settings indexSettings) { + } + + /** + * Called after the index shard has been deleted from disk. + * + * Note: this method is only called if the deletion of the shard did finish without an exception + * + * @param shardId The shard id + * @param indexSettings the shards index settings + */ + default void afterIndexShardDeleted(ShardId shardId, Settings indexSettings) { + } + + /** + * Called after the index has been deleted. + * This listener method is invoked after {@link #afterIndexClosed(org.elasticsearch.index.Index, org.elasticsearch.common.settings.Settings)} + * when an index is deleted + * + * @param index The index + */ + default void afterIndexDeleted(Index index, Settings indexSettings) { + + } + + /** + * Called before the index gets deleted. + * This listener method is invoked after + * {@link #beforeIndexClosed(org.elasticsearch.index.IndexService)} when an index is deleted + * + * @param indexService The index service + */ + default void beforeIndexDeleted(IndexService indexService) { + + } + + /** + * Called on the Master node only before the {@link IndexService} instances is created to simulate an index creation. + * This happens right before the index and it's metadata is registered in the cluster state + */ + default void beforeIndexAddedToCluster(Index index, Settings indexSettings) { + } +} diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java index dff59e9b244..5603001a293 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineConfig; import java.io.IOException; @@ -54,13 +53,11 @@ public class IndexSearcherWrapper { } /** - * @param engineConfig The engine config which can be used to get the query cache and query cache policy from - * when creating a new index searcher * @param searcher The provided index searcher to be wrapped to add custom functionality * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed * the provided index searcher */ - protected IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws IOException { + protected IndexSearcher wrap(IndexSearcher searcher) throws IOException { return searcher; } /** @@ -69,7 +66,7 @@ public class IndexSearcherWrapper { * * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search) */ - public final Engine.Searcher wrap(EngineConfig engineConfig, Engine.Searcher engineSearcher) throws IOException { + public final Engine.Searcher wrap(Engine.Searcher engineSearcher) throws IOException { final ElasticsearchDirectoryReader elasticsearchDirectoryReader = ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(engineSearcher.getDirectoryReader()); if (elasticsearchDirectoryReader == null) { throw new IllegalStateException("Can't wrap non elasticsearch directory reader"); @@ -87,14 +84,15 @@ public class IndexSearcherWrapper { } } + final IndexSearcher origIndexSearcher = engineSearcher.searcher(); final IndexSearcher innerIndexSearcher = new IndexSearcher(reader); - innerIndexSearcher.setQueryCache(engineConfig.getQueryCache()); - innerIndexSearcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy()); - innerIndexSearcher.setSimilarity(engineConfig.getSimilarity()); + innerIndexSearcher.setQueryCache(origIndexSearcher.getQueryCache()); + innerIndexSearcher.setQueryCachingPolicy(origIndexSearcher.getQueryCachingPolicy()); + innerIndexSearcher.setSimilarity(origIndexSearcher.getSimilarity(true)); // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times - final IndexSearcher indexSearcher = wrap(engineConfig, innerIndexSearcher); + final IndexSearcher indexSearcher = wrap(innerIndexSearcher); if (reader == nonClosingReaderWrapper && indexSearcher == innerIndexSearcher) { return engineSearcher; } else { diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 7caa98b6104..fcf204603f5 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -19,29 +19,25 @@ package org.elasticsearch.index.shard; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.*; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.util.CloseableThreadLocal; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.support.LoggerMessageFormat; @@ -51,14 +47,16 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.Callback; import org.elasticsearch.common.util.concurrent.AbstractRefCounted; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.gateway.MetaDataStateFormat; -import org.elasticsearch.index.IndexServicesProvider; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.cache.IndexCache; -import org.elasticsearch.index.cache.IndexCacheModule; import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.cache.request.ShardRequestCache; @@ -76,17 +74,15 @@ import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.percolator.PercolateStats; import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; -import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.search.stats.ShardSearchStats; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.snapshots.IndexShardRepository; -import org.elasticsearch.index.store.Store.MetadataSnapshot; import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.Store.MetadataSnapshot; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.suggest.stats.ShardSuggestMetric; @@ -99,13 +95,12 @@ import org.elasticsearch.index.translog.TranslogWriter; import org.elasticsearch.index.warmer.ShardIndexWarmerService; import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.indices.IndicesWarmer; -import org.elasticsearch.indices.InternalIndicesLifecycle; import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.indices.recovery.RecoveryFailedException; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.percolator.PercolatorService; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat; +import org.elasticsearch.search.suggest.completion.CompletionFieldStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.threadpool.ThreadPool; @@ -121,12 +116,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -public class IndexShard extends AbstractIndexShardComponent implements IndexSettingsService.Listener { +public class IndexShard extends AbstractIndexShardComponent { private final ThreadPool threadPool; private final MapperService mapperService; private final IndexCache indexCache; - private final InternalIndicesLifecycle indicesLifecycle; private final Store store; private final MergeSchedulerConfig mergeSchedulerConfig; private final ShardIndexingService indexingService; @@ -150,6 +144,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett private final TranslogConfig translogConfig; private final MergePolicyConfig mergePolicyConfig; private final IndicesQueryCache indicesQueryCache; + private final IndexEventListener indexEventListener; + private final IndexSettings idxSettings; + private final NodeServicesProvider provider; private TimeValue refreshInterval; @@ -167,7 +164,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett private final MeanMetric refreshMetric = new MeanMetric(); private final MeanMetric flushMetric = new MeanMetric(); - private final ShardEngineFailListener failedEngineListener = new ShardEngineFailListener(); + private final ShardEventListener shardEventListener = new ShardEventListener(); private volatile boolean flushOnClose = true; private volatile int flushThresholdOperations; private volatile ByteSizeValue flushThresholdSize; @@ -191,67 +188,71 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett private final IndexSearcherWrapper searcherWrapper; - /** True if this shard is still indexing (recently) and false if we've been idle for long enough (as periodically checked by {@link - * IndexingMemoryController}). */ + /** + * True if this shard is still indexing (recently) and false if we've been idle for long enough (as periodically checked by {@link + * IndexingMemoryController}). + */ private final AtomicBoolean active = new AtomicBoolean(); - - private volatile long lastWriteNS; private final IndexingMemoryController indexingMemoryController; - @Inject - public IndexShard(ShardId shardId, @IndexSettings Settings indexSettings, ShardPath path, Store store, IndexServicesProvider provider) { + public IndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache, + MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService, + @Nullable EngineFactory engineFactory, + IndexEventListener indexEventListener, IndexSearcherWrapper indexSearcherWrapper, NodeServicesProvider provider) { super(shardId, indexSettings); - this.codecService = provider.getCodecService(); + final Settings settings = indexSettings.getSettings(); + this.idxSettings = indexSettings; + this.codecService = new CodecService(mapperService, logger); this.warmer = provider.getWarmer(); this.deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - this.similarityService = provider.getSimilarityService(); + this.similarityService = similarityService; Objects.requireNonNull(store, "Store must be provided to the index shard"); - this.engineFactory = provider.getFactory(); - this.indicesLifecycle = (InternalIndicesLifecycle) provider.getIndicesLifecycle(); + this.engineFactory = engineFactory == null ? new InternalEngineFactory() : engineFactory; this.store = store; + this.indexEventListener = indexEventListener; this.mergeSchedulerConfig = new MergeSchedulerConfig(indexSettings); this.threadPool = provider.getThreadPool(); - this.mapperService = provider.getMapperService(); - this.indexCache = provider.getIndexCache(); + this.mapperService = mapperService; + this.indexCache = indexCache; this.indexingService = new ShardIndexingService(shardId, indexSettings); - this.getService = new ShardGetService(this, mapperService); - this.termVectorsService = provider.getTermVectorsService(); - this.searchService = new ShardSearchStats(indexSettings); + this.getService = new ShardGetService(indexSettings, this, mapperService); + this.termVectorsService = provider.getTermVectorsService(); + this.searchService = new ShardSearchStats(settings); this.shardWarmerService = new ShardIndexWarmerService(shardId, indexSettings); - this.indicesQueryCache = provider.getIndicesQueryCache(); + this.indicesQueryCache = provider.getIndicesQueryCache(); this.shardQueryCache = new ShardRequestCache(shardId, indexSettings); this.shardFieldData = new ShardFieldData(); - this.indexFieldDataService = provider.getIndexFieldDataService(); + this.indexFieldDataService = indexFieldDataService; this.shardBitsetFilterCache = new ShardBitsetFilterCache(shardId, indexSettings); state = IndexShardState.CREATED; - this.refreshInterval = indexSettings.getAsTime(INDEX_REFRESH_INTERVAL, EngineConfig.DEFAULT_REFRESH_INTERVAL); - this.flushOnClose = indexSettings.getAsBoolean(INDEX_FLUSH_ON_CLOSE, true); + this.refreshInterval = settings.getAsTime(INDEX_REFRESH_INTERVAL, EngineConfig.DEFAULT_REFRESH_INTERVAL); + this.flushOnClose = settings.getAsBoolean(INDEX_FLUSH_ON_CLOSE, true); this.path = path; - this.mergePolicyConfig = new MergePolicyConfig(logger, indexSettings); + this.mergePolicyConfig = new MergePolicyConfig(logger, settings); /* create engine config */ - logger.debug("state: [CREATED]"); - this.checkIndexOnStartup = indexSettings.get("index.shard.check_on_startup", "false"); - this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, getFromSettings(logger, indexSettings, Translog.Durabilty.REQUEST), - provider.getBigArrays(), threadPool); + this.checkIndexOnStartup = settings.get("index.shard.check_on_startup", "false"); + this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, getFromSettings(logger, settings, Translog.Durabilty.REQUEST), + provider.getBigArrays(), threadPool); final QueryCachingPolicy cachingPolicy; // the query cache is a node-level thing, however we want the most popular filters // to be computed on a per-shard basis - if (indexSettings.getAsBoolean(IndexCacheModule.QUERY_CACHE_EVERYTHING, false)) { + if (settings.getAsBoolean(IndexModule.QUERY_CACHE_EVERYTHING, false)) { cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE; } else { cachingPolicy = new UsageTrackingQueryCachingPolicy(); } + this.engineConfig = newEngineConfig(translogConfig, cachingPolicy); - this.flushThresholdOperations = indexSettings.getAsInt(INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, indexSettings.getAsInt("index.translog.flush_threshold", Integer.MAX_VALUE)); - this.flushThresholdSize = indexSettings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB)); - this.disableFlush = indexSettings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false); + this.flushThresholdOperations = settings.getAsInt(INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, settings.getAsInt("index.translog.flush_threshold", Integer.MAX_VALUE)); + this.flushThresholdSize = settings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB)); + this.disableFlush = settings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false); this.indexShardOperationCounter = new IndexShardOperationCounter(logger, shardId); this.indexingMemoryController = provider.getIndexingMemoryController(); - - this.searcherWrapper = provider.getIndexSearcherWrapper(); - this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, provider.getQueryParserService(), indexingService, mapperService, indexFieldDataService); + this.provider = provider; + this.searcherWrapper = indexSearcherWrapper; + this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, indexingService, mapperService, newQueryShardContext(), indexFieldDataService); if (mapperService.hasMapping(PercolatorService.TYPE_NAME)) { percolatorQueriesRegistry.enableRealTimePercolator(); } @@ -261,6 +262,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return this.store; } + public IndexSettings getIndexSettings() { + return idxSettings; + } + /** returns true if this shard supports indexing (i.e., write) operations. */ public boolean canIndex() { return true; @@ -365,12 +370,12 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } } if (movedToStarted) { - indicesLifecycle.afterIndexShardStarted(this); + indexEventListener.afterIndexShardStarted(this); } } } this.shardRouting = newRouting; - indicesLifecycle.shardRoutingChanged(this, currentRouting, newRouting); + indexEventListener.shardRoutingChanged(this, currentRouting, newRouting); } finally { if (persistState) { persistMetadata(newRouting, currentRouting); @@ -381,8 +386,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett /** * Marks the shard as recovering based on a recovery state, fails with exception is recovering is not allowed to be set. */ - public IndexShardState recovering(String reason, RecoveryState recoveryState) throws IndexShardStartedException, - IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException { + public IndexShardState markAsRecovering(String reason, RecoveryState recoveryState) throws IndexShardStartedException, + IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException { synchronized (mutex) { if (state == IndexShardState.CLOSED) { throw new IndexShardClosedException(shardId); @@ -429,13 +434,25 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett logger.debug("state: [{}]->[{}], reason [{}]", state, newState, reason); IndexShardState previousState = state; state = newState; - this.indicesLifecycle.indexShardStateChanged(this, previousState, reason); + this.indexEventListener.indexShardStateChanged(this, previousState, newState, reason); return previousState; } - public Engine.Index prepareIndex(SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin) { + public Engine.Index prepareIndexOnPrimary(SourceToParse source, long version, VersionType versionType) { try { - return prepareIndex(docMapper(source.type()), source, version, versionType, origin); + if (shardRouting.primary() == false) { + throw new IllegalIndexShardStateException(shardId, state, "shard is not a primary"); + } + return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.PRIMARY); + } catch (Throwable t) { + verifyNotClosed(t); + throw t; + } + } + + public Engine.Index prepareIndexOnReplica(SourceToParse source, long version, VersionType versionType) { + try { + return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.REPLICA); } catch (Throwable t) { verifyNotClosed(t); throw t; @@ -457,7 +474,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett */ public boolean index(Engine.Index index) { ensureWriteAllowed(index); - markLastWrite(index); + markLastWrite(); index = indexingService.preIndex(index); final boolean created; try { @@ -475,15 +492,27 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return created; } - public Engine.Delete prepareDelete(String type, String id, long version, VersionType versionType, Engine.Operation.Origin origin) { - long startTime = System.nanoTime(); + public Engine.Delete prepareDeleteOnPrimary(String type, String id, long version, VersionType versionType) { + if (shardRouting.primary() == false) { + throw new IllegalIndexShardStateException(shardId, state, "shard is not a primary"); + } final DocumentMapper documentMapper = docMapper(type).getDocumentMapper(); - return new Engine.Delete(type, id, documentMapper.uidMapper().term(Uid.createUid(type, id)), version, versionType, origin, startTime, false); + return prepareDelete(type, id, documentMapper.uidMapper().term(Uid.createUid(type, id)), version, versionType, Engine.Operation.Origin.PRIMARY); + } + + public Engine.Delete prepareDeleteOnReplica(String type, String id, long version, VersionType versionType) { + final DocumentMapper documentMapper = docMapper(type).getDocumentMapper(); + return prepareDelete(type, id, documentMapper.uidMapper().term(Uid.createUid(type, id)), version, versionType, Engine.Operation.Origin.REPLICA); + } + + static Engine.Delete prepareDelete(String type, String id, Term uid, long version, VersionType versionType, Engine.Operation.Origin origin) { + long startTime = System.nanoTime(); + return new Engine.Delete(type, id, uid, version, versionType, origin, startTime, false); } public void delete(Engine.Delete delete) { ensureWriteAllowed(delete); - markLastWrite(delete); + markLastWrite(); delete = indexingService.preDelete(delete); try { if (logger.isTraceEnabled()) { @@ -508,7 +537,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett verifyNotClosed(); // nocommit OK to throw EngineClosedExc? long ramBytesUsed = getEngine().indexBufferRAMBytesUsed(); - indexingMemoryController.addRefreshingBytes(shardId, ramBytesUsed); + indexingMemoryController.addRefreshingBytes(this, ramBytesUsed); try { if (logger.isTraceEnabled()) { logger.trace("refresh with source: {} indexBufferRAMBytesUsed={}", source, ramBytesUsed); @@ -517,7 +546,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett getEngine().refresh(source); refreshMetric.inc(System.nanoTime() - time); } finally { - indexingMemoryController.removeRefreshingBytes(shardId, ramBytesUsed); + indexingMemoryController.removeRefreshingBytes(this, ramBytesUsed); } } @@ -530,11 +559,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } public DocsStats docStats() { - final Engine.Searcher searcher = acquireSearcher("doc_stats"); - try { + try (Engine.Searcher searcher = acquireSearcher("doc_stats")) { return new DocsStats(searcher.reader().numDocs(), searcher.reader().numDeletedDocs()); - } finally { - searcher.close(); } } @@ -613,15 +639,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett public CompletionStats completionStats(String... fields) { CompletionStats completionStats = new CompletionStats(); - final Engine.Searcher currentSearcher = acquireSearcher("completion_stats"); - try { - PostingsFormat postingsFormat = PostingsFormat.forName(Completion090PostingsFormat.CODEC_NAME); - if (postingsFormat instanceof Completion090PostingsFormat) { - Completion090PostingsFormat completionPostingsFormat = (Completion090PostingsFormat) postingsFormat; - completionStats.add(completionPostingsFormat.completionStats(currentSearcher.reader(), fields)); - } - } finally { - currentSearcher.close(); + try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) { + completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields)); } return completionStats; } @@ -650,12 +669,13 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } - public void optimize(OptimizeRequest optimize) throws IOException { + public void forceMerge(ForceMergeRequest forceMerge) throws IOException { verifyStarted(); if (logger.isTraceEnabled()) { - logger.trace("optimize with {}", optimize); + logger.trace("force merge with {}", forceMerge); } - getEngine().forceMerge(optimize.flush(), optimize.maxNumSegments(), optimize.onlyExpungeDeletes(), false, false); + getEngine().forceMerge(forceMerge.flush(), forceMerge.maxNumSegments(), + forceMerge.onlyExpungeDeletes(), false, false); } /** @@ -667,10 +687,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett logger.trace("upgrade with {}", upgrade); } org.apache.lucene.util.Version previousVersion = minimumCompatibleVersion(); - // we just want to upgrade the segments, not actually optimize to a single segment + // we just want to upgrade the segments, not actually forge merge to a single segment getEngine().forceMerge(true, // we need to flush at the end to make sure the upgrade is durable - Integer.MAX_VALUE, // we just want to upgrade the segments, not actually optimize to a single segment - false, true, upgrade.upgradeOnlyAncientSegments()); + Integer.MAX_VALUE, // we just want to upgrade the segments, not actually optimize to a single segment + false, true, upgrade.upgradeOnlyAncientSegments()); org.apache.lucene.util.Version version = minimumCompatibleVersion(); if (logger.isTraceEnabled()) { logger.trace("upgraded segment {} from version {} to version {}", previousVersion, version); @@ -686,7 +706,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett luceneVersion = segment.getVersion(); } } - return luceneVersion == null ? Version.indexCreated(indexSettings).luceneVersion : luceneVersion; + return luceneVersion == null ? idxSettings.getIndexVersionCreated().luceneVersion : luceneVersion; } /** @@ -729,7 +749,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett final Engine.Searcher searcher = engine.acquireSearcher(source); boolean success = false; try { - final Engine.Searcher wrappedSearcher = searcherWrapper == null ? searcher : searcherWrapper.wrap(engineConfig, searcher); + final Engine.Searcher wrappedSearcher = searcherWrapper == null ? searcher : searcherWrapper.wrap(searcher); assert wrappedSearcher != null; success = true; return wrappedSearcher; @@ -760,7 +780,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett engine.flushAndClose(); } } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times - IOUtils.close(engine, percolatorQueriesRegistry); + IOUtils.close(engine, percolatorQueriesRegistry, queryShardContextCache); } } } @@ -900,7 +920,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett public boolean ignoreRecoveryAttempt() { IndexShardState state = state(); // one time volatile read return state == IndexShardState.POST_RECOVERY || state == IndexShardState.RECOVERING || state == IndexShardState.STARTED || - state == IndexShardState.RELOCATED || state == IndexShardState.CLOSED; + state == IndexShardState.RELOCATED || state == IndexShardState.CLOSED; } public void readAllowed() throws IllegalIndexShardStateException { @@ -910,14 +930,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } } - /** Returns timestamp of last indexing operation */ - public long getLastWriteNS() { - return lastWriteNS; - } - - /** Records timestamp of the last write operation, possibly switching {@code active} to true if we were inactive. */ - private void markLastWrite(Engine.Operation op) { - lastWriteNS = op.startTime(); + /** Sets {@code active} to true if we were inactive. */ + private void markLastWrite() { active.set(true); } @@ -978,10 +992,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } } - public void addFailedEngineListener(Engine.FailedEngineListener failedEngineListener) { - this.failedEngineListener.delegates.add(failedEngineListener); - } - public long getIndexBufferRAMBytesUsed() { Engine engine = getEngineOrNull(); if (engine == null) { @@ -994,20 +1004,27 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } } + public void addShardFailureCallback(Callback onShardFailure) { + this.shardEventListener.delegates.add(onShardFailure); + } + /** Called by {@link IndexingMemoryController} to check whether more than {@code inactiveTimeNS} has passed since the last * indexing operation, and become inactive (reducing indexing and translog buffers to tiny values) if so. */ public void checkIdle(long inactiveTimeNS) { - if (System.nanoTime() - lastWriteNS >= inactiveTimeNS) { + Engine engineOrNull = getEngineOrNull(); + if (engineOrNull != null && System.nanoTime() - engineOrNull.getLastWriteNanos() >= inactiveTimeNS) { boolean wasActive = active.getAndSet(false); if (wasActive) { logger.debug("shard is now inactive"); - indicesLifecycle.onShardInactive(this); + indexEventListener.onShardInactive(this); } } } - /** Returns {@code true} if this shard is active (has seen indexing ops in the last {@link - * IndexingMemoryController#SHARD_INACTIVE_TIME_SETTING} (default 5 minutes), else {@code false}. */ + /** + * Returns {@code true} if this shard is active (has seen indexing ops in the last {@link + * IndexingMemoryController#SHARD_INACTIVE_TIME_SETTING} (default 5 minutes), else {@code false}. + */ public boolean getActive() { return active.get(); } @@ -1032,19 +1049,19 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return path; } - public boolean recoverFromStore(ShardRouting shard, DiscoveryNode localNode) { + public boolean recoverFromStore(DiscoveryNode localNode) { // we are the first primary, recover from the gateway // if its post api allocation, the index should exists - assert shard.primary() : "recover from store only makes sense if the shard is a primary shard"; - final boolean shouldExist = shard.allocatedPostIndexCreate(); + assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard"; + final boolean shouldExist = shardRouting.allocatedPostIndexCreate(); StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); return storeRecovery.recoverFromStore(this, shouldExist, localNode); } - public boolean restoreFromRepository(ShardRouting shard, IndexShardRepository repository, DiscoveryNode locaNode) { - assert shard.primary() : "recover from store only makes sense if the shard is a primary shard"; + public boolean restoreFromRepository(IndexShardRepository repository, DiscoveryNode localNode) { + assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard"; StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); - return storeRecovery.recoverFromRepository(this, repository, locaNode); + return storeRecovery.recoverFromRepository(this, repository, localNode); } /** @@ -1058,7 +1075,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett try { Translog translog = engine.getTranslog(); return translog.totalOperations() > flushThresholdOperations || translog.sizeInBytes() > flushThresholdSize.bytes(); - } catch (AlreadyClosedException ex) { + } catch (AlreadyClosedException | EngineClosedException ex) { // that's fine we are already close - no need to flush } } @@ -1066,7 +1083,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return false; } - @Override public void onRefreshSettings(Settings settings) { boolean change = false; synchronized (mutex) { @@ -1189,6 +1205,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return percolatorQueriesRegistry.stats(); } + public IndexEventListener getIndexEventListener() { + return indexEventListener; + } + /** * Asynchronously refreshes the engine for new search operations to reflect the latest * changes. @@ -1341,21 +1361,24 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return engine; } - /** NOTE: returns null if engine is not yet started (e.g. recovery phase 1, copying over index files, is still running), or if engine is - * closed. */ + /** + * NOTE: returns null if engine is not yet started (e.g. recovery phase 1, copying over index files, is still running), or if engine is + * closed. + */ protected Engine getEngineOrNull() { return this.currentEngineReference.get(); } - class ShardEngineFailListener implements Engine.FailedEngineListener { - private final CopyOnWriteArrayList delegates = new CopyOnWriteArrayList<>(); + class ShardEventListener implements Engine.EventListener { + private final CopyOnWriteArrayList> delegates = new CopyOnWriteArrayList<>(); // called by the current engine @Override - public void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable failure) { - for (Engine.FailedEngineListener listener : delegates) { + public void onFailedEngine(String reason, @Nullable Throwable failure) { + final ShardFailure shardFailure = new ShardFailure(shardRouting, reason, failure, getIndexUUID()); + for (Callback listener : delegates) { try { - listener.onFailedEngine(shardId, reason, failure); + listener.handle(shardFailure); } catch (Exception e) { logger.warn("exception while notifying engine failure", e); } @@ -1398,12 +1421,12 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett writeReason = "routing changed from " + currentRouting + " to " + newRouting; } else { logger.trace("skip writing shard state, has been written before; previous version: [" + - currentRouting.version() + "] current version [" + newRouting.version() + "]"); + currentRouting.version() + "] current version [" + newRouting.version() + "]"); assert currentRouting.version() <= newRouting.version() : "version should not go backwards for shardID: " + shardId + - " previous version: [" + currentRouting.version() + "] current version [" + newRouting.version() + "]"; + " previous version: [" + currentRouting.version() + "] current version [" + newRouting.version() + "]"; return; } - final ShardStateMetaData newShardStateMetadata = new ShardStateMetaData(newRouting.version(), newRouting.primary(), getIndexUUID()); + final ShardStateMetaData newShardStateMetadata = new ShardStateMetaData(newRouting.version(), newRouting.primary(), getIndexUUID(), newRouting.allocationId()); logger.trace("{} writing shard state, reason [{}]", shardId, writeReason); ShardStateMetaData.FORMAT.write(newShardStateMetadata, newShardStateMetadata.version, shardPath().getShardStatePath()); } catch (IOException e) { // this is how we used to handle it.... :( @@ -1415,10 +1438,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } private String getIndexUUID() { - assert indexSettings.get(IndexMetaData.SETTING_INDEX_UUID) != null - || indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).before(Version.V_0_90_6) : - "version: " + indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) + " uuid: " + indexSettings.get(IndexMetaData.SETTING_INDEX_UUID); - return indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); + return indexSettings.getUUID(); } private DocumentMapperForType docMapper(String type) { @@ -1433,9 +1453,11 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett recoveryState.getTranslog().incrementRecoveredOperations(); } }; + final Engine.Warmer engineWarmer = (searcher, toLevel) -> warmer.warm(searcher, this, idxSettings, toLevel); return new EngineConfig(shardId, - threadPool, indexingService, indexSettings, warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig, - mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig); + threadPool, indexingService, indexSettings, engineWarmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig, + mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, shardEventListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig, + idxSettings.getSettings().getAsTime(IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING, TimeValue.timeValueMinutes(5))); // nocommit } private static class IndexShardOperationCounter extends AbstractRefCounted { @@ -1549,4 +1571,47 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return false; } + /** + * Simple struct encapsulating a shard failure + * + * @see IndexShard#addShardFailureCallback(Callback) + */ + public static final class ShardFailure { + public final ShardRouting routing; + public final String reason; + @Nullable + public final Throwable cause; + public final String indexUUID; + + public ShardFailure(ShardRouting routing, String reason, @Nullable Throwable cause, String indexUUID) { + this.routing = routing; + this.reason = reason; + this.cause = cause; + this.indexUUID = indexUUID; + } + } + + private CloseableThreadLocal queryShardContextCache = new CloseableThreadLocal() { + // TODO We should get rid of this threadlocal but I think it should be a sep change + @Override + protected QueryShardContext initialValue() { + return newQueryShardContext(); + } + }; + + private QueryShardContext newQueryShardContext() { + return new QueryShardContext(idxSettings, provider.getClient(), indexCache.bitsetFilterCache(), indexFieldDataService, mapperService, similarityService, provider.getScriptService(), provider.getIndicesQueriesRegistry()); + } + + /** + * Returns a threadlocal QueryShardContext for this shard. + */ + public QueryShardContext getQueryShardContext() { + return queryShardContextCache.get(); + } + + EngineFactory getEngineFactory() { + return engineFactory; + } + } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShardComponent.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShardComponent.java index 38886032a7c..a1665a7d5c1 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShardComponent.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShardComponent.java @@ -19,7 +19,7 @@ package org.elasticsearch.index.shard; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; /** * @@ -28,5 +28,5 @@ public interface IndexShardComponent { ShardId shardId(); - Settings indexSettings(); + IndexSettings indexSettings(); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/MergePolicyConfig.java b/core/src/main/java/org/elasticsearch/index/shard/MergePolicyConfig.java index c664d3a3794..0a9315dbc25 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/MergePolicyConfig.java +++ b/core/src/main/java/org/elasticsearch/index/shard/MergePolicyConfig.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.index.settings.IndexSettingsService; /** * A shard in elasticsearch is a Lucene index, and a Lucene index is broken @@ -65,15 +64,15 @@ import org.elasticsearch.index.settings.IndexSettingsService; * *
  • index.merge.policy.max_merge_at_once_explicit: * - * Maximum number of segments to be merged at a time, during optimize or + * Maximum number of segments to be merged at a time, during force merge or * expungeDeletes. Default is 30. * *
  • index.merge.policy.max_merged_segment: * * Maximum sized segment to produce during normal merging (not explicit - * optimize). This setting is approximate: the estimate of the merged segment - * size is made by summing sizes of to-be-merged segments (compensating for - * percent deleted docs). Default is 5gb. + * force merge). This setting is approximate: the estimate of the merged + * segment size is made by summing sizes of to-be-merged segments + * (compensating for percent deleted docs). Default is 5gb. * *
  • index.merge.policy.segments_per_tier: * @@ -114,7 +113,7 @@ import org.elasticsearch.index.settings.IndexSettingsService; * call for the index (try and aim to issue it on a low traffic time). */ -public final class MergePolicyConfig implements IndexSettingsService.Listener{ +public final class MergePolicyConfig { private final TieredMergePolicy mergePolicy = new TieredMergePolicy(); private final ESLogger logger; private final boolean mergesEnabled; @@ -185,7 +184,6 @@ public final class MergePolicyConfig implements IndexSettingsService.Listener{ return mergesEnabled ? mergePolicy : NoMergePolicy.INSTANCE; } - @Override public void onRefreshSettings(Settings settings) { final double oldExpungeDeletesPctAllowed = mergePolicy.getForceMergeDeletesPctAllowed(); final double expungeDeletesPctAllowed = settings.getAsDouble(INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED, oldExpungeDeletesPctAllowed); diff --git a/core/src/main/java/org/elasticsearch/index/shard/MergeSchedulerConfig.java b/core/src/main/java/org/elasticsearch/index/shard/MergeSchedulerConfig.java index f061a95f2af..c329722a135 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/MergeSchedulerConfig.java +++ b/core/src/main/java/org/elasticsearch/index/shard/MergeSchedulerConfig.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.shard; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.index.IndexSettings; /** * The merge scheduler (ConcurrentMergeScheduler) controls the execution of @@ -61,11 +62,12 @@ public final class MergeSchedulerConfig { private volatile int maxMergeCount; private final boolean notifyOnMergeFailure; - public MergeSchedulerConfig(Settings indexSettings) { - maxThreadCount = indexSettings.getAsInt(MAX_THREAD_COUNT, Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(indexSettings) / 2))); - maxMergeCount = indexSettings.getAsInt(MAX_MERGE_COUNT, maxThreadCount + 5); - this.autoThrottle = indexSettings.getAsBoolean(AUTO_THROTTLE, true); - notifyOnMergeFailure = indexSettings.getAsBoolean(NOTIFY_ON_MERGE_FAILURE, true); + public MergeSchedulerConfig(IndexSettings indexSettings) { + final Settings settings = indexSettings.getSettings(); + maxThreadCount = settings.getAsInt(MAX_THREAD_COUNT, Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(settings) / 2))); + maxMergeCount = settings.getAsInt(MAX_MERGE_COUNT, maxThreadCount + 5); + this.autoThrottle = settings.getAsBoolean(AUTO_THROTTLE, true); + notifyOnMergeFailure = settings.getAsBoolean(NOTIFY_ON_MERGE_FAILURE, true); } /** diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java index 8bdf1fb5382..50a16fa1cee 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java @@ -21,12 +21,17 @@ package org.elasticsearch.index.shard; import java.io.IOException; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexServicesProvider; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.TranslogStats; @@ -38,8 +43,9 @@ import org.elasticsearch.index.translog.TranslogStats; */ public final class ShadowIndexShard extends IndexShard { - public ShadowIndexShard(ShardId shardId, @IndexSettings Settings indexSettings, ShardPath path, Store store, IndexServicesProvider provider) throws IOException { - super(shardId, indexSettings, path, store, provider); + public ShadowIndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache, MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService, @Nullable EngineFactory engineFactory, + IndexEventListener indexEventListener, IndexSearcherWrapper wrapper, NodeServicesProvider provider) throws IOException { + super(shardId, indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory, indexEventListener, wrapper, provider); } /** diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java index b2b26c12d19..d940d1a93cd 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java @@ -20,12 +20,10 @@ package org.elasticsearch.index.shard; import org.apache.lucene.util.IOUtils; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import java.io.IOException; import java.nio.file.FileStore; @@ -116,8 +114,8 @@ public final class ShardPath { * directories with a valid shard state exist the one with the highest version will be used. * Note: this method resolves custom data locations for the shard. */ - public static ShardPath loadShardPath(ESLogger logger, NodeEnvironment env, ShardId shardId, @IndexSettings Settings indexSettings) throws IOException { - final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); + public static ShardPath loadShardPath(ESLogger logger, NodeEnvironment env, ShardId shardId, IndexSettings indexSettings) throws IOException { + final String indexUUID = indexSettings.getUUID(); final Path[] paths = env.availableShardPaths(shardId); Path loadedPath = null; for (Path path : paths) { @@ -140,13 +138,13 @@ public final class ShardPath { } else { final Path dataPath; final Path statePath = loadedPath; - if (NodeEnvironment.hasCustomDataPath(indexSettings)) { + if (indexSettings.hasCustomDataPath()) { dataPath = env.resolveCustomLocation(indexSettings, shardId); } else { dataPath = statePath; } logger.debug("{} loaded data path [{}], state path [{}]", shardId, dataPath, statePath); - return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId); + return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, indexUUID, shardId); } } @@ -154,8 +152,8 @@ public final class ShardPath { * This method tries to delete left-over shards where the index name has been reused but the UUID is different * to allow the new shard to be allocated. */ - public static void deleteLeftoverShardDirectory(ESLogger logger, NodeEnvironment env, ShardLock lock, @IndexSettings Settings indexSettings) throws IOException { - final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); + public static void deleteLeftoverShardDirectory(ESLogger logger, NodeEnvironment env, ShardLock lock, IndexSettings indexSettings) throws IOException { + final String indexUUID = indexSettings.getUUID(); final Path[] paths = env.availableShardPaths(lock.getShardId()); for (Path path : paths) { ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path); @@ -198,13 +196,13 @@ public final class ShardPath { return reservedBytes; } - public static ShardPath selectNewPathForShard(NodeEnvironment env, ShardId shardId, @IndexSettings Settings indexSettings, + public static ShardPath selectNewPathForShard(NodeEnvironment env, ShardId shardId, IndexSettings indexSettings, long avgShardSizeInBytes, Map dataPathToShardCount) throws IOException { final Path dataPath; final Path statePath; - if (NodeEnvironment.hasCustomDataPath(indexSettings)) { + if (indexSettings.hasCustomDataPath()) { dataPath = env.resolveCustomLocation(indexSettings, shardId); statePath = env.nodePaths()[0].resolve(shardId); } else { @@ -244,9 +242,8 @@ public final class ShardPath { dataPath = statePath; } - final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); - - return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId); + final String indexUUID = indexSettings.getUUID(); + return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, indexUUID, shardId); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java b/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java index 5c61496b987..3cee89f84be 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java @@ -20,6 +20,9 @@ package org.elasticsearch.index.shard; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.AllocationId; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -36,17 +39,21 @@ public final class ShardStateMetaData { private static final String SHARD_STATE_FILE_PREFIX = "state-"; private static final String PRIMARY_KEY = "primary"; private static final String VERSION_KEY = "version"; - private static final String INDEX_UUID_KEY = "index_uuid" ; + private static final String INDEX_UUID_KEY = "index_uuid"; + private static final String ALLOCATION_ID_KEY = "allocation_id"; public final long version; public final String indexUUID; public final boolean primary; + @Nullable + public final AllocationId allocationId; // can be null if we read from legacy format (see fromXContent and MultiDataPathUpgrader) - public ShardStateMetaData(long version, boolean primary, String indexUUID) { + public ShardStateMetaData(long version, boolean primary, String indexUUID, AllocationId allocationId) { assert indexUUID != null; this.version = version; this.primary = primary; this.indexUUID = indexUUID; + this.allocationId = allocationId; } @Override @@ -69,21 +76,25 @@ public final class ShardStateMetaData { if (indexUUID != null ? !indexUUID.equals(that.indexUUID) : that.indexUUID != null) { return false; } + if (allocationId != null ? !allocationId.equals(that.allocationId) : that.allocationId != null) { + return false; + } return true; } @Override public int hashCode() { - int result = (int) (version ^ (version >>> 32)); + int result = Long.hashCode(version); result = 31 * result + (indexUUID != null ? indexUUID.hashCode() : 0); + result = 31 * result + (allocationId != null ? allocationId.hashCode() : 0); result = 31 * result + (primary ? 1 : 0); return result; } @Override public String toString() { - return "version [" + version + "], primary [" + primary + "]"; + return "version [" + version + "], primary [" + primary + "], allocation [" + allocationId + "]"; } public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(XContentType.JSON, SHARD_STATE_FILE_PREFIX) { @@ -100,6 +111,9 @@ public final class ShardStateMetaData { builder.field(VERSION_KEY, shardStateMetaData.version); builder.field(PRIMARY_KEY, shardStateMetaData.primary); builder.field(INDEX_UUID_KEY, shardStateMetaData.indexUUID); + if (shardStateMetaData.allocationId != null) { + builder.field(ALLOCATION_ID_KEY, shardStateMetaData.allocationId); + } } @Override @@ -112,6 +126,7 @@ public final class ShardStateMetaData { Boolean primary = null; String currentFieldName = null; String indexUUID = IndexMetaData.INDEX_UUID_NA_VALUE; + AllocationId allocationId = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -125,6 +140,12 @@ public final class ShardStateMetaData { } else { throw new CorruptStateException("unexpected field in shard state [" + currentFieldName + "]"); } + } else if (token == XContentParser.Token.START_OBJECT) { + if (ALLOCATION_ID_KEY.equals(currentFieldName)) { + allocationId = AllocationId.fromXContent(parser); + } else { + throw new CorruptStateException("unexpected object in shard state [" + currentFieldName + "]"); + } } else { throw new CorruptStateException("unexpected token in shard state [" + token.name() + "]"); } @@ -135,7 +156,7 @@ public final class ShardStateMetaData { if (version == -1) { throw new CorruptStateException("missing value for [version] in shard state"); } - return new ShardStateMetaData(version, primary, indexUUID); + return new ShardStateMetaData(version, primary, indexUUID, allocationId); } }; } diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index 9059c162680..0fc166cacc5 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -72,13 +72,6 @@ final class StoreRecovery { if (indexShard.routingEntry().restoreSource() != null) { throw new IllegalStateException("can't recover - restore source is not null"); } - try { - final RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), indexShard.routingEntry().primary(), RecoveryState.Type.STORE, localNode, localNode); - indexShard.recovering("from store", recoveryState); - } catch (IllegalIndexShardStateException e) { - // that's fine, since we might be called concurrently, just ignore this, we are already recovering - return false; - } return executeRecovery(indexShard, () -> { logger.debug("starting recovery from store ..."); internalRecoverFromStore(indexShard, indexShouldExists); @@ -101,13 +94,6 @@ final class StoreRecovery { if (shardRouting.restoreSource() == null) { throw new IllegalStateException("can't restore - restore source is null"); } - try { - final RecoveryState recoveryState = new RecoveryState(shardId, shardRouting.primary(), RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), localNode); - indexShard.recovering("from snapshot", recoveryState); - } catch (IllegalIndexShardStateException e) { - // that's fine, since we might be called concurrently, just ignore this, we are already recovering - return false; - } return executeRecovery(indexShard, () -> { logger.debug("restoring from {} ...", shardRouting.restoreSource()); restore(indexShard, repository); diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index 68c552d4419..ac46f6725de 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -67,7 +67,7 @@ public class TranslogRecoveryPerformer { numOps++; } } catch (Throwable t) { - throw new BatchOperationException(shardId, "failed to apply batch translog operation [" + t.getMessage() + "]", numOps, t); + throw new BatchOperationException(shardId, "failed to apply batch translog operation", numOps, t); } return numOps; } diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java deleted file mode 100644 index 29312f2557b..00000000000 --- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.similarity; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.BiFunction; - -/** - * {@link SimilarityModule} is responsible gathering registered and configured {@link SimilarityProvider} - * implementations and making them available through the {@link SimilarityService}. - * - * New {@link SimilarityProvider} implementations can be registered through {@link #addSimilarity(String, BiFunction)} - * while existing Providers can be referenced through Settings under the {@link #SIMILARITY_SETTINGS_PREFIX} prefix - * along with the "type" value. For example, to reference the {@link BM25SimilarityProvider}, the configuration - * "index.similarity.my_similarity.type : "BM25" can be used. - */ -public class SimilarityModule extends AbstractModule { - - public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity"; - - private final Settings settings; - private final Map> similarities = new HashMap<>(); - private final Index index; - - public SimilarityModule(Index index, Settings settings) { - this.settings = settings; - this.index = index; - } - - /** - * Registers the given {@link SimilarityProvider} with the given name - * - * @param name Name of the SimilarityProvider - * @param similarity SimilarityProvider to register - */ - public void addSimilarity(String name, BiFunction similarity) { - if (similarities.containsKey(name) || SimilarityService.BUILT_IN.containsKey(name)) { - throw new IllegalArgumentException("similarity for name: [" + name + " is already registered"); - } - similarities.put(name, similarity); - } - - @Override - protected void configure() { - SimilarityService service = new SimilarityService(index, settings, new HashMap<>(similarities)); - bind(SimilarityService.class).toInstance(service); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index a77a2de4dff..1d08683f47b 100644 --- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -21,30 +21,26 @@ package org.elasticsearch.index.similarity; import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; import org.apache.lucene.search.similarities.Similarity; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.function.BiFunction; -/** - * - */ -public class SimilarityService extends AbstractIndexComponent { +public final class SimilarityService extends AbstractIndexComponent { public final static String DEFAULT_SIMILARITY = "default"; private final Similarity defaultSimilarity; private final Similarity baseSimilarity; private final Map similarities; static final Map> DEFAULTS; - static final Map> BUILT_IN; + public static final Map> BUILT_IN; static { Map> defaults = new HashMap<>(); Map> buildIn = new HashMap<>(); @@ -59,19 +55,11 @@ public class SimilarityService extends AbstractIndexComponent { DEFAULTS = Collections.unmodifiableMap(defaults); BUILT_IN = Collections.unmodifiableMap(buildIn); } - public SimilarityService(Index index) { - this(index, Settings.Builder.EMPTY_SETTINGS); - } - public SimilarityService(Index index, Settings settings) { - this(index, settings, Collections.EMPTY_MAP); - } - - @Inject - public SimilarityService(Index index, @IndexSettings Settings indexSettings, Map> similarities) { - super(index, indexSettings); + public SimilarityService(IndexSettings indexSettings, Map> similarities) { + super(indexSettings); Map providers = new HashMap<>(similarities.size()); - Map similaritySettings = indexSettings.getGroups(SimilarityModule.SIMILARITY_SETTINGS_PREFIX); + Map similaritySettings = this.indexSettings.getSettings().getGroups(IndexModule.SIMILARITY_SETTINGS_PREFIX); for (Map.Entry entry : similaritySettings.entrySet()) { String name = entry.getKey(); Settings settings = entry.getValue(); diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotException.java b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotException.java index 741350966a5..7ace0303f67 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotException.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotException.java @@ -33,6 +33,11 @@ public class IndexShardSnapshotException extends ElasticsearchException { this(shardId, msg, null); } + public IndexShardSnapshotException(ShardId shardId, Throwable cause) { + super(cause); + setShard(shardId); + } + public IndexShardSnapshotException(ShardId shardId, String msg, Throwable cause) { super(msg, cause); setShard(shardId); diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotFailedException.java b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotFailedException.java index bfb755c9e14..7b7fc68d4d4 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotFailedException.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotFailedException.java @@ -32,6 +32,10 @@ public class IndexShardSnapshotFailedException extends IndexShardSnapshotExcepti super(shardId, msg); } + public IndexShardSnapshotFailedException(ShardId shardId, Throwable cause) { + super(shardId, cause); + } + public IndexShardSnapshotFailedException(ShardId shardId, String msg, Throwable cause) { super(shardId, msg, cause); } diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java index d90a869f5b3..674d1085660 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java @@ -191,7 +191,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements if (e instanceof IndexShardSnapshotFailedException) { throw (IndexShardSnapshotFailedException) e; } else { - throw new IndexShardSnapshotFailedException(shardId, e.getMessage(), e); + throw new IndexShardSnapshotFailedException(shardId, e); } } } @@ -373,7 +373,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements } catch (IOException e) { // We cannot delete index file - this is fatal, we cannot continue, otherwise we might end up // with references to non-existing files - throw new IndexShardSnapshotFailedException(shardId, "error deleting index files during cleanup, reason: " + e.getMessage(), e); + throw new IndexShardSnapshotFailedException(shardId, "error deleting index files during cleanup", e); } blobsToDelete = new ArrayList<>(); diff --git a/core/src/main/java/org/elasticsearch/index/store/DirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/DirectoryService.java index fbf25649b74..90f9ed92712 100644 --- a/core/src/main/java/org/elasticsearch/index/store/DirectoryService.java +++ b/core/src/main/java/org/elasticsearch/index/store/DirectoryService.java @@ -20,8 +20,7 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; @@ -31,7 +30,7 @@ import java.io.IOException; */ public abstract class DirectoryService extends AbstractIndexShardComponent { - protected DirectoryService(ShardId shardId, @IndexSettings Settings indexSettings) { + protected DirectoryService(ShardId shardId, IndexSettings indexSettings) { super(shardId, indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java index 2d12fab1637..60752dd774b 100644 --- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java +++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java @@ -19,23 +19,14 @@ package org.elasticsearch.index.store; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FileSwitchDirectory; -import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.MMapDirectory; -import org.apache.lucene.store.NIOFSDirectory; -import org.apache.lucene.store.NativeFSLockFactory; -import org.apache.lucene.store.RateLimitedFSDirectory; -import org.apache.lucene.store.SimpleFSDirectory; -import org.apache.lucene.store.SimpleFSLockFactory; -import org.apache.lucene.store.StoreRateLimiting; +import org.apache.lucene.store.*; import org.apache.lucene.util.Constants; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; import java.io.IOException; @@ -54,7 +45,7 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim private final ShardPath path; @Inject - public FsDirectoryService(@IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath path) { + public FsDirectoryService(IndexSettings indexSettings, IndexStore indexStore, ShardPath path) { super(path.getShardId(), indexSettings); this.path = path; this.indexStore = indexStore; @@ -70,8 +61,9 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim return indexStore.rateLimiting(); } - public static LockFactory buildLockFactory(@IndexSettings Settings indexSettings) { - String fsLock = indexSettings.get("index.store.fs.lock", indexSettings.get("index.store.fs.fs_lock", "native")); + public static LockFactory buildLockFactory(IndexSettings indexSettings) { + final Settings settings = indexSettings.getSettings(); + String fsLock = settings.get("index.store.fs.lock", settings.get("index.store.fs.fs_lock", "native")); LockFactory lockFactory; if (fsLock.equals("native")) { lockFactory = NativeFSLockFactory.INSTANCE; @@ -110,18 +102,18 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { - final String storeType = indexSettings.get(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.DEFAULT.getSettingsKey()); - if (IndexStoreModule.Type.FS.match(storeType) || IndexStoreModule.Type.DEFAULT.match(storeType)) { + final String storeType = indexSettings.getSettings().get(IndexModule.STORE_TYPE, IndexModule.Type.DEFAULT.getSettingsKey()); + if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) { final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults if (open instanceof MMapDirectory && Constants.WINDOWS == false) { return newDefaultDir(location, (MMapDirectory) open, lockFactory); } return open; - } else if (IndexStoreModule.Type.SIMPLEFS.match(storeType)) { + } else if (IndexModule.Type.SIMPLEFS.match(storeType)) { return new SimpleFSDirectory(location, lockFactory); - } else if (IndexStoreModule.Type.NIOFS.match(storeType)) { + } else if (IndexModule.Type.NIOFS.match(storeType)) { return new NIOFSDirectory(location, lockFactory); - } else if (IndexStoreModule.Type.MMAPFS.match(storeType)) { + } else if (IndexModule.Type.MMAPFS.match(storeType)) { return new MMapDirectory(location, lockFactory); } throw new IllegalArgumentException("No directory found for type [" + storeType + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/store/IndexStore.java b/core/src/main/java/org/elasticsearch/index/store/IndexStore.java index 3a23a09a652..ea6f59b0520 100644 --- a/core/src/main/java/org/elasticsearch/index/store/IndexStore.java +++ b/core/src/main/java/org/elasticsearch/index/store/IndexStore.java @@ -20,86 +20,41 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.StoreRateLimiting; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.indices.store.IndicesStore; - -import java.io.Closeable; - /** * */ -public class IndexStore extends AbstractIndexComponent implements Closeable { +public class IndexStore extends AbstractIndexComponent { public static final String INDEX_STORE_THROTTLE_TYPE = "index.store.throttle.type"; public static final String INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC = "index.store.throttle.max_bytes_per_sec"; - private final IndexSettingsService settingsService; - - class ApplySettings implements IndexSettingsService.Listener { - @Override - public void onRefreshSettings(Settings settings) { - String rateLimitingType = settings.get(INDEX_STORE_THROTTLE_TYPE, IndexStore.this.rateLimitingType); - if (!rateLimitingType.equals(IndexStore.this.rateLimitingType)) { - logger.info("updating index.store.throttle.type from [{}] to [{}]", IndexStore.this.rateLimitingType, rateLimitingType); - if (rateLimitingType.equalsIgnoreCase("node")) { - IndexStore.this.rateLimitingType = rateLimitingType; - IndexStore.this.nodeRateLimiting = true; - } else { - StoreRateLimiting.Type.fromString(rateLimitingType); - IndexStore.this.rateLimitingType = rateLimitingType; - IndexStore.this.nodeRateLimiting = false; - IndexStore.this.rateLimiting.setType(rateLimitingType); - } - } - - ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndexStore.this.rateLimitingThrottle); - if (!rateLimitingThrottle.equals(IndexStore.this.rateLimitingThrottle)) { - logger.info("updating index.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndexStore.this.rateLimitingThrottle, rateLimitingThrottle, IndexStore.this.rateLimitingType); - IndexStore.this.rateLimitingThrottle = rateLimitingThrottle; - IndexStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); - } - } - } - protected final IndicesStore indicesStore; - + protected final IndexStoreConfig indexStoreConfig; private volatile String rateLimitingType; private volatile ByteSizeValue rateLimitingThrottle; private volatile boolean nodeRateLimiting; private final StoreRateLimiting rateLimiting = new StoreRateLimiting(); - private final ApplySettings applySettings = new ApplySettings(); + public IndexStore(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig) { + super(indexSettings); + this.indexStoreConfig = indexStoreConfig; - @Inject - public IndexStore(Index index, @IndexSettings Settings indexSettings, IndexSettingsService settingsService, IndicesStore indicesStore) { - super(index, indexSettings); - this.indicesStore = indicesStore; - - this.rateLimitingType = indexSettings.get(INDEX_STORE_THROTTLE_TYPE, "none"); + this.rateLimitingType = indexSettings.getSettings().get(INDEX_STORE_THROTTLE_TYPE, "none"); if (rateLimitingType.equalsIgnoreCase("node")) { nodeRateLimiting = true; } else { nodeRateLimiting = false; rateLimiting.setType(rateLimitingType); } - this.rateLimitingThrottle = indexSettings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0)); + this.rateLimitingThrottle = indexSettings.getSettings().getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0)); rateLimiting.setMaxRate(rateLimitingThrottle); logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); - this.settingsService = settingsService; - this.settingsService.addListener(applySettings); - } - - @Override - public void close() { - settingsService.removeListener(applySettings); } /** @@ -107,7 +62,7 @@ public class IndexStore extends AbstractIndexComponent implements Closeable { * the node level one (defaults to the node level one). */ public StoreRateLimiting rateLimiting() { - return nodeRateLimiting ? indicesStore.rateLimiting() : this.rateLimiting; + return nodeRateLimiting ? indexStoreConfig.getNodeRateLimiter() : this.rateLimiting; } /** @@ -116,4 +71,27 @@ public class IndexStore extends AbstractIndexComponent implements Closeable { public DirectoryService newDirectoryService(ShardPath path) { return new FsDirectoryService(indexSettings, this, path); } + + public void onRefreshSettings(Settings settings) { + String rateLimitingType = settings.get(INDEX_STORE_THROTTLE_TYPE, IndexStore.this.rateLimitingType); + if (!rateLimitingType.equals(IndexStore.this.rateLimitingType)) { + logger.info("updating index.store.throttle.type from [{}] to [{}]", IndexStore.this.rateLimitingType, rateLimitingType); + if (rateLimitingType.equalsIgnoreCase("node")) { + IndexStore.this.rateLimitingType = rateLimitingType; + IndexStore.this.nodeRateLimiting = true; + } else { + StoreRateLimiting.Type.fromString(rateLimitingType); + IndexStore.this.rateLimitingType = rateLimitingType; + IndexStore.this.nodeRateLimiting = false; + IndexStore.this.rateLimiting.setType(rateLimitingType); + } + } + + ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndexStore.this.rateLimitingThrottle); + if (!rateLimitingThrottle.equals(IndexStore.this.rateLimitingThrottle)) { + logger.info("updating index.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndexStore.this.rateLimitingThrottle, rateLimitingThrottle, IndexStore.this.rateLimitingType); + IndexStore.this.rateLimitingThrottle = rateLimitingThrottle; + IndexStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); + } + } } diff --git a/core/src/main/java/org/elasticsearch/index/store/IndexStoreConfig.java b/core/src/main/java/org/elasticsearch/index/store/IndexStoreConfig.java new file mode 100644 index 00000000000..1bd023abdb0 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/store/IndexStoreConfig.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.store; + +import org.apache.lucene.store.StoreRateLimiting; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.node.settings.NodeSettingsService; + +/** + * IndexStoreConfig encapsulates node / cluster level configuration for index level {@link IndexStore} instances. + * For instance it maintains the node level rate limiter configuration: updates to the cluster that disable or enable + * {@value #INDICES_STORE_THROTTLE_TYPE} or {@value #INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC} are reflected immediately + * on all referencing {@link IndexStore} instances + */ +public class IndexStoreConfig implements NodeSettingsService.Listener { + + /** + * Configures the node / cluster level throttle type. See {@link StoreRateLimiting.Type}. + */ + public static final String INDICES_STORE_THROTTLE_TYPE = "indices.store.throttle.type"; + /** + * Configures the node / cluster level throttle intensity. The default is 10240 MB + */ + public static final String INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC = "indices.store.throttle.max_bytes_per_sec"; + private volatile String rateLimitingType; + private volatile ByteSizeValue rateLimitingThrottle; + private final StoreRateLimiting rateLimiting = new StoreRateLimiting(); + private final ESLogger logger; + public IndexStoreConfig(Settings settings) { + logger = Loggers.getLogger(IndexStoreConfig.class, settings); + // we don't limit by default (we default to CMS's auto throttle instead): + this.rateLimitingType = settings.get("indices.store.throttle.type", StoreRateLimiting.Type.NONE.name()); + rateLimiting.setType(rateLimitingType); + this.rateLimitingThrottle = settings.getAsBytesSize("indices.store.throttle.max_bytes_per_sec", new ByteSizeValue(0)); + rateLimiting.setMaxRate(rateLimitingThrottle); + logger.debug("using indices.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); + } + + /** + * Returns the node level rate limiter + */ + public StoreRateLimiting getNodeRateLimiter(){ + return rateLimiting; + } + + @Override + public void onRefreshSettings(Settings settings) { + String rateLimitingType = settings.get(INDICES_STORE_THROTTLE_TYPE, this.rateLimitingType); + // try and parse the type + StoreRateLimiting.Type.fromString(rateLimitingType); + if (!rateLimitingType.equals(this.rateLimitingType)) { + logger.info("updating indices.store.throttle.type from [{}] to [{}]", this.rateLimitingType, rateLimitingType); + this.rateLimitingType = rateLimitingType; + this.rateLimiting.setType(rateLimitingType); + } + + ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, this.rateLimitingThrottle); + if (!rateLimitingThrottle.equals(this.rateLimitingThrottle)) { + logger.info("updating indices.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", this.rateLimitingThrottle, rateLimitingThrottle, this.rateLimitingType); + this.rateLimitingThrottle = rateLimitingThrottle; + this.rateLimiting.setMaxRate(rateLimitingThrottle); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/store/IndexStoreModule.java b/core/src/main/java/org/elasticsearch/index/store/IndexStoreModule.java deleted file mode 100644 index 84c856d1701..00000000000 --- a/core/src/main/java/org/elasticsearch/index/store/IndexStoreModule.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.store; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.settings.Settings; - -import java.util.HashMap; -import java.util.Map; -import java.util.Locale; - -/** - * - */ -public class IndexStoreModule extends AbstractModule { - - public static final String STORE_TYPE = "index.store.type"; - - private final Settings settings; - private final Map> storeTypes = new HashMap<>(); - - public enum Type { - NIOFS, - MMAPFS, - SIMPLEFS, - FS, - DEFAULT; - - public String getSettingsKey() { - return this.name().toLowerCase(Locale.ROOT); - } - /** - * Returns true iff this settings matches the type. - */ - public boolean match(String setting) { - return getSettingsKey().equals(setting); - } - } - - public IndexStoreModule(Settings settings) { - this.settings = settings; - } - - public void addIndexStore(String type, Class clazz) { - storeTypes.put(type, clazz); - } - - private static boolean isBuiltinType(String storeType) { - for (Type type : Type.values()) { - if (type.match(storeType)) { - return true; - } - } - return false; - } - - @Override - protected void configure() { - final String storeType = settings.get(STORE_TYPE); - if (storeType == null || isBuiltinType(storeType)) { - bind(IndexStore.class).asEagerSingleton(); - } else { - Class clazz = storeTypes.get(storeType); - if (clazz == null) { - throw new IllegalArgumentException("Unknown store type [" + storeType + "]"); - } - bind(IndexStore.class).to(clazz).asEagerSingleton(); - } - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java index c2b55ac0032..729e2b65b2a 100644 --- a/core/src/main/java/org/elasticsearch/index/store/Store.java +++ b/core/src/main/java/org/elasticsearch/index/store/Store.java @@ -20,31 +20,9 @@ package org.elasticsearch.index.store; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.IndexNotFoundException; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.SegmentCommitInfo; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.BufferedChecksum; -import org.apache.lucene.store.ByteArrayDataInput; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FilterDirectory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Lock; -import org.apache.lucene.store.SimpleFSDirectory; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.Version; +import org.apache.lucene.index.*; +import org.apache.lucene.store.*; +import org.apache.lucene.util.*; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Strings; @@ -69,25 +47,15 @@ import org.elasticsearch.common.util.concurrent.AbstractRefCounted; import org.elasticsearch.common.util.concurrent.RefCounted; import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.env.ShardLock; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; -import java.io.Closeable; -import java.io.EOFException; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.zip.Adler32; @@ -142,17 +110,18 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref } }; - public Store(ShardId shardId, @IndexSettings Settings indexSettings, DirectoryService directoryService, ShardLock shardLock) throws IOException { + public Store(ShardId shardId, IndexSettings indexSettings, DirectoryService directoryService, ShardLock shardLock) throws IOException { this(shardId, indexSettings, directoryService, shardLock, OnClose.EMPTY); } @Inject - public Store(ShardId shardId, @IndexSettings Settings indexSettings, DirectoryService directoryService, ShardLock shardLock, OnClose onClose) throws IOException { + public Store(ShardId shardId, IndexSettings indexSettings, DirectoryService directoryService, ShardLock shardLock, OnClose onClose) throws IOException { super(shardId, indexSettings); - this.directory = new StoreDirectory(directoryService.newDirectory(), Loggers.getLogger("index.store.deletes", indexSettings, shardId)); + final Settings settings = indexSettings.getSettings(); + this.directory = new StoreDirectory(directoryService.newDirectory(), Loggers.getLogger("index.store.deletes", settings, shardId)); this.shardLock = shardLock; this.onClose = onClose; - final TimeValue refreshInterval = indexSettings.getAsTime(INDEX_STORE_STATS_REFRESH_INTERVAL, TimeValue.timeValueSeconds(10)); + final TimeValue refreshInterval = settings.getAsTime(INDEX_STORE_STATS_REFRESH_INTERVAL, TimeValue.timeValueSeconds(10)); this.statsCache = new StoreStatsCache(refreshInterval, directory, directoryService); logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval); diff --git a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java b/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java index 2ced4a8a058..750d7de7b22 100644 --- a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java +++ b/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java @@ -19,13 +19,8 @@ package org.elasticsearch.index.suggest.stats; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.AbstractIndexShardComponent; -import org.elasticsearch.index.shard.ShardId; import java.util.concurrent.TimeUnit; diff --git a/core/src/main/java/org/elasticsearch/index/translog/BufferingTranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/BufferingTranslogWriter.java index 05b5b917879..2ea33127633 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/BufferingTranslogWriter.java +++ b/core/src/main/java/org/elasticsearch/index/translog/BufferingTranslogWriter.java @@ -47,22 +47,28 @@ public final class BufferingTranslogWriter extends TranslogWriter { @Override public Translog.Location add(BytesReference data) throws IOException { try (ReleasableLock lock = writeLock.acquire()) { - operationCounter++; + ensureOpen(); final long offset = totalOffset; if (data.length() >= buffer.length) { flush(); // we use the channel to write, since on windows, writing to the RAF might not be reflected // when reading through the channel - data.writeTo(channel); + try { + data.writeTo(channel); + } catch (Throwable ex) { + closeWithTragicEvent(ex); + throw ex; + } writtenOffset += data.length(); totalOffset += data.length(); - return new Translog.Location(generation, offset, data.length()); + } else { + if (data.length() > buffer.length - bufferCount) { + flush(); + } + data.writeTo(bufferOs); + totalOffset += data.length(); } - if (data.length() > buffer.length - bufferCount) { - flush(); - } - data.writeTo(bufferOs); - totalOffset += data.length(); + operationCounter++; return new Translog.Location(generation, offset, data.length()); } } @@ -70,10 +76,17 @@ public final class BufferingTranslogWriter extends TranslogWriter { protected final void flush() throws IOException { assert writeLock.isHeldByCurrentThread(); if (bufferCount > 0) { + ensureOpen(); // we use the channel to write, since on windows, writing to the RAF might not be reflected // when reading through the channel - Channels.writeToChannel(buffer, 0, bufferCount, channel); - writtenOffset += bufferCount; + final int bufferSize = bufferCount; + try { + Channels.writeToChannel(buffer, 0, bufferSize, channel); + } catch (Throwable ex) { + closeWithTragicEvent(ex); + throw ex; + } + writtenOffset += bufferSize; bufferCount = 0; } } @@ -101,18 +114,31 @@ public final class BufferingTranslogWriter extends TranslogWriter { } @Override - public void sync() throws IOException { - if (!syncNeeded()) { - return; - } - synchronized (this) { - try (ReleasableLock lock = writeLock.acquire()) { - flush(); - lastSyncedOffset = totalOffset; + public synchronized void sync() throws IOException { + if (syncNeeded()) { + ensureOpen(); // this call gives a better exception that the incRef if we are closed by a tragic event + channelReference.incRef(); + try { + final long offsetToSync; + final int opsCounter; + try (ReleasableLock lock = writeLock.acquire()) { + flush(); + offsetToSync = totalOffset; + opsCounter = operationCounter; + } + // we can do this outside of the write lock but we have to protect from + // concurrent syncs + ensureOpen(); // just for kicks - the checkpoint happens or not either way + try { + checkpoint(offsetToSync, opsCounter, channelReference); + } catch (Throwable ex) { + closeWithTragicEvent(ex); + throw ex; + } + lastSyncedOffset = offsetToSync; + } finally { + channelReference.decRef(); } - // we can do this outside of the write lock but we have to protect from - // concurrent syncs - checkpoint(lastSyncedOffset, operationCounter, channelReference); } } diff --git a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index 9b73d0346a5..cd0f94567f3 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -90,4 +90,24 @@ class Checkpoint { } } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Checkpoint that = (Checkpoint) o; + + if (offset != that.offset) return false; + if (numOps != that.numOps) return false; + return generation == that.generation; + + } + + @Override + public int hashCode() { + int result = Long.hashCode(offset); + result = 31 * result + numOps; + result = 31 * result + Long.hashCode(generation); + return result; + } } diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java index 64c9c456611..9ad8715ed0c 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -115,7 +115,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC private final Path location; private TranslogWriter current; private volatile ImmutableTranslogReader currentCommittingTranslog; - private long lastCommittedTranslogFileGeneration = -1; // -1 is safe as it will not cause an translog deletion. + private volatile long lastCommittedTranslogFileGeneration = -1; // -1 is safe as it will not cause an translog deletion. private final AtomicBoolean closed = new AtomicBoolean(); private final TranslogConfig config; private final String translogUUID; @@ -129,11 +129,11 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC }; - /** * Creates a new Translog instance. This method will create a new transaction log unless the given {@link TranslogConfig} has * a non-null {@link org.elasticsearch.index.translog.Translog.TranslogGeneration}. If the generation is null this method * us destructive and will delete all files in the translog path given. + * * @see TranslogConfig#getTranslogPath() */ public Translog(TranslogConfig config) throws IOException { @@ -141,7 +141,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC this.config = config; TranslogGeneration translogGeneration = config.getTranslogGeneration(); - if (translogGeneration == null || translogGeneration.translogUUID == null) { // legacy case + if (translogGeneration == null || translogGeneration.translogUUID == null) { // legacy case translogUUID = Strings.randomBase64UUID(); } else { translogUUID = translogGeneration.translogUUID; @@ -166,7 +166,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC current = createWriter(checkpoint.generation + 1); this.lastCommittedTranslogFileGeneration = translogGeneration.translogFileGeneration; } else { - this.recoveredTranslogs = Collections.EMPTY_LIST; + this.recoveredTranslogs = Collections.emptyList(); IOUtils.rm(location); logger.debug("wipe translog location - creating new translog"); Files.createDirectories(location); @@ -186,11 +186,12 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC } /** recover all translog files found on disk */ - private ArrayList recoverFromFiles(TranslogGeneration translogGeneration, Checkpoint checkpoint) throws IOException { + private final ArrayList recoverFromFiles(TranslogGeneration translogGeneration, Checkpoint checkpoint) throws IOException { boolean success = false; ArrayList foundTranslogs = new ArrayList<>(); + final Path tempFile = Files.createTempFile(location, TRANSLOG_FILE_PREFIX, TRANSLOG_FILE_SUFFIX); // a temp file to copy checkpoint to - note it must be in on the same FS otherwise atomic move won't work + boolean tempFileRenamed = false; try (ReleasableLock lock = writeLock.acquire()) { - logger.debug("open uncommitted translog checkpoint {}", checkpoint); final String checkpointTranslogFile = getFilename(checkpoint.generation); for (long i = translogGeneration.translogFileGeneration; i < checkpoint.generation; i++) { @@ -204,14 +205,33 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC } foundTranslogs.add(openReader(location.resolve(checkpointTranslogFile), checkpoint)); Path commitCheckpoint = location.resolve(getCommitCheckpointFileName(checkpoint.generation)); - Files.copy(location.resolve(CHECKPOINT_FILE_NAME), commitCheckpoint); - IOUtils.fsync(commitCheckpoint, false); - IOUtils.fsync(commitCheckpoint.getParent(), true); + if (Files.exists(commitCheckpoint)) { + Checkpoint checkpointFromDisk = Checkpoint.read(commitCheckpoint); + if (checkpoint.equals(checkpointFromDisk) == false) { + throw new IllegalStateException("Checkpoint file " + commitCheckpoint.getFileName() + " already exists but has corrupted content expected: " + checkpoint + " but got: " + checkpointFromDisk); + } + } else { + // we first copy this into the temp-file and then fsync it followed by an atomic move into the target file + // that way if we hit a disk-full here we are still in an consistent state. + Files.copy(location.resolve(CHECKPOINT_FILE_NAME), tempFile, StandardCopyOption.REPLACE_EXISTING); + IOUtils.fsync(tempFile, false); + Files.move(tempFile, commitCheckpoint, StandardCopyOption.ATOMIC_MOVE); + tempFileRenamed = true; + // we only fsync the directory the tempFile was already fsynced + IOUtils.fsync(commitCheckpoint.getParent(), true); + } success = true; } finally { if (success == false) { IOUtils.closeWhileHandlingException(foundTranslogs); } + if (tempFileRenamed == false) { + try { + Files.delete(tempFile); + } catch (IOException ex) { + logger.warn("failed to delete temp file {}", ex, tempFile); + } + } } return foundTranslogs; } @@ -237,7 +257,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC /** * Extracts the translog generation from a file name. * - * @throw IllegalArgumentException if the path doesn't match the expected pattern. + * @throws IllegalArgumentException if the path doesn't match the expected pattern. */ public static long parseIdFromFileName(Path translogFile) { final String fileName = translogFile.getFileName().toString(); @@ -252,7 +272,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC throw new IllegalArgumentException("can't parse id from file: " + fileName); } - boolean isOpen() { + /** Returns {@code true} if this {@code Translog} is still open. */ + public boolean isOpen() { return closed.get() == false; } @@ -261,10 +282,14 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC if (closed.compareAndSet(false, true)) { try (ReleasableLock lock = writeLock.acquire()) { try { - IOUtils.close(current, currentCommittingTranslog); + current.sync(); } finally { - IOUtils.close(recoveredTranslogs); - recoveredTranslogs.clear(); + try { + IOUtils.close(current, currentCommittingTranslog); + } finally { + IOUtils.close(recoveredTranslogs); + recoveredTranslogs.clear(); + } } } finally { FutureUtils.cancel(syncScheduler); @@ -324,11 +349,10 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC } - TranslogWriter createWriter(long fileGeneration) throws IOException { TranslogWriter newFile; try { - newFile = TranslogWriter.create(config.getType(), shardId, translogUUID, fileGeneration, location.resolve(getFilename(fileGeneration)), new OnCloseRunnable(), config.getBufferSizeBytes()); + newFile = TranslogWriter.create(config.getType(), shardId, translogUUID, fileGeneration, location.resolve(getFilename(fileGeneration)), new OnCloseRunnable(), config.getBufferSizeBytes(), getChannelFactory()); } catch (IOException e) { throw new TranslogException(shardId, "failed to create new translog file", e); } @@ -367,7 +391,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC * @see Index * @see org.elasticsearch.index.translog.Translog.Delete */ - public Location add(Operation operation) throws TranslogException { + public Location add(Operation operation) throws IOException { final ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(bigArrays); try { final BufferedChecksumStreamOutput checksumStreamOutput = new BufferedChecksumStreamOutput(out); @@ -381,6 +405,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC out.seek(end); final ReleasablePagedBytesReference bytes = out.bytes(); try (ReleasableLock lock = readLock.acquire()) { + ensureOpen(); Location location = current.add(bytes); if (config.isSyncOnEachOperation()) { current.sync(); @@ -388,6 +413,15 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC assert current.assertBytesAtLocation(location, bytes); return location; } + } catch (AlreadyClosedException | IOException ex) { + if (current.getTragicException() != null) { + try { + close(); + } catch (Exception inner) { + ex.addSuppressed(inner); + } + } + throw ex; } catch (Throwable e) { throw new TranslogException(shardId, "Failed to write operation [" + operation + "]", e); } finally { @@ -400,6 +434,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC * Snapshots are fixed in time and will not be updated with future operations. */ public Snapshot newSnapshot() { + ensureOpen(); try (ReleasableLock lock = readLock.acquire()) { ArrayList toOpen = new ArrayList<>(); toOpen.addAll(recoveredTranslogs); @@ -464,6 +499,15 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC if (closed.get() == false) { current.sync(); } + } catch (AlreadyClosedException | IOException ex) { + if (current.getTragicException() != null) { + try { + close(); + } catch (Exception inner) { + ex.addSuppressed(inner); + } + } + throw ex; } } @@ -485,11 +529,13 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC /** * Ensures that the given location has be synced / written to the underlying storage. + * * @return Returns true iff this call caused an actual sync operation otherwise false */ public boolean ensureSynced(Location location) throws IOException { try (ReleasableLock lock = readLock.acquire()) { if (location.generation == current.generation) { // if we have a new one it's already synced + ensureOpen(); return current.syncUpTo(location.translogLocation + location.size); } } @@ -518,31 +564,29 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC private final class OnCloseRunnable implements Callback { @Override public void handle(ChannelReference channelReference) { - try (ReleasableLock lock = writeLock.acquire()) { - if (isReferencedGeneration(channelReference.getGeneration()) == false) { - Path translogPath = channelReference.getPath(); - assert channelReference.getPath().getParent().equals(location) : "translog files must be in the location folder: " + location + " but was: " + translogPath; - // if the given translogPath is not the current we can safely delete the file since all references are released - logger.trace("delete translog file - not referenced and not current anymore {}", translogPath); - IOUtils.deleteFilesIgnoringExceptions(translogPath); - IOUtils.deleteFilesIgnoringExceptions(translogPath.resolveSibling(getCommitCheckpointFileName(channelReference.getGeneration()))); + if (isReferencedGeneration(channelReference.getGeneration()) == false) { + Path translogPath = channelReference.getPath(); + assert channelReference.getPath().getParent().equals(location) : "translog files must be in the location folder: " + location + " but was: " + translogPath; + // if the given translogPath is not the current we can safely delete the file since all references are released + logger.trace("delete translog file - not referenced and not current anymore {}", translogPath); + IOUtils.deleteFilesIgnoringExceptions(translogPath); + IOUtils.deleteFilesIgnoringExceptions(translogPath.resolveSibling(getCommitCheckpointFileName(channelReference.getGeneration()))); - } - try (DirectoryStream stream = Files.newDirectoryStream(location)) { - for (Path path : stream) { - Matcher matcher = PARSE_STRICT_ID_PATTERN.matcher(path.getFileName().toString()); - if (matcher.matches()) { - long generation = Long.parseLong(matcher.group(1)); - if (isReferencedGeneration(generation) == false) { - logger.trace("delete translog file - not referenced and not current anymore {}", path); - IOUtils.deleteFilesIgnoringExceptions(path); - IOUtils.deleteFilesIgnoringExceptions(path.resolveSibling(getCommitCheckpointFileName(channelReference.getGeneration()))); - } + } + try (DirectoryStream stream = Files.newDirectoryStream(location)) { + for (Path path : stream) { + Matcher matcher = PARSE_STRICT_ID_PATTERN.matcher(path.getFileName().toString()); + if (matcher.matches()) { + long generation = Long.parseLong(matcher.group(1)); + if (isReferencedGeneration(generation) == false) { + logger.trace("delete translog file - not referenced and not current anymore {}", path); + IOUtils.deleteFilesIgnoringExceptions(path); + IOUtils.deleteFilesIgnoringExceptions(path.resolveSibling(getCommitCheckpointFileName(channelReference.getGeneration()))); } } - } catch (IOException e) { - logger.warn("failed to delete unreferenced translog files", e); } + } catch (IOException e) { + logger.warn("failed to delete unreferenced translog files", e); } } } @@ -552,7 +596,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC * and updated with any future translog. */ public static final class View implements Closeable { - public static final Translog.View EMPTY_VIEW = new View(Collections.EMPTY_LIST, null); + public static final Translog.View EMPTY_VIEW = new View(Collections.emptyList(), null); boolean closed; // last in this list is always FsTranslog.current @@ -726,21 +770,29 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } Location location = (Location) o; - if (generation != location.generation) return false; - if (translogLocation != location.translogLocation) return false; + if (generation != location.generation) { + return false; + } + if (translogLocation != location.translogLocation) { + return false; + } return size == location.size; } @Override public int hashCode() { - int result = (int) (generation ^ (generation >>> 32)); - result = 31 * result + (int) (translogLocation ^ (translogLocation >>> 32)); + int result = Long.hashCode(generation); + result = 31 * result + Long.hashCode(translogLocation); result = 31 * result + size; return result; } @@ -998,13 +1050,13 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC public int hashCode() { int result = id.hashCode(); result = 31 * result + type.hashCode(); - result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + Long.hashCode(version); result = 31 * result + versionType.hashCode(); result = 31 * result + source.hashCode(); result = 31 * result + (routing != null ? routing.hashCode() : 0); result = 31 * result + (parent != null ? parent.hashCode() : 0); - result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); - result = 31 * result + (int) (ttl ^ (ttl >>> 32)); + result = 31 * result + Long.hashCode(timestamp); + result = 31 * result + Long.hashCode(ttl); return result; } @@ -1066,7 +1118,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC } @Override - public Source getSource(){ + public Source getSource() { throw new IllegalStateException("trying to read doc source from delete operation"); } @@ -1112,7 +1164,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC @Override public int hashCode() { int result = uid.hashCode(); - result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + Long.hashCode(version); result = 31 * result + versionType.hashCode(); return result; } @@ -1175,7 +1227,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC // to prevent this unfortunately. in.mark(opSize); - in.skip(opSize-4); + in.skip(opSize - 4); verifyChecksum(in); in.reset(); } @@ -1227,7 +1279,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC out.writeByte(op.opType().id()); op.writeTo(out); long checksum = out.getChecksum(); - out.writeInt((int)checksum); + out.writeInt((int) checksum); } /** @@ -1250,12 +1302,13 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC @Override public void prepareCommit() throws IOException { - ensureOpen(); try (ReleasableLock lock = writeLock.acquire()) { + ensureOpen(); if (currentCommittingTranslog != null) { throw new IllegalStateException("already committing a translog with generation: " + currentCommittingTranslog.getGeneration()); } final TranslogWriter oldCurrent = current; + oldCurrent.ensureOpen(); oldCurrent.sync(); currentCommittingTranslog = current.immutableReader(); Path checkpoint = location.resolve(CHECKPOINT_FILE_NAME); @@ -1283,9 +1336,9 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC @Override public void commit() throws IOException { - ensureOpen(); ImmutableTranslogReader toClose = null; try (ReleasableLock lock = writeLock.acquire()) { + ensureOpen(); if (currentCommittingTranslog == null) { prepareCommit(); } @@ -1351,7 +1404,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC private void ensureOpen() { if (closed.get()) { - throw new AlreadyClosedException("translog is already closed"); + throw new AlreadyClosedException("translog is already closed", current.getTragicException()); } } @@ -1362,4 +1415,15 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC return outstandingViews.size(); } + TranslogWriter.ChannelFactory getChannelFactory() { + return TranslogWriter.ChannelFactory.DEFAULT; + } + + /** If this {@code Translog} was closed as a side-effect of a tragic exception, + * e.g. disk full while flushing a new segment, this returns the root cause exception. + * Otherwise (no tragic exception has occurred) it returns null. */ + public Throwable getTragicException() { + return current.getTragicException(); + } + } diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java index c831eb5aafb..e8a8d1803ee 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java +++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java @@ -20,11 +20,10 @@ package org.elasticsearch.index.translog; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog.TranslogGeneration; import org.elasticsearch.indices.memory.IndexingMemoryController; @@ -41,7 +40,6 @@ public final class TranslogConfig { public static final String INDEX_TRANSLOG_DURABILITY = "index.translog.durability"; public static final String INDEX_TRANSLOG_FS_TYPE = "index.translog.fs.type"; - public static final String INDEX_TRANSLOG_BUFFER_SIZE = "index.translog.fs.buffer_size"; public static final String INDEX_TRANSLOG_SYNC_INTERVAL = "index.translog.sync_interval"; private final TimeValue syncInterval; @@ -52,7 +50,7 @@ public final class TranslogConfig { private volatile TranslogGeneration translogGeneration; private volatile Translog.Durabilty durabilty = Translog.Durabilty.REQUEST; private volatile TranslogWriter.Type type; - private final Settings indexSettings; + private final IndexSettings indexSettings; private final ShardId shardId; private final Path translogPath; @@ -65,17 +63,17 @@ public final class TranslogConfig { * @param bigArrays a bigArrays instance used for temporarily allocating write operations * @param threadPool a {@link ThreadPool} to schedule async sync durability */ - public TranslogConfig(ShardId shardId, Path translogPath, @IndexSettings Settings indexSettings, Translog.Durabilty durabilty, BigArrays bigArrays, @Nullable ThreadPool threadPool) { + public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSettings, Translog.Durabilty durabilty, BigArrays bigArrays, @Nullable ThreadPool threadPool) { this.indexSettings = indexSettings; this.shardId = shardId; this.translogPath = translogPath; this.durabilty = durabilty; this.threadPool = threadPool; this.bigArrays = bigArrays; - this.type = TranslogWriter.Type.fromString(indexSettings.get(INDEX_TRANSLOG_FS_TYPE, TranslogWriter.Type.BUFFERED.name())); - this.bufferSizeBytes = (int) indexSettings.getAsBytesSize(INDEX_TRANSLOG_BUFFER_SIZE, IndexingMemoryController.SHARD_TRANSLOG_BUFFER).bytes(); + this.type = TranslogWriter.Type.fromString(indexSettings.getSettings().get(INDEX_TRANSLOG_FS_TYPE, TranslogWriter.Type.BUFFERED.name())); + this.bufferSizeBytes = (int) IndexingMemoryController.SHARD_TRANSLOG_BUFFER.bytes(); - syncInterval = indexSettings.getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5)); + syncInterval = indexSettings.getSettings().getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5)); if (syncInterval.millis() > 0 && threadPool != null) { syncOnEachOperation = false; } else if (syncInterval.millis() == 0) { @@ -128,7 +126,7 @@ public final class TranslogConfig { } /** - * Retruns the current translog buffer size. + * Returns the current translog buffer size. */ public int getBufferSizeBytes() { return bufferSizeBytes; @@ -142,9 +140,9 @@ public final class TranslogConfig { } /** - * Returns the current index settings + * Returns the index indexSettings */ - public Settings getIndexSettings() { + public IndexSettings getIndexSettings() { return indexSettings; } diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogReader.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogReader.java index 590bc319057..d7077fd90ad 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/TranslogReader.java +++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogReader.java @@ -140,16 +140,16 @@ public abstract class TranslogReader implements Closeable, Comparable onClose, int bufferSize) throws IOException { + public static TranslogWriter create(Type type, ShardId shardId, String translogUUID, long fileGeneration, Path file, Callback onClose, int bufferSize, ChannelFactory channelFactory) throws IOException { final BytesRef ref = new BytesRef(translogUUID); final int headerLength = CodecUtil.headerLength(TRANSLOG_CODEC) + ref.length + RamUsageEstimator.NUM_BYTES_INT; - final FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW); + final FileChannel channel = channelFactory.open(file); try { // This OutputStreamDataOutput is intentionally not closed because // closing it will close the FileChannel @@ -90,6 +94,12 @@ public class TranslogWriter extends TranslogReader { throw throwable; } } + /** If this {@code TranslogWriter} was closed as a side-effect of a tragic exception, + * e.g. disk full while flushing a new segment, this returns the root cause exception. + * Otherwise (no tragic exception has occurred) it returns null. */ + public Throwable getTragicException() { + return tragedy; + } public enum Type { @@ -118,18 +128,33 @@ public class TranslogWriter extends TranslogReader { } } + protected final void closeWithTragicEvent(Throwable throwable) throws IOException { + try (ReleasableLock lock = writeLock.acquire()) { + if (tragedy == null) { + tragedy = throwable; + } else { + tragedy.addSuppressed(throwable); + } + close(); + } + } /** * add the given bytes to the translog and return the location they were written at */ public Translog.Location add(BytesReference data) throws IOException { - ensureOpen(); final long position; try (ReleasableLock lock = writeLock.acquire()) { + ensureOpen(); position = writtenOffset; - data.writeTo(channel); + try { + data.writeTo(channel); + } catch (Throwable e) { + closeWithTragicEvent(e); + throw e; + } writtenOffset = writtenOffset + data.length(); - operationCounter = operationCounter + 1; + operationCounter++;; } return new Translog.Location(generation, position, data.length()); } @@ -143,12 +168,13 @@ public class TranslogWriter extends TranslogReader { /** * write all buffered ops to disk and fsync file */ - public void sync() throws IOException { + public synchronized void sync() throws IOException { // synchronized to ensure only one sync happens a time // check if we really need to sync here... if (syncNeeded()) { try (ReleasableLock lock = writeLock.acquire()) { + ensureOpen(); + checkpoint(writtenOffset, operationCounter, channelReference); lastSyncedOffset = writtenOffset; - checkpoint(lastSyncedOffset, operationCounter, channelReference); } } } @@ -200,9 +226,9 @@ public class TranslogWriter extends TranslogReader { * returns a new immutable reader which only exposes the current written operation * */ public ImmutableTranslogReader immutableReader() throws TranslogException { - ensureOpen(); if (channelReference.tryIncRef()) { try (ReleasableLock lock = writeLock.acquire()) { + ensureOpen(); flush(); ImmutableTranslogReader reader = new ImmutableTranslogReader(this.generation, channelReference, firstOperationOffset, writtenOffset, operationCounter); channelReference.incRef(); // for new reader @@ -262,15 +288,6 @@ public class TranslogWriter extends TranslogReader { return false; } - @Override - protected final void doClose() throws IOException { - try (ReleasableLock lock = writeLock.acquire()) { - sync(); - } finally { - super.doClose(); - } - } - @Override protected void readBytes(ByteBuffer buffer, long position) throws IOException { try (ReleasableLock lock = readLock.acquire()) { @@ -288,4 +305,20 @@ public class TranslogWriter extends TranslogReader { Checkpoint checkpoint = new Checkpoint(syncPosition, numOperations, generation); Checkpoint.write(checkpointFile, checkpoint, options); } + + static class ChannelFactory { + + static final ChannelFactory DEFAULT = new ChannelFactory(); + + // only for testing until we have a disk-full FileSystemt + public FileChannel open(Path file) throws IOException { + return FileChannel.open(file, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW); + } + } + + protected final void ensureOpen() { + if (isClosed()) { + throw new AlreadyClosedException("translog [" + getGeneration() + "] is already closed", tragedy); + } + } } diff --git a/core/src/main/java/org/elasticsearch/index/warmer/ShardIndexWarmerService.java b/core/src/main/java/org/elasticsearch/index/warmer/ShardIndexWarmerService.java index f9b33769f81..57decb25f56 100644 --- a/core/src/main/java/org/elasticsearch/index/warmer/ShardIndexWarmerService.java +++ b/core/src/main/java/org/elasticsearch/index/warmer/ShardIndexWarmerService.java @@ -19,12 +19,10 @@ package org.elasticsearch.index.warmer; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; @@ -37,7 +35,7 @@ public class ShardIndexWarmerService extends AbstractIndexShardComponent { private final CounterMetric current = new CounterMetric(); private final MeanMetric warmerMetric = new MeanMetric(); - public ShardIndexWarmerService(ShardId shardId, @IndexSettings Settings indexSettings) { + public ShardIndexWarmerService(ShardId shardId, IndexSettings indexSettings) { super(shardId, indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesLifecycle.java b/core/src/main/java/org/elasticsearch/indices/IndicesLifecycle.java deleted file mode 100644 index 8c761dfe898..00000000000 --- a/core/src/main/java/org/elasticsearch/indices/IndicesLifecycle.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices; - -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.shard.IndexShardState; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.settings.IndexSettings; - -/** - * A global component allowing to register for lifecycle of an index (create/closed) and - * an index shard (created/closed). - */ -public interface IndicesLifecycle { - - /** - * Add a listener. - */ - void addListener(Listener listener); - - /** - * Remove a listener. - */ - void removeListener(Listener listener); - - /** - * A listener for index and index shard lifecycle events (create/closed). - */ - public abstract static class Listener { - - /** - * Called when the shard routing has changed state. - * - * @param indexShard The index shard - * @param oldRouting The old routing state (can be null) - * @param newRouting The new routing state - */ - public void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting oldRouting, ShardRouting newRouting) { - - } - - /** - * Called on the Master node only before the index is created - */ - public void beforeIndexAddedToCluster(Index index, @IndexSettings Settings indexSettings) { - - } - - /** - * Called before the index gets created. Note that this is also called - * when the index is created on data nodes - */ - public void beforeIndexCreated(Index index, @IndexSettings Settings indexSettings) { - - } - - /** - * Called after the index has been created. - */ - public void afterIndexCreated(IndexService indexService) { - - } - - /** - * Called before the index shard gets created. - */ - public void beforeIndexShardCreated(ShardId shardId, @IndexSettings Settings indexSettings) { - - } - - /** - * Called after the index shard has been created. - */ - public void afterIndexShardCreated(IndexShard indexShard) { - - } - - /** - * Called after the index shard has been started. - */ - public void afterIndexShardStarted(IndexShard indexShard) { - - } - - /** - * Called before the index get closed. - * - * @param indexService The index service - */ - public void beforeIndexClosed(IndexService indexService) { - - } - - /** - * Called after the index has been closed. - * - * @param index The index - */ - public void afterIndexClosed(Index index, @IndexSettings Settings indexSettings) { - - } - - /** - * Called before the index shard gets closed. - * - * @param indexShard The index shard - */ - public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - - } - - /** - * Called after the index shard has been closed. - * - * @param shardId The shard id - */ - public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - - } - - /** - * Called before the index shard gets deleted from disk - * Note: this method is only executed on the first attempt of deleting the shard. Retries are will not invoke - * this method. - * @param shardId The shard id - * @param indexSettings the shards index settings - */ - public void beforeIndexShardDeleted(ShardId shardId, @IndexSettings Settings indexSettings) { - } - - /** - * Called after the index shard has been deleted from disk. - * - * Note: this method is only called if the deletion of the shard did finish without an exception - * - * @param shardId The shard id - * @param indexSettings the shards index settings - */ - public void afterIndexShardDeleted(ShardId shardId, @IndexSettings Settings indexSettings) { - } - - /** - * Called after a shard's {@link org.elasticsearch.index.shard.IndexShardState} changes. - * The order of concurrent events is preserved. The execution must be lightweight. - * - * @param indexShard the shard the new state was applied to - * @param previousState the previous index shard state if there was one, null otherwise - * @param currentState the new shard state - * @param reason the reason for the state change if there is one, null otherwise - */ - public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) { - - } - - /** - * Called after the index has been deleted. - * This listener method is invoked after {@link #afterIndexClosed(org.elasticsearch.index.Index, org.elasticsearch.common.settings.Settings)} - * when an index is deleted - * - * @param index The index - */ - public void afterIndexDeleted(Index index, @IndexSettings Settings indexSettings) { - - } - - /** - * Called before the index gets deleted. - * This listener method is invoked after - * {@link #beforeIndexClosed(org.elasticsearch.index.IndexService)} when an index is deleted - * - * @param indexService The index service - */ - public void beforeIndexDeleted(IndexService indexService) { - - } - - /** - * Called when a shard is marked as inactive - * - * @param indexShard The shard that was marked inactive - */ - public void onShardInactive(IndexShard indexShard) { - - } - } - -} diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java index ff9bd334d9f..6878002c015 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -19,25 +19,30 @@ package org.elasticsearch.indices; -import org.apache.lucene.analysis.hunspell.Dictionary; import org.elasticsearch.action.update.UpdateHelper; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.ExtensionPoint; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.core.*; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.internal.*; +import org.elasticsearch.index.mapper.ip.IpFieldMapper; +import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryParser; -import org.elasticsearch.index.query.MoreLikeThisQueryParser; import org.elasticsearch.index.termvectors.TermVectorsService; -import org.elasticsearch.indices.analysis.HunspellService; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.cache.request.IndicesRequestCache; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener; import org.elasticsearch.indices.flush.SyncedFlushService; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.indices.recovery.RecoverySettings; @@ -47,23 +52,30 @@ import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.indices.ttl.IndicesTTLService; +import java.util.LinkedHashMap; +import java.util.Map; + /** * Configures classes and services that are shared by indices on each node. */ public class IndicesModule extends AbstractModule { - private final Settings settings; private final ExtensionPoint.ClassSet queryParsers = new ExtensionPoint.ClassSet<>("query_parser", QueryParser.class); - private final ExtensionPoint.InstanceMap hunspellDictionaries - = new ExtensionPoint.InstanceMap<>("hunspell_dictionary", String.class, Dictionary.class); - public IndicesModule(Settings settings) { - this.settings = settings; + private final Map mapperParsers + = new LinkedHashMap<>(); + // Use a LinkedHashMap for metadataMappers because iteration order matters + private final Map metadataMapperParsers + = new LinkedHashMap<>(); + + public IndicesModule() { registerBuiltinQueryParsers(); + registerBuiltInMappers(); + registerBuiltInMetadataMappers(); } - + private void registerBuiltinQueryParsers() { registerQueryParser(MatchQueryParser.class); registerQueryParser(MultiMatchQueryParser.class); @@ -107,10 +119,7 @@ public class IndicesModule extends AbstractModule { registerQueryParser(GeoBoundingBoxQueryParser.class); registerQueryParser(GeohashCellQuery.Parser.class); registerQueryParser(GeoPolygonQueryParser.class); - registerQueryParser(QueryFilterParser.class); - registerQueryParser(NotQueryParser.class); registerQueryParser(ExistsQueryParser.class); - registerQueryParser(MissingQueryParser.class); registerQueryParser(MatchNoneQueryParser.class); if (ShapesAvailability.JTS_AVAILABLE) { @@ -118,20 +127,78 @@ public class IndicesModule extends AbstractModule { } } + private void registerBuiltInMappers() { + registerMapper(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser()); + registerMapper(ShortFieldMapper.CONTENT_TYPE, new ShortFieldMapper.TypeParser()); + registerMapper(IntegerFieldMapper.CONTENT_TYPE, new IntegerFieldMapper.TypeParser()); + registerMapper(LongFieldMapper.CONTENT_TYPE, new LongFieldMapper.TypeParser()); + registerMapper(FloatFieldMapper.CONTENT_TYPE, new FloatFieldMapper.TypeParser()); + registerMapper(DoubleFieldMapper.CONTENT_TYPE, new DoubleFieldMapper.TypeParser()); + registerMapper(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()); + registerMapper(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser()); + registerMapper(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser()); + registerMapper(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser()); + registerMapper(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); + registerMapper(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); + registerMapper(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); + registerMapper(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); + registerMapper(TypeParsers.MULTI_FIELD_CONTENT_TYPE, TypeParsers.multiFieldConverterTypeParser); + registerMapper(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); + registerMapper(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); + + if (ShapesAvailability.JTS_AVAILABLE) { + registerMapper(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); + } + } + + private void registerBuiltInMetadataMappers() { + // NOTE: the order is important + + // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination + registerMetadataMapper(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()); + registerMetadataMapper(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); + registerMetadataMapper(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); + registerMetadataMapper(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser()); + registerMetadataMapper(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser()); + registerMetadataMapper(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser()); + registerMetadataMapper(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); + registerMetadataMapper(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser()); + registerMetadataMapper(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser()); + registerMetadataMapper(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()); + registerMetadataMapper(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser()); + // _field_names is not registered here, see #getMapperRegistry: we need to register it + // last so that it can see all other mappers, including those coming from plugins + } + public void registerQueryParser(Class queryParser) { queryParsers.registerExtension(queryParser); } - public void registerHunspellDictionary(String name, Dictionary dictionary) { - hunspellDictionaries.registerExtension(name, dictionary); + /** + * Register a mapper for the given type. + */ + public synchronized void registerMapper(String type, Mapper.TypeParser parser) { + if (mapperParsers.containsKey(type)) { + throw new IllegalArgumentException("A mapper is already registered for type [" + type + "]"); + } + mapperParsers.put(type, parser); + } + + /** + * Register a root mapper under the given name. + */ + public synchronized void registerMetadataMapper(String name, MetadataFieldMapper.TypeParser parser) { + if (metadataMapperParsers.containsKey(name)) { + throw new IllegalArgumentException("A mapper is already registered for metadata mapper [" + name + "]"); + } + metadataMapperParsers.put(name, parser); } @Override protected void configure() { bindQueryParsersExtension(); - bindHunspellExtension(); + bindMapperExtension(); - bind(IndicesLifecycle.class).to(InternalIndicesLifecycle.class).asEagerSingleton(); bind(IndicesService.class).asEagerSingleton(); bind(RecoverySettings.class).asEagerSingleton(); bind(RecoveryTarget.class).asEagerSingleton(); @@ -150,16 +217,28 @@ public class IndicesModule extends AbstractModule { bind(MetaDataIndexUpgradeService.class).asEagerSingleton(); bind(IndicesFieldDataCacheListener.class).asEagerSingleton(); bind(TermVectorsService.class).asEagerSingleton(); + bind(NodeServicesProvider.class).asEagerSingleton(); + } + + // public for testing + public synchronized MapperRegistry getMapperRegistry() { + // NOTE: we register _field_names here so that it has a chance to see all other + // mappers, including from plugins + if (metadataMapperParsers.containsKey(FieldNamesFieldMapper.NAME)) { + throw new IllegalStateException("Metadata mapper [" + FieldNamesFieldMapper.NAME + "] is already registered"); + } + final Map metadataMapperParsers + = new LinkedHashMap<>(this.metadataMapperParsers); + metadataMapperParsers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()); + return new MapperRegistry(mapperParsers, metadataMapperParsers); + } + + protected void bindMapperExtension() { + bind(MapperRegistry.class).toInstance(getMapperRegistry()); } protected void bindQueryParsersExtension() { queryParsers.bind(binder()); bind(IndicesQueriesRegistry.class).asEagerSingleton(); } - - protected void bindHunspellExtension() { - hunspellDictionaries.bind(binder()); - bind(HunspellService.class).asEagerSingleton(); - bind(IndicesAnalysisService.class).asEagerSingleton(); - } } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index dedfb4b1ff9..dead72aee8b 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -28,16 +28,13 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.inject.CreationException; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.Injectors; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -45,118 +42,80 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; import org.elasticsearch.gateway.MetaDataStateFormat; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexModule; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.analysis.AnalysisModule; -import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.cache.IndexCache; -import org.elasticsearch.index.cache.IndexCacheModule; -import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.*; +import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.indexing.IndexingStats; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsModule; import org.elasticsearch.index.shard.IllegalIndexShardStateException; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.similarity.SimilarityModule; -import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.index.store.IndexStoreModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; -import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.index.store.IndexStoreConfig; +import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.plugins.PluginsService; -import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; +import java.util.function.Predicate; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; /** * */ -public class IndicesService extends AbstractLifecycleComponent implements Iterable { +public class IndicesService extends AbstractLifecycleComponent implements Iterable, IndexService.ShardStoreDeleter { public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout"; - - private final InternalIndicesLifecycle indicesLifecycle; - - private final IndicesAnalysisService indicesAnalysisService; - - private final Injector injector; - private final PluginsService pluginsService; private final NodeEnvironment nodeEnv; private final TimeValue shardsClosedTimeout; - - private volatile Map indices = emptyMap(); - - static class IndexServiceInjectorPair { - private final IndexService indexService; - private final Injector injector; - - public IndexServiceInjectorPair(IndexService indexService, Injector injector) { - this.indexService = indexService; - this.injector = injector; - } - - public IndexService getIndexService() { - return indexService; - } - - public Injector getInjector() { - return injector; - } - } - + private final AnalysisRegistry analysisRegistry; + private final IndicesQueriesRegistry indicesQueriesRegistry; + private final ClusterService clusterService; + private final IndexNameExpressionResolver indexNameExpressionResolver; + private volatile Map indices = emptyMap(); private final Map> pendingDeletes = new HashMap<>(); - private final OldShardsStats oldShardsStats = new OldShardsStats(); - - @Inject - public IndicesService(Settings settings, IndicesLifecycle indicesLifecycle, IndicesAnalysisService indicesAnalysisService, Injector injector, NodeEnvironment nodeEnv) { - super(settings); - this.indicesLifecycle = (InternalIndicesLifecycle) indicesLifecycle; - this.indicesAnalysisService = indicesAnalysisService; - this.injector = injector; - this.pluginsService = injector.getInstance(PluginsService.class); - this.indicesLifecycle.addListener(oldShardsStats); - this.nodeEnv = nodeEnv; - this.shardsClosedTimeout = settings.getAsTime(INDICES_SHARDS_CLOSED_TIMEOUT, new TimeValue(1, TimeUnit.DAYS)); - } + private final IndexStoreConfig indexStoreConfig; + private final MapperRegistry mapperRegistry; @Override protected void doStart() { } + @Inject + public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv, + NodeSettingsService nodeSettingsService, AnalysisRegistry analysisRegistry, + IndicesQueriesRegistry indicesQueriesRegistry, IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService, MapperRegistry mapperRegistry) { + super(settings); + this.pluginsService = pluginsService; + this.nodeEnv = nodeEnv; + this.shardsClosedTimeout = settings.getAsTime(INDICES_SHARDS_CLOSED_TIMEOUT, new TimeValue(1, TimeUnit.DAYS)); + this.indexStoreConfig = new IndexStoreConfig(settings); + this.analysisRegistry = analysisRegistry; + this.indicesQueriesRegistry = indicesQueriesRegistry; + this.clusterService = clusterService; + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.mapperRegistry = mapperRegistry; + nodeSettingsService.addListener(indexStoreConfig); + } + @Override protected void doStop() { ExecutorService indicesStopExecutor = Executors.newFixedThreadPool(5, EsExecutors.daemonThreadFactory("indices_shutdown")); @@ -165,16 +124,13 @@ public class IndicesService extends AbstractLifecycleComponent i Set indices = new HashSet<>(this.indices.keySet()); final CountDownLatch latch = new CountDownLatch(indices.size()); for (final String index : indices) { - indicesStopExecutor.execute(new Runnable() { - @Override - public void run() { - try { - removeIndex(index, "shutdown", false); - } catch (Throwable e) { - logger.warn("failed to remove index on stop [" + index + "]", e); - } finally { - latch.countDown(); - } + indicesStopExecutor.execute(() -> { + try { + removeIndex(index, "shutdown", false); + } catch (Throwable e) { + logger.warn("failed to remove index on stop [" + index + "]", e); + } finally { + latch.countDown(); } }); } @@ -191,12 +147,7 @@ public class IndicesService extends AbstractLifecycleComponent i @Override protected void doClose() { - IOUtils.closeWhileHandlingException(injector.getInstance(RecoverySettings.class), - indicesAnalysisService); - } - - public IndicesLifecycle indicesLifecycle() { - return this.indicesLifecycle; + IOUtils.closeWhileHandlingException(analysisRegistry); } /** @@ -241,8 +192,7 @@ public class IndicesService extends AbstractLifecycleComponent i } Map> statsByShard = new HashMap<>(); - for (IndexServiceInjectorPair value : indices.values()) { - IndexService indexService = value.getIndexService(); + for (IndexService indexService : indices.values()) { for (IndexShard indexShard : indexService) { try { if (indexShard.routingEntry() == null) { @@ -273,7 +223,7 @@ public class IndicesService extends AbstractLifecycleComponent i @Override public Iterator iterator() { - return indices.values().stream().map((p) -> p.getIndexService()).iterator(); + return indices.values().iterator(); } public boolean hasIndex(String index) { @@ -286,12 +236,7 @@ public class IndicesService extends AbstractLifecycleComponent i */ @Nullable public IndexService indexService(String index) { - IndexServiceInjectorPair indexServiceInjectorPair = indices.get(index); - if (indexServiceInjectorPair == null) { - return null; - } else { - return indexServiceInjectorPair.getIndexService(); - } + return indices.get(index); } /** @@ -305,58 +250,53 @@ public class IndicesService extends AbstractLifecycleComponent i return indexService; } - public synchronized IndexService createIndex(IndexMetaData indexMetaData) { + + + /** + * Creates a new {@link IndexService} for the given metadata. + * @param indexMetaData the index metadata to create the index for + * @param builtInListeners a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with the per-index listeners + * @throws IndexAlreadyExistsException if the index already exists. + */ + public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List builtInListeners) throws IOException { if (!lifecycle.started()) { throw new IllegalStateException("Can't create an index [" + indexMetaData.getIndex() + "], node is closed"); } - final Settings settings = indexMetaData.getSettings(); + final String indexName = indexMetaData.getIndex(); + final Predicate indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(indexName, indexExpression, clusterService.state()); + final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, Collections.emptyList(), indexNameMatcher); Index index = new Index(indexMetaData.getIndex()); if (indices.containsKey(index.name())) { throw new IndexAlreadyExistsException(index); } - - indicesLifecycle.beforeIndexCreated(index, settings); - logger.debug("creating Index [{}], shards [{}]/[{}{}]", indexMetaData.getIndex(), - settings.get(SETTING_NUMBER_OF_SHARDS), - settings.get(SETTING_NUMBER_OF_REPLICAS), - IndexMetaData.isIndexUsingShadowReplicas(settings) ? "s" : ""); + idxSettings.getNumberOfShards(), + idxSettings.getNumberOfReplicas(), + idxSettings.isShadowReplicaIndex() ? "s" : ""); - Settings indexSettings = settingsBuilder() - .put(this.settings) - .put(indexMetaData.getSettings()) - .build(); - - ModulesBuilder modules = new ModulesBuilder(); - modules.add(new IndexNameModule(index)); - modules.add(new IndexSettingsModule(index, indexSettings)); - // plugin modules must be added here, before others or we can get crazy injection errors... - for (Module pluginModule : pluginsService.indexModules(indexSettings)) { - modules.add(pluginModule); + final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry); + pluginsService.onIndexModule(indexModule); + for (IndexEventListener listener : builtInListeners) { + indexModule.addIndexEventListener(listener); } - modules.add(new IndexStoreModule(indexSettings)); - modules.add(new AnalysisModule(indexSettings, indicesAnalysisService)); - modules.add(new SimilarityModule(index, indexSettings)); - modules.add(new IndexCacheModule(indexSettings)); - modules.add(new IndexModule(indexMetaData)); - pluginsService.processModules(modules); - - Injector indexInjector; + indexModule.addIndexEventListener(oldShardsStats); + final IndexEventListener listener = indexModule.freeze(); + listener.beforeIndexCreated(index, idxSettings.getSettings()); + final IndexService indexService = indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, mapperRegistry); + boolean success = false; try { - indexInjector = modules.createChildInjector(injector); - } catch (CreationException e) { - throw new IndexCreationException(index, Injectors.getFirstErrorFailure(e)); - } catch (Throwable e) { - throw new IndexCreationException(index, e); + assert indexService.getIndexEventListener() == listener; + listener.afterIndexCreated(indexService); + indices = newMapBuilder(indices).put(index.name(), indexService).immutableMap(); + success = true; + return indexService; + } finally { + if (success == false) { + indexService.close("plugins_failed", true); + } } - IndexService indexService = indexInjector.getInstance(IndexService.class); - - indicesLifecycle.afterIndexCreated(indexService); - - indices = newMapBuilder(indices).put(index.name(), new IndexServiceInjectorPair(indexService, indexInjector)).immutableMap(); - return indexService; } /** @@ -372,50 +312,30 @@ public class IndicesService extends AbstractLifecycleComponent i private void removeIndex(String index, String reason, boolean delete) { try { final IndexService indexService; - final Injector indexInjector; + final IndexEventListener listener; synchronized (this) { if (indices.containsKey(index) == false) { return; } logger.debug("[{}] closing ... (reason [{}])", index, reason); - Map newIndices = new HashMap<>(indices); - IndexServiceInjectorPair remove = newIndices.remove(index); - indexService = remove.getIndexService(); - indexInjector = remove.getInjector(); + Map newIndices = new HashMap<>(indices); + indexService = newIndices.remove(index); indices = unmodifiableMap(newIndices); + listener = indexService.getIndexEventListener(); } - indicesLifecycle.beforeIndexClosed(indexService); + listener.beforeIndexClosed(indexService); if (delete) { - indicesLifecycle.beforeIndexDeleted(indexService); + listener.beforeIndexDeleted(indexService); } - Stream closeables = pluginsService.indexServices().stream().map(p -> indexInjector.getInstance(p)); - IOUtils.close(closeables::iterator); - logger.debug("[{}] closing index service (reason [{}])", index, reason); indexService.close(reason, delete); - - logger.debug("[{}] closing index cache (reason [{}])", index, reason); - indexInjector.getInstance(IndexCache.class).close(); - logger.debug("[{}] clearing index field data (reason [{}])", index, reason); - indexInjector.getInstance(IndexFieldDataService.class).clear(); - logger.debug("[{}] closing analysis service (reason [{}])", index, reason); - indexInjector.getInstance(AnalysisService.class).close(); - - logger.debug("[{}] closing mapper service (reason [{}])", index, reason); - indexInjector.getInstance(MapperService.class).close(); - logger.debug("[{}] closing index query parser service (reason [{}])", index, reason); - indexInjector.getInstance(IndexQueryParserService.class).close(); - - logger.debug("[{}] closing index service (reason [{}])", index, reason); - indexInjector.getInstance(IndexStore.class).close(); - logger.debug("[{}] closed... (reason [{}])", index, reason); - indicesLifecycle.afterIndexClosed(indexService.index(), indexService.settingsService().getSettings()); + listener.afterIndexClosed(indexService.index(), indexService.getIndexSettings().getSettings()); if (delete) { - final Settings indexSettings = indexService.getIndexSettings(); - indicesLifecycle.afterIndexDeleted(indexService.index(), indexSettings); + final IndexSettings indexSettings = indexService.getIndexSettings(); + listener.afterIndexDeleted(indexService.index(), indexSettings.getSettings()); // now we are done - try to wipe data on disk if possible deleteIndexStore(reason, indexService.index(), indexSettings, false); } @@ -424,7 +344,7 @@ public class IndicesService extends AbstractLifecycleComponent i } } - static class OldShardsStats extends IndicesLifecycle.Listener { + static class OldShardsStats implements IndexEventListener { final SearchStats searchStats = new SearchStats(); final GetStats getStats = new GetStats(); @@ -435,16 +355,15 @@ public class IndicesService extends AbstractLifecycleComponent i final RecoveryStats recoveryStats = new RecoveryStats(); @Override - public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { + public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { if (indexShard != null) { - getStats.add(indexShard.getStats()); - indexingStats.add(indexShard.indexingStats(), false); - searchStats.add(indexShard.searchStats(), false); - mergeStats.add(indexShard.mergeStats()); - refreshStats.add(indexShard.refreshStats()); - flushStats.add(indexShard.flushStats()); - recoveryStats.addAsOld(indexShard.recoveryStats()); + getStats.addTotals(indexShard.getStats()); + indexingStats.addTotals(indexShard.indexingStats()); + searchStats.addTotals(indexShard.searchStats()); + mergeStats.addTotals(indexShard.mergeStats()); + refreshStats.addTotals(indexShard.refreshStats()); + flushStats.addTotals(indexShard.flushStats()); + recoveryStats.addTotals(indexShard.recoveryStats()); } } } @@ -472,7 +391,7 @@ public class IndicesService extends AbstractLifecycleComponent i } deleteIndexStore(reason, metaData, clusterState, true); } catch (IOException e) { - logger.warn("[{}] failed to delete closed index", e, metaData.index()); + logger.warn("[{}] failed to delete closed index", e, metaData.getIndex()); } } } @@ -484,9 +403,9 @@ public class IndicesService extends AbstractLifecycleComponent i public void deleteIndexStore(String reason, IndexMetaData metaData, ClusterState clusterState, boolean closed) throws IOException { if (nodeEnv.hasNodeFile()) { synchronized (this) { - String indexName = metaData.index(); + String indexName = metaData.getIndex(); if (indices.containsKey(indexName)) { - String localUUid = indices.get(indexName).getIndexService().indexUUID(); + String localUUid = indices.get(indexName).indexUUID(); throw new IllegalStateException("Can't delete index store for [" + indexName + "] - it's still part of the indices service [" + localUUid + "] [" + metaData.getIndexUUID() + "]"); } if (clusterState.metaData().hasIndex(indexName) && (clusterState.nodes().localNode().masterNode() == true)) { @@ -496,13 +415,13 @@ public class IndicesService extends AbstractLifecycleComponent i throw new IllegalStateException("Can't delete closed index store for [" + indexName + "] - it's still part of the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]"); } } - Index index = new Index(metaData.index()); - final Settings indexSettings = buildIndexSettings(metaData); + Index index = new Index(metaData.getIndex()); + final IndexSettings indexSettings = buildIndexSettings(metaData); deleteIndexStore(reason, index, indexSettings, closed); } } - private void deleteIndexStore(String reason, Index index, Settings indexSettings, boolean closed) throws IOException { + private void deleteIndexStore(String reason, Index index, IndexSettings indexSettings, boolean closed) throws IOException { boolean success = false; try { // we are trying to delete the index store here - not a big deal if the lock can't be obtained @@ -534,7 +453,7 @@ public class IndicesService extends AbstractLifecycleComponent i * @param indexSettings the shards index settings. * @throws IOException if an IOException occurs */ - public void deleteShardStore(String reason, ShardLock lock, Settings indexSettings) throws IOException { + public void deleteShardStore(String reason, ShardLock lock, IndexSettings indexSettings) throws IOException { ShardId shardId = lock.getShardId(); logger.trace("{} deleting shard reason [{}]", shardId, reason); nodeEnv.deleteShardDirectoryUnderLock(lock, indexSettings); @@ -542,7 +461,7 @@ public class IndicesService extends AbstractLifecycleComponent i /** * This method deletes the shard contents on disk for the given shard ID. This method will fail if the shard deleting - * is prevented by {@link #canDeleteShardContent(org.elasticsearch.index.shard.ShardId, org.elasticsearch.cluster.metadata.IndexMetaData)} + * is prevented by {@link #canDeleteShardContent(ShardId, IndexSettings)} * of if the shards lock can not be acquired. * * On data nodes, if the deleted shard is the last shard folder in its index, the method will attempt to remove the index folder as well. @@ -555,7 +474,7 @@ public class IndicesService extends AbstractLifecycleComponent i public void deleteShardStore(String reason, ShardId shardId, ClusterState clusterState) throws IOException { final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndex()); - final Settings indexSettings = buildIndexSettings(metaData); + final IndexSettings indexSettings = buildIndexSettings(metaData); if (canDeleteShardContent(shardId, indexSettings) == false) { throw new IllegalStateException("Can't delete shard " + shardId); } @@ -583,15 +502,15 @@ public class IndicesService extends AbstractLifecycleComponent i * given index. If the index uses a shared filesystem this method always * returns false. * @param index {@code Index} to check whether deletion is allowed - * @param indexSettings {@code Settings} for the given index + * @param indexSettings {@code IndexSettings} for the given index * @return true if the index can be deleted on this node */ - public boolean canDeleteIndexContents(Index index, Settings indexSettings, boolean closed) { - final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(index.name()); + public boolean canDeleteIndexContents(Index index, IndexSettings indexSettings, boolean closed) { + final IndexService indexService = this.indices.get(index.name()); // Closed indices may be deleted, even if they are on a shared // filesystem. Since it is closed we aren't deleting it for relocation - if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false || closed) { - if (indexServiceInjectorPair == null && nodeEnv.hasNodeFile()) { + if (indexSettings.isOnSharedFilesystem() == false || closed) { + if (indexService == null && nodeEnv.hasNodeFile()) { return true; } } else { @@ -610,25 +529,16 @@ public class IndicesService extends AbstractLifecycleComponent i * * * @param shardId the shard to delete. - * @param metaData the shards index metadata. This is required to access the indexes settings etc. + * @param indexSettings the shards's relevant {@link IndexSettings}. This is required to access the indexes settings etc. */ - public boolean canDeleteShardContent(ShardId shardId, IndexMetaData metaData) { - // we need the metadata here since we have to build the complete settings - // to decide where the shard content lives. In the future we might even need more info here ie. for shadow replicas - // The plan was to make it harder to miss-use and ask for metadata instead of simple settings - assert shardId.getIndex().equals(metaData.getIndex()); - final Settings indexSettings = buildIndexSettings(metaData); - return canDeleteShardContent(shardId, indexSettings); - } - - private boolean canDeleteShardContent(ShardId shardId, @IndexSettings Settings indexSettings) { - final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(shardId.getIndex()); - if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false) { - if (indexServiceInjectorPair != null && nodeEnv.hasNodeFile()) { - final IndexService indexService = indexServiceInjectorPair.getIndexService(); + public boolean canDeleteShardContent(ShardId shardId, IndexSettings indexSettings) { + assert shardId.getIndex().equals(indexSettings.getIndex().name()); + final IndexService indexService = this.indices.get(shardId.getIndex()); + if (indexSettings.isOnSharedFilesystem() == false) { + if (indexService != null && nodeEnv.hasNodeFile()) { return indexService.hasShard(shardId.id()) == false; } else if (nodeEnv.hasNodeFile()) { - if (NodeEnvironment.hasCustomDataPath(indexSettings)) { + if (indexSettings.hasCustomDataPath()) { return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId)); } else { return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId)); @@ -640,20 +550,17 @@ public class IndicesService extends AbstractLifecycleComponent i return false; } - private Settings buildIndexSettings(IndexMetaData metaData) { + private IndexSettings buildIndexSettings(IndexMetaData metaData) { // play safe here and make sure that we take node level settings into account. // we might run on nodes where we use shard FS and then in the future don't delete // actual content. - Settings.Builder builder = settingsBuilder(); - builder.put(settings); - builder.put(metaData.getSettings()); - return builder.build(); + return new IndexSettings(metaData, settings, Collections.emptyList()); } /** * Adds a pending delete for the given index shard. */ - public void addPendingDelete(ShardId shardId, @IndexSettings Settings settings) { + public void addPendingDelete(ShardId shardId, IndexSettings settings) { if (shardId == null) { throw new IllegalArgumentException("shardId must not be null"); } @@ -667,7 +574,7 @@ public class IndicesService extends AbstractLifecycleComponent i /** * Adds a pending delete for the given index. */ - public void addPendingDelete(Index index, @IndexSettings Settings settings) { + public void addPendingDelete(Index index, IndexSettings settings) { PendingDelete pendingDelete = new PendingDelete(index, settings); addPendingDelete(index, pendingDelete); } @@ -686,13 +593,13 @@ public class IndicesService extends AbstractLifecycleComponent i private static final class PendingDelete implements Comparable { final String index; final int shardId; - final Settings settings; + final IndexSettings settings; final boolean deleteIndex; /** * Creates a new pending delete of an index */ - public PendingDelete(ShardId shardId, Settings settings) { + public PendingDelete(ShardId shardId, IndexSettings settings) { this.index = shardId.getIndex(); this.shardId = shardId.getId(); this.settings = settings; @@ -702,7 +609,7 @@ public class IndicesService extends AbstractLifecycleComponent i /** * Creates a new pending delete of a shard */ - public PendingDelete(Index index, Settings settings) { + public PendingDelete(Index index, IndexSettings settings) { this.index = index.getName(); this.shardId = -1; this.settings = settings; @@ -734,7 +641,7 @@ public class IndicesService extends AbstractLifecycleComponent i * @param index the index to process the pending deletes for * @param timeout the timeout used for processing pending deletes */ - public void processPendingDeletes(Index index, @IndexSettings Settings indexSettings, TimeValue timeout) throws IOException { + public void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeout) throws IOException, InterruptedException { logger.debug("{} processing pending deletes", index); final long startTimeNS = System.nanoTime(); final List shardLocks = nodeEnv.lockAllForIndex(index, indexSettings, timeout.millis()); @@ -786,14 +693,9 @@ public class IndicesService extends AbstractLifecycleComponent i } if (remove.isEmpty() == false) { logger.warn("{} still pending deletes present for shards {} - retrying", index, remove.toString()); - try { - Thread.sleep(sleepTime); - sleepTime = Math.min(maxSleepTimeMs, sleepTime * 2); // increase the sleep time gradually - logger.debug("{} schedule pending delete retry after {} ms", index, sleepTime); - } catch (InterruptedException e) { - Thread.interrupted(); - return; - } + Thread.sleep(sleepTime); + sleepTime = Math.min(maxSleepTimeMs, sleepTime * 2); // increase the sleep time gradually + logger.debug("{} schedule pending delete retry after {} ms", index, sleepTime); } } while ((System.nanoTime() - startTimeNS) < timeout.nanos()); } @@ -811,4 +713,16 @@ public class IndicesService extends AbstractLifecycleComponent i return deleteList.size(); } } + + /** + * Returns this nodes {@link IndicesQueriesRegistry} + */ + public IndicesQueriesRegistry getIndicesQueryRegistry() { + return indicesQueriesRegistry; + } + + public AnalysisRegistry getAnalysis() { + return analysisRegistry; + } + } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java b/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java index 4f6f238ef6d..30ec403942f 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java @@ -19,23 +19,21 @@ package org.elasticsearch.indices; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; /** @@ -46,68 +44,45 @@ public final class IndicesWarmer extends AbstractComponent { private final ThreadPool threadPool; - private final ClusterService clusterService; - - private final IndicesService indicesService; - private final CopyOnWriteArrayList listeners = new CopyOnWriteArrayList<>(); @Inject - public IndicesWarmer(Settings settings, ThreadPool threadPool, ClusterService clusterService, IndicesService indicesService) { + public IndicesWarmer(Settings settings, ThreadPool threadPool) { super(settings); this.threadPool = threadPool; - this.clusterService = clusterService; - this.indicesService = indicesService; } public void addListener(Listener listener) { listeners.add(listener); } - public void removeListener(Listener listener) { listeners.remove(listener); } - public void warmNewReaders(final WarmerContext context) { - warmInternal(context, false); - } - - public void warmTopReader(WarmerContext context) { - warmInternal(context, true); - } - - private void warmInternal(final WarmerContext context, boolean topReader) { - final IndexMetaData indexMetaData = clusterService.state().metaData().index(context.shardId().index().name()); - if (indexMetaData == null) { + public void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings, boolean isTopReader) { + if (shard.state() == IndexShardState.CLOSED) { return; } - if (!indexMetaData.settings().getAsBoolean(INDEX_WARMER_ENABLED, settings.getAsBoolean(INDEX_WARMER_ENABLED, true))) { - return; - } - IndexService indexService = indicesService.indexService(context.shardId().index().name()); - if (indexService == null) { - return; - } - final IndexShard indexShard = indexService.getShardOrNull(context.shardId().id()); - if (indexShard == null) { + final Settings indexSettings = settings.getSettings(); + if (!indexSettings.getAsBoolean(INDEX_WARMER_ENABLED, settings.getNodeSettings().getAsBoolean(INDEX_WARMER_ENABLED, true))) { return; } if (logger.isTraceEnabled()) { - if (topReader) { - logger.trace("[{}][{}] top warming [{}]", context.shardId().index().name(), context.shardId().id(), context); + if (isTopReader) { + logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader()); } else { - logger.trace("[{}][{}] warming [{}]", context.shardId().index().name(), context.shardId().id(), context); + logger.trace("{} warming [{}]", shard.shardId(), searcher.reader()); } } - indexShard.warmerService().onPreWarm(); + shard.warmerService().onPreWarm(); long time = System.nanoTime(); final List terminationHandles = new ArrayList<>(); // get a handle on pending tasks for (final Listener listener : listeners) { - if (topReader) { - terminationHandles.add(listener.warmTopReader(indexShard, indexMetaData, context, threadPool)); + if (isTopReader) { + terminationHandles.add(listener.warmTopReader(shard, searcher)); } else { - terminationHandles.add(listener.warmNewReaders(indexShard, indexMetaData, context, threadPool)); + terminationHandles.add(listener.warmNewReaders(shard, searcher)); } } // wait for termination @@ -116,7 +91,7 @@ public final class IndicesWarmer extends AbstractComponent { terminationHandle.awaitTermination(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - if (topReader) { + if (isTopReader) { logger.warn("top warming has been interrupted", e); } else { logger.warn("warming has been interrupted", e); @@ -125,69 +100,36 @@ public final class IndicesWarmer extends AbstractComponent { } } long took = System.nanoTime() - time; - indexShard.warmerService().onPostWarm(took); - if (indexShard.warmerService().logger().isTraceEnabled()) { - if (topReader) { - indexShard.warmerService().logger().trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS)); + shard.warmerService().onPostWarm(took); + if (shard.warmerService().logger().isTraceEnabled()) { + if (isTopReader) { + shard.warmerService().logger().trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS)); } else { - indexShard.warmerService().logger().trace("warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS)); + shard.warmerService().logger().trace("warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS)); } } } + /** + * Returns an executor for async warmer tasks + */ + public Executor getExecutor() { + return threadPool.executor(ThreadPool.Names.WARMER); + } + /** A handle on the execution of warm-up action. */ public interface TerminationHandle { - public static TerminationHandle NO_WAIT = new TerminationHandle() { - @Override - public void awaitTermination() {} - }; + TerminationHandle NO_WAIT = () -> {}; /** Wait until execution of the warm-up action completes. */ void awaitTermination() throws InterruptedException; } - public static abstract class Listener { - - public String executor() { - return ThreadPool.Names.WARMER; - } - + public interface Listener { /** Queue tasks to warm-up the given segments and return handles that allow to wait for termination of the execution of those tasks. */ - public abstract TerminationHandle warmNewReaders(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool); + TerminationHandle warmNewReaders(IndexShard indexShard, Engine.Searcher searcher); - public abstract TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool); + TerminationHandle warmTopReader(IndexShard indexShard, Engine.Searcher searcher); } - public static final class WarmerContext { - - private final ShardId shardId; - private final Engine.Searcher searcher; - - public WarmerContext(ShardId shardId, Engine.Searcher searcher) { - this.shardId = shardId; - this.searcher = searcher; - } - - public ShardId shardId() { - return shardId; - } - - /** Return a searcher instance that only wraps the segments to warm. */ - public Engine.Searcher searcher() { - return searcher; - } - - public IndexReader reader() { - return searcher.reader(); - } - - public DirectoryReader getDirectoryReader() { - return searcher.getDirectoryReader(); - } - - @Override - public String toString() { - return "WarmerContext: " + searcher.reader(); - } - } } diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java new file mode 100644 index 00000000000..350678da117 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -0,0 +1,213 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.analysis; + +import org.apache.lucene.analysis.hunspell.Dictionary; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.*; + +import java.io.IOException; +import java.util.*; + +/** + * The AnalysisModule is the main extension point for node and index level analysis components. The lucene classes + * {@link org.apache.lucene.analysis.Analyzer}, {@link org.apache.lucene.analysis.TokenFilter}, {@link org.apache.lucene.analysis.Tokenizer} + * and {@link org.apache.lucene.analysis.CharFilter} can be extended in plugins and registered on node startup when the analysis module + * gets loaded. Since elasticsearch needs to create multiple instances for different configurations dedicated factories need to be provided for + * each of the components: + *
      + *
    • {@link org.apache.lucene.analysis.Analyzer} can be exposed via {@link AnalyzerProvider} and registered on {@link #registerAnalyzer(String, AnalysisProvider)}
      • + *
    • {@link org.apache.lucene.analysis.TokenFilter} can be exposed via {@link TokenFilterFactory} and registered on {@link #registerTokenFilter(String, AnalysisProvider)}
      • + *
    • {@link org.apache.lucene.analysis.Tokenizer} can be exposed via {@link TokenizerFactory} and registered on {@link #registerTokenizer(String, AnalysisProvider)}
      • + *
    • {@link org.apache.lucene.analysis.CharFilter} can be exposed via {@link CharFilterFactory} and registered on {@link #registerCharFilter(String, AnalysisProvider)}
      • + *
    + * + * The {@link org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider} is only a functional interface that allows to register factory constructors directly like the plugin example below: + * 
    + *     public class MyAnalysisPlugin extends Plugin {
+ *       \@Override
+ *       public String name() {
+ *         return "analysis-my-plugin";
+ *       }
+ *
+ *       \@Override
+ *       public String description() {
+ *         return "my very fast and efficient analyzer";
+ *       }
+ *
+ *       public void onModule(AnalysisModule module) {
+ *         module.registerAnalyzer("my-analyzer-name", MyAnalyzer::new);
+ *       }
+ *     }
+ *
    + */ +public final class AnalysisModule extends AbstractModule { + + static { + Settings build = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetaData metaData = IndexMetaData.builder("_na_").settings(build).build(); + NA_INDEX_SETTINGS = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList()); + } + private static final IndexSettings NA_INDEX_SETTINGS; + private final Environment environment; + private final Map> charFilters = new HashMap<>(); + private final Map> tokenFilters = new HashMap<>(); + private final Map> tokenizers = new HashMap<>(); + private final Map> analyzers = new HashMap<>(); + private final Map knownDictionaries = new HashMap<>(); + + /** + * Creates a new AnalysisModule + */ + public AnalysisModule(Environment environment) { + this.environment = environment; + } + + /** + * Registers a new {@link AnalysisProvider} to create + * {@link CharFilterFactory} instance per node as well as per index. + */ + public void registerCharFilter(String name, AnalysisProvider charFilter) { + if (charFilter == null) { + throw new IllegalArgumentException("char_filter provider must not be null"); + } + if (charFilters.putIfAbsent(name, charFilter) != null) { + throw new IllegalArgumentException("char_filter provider for name " + name + " already registered"); + } + } + + /** + * Registers a new {@link AnalysisProvider} to create + * {@link TokenFilterFactory} instance per node as well as per index. + */ + public void registerTokenFilter(String name, AnalysisProvider tokenFilter) { + if (tokenFilter == null) { + throw new IllegalArgumentException("token_filter provider must not be null"); + } + if (tokenFilters.putIfAbsent(name, tokenFilter) != null) { + throw new IllegalArgumentException("token_filter provider for name " + name + " already registered"); + } + } + + /** + * Registers a new {@link AnalysisProvider} to create + * {@link TokenizerFactory} instance per node as well as per index. + */ + public void registerTokenizer(String name, AnalysisProvider tokenizer) { + if (tokenizer == null) { + throw new IllegalArgumentException("tokenizer provider must not be null"); + } + if (tokenizers.putIfAbsent(name, tokenizer) != null) { + throw new IllegalArgumentException("tokenizer provider for name " + name + " already registered"); + } + } + + /** + * Registers a new {@link AnalysisProvider} to create + * {@link AnalyzerProvider} instance per node as well as per index. + */ + public void registerAnalyzer(String name, AnalysisProvider analyzer) { + if (analyzer == null) { + throw new IllegalArgumentException("analyzer provider must not be null"); + } + if (analyzers.putIfAbsent(name, analyzer) != null) { + throw new IllegalArgumentException("analyzer provider for name " + name + " already registered"); + } + } + + /** + * Registers a new hunspell {@link Dictionary} that can be referenced by the given name in + * hunspell analysis configuration. + */ + public void registerHunspellDictionary(String name, Dictionary dictionary) { + if (knownDictionaries.putIfAbsent(name, dictionary) != null) { + throw new IllegalArgumentException("dictionary for [" + name + "] is already registered"); + } + } + + @Override + protected void configure() { + try { + HunspellService service = new HunspellService(environment.settings(), environment, knownDictionaries); + AnalysisRegistry registry = new AnalysisRegistry(service, environment, charFilters, tokenFilters, tokenizers, analyzers); + bind(HunspellService.class).toInstance(service); + bind(AnalysisRegistry.class).toInstance(registry); + } catch (IOException e) { + throw new ElasticsearchException("failed to load hunspell service", e); + } + } + + /** + * AnalysisProvider is the basic factory interface for registering analysis components like: + *
      + *
    • {@link TokenizerFactory} - see {@link AnalysisModule#registerTokenizer(String, AnalysisProvider)}
      • + *
    • {@link CharFilterFactory} - see {@link AnalysisModule#registerCharFilter(String, AnalysisProvider)}
      • + *
    • {@link AnalyzerProvider} - see {@link AnalysisModule#registerAnalyzer(String, AnalysisProvider)}
      • + *
    • {@link TokenFilterFactory}- see {@link AnalysisModule#registerTokenFilter(String, AnalysisProvider)} )}
      • + *
    + */ + public interface AnalysisProvider { + + /** + * Creates a new analysis provider. + * @param indexSettings the index settings for the index this provider is created for + * @param environment the nodes environment to load resources from persistent storage + * @param name the name of the analysis component + * @param settings the component specific settings without context prefixes + * @return a new provider instance + * @throws IOException if an {@link IOException} occurs + */ + T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException; + + /** + * Creates a new global scope analysis provider without index specific settings not settings for the provider itself. + * This can be used to get a default instance of an analysis factory without binding to an index. + * + * @param environment the nodes environment to load resources from persistent storage + * @param name the name of the analysis component + * @return a new provider instance + * @throws IOException if an {@link IOException} occurs + * @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns true + */ + default T get(Environment environment, String name) throws IOException { + if (requiresAnalysisSettings()) { + throw new IllegalArgumentException("Analysis settings required - can't instantiate analysis factory"); + } + return get(NA_INDEX_SETTINGS, environment, name, NA_INDEX_SETTINGS.getSettings()); + } + + /** + * If true the analysis component created by this provider requires certain settings to be instantiated. + * it can't be created with defaults. The default is false. + */ + default boolean requiresAnalysisSettings() { + return false; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java index 1a48c5c3ca5..1ae41560067 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java @@ -22,7 +22,6 @@ import org.apache.lucene.analysis.hunspell.Dictionary; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -76,7 +75,6 @@ public class HunspellService extends AbstractComponent { private final Path hunspellDir; private final Function loadingFunction; - @Inject public HunspellService(final Settings settings, final Environment env, final Map knownDictionaries) throws IOException { super(settings); this.knownDictionaries = Collections.unmodifiableMap(knownDictionaries); diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/IndicesAnalysisService.java b/core/src/main/java/org/elasticsearch/indices/analysis/IndicesAnalysisService.java deleted file mode 100644 index 9acdce3f8ab..00000000000 --- a/core/src/main/java/org/elasticsearch/indices/analysis/IndicesAnalysisService.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.analysis; - -import org.apache.lucene.analysis.Analyzer; -import org.elasticsearch.Version; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.index.analysis.*; - -import java.io.Closeable; -import java.util.Locale; -import java.util.Map; - -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; - -/** - * A node level registry of analyzers, to be reused by different indices which use default analyzers. - */ -public class IndicesAnalysisService extends AbstractComponent implements Closeable { - - private final Map analyzerProviderFactories = ConcurrentCollections.newConcurrentMap(); - private final Map tokenizerFactories = ConcurrentCollections.newConcurrentMap(); - private final Map tokenFilterFactories = ConcurrentCollections.newConcurrentMap(); - private final Map charFilterFactories = ConcurrentCollections.newConcurrentMap(); - - public IndicesAnalysisService() { - super(EMPTY_SETTINGS); - } - - @Inject - public IndicesAnalysisService(Settings settings) { - super(settings); - - // Analyzers - for (PreBuiltAnalyzers preBuiltAnalyzerEnum : PreBuiltAnalyzers.values()) { - String name = preBuiltAnalyzerEnum.name().toLowerCase(Locale.ROOT); - analyzerProviderFactories.put(name, new PreBuiltAnalyzerProviderFactory(name, AnalyzerScope.INDICES, preBuiltAnalyzerEnum.getAnalyzer(Version.CURRENT))); - } - - // Tokenizers - for (PreBuiltTokenizers preBuiltTokenizer : PreBuiltTokenizers.values()) { - String name = preBuiltTokenizer.name().toLowerCase(Locale.ROOT); - tokenizerFactories.put(name, new PreBuiltTokenizerFactoryFactory(preBuiltTokenizer.getTokenizerFactory(Version.CURRENT))); - } - - // Tokenizer aliases - tokenizerFactories.put("nGram", new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.NGRAM.getTokenizerFactory(Version.CURRENT))); - tokenizerFactories.put("edgeNGram", new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.EDGE_NGRAM.getTokenizerFactory(Version.CURRENT))); - - - // Token filters - for (PreBuiltTokenFilters preBuiltTokenFilter : PreBuiltTokenFilters.values()) { - String name = preBuiltTokenFilter.name().toLowerCase(Locale.ROOT); - tokenFilterFactories.put(name, new PreBuiltTokenFilterFactoryFactory(preBuiltTokenFilter.getTokenFilterFactory(Version.CURRENT))); - } - // Token filter aliases - tokenFilterFactories.put("nGram", new PreBuiltTokenFilterFactoryFactory(PreBuiltTokenFilters.NGRAM.getTokenFilterFactory(Version.CURRENT))); - tokenFilterFactories.put("edgeNGram", new PreBuiltTokenFilterFactoryFactory(PreBuiltTokenFilters.EDGE_NGRAM.getTokenFilterFactory(Version.CURRENT))); - - - // Char Filters - for (PreBuiltCharFilters preBuiltCharFilter : PreBuiltCharFilters.values()) { - String name = preBuiltCharFilter.name().toLowerCase(Locale.ROOT); - charFilterFactories.put(name, new PreBuiltCharFilterFactoryFactory(preBuiltCharFilter.getCharFilterFactory(Version.CURRENT))); - } - // Char filter aliases - charFilterFactories.put("htmlStrip", new PreBuiltCharFilterFactoryFactory(PreBuiltCharFilters.HTML_STRIP.getCharFilterFactory(Version.CURRENT))); - } - - public boolean hasCharFilter(String name) { - return charFilterFactoryFactory(name) != null; - } - - public Map charFilterFactories() { - return charFilterFactories; - } - - public CharFilterFactoryFactory charFilterFactoryFactory(String name) { - return charFilterFactories.get(name); - } - - public boolean hasTokenFilter(String name) { - return tokenFilterFactoryFactory(name) != null; - } - - public Map tokenFilterFactories() { - return tokenFilterFactories; - } - - public TokenFilterFactoryFactory tokenFilterFactoryFactory(String name) { - return tokenFilterFactories.get(name); - } - - public boolean hasTokenizer(String name) { - return tokenizerFactoryFactory(name) != null; - } - - public Map tokenizerFactories() { - return tokenizerFactories; - } - - public TokenizerFactoryFactory tokenizerFactoryFactory(String name) { - return tokenizerFactories.get(name); - } - - public Map analyzerProviderFactories() { - return analyzerProviderFactories; - } - - public PreBuiltAnalyzerProviderFactory analyzerProviderFactory(String name) { - return analyzerProviderFactories.get(name); - } - - public boolean hasAnalyzer(String name) { - return analyzerProviderFactories.containsKey(name); - } - - public Analyzer analyzer(String name) { - PreBuiltAnalyzerProviderFactory analyzerProviderFactory = analyzerProviderFactory(name); - if (analyzerProviderFactory == null) { - return null; - } - return analyzerProviderFactory.analyzer(); - } - - @Override - public void close() { - for (PreBuiltAnalyzerProviderFactory analyzerProviderFactory : analyzerProviderFactories.values()) { - try { - analyzerProviderFactory.analyzer().close(); - } catch (Exception e) { - // ignore - } - } - } -} diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java index 31908eb79f2..ddda8a08745 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java @@ -36,7 +36,7 @@ public class PreBuiltCacheFactory { * LUCENE Exactly one version for each lucene version is stored. Useful to prevent different analyzers with the same version * ELASTICSEARCH Exactly one version per elasticsearch version is stored. Useful if you change an analyzer between elasticsearch releases, when the lucene version does not change */ - static enum CachingStrategy { ONE, LUCENE, ELASTICSEARCH }; + public enum CachingStrategy { ONE, LUCENE, ELASTICSEARCH }; public interface PreBuiltCache { T get(Version version); diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/AllCircuitBreakerStats.java b/core/src/main/java/org/elasticsearch/indices/breaker/AllCircuitBreakerStats.java index eda94f85310..693a6f5b453 100644 --- a/core/src/main/java/org/elasticsearch/indices/breaker/AllCircuitBreakerStats.java +++ b/core/src/main/java/org/elasticsearch/indices/breaker/AllCircuitBreakerStats.java @@ -57,7 +57,7 @@ public class AllCircuitBreakerStats implements Streamable, ToXContent { } public static AllCircuitBreakerStats readOptionalAllCircuitBreakerStats(StreamInput in) throws IOException { - AllCircuitBreakerStats stats = in.readOptionalStreamable(new AllCircuitBreakerStats()); + AllCircuitBreakerStats stats = in.readOptionalStreamable(AllCircuitBreakerStats::new); return stats; } diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerStats.java b/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerStats.java index 9b5313ce6ae..8d3043c9ccc 100644 --- a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerStats.java +++ b/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerStats.java @@ -74,7 +74,7 @@ public class CircuitBreakerStats implements Streamable, ToXContent { } public static CircuitBreakerStats readOptionalCircuitBreakerStats(StreamInput in) throws IOException { - CircuitBreakerStats stats = in.readOptionalStreamable(new CircuitBreakerStats()); + CircuitBreakerStats stats = in.readOptionalStreamable(CircuitBreakerStats::new); return stats; } diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index 12cf8652ccb..33f3c127d67 100644 --- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.node.settings.NodeSettingsService; @@ -40,6 +41,8 @@ import java.util.concurrent.atomic.AtomicLong; */ public class HierarchyCircuitBreakerService extends CircuitBreakerService { + private static final String CHILD_LOGGER_PREFIX = "org.elasticsearch.indices.breaker."; + private final ConcurrentMap breakers = new ConcurrentHashMap(); // Old pre-1.4.0 backwards compatible settings @@ -237,7 +240,8 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { } else { CircuitBreaker oldBreaker; CircuitBreaker breaker = new ChildMemoryCircuitBreaker(breakerSettings, - logger, this, breakerSettings.getName()); + Loggers.getLogger(CHILD_LOGGER_PREFIX + breakerSettings.getName()), + this, breakerSettings.getName()); for (;;) { oldBreaker = breakers.putIfAbsent(breakerSettings.getName(), breaker); @@ -245,7 +249,9 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { return; } breaker = new ChildMemoryCircuitBreaker(breakerSettings, - (ChildMemoryCircuitBreaker)oldBreaker, logger, this, breakerSettings.getName()); + (ChildMemoryCircuitBreaker)oldBreaker, + Loggers.getLogger(CHILD_LOGGER_PREFIX + breakerSettings.getName()), + this, breakerSettings.getName()); if (breakers.replace(breakerSettings.getName(), oldBreaker, breaker)) { return; diff --git a/core/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java index 30cd6de1233..23b4bc84c44 100644 --- a/core/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java +++ b/core/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java @@ -21,6 +21,7 @@ package org.elasticsearch.indices.cache.query; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.LRUQueryCache; import org.apache.lucene.search.Query; @@ -149,18 +150,23 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, protected void onDocIdSetEviction(Object readerCoreKey, int numEntries, long sumRamBytesUsed) { assert Thread.holdsLock(this); super.onDocIdSetEviction(readerCoreKey, numEntries, sumRamBytesUsed); - // We can't use ShardCoreKeyMap here because its core closed - // listener is called before the listener of the cache which - // triggers this eviction. So instead we use use stats2 that - // we only evict when nothing is cached anymore on the segment - // instead of relying on close listeners - final StatsAndCount statsAndCount = stats2.get(readerCoreKey); - final Stats shardStats = statsAndCount.stats; - shardStats.cacheSize -= numEntries; - shardStats.ramBytesUsed -= sumRamBytesUsed; - statsAndCount.count -= numEntries; - if (statsAndCount.count == 0) { - stats2.remove(readerCoreKey); + // onDocIdSetEviction might sometimes be called with a number + // of entries equal to zero if the cache for the given segment + // was already empty when the close listener was called + if (numEntries > 0) { + // We can't use ShardCoreKeyMap here because its core closed + // listener is called before the listener of the cache which + // triggers this eviction. So instead we use use stats2 that + // we only evict when nothing is cached anymore on the segment + // instead of relying on close listeners + final StatsAndCount statsAndCount = stats2.get(readerCoreKey); + final Stats shardStats = statsAndCount.stats; + shardStats.cacheSize -= numEntries; + shardStats.ramBytesUsed -= sumRamBytesUsed; + statsAndCount.count -= numEntries; + if (statsAndCount.count == 0) { + stats2.remove(readerCoreKey); + } } } @@ -256,6 +262,12 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, shardKeyMap.add(context.reader()); return in.scorer(context); } + + @Override + public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { + shardKeyMap.add(context.reader()); + return in.bulkScorer(context); + } } /** Clear all entries that belong to the given index. */ diff --git a/core/src/main/java/org/elasticsearch/indices/cache/query/terms/TermsLookup.java b/core/src/main/java/org/elasticsearch/indices/cache/query/terms/TermsLookup.java index 92727713efe..62c0011312d 100644 --- a/core/src/main/java/org/elasticsearch/indices/cache/query/terms/TermsLookup.java +++ b/core/src/main/java/org/elasticsearch/indices/cache/query/terms/TermsLookup.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.TermsQueryBuilder; import java.io.IOException; import java.util.Objects; @@ -49,13 +50,13 @@ public class TermsLookup implements Writeable, ToXContent { public TermsLookup(String index, String type, String id, String path) { if (id == null) { - throw new IllegalArgumentException("[terms] query lookup element requires specifying the id."); + throw new IllegalArgumentException("[" + TermsQueryBuilder.NAME + "] query lookup element requires specifying the id."); } if (type == null) { - throw new IllegalArgumentException("[terms] query lookup element requires specifying the type."); + throw new IllegalArgumentException("[" + TermsQueryBuilder.NAME + "] query lookup element requires specifying the type."); } if (path == null) { - throw new IllegalArgumentException("[terms] query lookup element requires specifying the path."); + throw new IllegalArgumentException("[" + TermsQueryBuilder.NAME + "] query lookup element requires specifying the path."); } this.index = index; this.type = type; @@ -122,9 +123,11 @@ public class TermsLookup implements Writeable, ToXContent { path = parser.text(); break; default: - throw new ParsingException(parser.getTokenLocation(), "[terms] query does not support [" + currentFieldName + throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "] within lookup element"); } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]"); } } return new TermsLookup(index, type, id, path).routing(routing); diff --git a/core/src/main/java/org/elasticsearch/indices/cache/request/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/cache/request/IndicesRequestCache.java index 7c42aef4788..6628252d8eb 100644 --- a/core/src/main/java/org/elasticsearch/indices/cache/request/IndicesRequestCache.java +++ b/core/src/main/java/org/elasticsearch/indices/cache/request/IndicesRequestCache.java @@ -47,13 +47,9 @@ import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.threadpool.ThreadPool; -import java.io.IOException; import java.util.*; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -import static org.elasticsearch.common.Strings.hasLength; /** * The indices request cache allows to cache a shard level request stage responses, helping with improving @@ -83,7 +79,6 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis @Deprecated public static final String DEPRECATED_INDICES_CACHE_QUERY_SIZE = "indices.cache.query.size"; public static final String INDICES_CACHE_QUERY_EXPIRE = "indices.requests.cache.expire"; - public static final String INDICES_CACHE_QUERY_CONCURRENCY_LEVEL = "indices.requests.cache.concurrency_level"; private static final Set CACHEABLE_SEARCH_TYPES = EnumSet.of(SearchType.QUERY_THEN_FETCH, SearchType.QUERY_AND_FETCH); @@ -100,7 +95,6 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis //TODO make these changes configurable on the cluster level private final String size; private final TimeValue expire; - private final int concurrencyLevel; private volatile Cache cache; @@ -126,11 +120,6 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis this.size = size; this.expire = settings.getAsTime(INDICES_CACHE_QUERY_EXPIRE, null); - // defaults to 4, but this is a busy map for all indices, increase it a bit by default - this.concurrencyLevel = settings.getAsInt(INDICES_CACHE_QUERY_CONCURRENCY_LEVEL, 16); - if (concurrencyLevel <= 0) { - throw new IllegalArgumentException("concurrency_level must be > 0 but was: " + concurrencyLevel); - } buildCache(); this.reaper = new Reaper(); @@ -189,8 +178,7 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis * Can the shard request be cached at all? */ public boolean canCache(ShardSearchRequest request, SearchContext context) { - // TODO: for now, template is not supported, though we could use the generated bytes as the key - if (hasLength(request.templateSource())) { + if (request.template() != null) { return false; } @@ -213,7 +201,7 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis } // if not explicitly set in the request, use the index setting, if not, use the request if (request.requestCache() == null) { - if (!isCacheEnabled(index.settings(), Boolean.FALSE)) { + if (!isCacheEnabled(index.getSettings(), Boolean.FALSE)) { return false; } } else if (!request.requestCache()) { @@ -356,7 +344,7 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis @Override public int hashCode() { int result = shard.hashCode(); - result = 31 * result + (int) (readerVersion ^ (readerVersion >>> 32)); + result = 31 * result + Long.hashCode(readerVersion); result = 31 * result + value.hashCode(); return result; } @@ -391,7 +379,7 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis @Override public int hashCode() { int result = indexShard.hashCode(); - result = 31 * result + (int) (readerVersion ^ (readerVersion >>> 32)); + result = 31 * result + Long.hashCode(readerVersion); return result; } } diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 1da88f7f488..64ff6c74587 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction; import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction; +import org.elasticsearch.cluster.action.shard.NoOpShardStateActionListener; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -41,27 +42,29 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.Callback; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexShardAlreadyExistsException; -import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.shard.*; import org.elasticsearch.index.snapshots.IndexShardRepository; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.flush.SyncedFlushService; +import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.indices.recovery.RecoveryFailedException; +import org.elasticsearch.indices.recovery.RecoverySource; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.search.SearchService; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.threadpool.ThreadPool; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentMap; /** @@ -76,6 +79,9 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent buildInIndexListener; @Inject public IndicesClusterStateService(Settings settings, IndicesService indicesService, ClusterService clusterService, ThreadPool threadPool, RecoveryTarget recoveryTarget, ShardStateAction shardStateAction, NodeIndexDeletedAction nodeIndexDeletedAction, - NodeMappingRefreshAction nodeMappingRefreshAction, RepositoriesService repositoriesService, RestoreService restoreService) { + NodeMappingRefreshAction nodeMappingRefreshAction, + RepositoriesService repositoriesService, RestoreService restoreService, + SearchService searchService, SyncedFlushService syncedFlushService, + RecoverySource recoverySource, NodeServicesProvider nodeServicesProvider, IndexingMemoryController indexingMemoryController) { super(settings); + this.buildInIndexListener = Arrays.asList(recoverySource, recoveryTarget, searchService, syncedFlushService, indexingMemoryController); this.indicesService = indicesService; this.clusterService = clusterService; this.threadPool = threadPool; @@ -119,6 +130,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent typesToRefresh = new ArrayList<>(); - String index = indexMetaData.index(); + boolean requireRefresh = false; + String index = indexMetaData.getIndex(); IndexService indexService = indicesService.indexService(index); if (indexService == null) { // got deleted on us, ignore (closing the node) @@ -348,31 +358,17 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent cursor : indexMetaData.mappings().values()) { + for (ObjectCursor cursor : indexMetaData.getMappings().values()) { MappingMetaData mappingMd = cursor.value; String mappingType = mappingMd.type(); CompressedXContent mappingSource = mappingMd.source(); - if (mappingType.equals(MapperService.DEFAULT_MAPPING)) { // we processed _default_ first - continue; - } - boolean requireRefresh = processMapping(index, mapperService, mappingType, mappingSource); - if (requireRefresh) { - typesToRefresh.add(mappingType); - } + requireRefresh |= processMapping(index, mapperService, mappingType, mappingSource); } - if (!typesToRefresh.isEmpty() && sendRefreshMapping) { + if (requireRefresh && sendRefreshMapping) { nodeMappingRefreshAction.nodeMappingRefresh(event.state(), - new NodeMappingRefreshAction.NodeMappingRefreshRequest(index, indexMetaData.indexUUID(), - typesToRefresh.toArray(new String[typesToRefresh.size()]), event.state().nodes().localNodeId()) + new NodeMappingRefreshAction.NodeMappingRefreshRequest(index, indexMetaData.getIndexUUID(), + event.state().nodes().localNodeId()) ); } } catch (Throwable t) { @@ -381,33 +377,28 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent(index, mappingType))) { - seenMappings.put(new Tuple<>(index, mappingType), true); - } - - // refresh mapping can happen for 2 reasons. The first is less urgent, and happens when the mapping on this - // node is ahead of what there is in the cluster state (yet an update-mapping has been sent to it already, - // it just hasn't been processed yet and published). Eventually, the mappings will converge, and the refresh - // mapping sent is more of a safe keeping (assuming the update mapping failed to reach the master, ...) - // the second case is where the parsing/merging of the mapping from the metadata doesn't result in the same + // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the // merge version of it, which it does when refreshing the mappings), and warn log it. boolean requiresRefresh = false; try { - if (!mapperService.hasMapping(mappingType)) { + DocumentMapper existingMapper = mapperService.documentMapper(mappingType); + + if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) { + String op = existingMapper == null ? "adding" : "updating"; if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) { - logger.debug("[{}] adding mapping [{}], source [{}]", index, mappingType, mappingSource.string()); + logger.debug("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string()); } else if (logger.isTraceEnabled()) { - logger.trace("[{}] adding mapping [{}], source [{}]", index, mappingType, mappingSource.string()); + logger.trace("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string()); } else { - logger.debug("[{}] adding mapping [{}] (source suppressed due to length, use TRACE level if needed)", index, mappingType); + logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index, op, mappingType); } // we don't apply default, since it has been applied when the mappings were parsed initially mapperService.merge(mappingType, mappingSource, false, true); @@ -415,24 +406,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent { - final RestoreSource restoreSource = shardRouting.restoreSource(); - final ShardId sId = indexShard.shardId(); try { - final boolean success; - if (restoreSource == null) { - // recover from filesystem store - success = indexShard.recoverFromStore(shardRouting, localNode); - } else { - // restore - final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository()); - try { - success = indexShard.restoreFromRepository(shardRouting, indexShardRepository, localNode); - } catch (Throwable t) { - if (Lucene.isCorruptionException(t)) { - restoreService.failRestore(restoreSource.snapshotId(), sId); - } - throw t; - } - if (success) { - restoreService.indexShardRestoreCompleted(restoreSource.snapshotId(), sId); - } - } - if (success) { + if (indexShard.recoverFromStore(nodes.localNode())) { shardStateAction.shardStarted(shardRouting, indexMetaData.getIndexUUID(), "after recovery from store"); } - } catch (Throwable e) { - handleRecoveryFailure(indexService, shardRouting, true, e); + } catch (Throwable t) { + handleRecoveryFailure(indexService, shardRouting, true, t); + } + + }); + } else { + // recover from a restore + final RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), shardRouting.primary(), + RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), nodes.localNode()); + indexShard.markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread + threadPool.generic().execute(() -> { + final ShardId sId = indexShard.shardId(); + try { + final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository()); + if (indexShard.restoreFromRepository(indexShardRepository, nodes.localNode())) { + restoreService.indexShardRestoreCompleted(restoreSource.snapshotId(), sId); + shardStateAction.shardStarted(shardRouting, indexMetaData.getIndexUUID(), "after recovery from repository"); + } + } catch (Throwable first) { + try { + if (Lucene.isCorruptionException(first)) { + restoreService.failRestore(restoreSource.snapshotId(), sId); + } + } catch (Throwable second) { + first.addSuppressed(second); + } finally { + handleRecoveryFailure(indexService, shardRouting, true, first); + } } }); } @@ -768,7 +734,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent tuple : seenMappings.keySet()) { - if (tuple.v1().equals(index)) { - seenMappings.remove(tuple); - } - } - } - private void deleteIndex(String index, String reason) { try { indicesService.deleteIndex(index, reason); } catch (Throwable e) { logger.warn("failed to delete index ({})", e, reason); } - // clear seen mappings as well - clearSeenMappings(index); } - private void failAndRemoveShard(ShardRouting shardRouting, IndexService indexService, boolean sendShardFailure, String message, @Nullable Throwable failure) { - if (indexService.hasShard(shardRouting.getId())) { + private void failAndRemoveShard(ShardRouting shardRouting, String indexUUID, @Nullable IndexService indexService, boolean sendShardFailure, String message, @Nullable Throwable failure) { + if (indexService != null && indexService.hasShard(shardRouting.getId())) { + // if the indexService is null we can't remove the shard, that's fine since we might have a failure + // when the index is remove and then we already removed the index service for that shard... try { indexService.removeShard(shardRouting.getId(), message); } catch (ShardNotFoundException e) { @@ -813,7 +769,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent { @Override - public void onFailedEngine(final ShardId shardId, final String reason, final @Nullable Throwable failure) { - ShardRouting shardRouting = null; - final IndexService indexService = indicesService.indexService(shardId.index().name()); - if (indexService != null) { - IndexShard indexShard = indexService.getShardOrNull(shardId.id()); - if (indexShard != null) { - shardRouting = indexShard.routingEntry(); - } - } - if (shardRouting == null) { - logger.warn("[{}][{}] engine failed, but can't find index shard. failure reason: [{}]", failure, - shardId.index().name(), shardId.id(), reason); - return; - } - final ShardRouting fShardRouting = shardRouting; - threadPool.generic().execute(new Runnable() { - @Override - public void run() { - synchronized (mutex) { - failAndRemoveShard(fShardRouting, indexService, true, "engine failure, reason [" + reason + "]", failure); - } + public void handle(final IndexShard.ShardFailure shardFailure) { + final IndexService indexService = indicesService.indexService(shardFailure.routing.shardId().index().name()); + final ShardRouting shardRouting = shardFailure.routing; + threadPool.generic().execute(() -> { + synchronized (mutex) { + failAndRemoveShard(shardRouting, shardFailure.indexUUID, indexService, true, "shard failure, reason [" + shardFailure.reason + "]", shardFailure.cause); } }); } diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java index 74940cfb5b7..73095f8ee5d 100644 --- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java +++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java @@ -55,7 +55,6 @@ import java.util.function.ToLongBiFunction; public class IndicesFieldDataCache extends AbstractComponent implements RemovalListener { public static final String FIELDDATA_CLEAN_INTERVAL_SETTING = "indices.fielddata.cache.cleanup_interval"; - public static final String FIELDDATA_CACHE_CONCURRENCY_LEVEL = "indices.fielddata.cache.concurrency_level"; public static final String INDICES_FIELDDATA_CACHE_SIZE_KEY = "indices.fielddata.cache.size"; diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java index c0e5dcdda55..ad264c2ac05 100644 --- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java +++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java @@ -41,11 +41,11 @@ import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardNotFoundException; import org.elasticsearch.indices.IndexClosedException; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.BaseTransportResponseHandler; @@ -63,7 +63,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; -public class SyncedFlushService extends AbstractComponent { +public class SyncedFlushService extends AbstractComponent implements IndexEventListener { private static final String PRE_SYNCED_FLUSH_ACTION_NAME = "internal:indices/flush/synced/pre"; private static final String SYNCED_FLUSH_ACTION_NAME = "internal:indices/flush/synced/sync"; @@ -85,25 +85,24 @@ public class SyncedFlushService extends AbstractComponent { transportService.registerRequestHandler(PRE_SYNCED_FLUSH_ACTION_NAME, PreSyncedFlushRequest::new, ThreadPool.Names.FLUSH, new PreSyncedFlushTransportHandler()); transportService.registerRequestHandler(SYNCED_FLUSH_ACTION_NAME, SyncedFlushRequest::new, ThreadPool.Names.FLUSH, new SyncedFlushTransportHandler()); transportService.registerRequestHandler(IN_FLIGHT_OPS_ACTION_NAME, InFlightOpsRequest::new, ThreadPool.Names.SAME, new InFlightOpCountTransportHandler()); - indicesService.indicesLifecycle().addListener(new IndicesLifecycle.Listener() { - @Override - public void onShardInactive(final IndexShard indexShard) { - // we only want to call sync flush once, so only trigger it when we are on a primary - if (indexShard.routingEntry().primary()) { - attemptSyncedFlush(indexShard.shardId(), new ActionListener() { - @Override - public void onResponse(ShardsSyncedFlushResult syncedFlushResult) { - logger.trace("{} sync flush on inactive shard returned successfully for sync_id: {}", syncedFlushResult.getShardId(), syncedFlushResult.syncId()); - } + } - @Override - public void onFailure(Throwable e) { - logger.debug("{} sync flush on inactive shard failed", e, indexShard.shardId()); - } - }); + @Override + public void onShardInactive(final IndexShard indexShard) { + // we only want to call sync flush once, so only trigger it when we are on a primary + if (indexShard.routingEntry().primary()) { + attemptSyncedFlush(indexShard.shardId(), new ActionListener() { + @Override + public void onResponse(ShardsSyncedFlushResult syncedFlushResult) { + logger.trace("{} sync flush on inactive shard returned successfully for sync_id: {}", syncedFlushResult.getShardId(), syncedFlushResult.syncId()); } - } - }); + + @Override + public void onFailure(Throwable e) { + logger.debug("{} sync flush on inactive shard failed", e, indexShard.shardId()); + } + }); + } } /** @@ -118,7 +117,7 @@ public class SyncedFlushService extends AbstractComponent { int numberOfShards = 0; for (String index : concreteIndices) { final IndexMetaData indexMetaData = state.metaData().index(index); - totalNumberOfShards += indexMetaData.totalNumberOfShards(); + totalNumberOfShards += indexMetaData.getTotalNumberOfShards(); numberOfShards += indexMetaData.getNumberOfShards(); results.put(index, Collections.synchronizedList(new ArrayList())); @@ -241,7 +240,7 @@ public class SyncedFlushService extends AbstractComponent { final IndexRoutingTable indexRoutingTable = state.routingTable().index(shardId.index().name()); if (indexRoutingTable == null) { IndexMetaData index = state.getMetaData().index(shardId.index().getName()); - if (index != null && index.state() == IndexMetaData.State.CLOSE) { + if (index != null && index.getState() == IndexMetaData.State.CLOSE) { throw new IndexClosedException(shardId.index()); } throw new IndexNotFoundException(shardId.index().getName()); diff --git a/core/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java b/core/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java new file mode 100644 index 00000000000..bcc4c09d3dd --- /dev/null +++ b/core/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.mapper; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * A registry for all field mappers. + */ +public final class MapperRegistry { + + private final Map mapperParsers; + private final Map metadataMapperParsers; + + public MapperRegistry(Map mapperParsers, + Map metadataMapperParsers) { + this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); + this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); + } + + /** + * Return a map of the mappers that have been registered. The + * returned map uses the type of the field as a key. + */ + public Map getMapperParsers() { + return mapperParsers; + } + + /** + * Return a map of the meta mappers that have been registered. The + * returned map uses the name of the field as a key. + */ + public Map getMetadataMapperParsers() { + return metadataMapperParsers; + } +} diff --git a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java index b3c55ea92ea..53c549efc77 100644 --- a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java @@ -20,9 +20,10 @@ package org.elasticsearch.indices.memory; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicLong; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -33,14 +34,15 @@ import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.EngineClosedException; import org.elasticsearch.index.engine.FlushNotAllowedEngineException; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.threadpool.ThreadPool; -public class IndexingMemoryController extends AbstractLifecycleComponent { +// nocommit what is IndexEventListener +public class IndexingMemoryController extends AbstractLifecycleComponent implements IndexEventListener { /** How much heap (% or bytes) we will share across all actively indexing shards on this node (default: 10%). */ public static final String INDEX_BUFFER_SIZE_SETTING = "indices.memory.index_buffer_size"; @@ -78,7 +80,7 @@ public class IndexingMemoryController extends AbstractLifecycleComponent refreshingBytes = new ConcurrentHashMap<>(); + private final Map refreshingBytes = new ConcurrentHashMap<>(); @Inject public IndexingMemoryController(Settings settings, ThreadPool threadPool, IndicesService indicesService) { @@ -122,13 +124,13 @@ public class IndexingMemoryController extends AbstractLifecycleComponent availableShards() { - ArrayList list = new ArrayList<>(); + protected List availableShards() { + List availableShards = new ArrayList<>(); for (IndexService indexService : indicesService) { - for (IndexShard indexShard : indexService) { - if (shardAvailable(indexShard)) { - list.add(indexShard.shardId()); + for (IndexShard shard : indexService) { + if (shardAvailable(shard)) { + availableShards.add(shard); } } } - return list; - } - - /** returns true if shard exists and is availabe for updates */ - protected boolean shardAvailable(ShardId shardId) { - return shardAvailable(getShard(shardId)); + return availableShards; } /** returns how much heap this shard is using for its indexing buffer */ - protected long getIndexBufferRAMBytesUsed(ShardId shardId) { - IndexShard shard = getShard(shardId); - if (shard == null) { - return 0; - } - + protected long getIndexBufferRAMBytesUsed(IndexShard shard) { return shard.getIndexBufferRAMBytesUsed(); } /** ask this shard to refresh, in the background, to free up heap */ - protected void refreshShardAsync(ShardId shardId) { - IndexShard shard = getShard(shardId); - if (shard != null) { - shard.refreshAsync("memory"); - } + protected void refreshShardAsync(IndexShard shard) { + shard.refreshAsync("memory"); } /** returns true if shard exists and is availabe for updates */ - protected boolean shardAvailable(@Nullable IndexShard shard) { + protected boolean shardAvailable(IndexShard shard) { // shadow replica doesn't have an indexing buffer - return shard != null && shard.canIndex() && CAN_UPDATE_INDEX_BUFFER_STATES.contains(shard.state()); - } - - /** ask this shard to check now whether it is inactive, and reduces its indexing and translog buffers if so. returns Boolean.TRUE if - * it did deactive, Boolean.FALSE if it did not, and null if the shard is unknown */ - protected void checkIdle(ShardId shardId, long inactiveTimeNS) { - final IndexShard shard = getShard(shardId); - if (shard != null) { - shard.checkIdle(inactiveTimeNS); - } - } - - /** gets an {@link IndexShard} instance for the given shard. returns null if the shard doesn't exist */ - protected IndexShard getShard(ShardId shardId) { - IndexService indexService = indicesService.indexService(shardId.index().name()); - if (indexService != null) { - IndexShard indexShard = indexService.getShardOrNull(shardId.id()); - return indexShard; - } - return null; + return shard.canIndex() && CAN_UPDATE_INDEX_BUFFER_STATES.contains(shard.state()); } /** check if any shards active status changed, now. */ @@ -230,11 +200,11 @@ public class IndexingMemoryController extends AbstractLifecycleComponent { final long bytesUsed; - final ShardId shardId; + final IndexShard shard; - public ShardAndBytesUsed(long bytesUsed, ShardId shardId) { + public ShardAndBytesUsed(long bytesUsed, IndexShard shard) { this.bytesUsed = bytesUsed; - this.shardId = shardId; + this.shard = shard; } @Override @@ -266,20 +236,19 @@ public class IndexingMemoryController extends AbstractLifecycleComponent queue = new PriorityQueue<>(); - for (ShardId shardId : availableShards()) { + for (IndexShard shard : availableShards()) { // nocommit explain why order is important here! - Long refreshingBytes = refreshingBytes.get(shardId); + Long bytes = refreshingBytes.get(shard); - long shardBytesUsed = getIndexBufferRAMBytesUsed(shardId); + long shardBytesUsed = getIndexBufferRAMBytesUsed(shard); - if (refreshingBytes != null) { + if (bytes != null) { // Only count up bytes not already being refreshed: - shardBytesUsed -= refreshingBytes; + shardBytesUsed -= bytes; // If the refresh completed just after we pulled refreshingBytes and before we pulled index buffer bytes, then we could // have a negative value here: @@ -316,15 +285,15 @@ public class IndexingMemoryController extends AbstractLifecycleComponent 0) { - queue.add(new ShardAndBytesUsed(shardBytesUsed, shardId)); + queue.add(new ShardAndBytesUsed(shardBytesUsed, shard)); } } while (totalBytesUsed > indexingBuffer.bytes() && queue.isEmpty() == false) { ShardAndBytesUsed largest = queue.poll(); - System.out.println("IMC: write " + largest.shardId + ": " + (largest.bytesUsed/1024./1024.) + " MB"); - logger.debug("refresh shard [{}] to free up its [{}] indexing buffer", largest.shardId, new ByteSizeValue(largest.bytesUsed)); - refreshShardAsync(largest.shardId); + System.out.println("IMC: write " + largest.shard.shardId() + ": " + (largest.bytesUsed/1024./1024.) + " MB"); + logger.debug("refresh shard [{}] to free up its [{}] indexing buffer", largest.shard.shardId(), new ByteSizeValue(largest.bytesUsed)); + refreshShardAsync(largest.shard); totalBytesUsed -= largest.bytesUsed; } } @@ -332,4 +301,16 @@ public class IndexingMemoryController extends AbstractLifecycleComponent 0"); + } + this.chunkSize = chunkSize; + } + class ApplySettings implements NodeSettingsService.Listener { @Override public void onRefreshSettings(Settings settings) { - ByteSizeValue maxSizePerSec = settings.getAsBytesSize(INDICES_RECOVERY_MAX_BYTES_PER_SEC, settings.getAsBytesSize(INDICES_RECOVERY_MAX_SIZE_PER_SEC, RecoverySettings.this.maxBytesPerSec)); + ByteSizeValue maxSizePerSec = settings.getAsBytesSize(INDICES_RECOVERY_MAX_BYTES_PER_SEC, RecoverySettings.this.maxBytesPerSec); if (!Objects.equals(maxSizePerSec, RecoverySettings.this.maxBytesPerSec)) { logger.info("updating [{}] from [{}] to [{}]", INDICES_RECOVERY_MAX_BYTES_PER_SEC, RecoverySettings.this.maxBytesPerSec, maxSizePerSec); RecoverySettings.this.maxBytesPerSec = maxSizePerSec; @@ -219,30 +190,6 @@ public class RecoverySettings extends AbstractComponent implements Closeable { } } - ByteSizeValue fileChunkSize = settings.getAsBytesSize(INDICES_RECOVERY_FILE_CHUNK_SIZE, RecoverySettings.this.fileChunkSize); - if (!fileChunkSize.equals(RecoverySettings.this.fileChunkSize)) { - logger.info("updating [indices.recovery.file_chunk_size] from [{}] to [{}]", RecoverySettings.this.fileChunkSize, fileChunkSize); - RecoverySettings.this.fileChunkSize = fileChunkSize; - } - - int translogOps = settings.getAsInt(INDICES_RECOVERY_TRANSLOG_OPS, RecoverySettings.this.translogOps); - if (translogOps != RecoverySettings.this.translogOps) { - logger.info("updating [indices.recovery.translog_ops] from [{}] to [{}]", RecoverySettings.this.translogOps, translogOps); - RecoverySettings.this.translogOps = translogOps; - } - - ByteSizeValue translogSize = settings.getAsBytesSize(INDICES_RECOVERY_TRANSLOG_SIZE, RecoverySettings.this.translogSize); - if (!translogSize.equals(RecoverySettings.this.translogSize)) { - logger.info("updating [indices.recovery.translog_size] from [{}] to [{}]", RecoverySettings.this.translogSize, translogSize); - RecoverySettings.this.translogSize = translogSize; - } - - boolean compress = settings.getAsBoolean(INDICES_RECOVERY_COMPRESS, RecoverySettings.this.compress); - if (compress != RecoverySettings.this.compress) { - logger.info("updating [indices.recovery.compress] from [{}] to [{}]", RecoverySettings.this.compress, compress); - RecoverySettings.this.compress = compress; - } - int concurrentStreams = settings.getAsInt(INDICES_RECOVERY_CONCURRENT_STREAMS, RecoverySettings.this.concurrentStreams); if (concurrentStreams != RecoverySettings.this.concurrentStreams) { logger.info("updating [indices.recovery.concurrent_streams] from [{}] to [{}]", RecoverySettings.this.concurrentStreams, concurrentStreams); diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java index 6ea41896e55..80c18ef3d63 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java @@ -21,7 +21,6 @@ package org.elasticsearch.indices.recovery; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; @@ -29,28 +28,22 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * The source recovery accepts recovery requests from other peer shards and start the recovery process from this * source shard to the target shard. */ -public class RecoverySource extends AbstractComponent { +public class RecoverySource extends AbstractComponent implements IndexEventListener{ public static class Actions { public static final String START_RECOVERY = "internal:index/shard/recovery/start_recovery"; @@ -72,21 +65,18 @@ public class RecoverySource extends AbstractComponent { this.transportService = transportService; this.indicesService = indicesService; this.clusterService = clusterService; - this.indicesService.indicesLifecycle().addListener(new IndicesLifecycle.Listener() { - @Override - public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - if (indexShard != null) { - ongoingRecoveries.cancel(indexShard, "shard is closed"); - } - } - }); - this.recoverySettings = recoverySettings; - transportService.registerRequestHandler(Actions.START_RECOVERY, StartRecoveryRequest::new, ThreadPool.Names.GENERIC, new StartRecoveryTransportRequestHandler()); } + @Override + public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, + Settings indexSettings) { + if (indexShard != null) { + ongoingRecoveries.cancel(indexShard, "shard is closed"); + } + } + private RecoveryResponse recover(final StartRecoveryRequest request) { final IndexService indexService = indicesService.indexServiceSafe(request.shardId().index().name()); final IndexShard shard = indexService.getShard(request.shardId().id()); @@ -117,7 +107,7 @@ public class RecoverySource extends AbstractComponent { logger.trace("[{}][{}] starting recovery to {}, mark_as_relocated {}", request.shardId().index().name(), request.shardId().id(), request.targetNode(), request.markAsRelocated()); final RecoverySourceHandler handler; - if (IndexMetaData.isOnSharedFilesystem(shard.indexSettings())) { + if (shard.indexSettings().isOnSharedFilesystem()) { handler = new SharedFSRecoverySourceHandler(shard, request, recoverySettings, transportService, logger); } else { handler = new RecoverySourceHandler(shard, request, recoverySettings, transportService, logger); diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 6ace3c6b433..4057af00841 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -36,10 +36,11 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.CancellableThreads.Interruptable; -import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.RecoveryEngineException; import org.elasticsearch.index.shard.*; import org.elasticsearch.index.store.Store; @@ -50,6 +51,7 @@ import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; @@ -58,6 +60,7 @@ import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.StreamSupport; @@ -78,9 +81,9 @@ public class RecoverySourceHandler { private final StartRecoveryRequest request; private final RecoverySettings recoverySettings; private final TransportService transportService; + private final int chunkSizeInBytes; protected final RecoveryResponse response; - private final TransportRequestOptions requestOptions; private final CancellableThreads cancellableThreads = new CancellableThreads() { @Override @@ -107,13 +110,8 @@ public class RecoverySourceHandler { this.transportService = transportService; this.indexName = this.request.shardId().index().name(); this.shardId = this.request.shardId().id(); - + this.chunkSizeInBytes = recoverySettings.getChunkSize().bytesAsInt(); this.response = new RecoveryResponse(); - this.requestOptions = TransportRequestOptions.options() - .withCompress(recoverySettings.compress()) - .withType(TransportRequestOptions.Type.RECOVERY) - .withTimeout(recoverySettings.internalActionTimeout()); - } /** @@ -218,7 +216,7 @@ public class RecoverySourceHandler { totalSize += md.length(); } List phase1Files = new ArrayList<>(diff.different.size() + diff.missing.size()); - phase1Files.addAll(diff.different); + phase1Files.addAll(diff.different); phase1Files.addAll(diff.missing); for (StoreFileMetaData md : phase1Files) { if (request.metadataSnapshot().asMap().containsKey(md.name())) { @@ -244,12 +242,12 @@ public class RecoverySourceHandler { response.phase1FileNames, response.phase1FileSizes, response.phase1ExistingFileNames, response.phase1ExistingFileSizes, translogView.totalOperations()); transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.FILES_INFO, recoveryInfoFilesRequest, - TransportRequestOptions.options().withTimeout(recoverySettings.internalActionTimeout()), + TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(), EmptyTransportResponseHandler.INSTANCE_SAME).txGet(); }); // How many bytes we've copied since we last called RateLimiter.pause final AtomicLong bytesSinceLastPause = new AtomicLong(); - final Function outputStreamFactories = (md) -> new RecoveryOutputStream(md, bytesSinceLastPause, translogView); + final Function outputStreamFactories = (md) -> new BufferedOutputStream(new RecoveryOutputStream(md, bytesSinceLastPause, translogView), chunkSizeInBytes); sendFiles(store, phase1Files.toArray(new StoreFileMetaData[phase1Files.size()]), outputStreamFactories); cancellableThreads.execute(() -> { // Send the CLEAN_FILES request, which takes all of the files that @@ -263,7 +261,7 @@ public class RecoverySourceHandler { try { transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.CLEAN_FILES, new RecoveryCleanFilesRequest(request.recoveryId(), shard.shardId(), recoverySourceMetadata, translogView.totalOperations()), - TransportRequestOptions.options().withTimeout(recoverySettings.internalActionTimeout()), + TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(), EmptyTransportResponseHandler.INSTANCE_SAME).txGet(); } catch (RemoteTransportException remoteException) { final IOException corruptIndexException; @@ -332,7 +330,7 @@ public class RecoverySourceHandler { // garbage collection (not the JVM's GC!) of tombstone deletes transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.PREPARE_TRANSLOG, new RecoveryPrepareForTranslogOperationsRequest(request.recoveryId(), request.shardId(), translogView.totalOperations()), - TransportRequestOptions.options().withTimeout(recoverySettings.internalActionTimeout()), EmptyTransportResponseHandler.INSTANCE_SAME).txGet(); + TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(), EmptyTransportResponseHandler.INSTANCE_SAME).txGet(); } }); @@ -390,7 +388,7 @@ public class RecoverySourceHandler { // during this time transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.FINALIZE, new RecoveryFinalizeRecoveryRequest(request.recoveryId(), request.shardId()), - TransportRequestOptions.options().withTimeout(recoverySettings.internalActionLongTimeout()), + TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionLongTimeout()).build(), EmptyTransportResponseHandler.INSTANCE_SAME).txGet(); } }); @@ -431,10 +429,11 @@ public class RecoverySourceHandler { throw new ElasticsearchException("failed to get next operation from translog", ex); } - final TransportRequestOptions recoveryOptions = TransportRequestOptions.options() - .withCompress(recoverySettings.compress()) + final TransportRequestOptions recoveryOptions = TransportRequestOptions.builder() + .withCompress(true) .withType(TransportRequestOptions.Type.RECOVERY) - .withTimeout(recoverySettings.internalActionLongTimeout()); + .withTimeout(recoverySettings.internalActionLongTimeout()) + .build(); if (operation == null) { logger.trace("[{}][{}] no translog operations to send to {}", @@ -450,9 +449,9 @@ public class RecoverySourceHandler { size += operation.estimateSize(); totalOperations++; - // Check if this request is past the size or bytes threshold, and + // Check if this request is past bytes threshold, and // if so, send it off - if (ops >= recoverySettings.translogOps() || size >= recoverySettings.translogSize().bytes()) { + if (size >= chunkSizeInBytes) { // don't throttle translog, since we lock for phase3 indexing, // so we need to move it as fast as possible. Note, since we @@ -536,7 +535,7 @@ public class RecoverySourceHandler { @Override public final void write(int b) throws IOException { - write(new byte[]{(byte) b}, 0, 1); + throw new UnsupportedOperationException("we can't send single bytes over the wire"); } @Override @@ -547,6 +546,11 @@ public class RecoverySourceHandler { } private void sendNextChunk(long position, BytesArray content, boolean lastChunk) throws IOException { + final TransportRequestOptions chunkSendOptions = TransportRequestOptions.builder() + .withCompress(false) // lucene files are already compressed and therefore compressing this won't really help much so we are safing the cpu for other things + .withType(TransportRequestOptions.Type.RECOVERY) + .withTimeout(recoverySettings.internalActionTimeout()) + .build(); cancellableThreads.execute(() -> { // Pause using the rate limiter, if desired, to throttle the recovery final long throttleTimeInNanos; @@ -576,7 +580,7 @@ public class RecoverySourceHandler { * see how many translog ops we accumulate while copying files across the network. A future optimization * would be in to restart file copy again (new deltas) if we have too many translog ops are piling up. */ - throttleTimeInNanos), requestOptions, EmptyTransportResponseHandler.INSTANCE_SAME).txGet(); + throttleTimeInNanos), chunkSendOptions, EmptyTransportResponseHandler.INSTANCE_SAME).txGet(); }); if (shard.state() == IndexShardState.CLOSED) { // check if the shard got closed on us throw new IndexShardClosedException(request.shardId()); @@ -669,9 +673,10 @@ public class RecoverySourceHandler { pool = recoverySettings.concurrentSmallFileStreamPool(); } Future future = pool.submit(() -> { - try (final OutputStream outputStream = outputStreamFactory.apply(md); - final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) { - Streams.copy(new InputStreamIndexInput(indexInput, md.length()), outputStream); + try (final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) { + // it's fine that we are only having the indexInput int he try/with block. The copy methods handles + // exceptions during close correctly and doesn't hide the original exception. + Streams.copy(new InputStreamIndexInput(indexInput, md.length()), outputStreamFactory.apply(md)); } return null; }); diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java index 022c326bf93..92bfc87218a 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java @@ -690,8 +690,8 @@ public class RecoveryState implements ToXContent, Streamable { @Override public int hashCode() { int result = name.hashCode(); - result = 31 * result + (int) (length ^ (length >>> 32)); - result = 31 * result + (int) (recovered ^ (recovered >>> 32)); + result = 31 * result + Long.hashCode(length); + result = 31 * result + Long.hashCode(recovered); result = 31 * result + (reused ? 1 : 0); return result; } diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryStatus.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryStatus.java index 6e9505f0777..0064021dd33 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryStatus.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryStatus.java @@ -22,6 +22,7 @@ package org.elasticsearch.indices.recovery; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; @@ -77,7 +78,7 @@ public class RecoveryStatus extends AbstractRefCounted { super("recovery_status"); this.recoveryId = idGenerator.incrementAndGet(); this.listener = listener; - this.logger = Loggers.getLogger(getClass(), indexShard.indexSettings(), indexShard.shardId()); + this.logger = Loggers.getLogger(getClass(), indexShard.indexSettings().getSettings(), indexShard.shardId()); this.indexShard = indexShard; this.sourceNode = sourceNode; this.shardId = indexShard.shardId(); @@ -175,7 +176,7 @@ public class RecoveryStatus extends AbstractRefCounted { listener.onRecoveryFailure(state(), e, sendShardFailure); } finally { try { - cancellableThreads.cancel("failed recovery [" + e.getMessage() + "]"); + cancellableThreads.cancel("failed recovery [" + ExceptionsHelper.stackTrace(e) + "]"); } finally { // release the initial reference. recovery files will be cleaned as soon as ref count goes to zero, potentially now decRef(); diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index d8880893a13..32e644ab7b8 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -45,10 +45,8 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.RecoveryEngineException; import org.elasticsearch.index.mapper.MapperException; -import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.*; import org.elasticsearch.index.store.Store; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; @@ -67,7 +65,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; * Note, it can be safely assumed that there will only be a single recovery per shard (index+id) and * not several of them (since we don't allocate several shard replicas to the same node). */ -public class RecoveryTarget extends AbstractComponent { +public class RecoveryTarget extends AbstractComponent implements IndexEventListener { public static class Actions { public static final String FILES_INFO = "internal:index/shard/recovery/filesInfo"; @@ -88,8 +86,7 @@ public class RecoveryTarget extends AbstractComponent { private final RecoveriesCollection onGoingRecoveries; @Inject - public RecoveryTarget(Settings settings, ThreadPool threadPool, TransportService transportService, - IndicesLifecycle indicesLifecycle, RecoverySettings recoverySettings, ClusterService clusterService) { + public RecoveryTarget(Settings settings, ThreadPool threadPool, TransportService transportService, RecoverySettings recoverySettings, ClusterService clusterService) { super(settings); this.threadPool = threadPool; this.transportService = transportService; @@ -103,16 +100,13 @@ public class RecoveryTarget extends AbstractComponent { transportService.registerRequestHandler(Actions.PREPARE_TRANSLOG, RecoveryPrepareForTranslogOperationsRequest::new, ThreadPool.Names.GENERIC, new PrepareForTranslogOperationsRequestHandler()); transportService.registerRequestHandler(Actions.TRANSLOG_OPS, RecoveryTranslogOperationsRequest::new, ThreadPool.Names.GENERIC, new TranslogOperationsRequestHandler()); transportService.registerRequestHandler(Actions.FINALIZE, RecoveryFinalizeRecoveryRequest::new, ThreadPool.Names.GENERIC, new FinalizeRecoveryRequestHandler()); + } - indicesLifecycle.addListener(new IndicesLifecycle.Listener() { - @Override - public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - if (indexShard != null) { - onGoingRecoveries.cancelRecoveriesForShard(shardId, "shard closed"); - } - } - }); + @Override + public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { + if (indexShard != null) { + onGoingRecoveries.cancelRecoveriesForShard(shardId, "shard closed"); + } } /** @@ -130,22 +124,23 @@ public class RecoveryTarget extends AbstractComponent { } public void startRecovery(final IndexShard indexShard, final RecoveryState.Type recoveryType, final DiscoveryNode sourceNode, final RecoveryListener listener) { - try { - RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), indexShard.routingEntry().primary(), recoveryType, sourceNode, clusterService.localNode()); - indexShard.recovering("from " + sourceNode, recoveryState); - } catch (IllegalIndexShardStateException e) { - // that's fine, since we might be called concurrently, just ignore this, we are already recovering - logger.debug("{} ignore recovery. already in recovering process, {}", indexShard.shardId(), e.getMessage()); - return; - } // create a new recovery status, and process... final long recoveryId = onGoingRecoveries.startRecovery(indexShard, sourceNode, listener, recoverySettings.activityTimeout()); threadPool.generic().execute(new RecoveryRunner(recoveryId)); } + protected void retryRecovery(final RecoveryStatus recoveryStatus, final Throwable reason, TimeValue retryAfter, final StartRecoveryRequest currentRequest) { + logger.trace("will retry recovery with id [{}] in [{}]", reason, recoveryStatus.recoveryId(), retryAfter); + retryRecovery(recoveryStatus, retryAfter, currentRequest); + } + protected void retryRecovery(final RecoveryStatus recoveryStatus, final String reason, TimeValue retryAfter, final StartRecoveryRequest currentRequest) { - logger.trace("will retrying recovery with id [{}] in [{}] (reason [{}])", recoveryStatus.recoveryId(), retryAfter, reason); + logger.trace("will retry recovery with id [{}] in [{}] (reason [{}])", recoveryStatus.recoveryId(), retryAfter, reason); + retryRecovery(recoveryStatus, retryAfter, currentRequest); + } + + private void retryRecovery(final RecoveryStatus recoveryStatus, TimeValue retryAfter, final StartRecoveryRequest currentRequest) { try { recoveryStatus.resetRecovery(); } catch (Throwable e) { @@ -213,11 +208,15 @@ public class RecoveryTarget extends AbstractComponent { } catch (CancellableThreads.ExecutionCancelledException e) { logger.trace("recovery cancelled", e); } catch (Throwable e) { - if (logger.isTraceEnabled()) { logger.trace("[{}][{}] Got exception on recovery", e, request.shardId().index().name(), request.shardId().id()); } Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof CancellableThreads.ExecutionCancelledException) { + // this can also come from the source wrapped in a RemoteTransportException + onGoingRecoveries.failRecovery(recoveryStatus.recoveryId(), new RecoveryFailedException(request, "source has canceled the recovery", cause), false); + return; + } if (cause instanceof RecoveryEngineException) { // unwrap an exception that was thrown as part of the recovery cause = cause.getCause(); @@ -238,7 +237,7 @@ public class RecoveryTarget extends AbstractComponent { } if (cause instanceof DelayRecoveryException) { - retryRecovery(recoveryStatus, cause.getMessage(), recoverySettings.retryDelayStateSync(), request); + retryRecovery(recoveryStatus, cause, recoverySettings.retryDelayStateSync(), request); return; } diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java index 123480e81de..e849580b2c4 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java @@ -69,7 +69,7 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler { // create a new IndexWriter logger.info("recovery failed for primary shadow shard, failing shard"); // pass the failure as null, as we want to ensure the store is not marked as corrupted - shard.failShard("primary relocation failed on shared filesystem caused by: [" + t.getMessage() + "]", null); + shard.failShard("primary relocation failed on shared filesystem", t); } else { logger.info("recovery failed on shared filesystem", t); } diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index b1cb507522e..45f2f91b0be 100644 --- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -19,7 +19,6 @@ package org.elasticsearch.indices.store; -import org.apache.lucene.store.StoreRateLimiting; import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -31,22 +30,21 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.concurrent.TimeUnit; @@ -57,89 +55,36 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class IndicesStore extends AbstractComponent implements ClusterStateListener, Closeable { - public static final String INDICES_STORE_THROTTLE_TYPE = "indices.store.throttle.type"; - public static final String INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC = "indices.store.throttle.max_bytes_per_sec"; + // TODO this class can be foled into either IndicesService and partially into IndicesClusterStateService there is no need for a seperate public service public static final String INDICES_STORE_DELETE_SHARD_TIMEOUT = "indices.store.delete.shard.timeout"; - public static final String ACTION_SHARD_EXISTS = "internal:index/shard/exists"; - private static final EnumSet ACTIVE_STATES = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED); - - class ApplySettings implements NodeSettingsService.Listener { - @Override - public void onRefreshSettings(Settings settings) { - String rateLimitingType = settings.get(INDICES_STORE_THROTTLE_TYPE, IndicesStore.this.rateLimitingType); - // try and parse the type - StoreRateLimiting.Type.fromString(rateLimitingType); - if (!rateLimitingType.equals(IndicesStore.this.rateLimitingType)) { - logger.info("updating indices.store.throttle.type from [{}] to [{}]", IndicesStore.this.rateLimitingType, rateLimitingType); - IndicesStore.this.rateLimitingType = rateLimitingType; - IndicesStore.this.rateLimiting.setType(rateLimitingType); - } - - ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndicesStore.this.rateLimitingThrottle); - if (!rateLimitingThrottle.equals(IndicesStore.this.rateLimitingThrottle)) { - logger.info("updating indices.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndicesStore.this.rateLimitingThrottle, rateLimitingThrottle, IndicesStore.this.rateLimitingType); - IndicesStore.this.rateLimitingThrottle = rateLimitingThrottle; - IndicesStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); - } - } - } - - private final NodeSettingsService nodeSettingsService; - private final IndicesService indicesService; - private final ClusterService clusterService; private final TransportService transportService; - private volatile String rateLimitingType; - private volatile ByteSizeValue rateLimitingThrottle; - private final StoreRateLimiting rateLimiting = new StoreRateLimiting(); - - private final ApplySettings applySettings = new ApplySettings(); - private TimeValue deleteShardTimeout; @Inject - public IndicesStore(Settings settings, NodeSettingsService nodeSettingsService, IndicesService indicesService, + public IndicesStore(Settings settings, IndicesService indicesService, ClusterService clusterService, TransportService transportService) { super(settings); - this.nodeSettingsService = nodeSettingsService; this.indicesService = indicesService; this.clusterService = clusterService; this.transportService = transportService; transportService.registerRequestHandler(ACTION_SHARD_EXISTS, ShardActiveRequest::new, ThreadPool.Names.SAME, new ShardActiveRequestHandler()); - - // we don't limit by default (we default to CMS's auto throttle instead): - this.rateLimitingType = settings.get("indices.store.throttle.type", StoreRateLimiting.Type.NONE.name()); - rateLimiting.setType(rateLimitingType); - this.rateLimitingThrottle = settings.getAsBytesSize("indices.store.throttle.max_bytes_per_sec", new ByteSizeValue(10240, ByteSizeUnit.MB)); - rateLimiting.setMaxRate(rateLimitingThrottle); - this.deleteShardTimeout = settings.getAsTime(INDICES_STORE_DELETE_SHARD_TIMEOUT, new TimeValue(30, TimeUnit.SECONDS)); - - logger.debug("using indices.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); - - nodeSettingsService.addListener(applySettings); clusterService.addLast(this); } IndicesStore() { super(Settings.EMPTY); - nodeSettingsService = null; indicesService = null; this.clusterService = null; this.transportService = null; } - - public StoreRateLimiting rateLimiting() { - return this.rateLimiting; - } - @Override public void close() { - nodeSettingsService.removeListener(applySettings); clusterService.remove(this); } @@ -154,11 +99,12 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe } for (IndexRoutingTable indexRoutingTable : event.state().routingTable()) { + IndexSettings indexSettings = new IndexSettings(event.state().getMetaData().index(indexRoutingTable.index()), settings, Collections.emptyList()); // Note, closed indices will not have any routing information, so won't be deleted for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { if (shardCanBeDeleted(event.state(), indexShardRoutingTable)) { ShardId shardId = indexShardRoutingTable.shardId(); - if (indicesService.canDeleteShardContent(shardId, event.state().getMetaData().index(shardId.getIndex()))) { + if (indicesService.canDeleteShardContent(shardId, indexSettings)) { deleteShardIfExistElseWhere(event.state(), indexShardRoutingTable); } } @@ -204,7 +150,6 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe return true; } - // TODO will have to ammend this for shadow replicas so we don't delete the shared copy... private void deleteShardIfExistElseWhere(ClusterState state, IndexShardRoutingTable indexShardRoutingTable) { List> requests = new ArrayList<>(indexShardRoutingTable.size()); String indexUUID = state.getMetaData().index(indexShardRoutingTable.shardId().getIndex()).getIndexUUID(); @@ -401,7 +346,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe } } - public static class ShardActiveRequest extends TransportRequest { + private static class ShardActiveRequest extends TransportRequest { protected TimeValue timeout = null; private ClusterName clusterName; private String indexUUID; diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java index ec5cc181aa3..d963ea24303 100644 --- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java +++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java @@ -42,11 +42,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.AsyncShardFetch; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.index.store.IndexStoreModule; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.indices.IndicesService; @@ -54,10 +55,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -169,11 +167,12 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction { private final FsService fsService; - @Inject - public MonitorService(Settings settings, JvmMonitorService jvmMonitorService, - OsService osService, ProcessService processService, JvmService jvmService, - FsService fsService) { + public MonitorService(Settings settings, NodeEnvironment nodeEnvironment, ThreadPool threadPool) throws IOException { super(settings); - this.jvmMonitorService = jvmMonitorService; - this.osService = osService; - this.processService = processService; - this.jvmService = jvmService; - this.fsService = fsService; + this.jvmMonitorService = new JvmMonitorService(settings, threadPool); + this.osService = new OsService(settings); + this.processService = new ProcessService(settings); + this.jvmService = new JvmService(settings); + this.fsService = new FsService(settings, nodeEnvironment); } public OsService osService() { diff --git a/core/src/main/java/org/elasticsearch/monitor/Probes.java b/core/src/main/java/org/elasticsearch/monitor/Probes.java new file mode 100644 index 00000000000..e31903c0bce --- /dev/null +++ b/core/src/main/java/org/elasticsearch/monitor/Probes.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.monitor; + +import java.lang.management.OperatingSystemMXBean; +import java.lang.reflect.Method; + +public class Probes { + public static short getLoadAndScaleToPercent(Method method, OperatingSystemMXBean osMxBean) { + if (method != null) { + try { + double load = (double) method.invoke(osMxBean); + if (load >= 0) { + return (short) (load * 100); + } + } catch (Throwable t) { + return -1; + } + } + return -1; + } +} diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java index 56bc352a5bc..dc1958f666b 100644 --- a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java +++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java @@ -20,7 +20,6 @@ package org.elasticsearch.monitor.fs; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeEnvironment.NodePath; @@ -31,7 +30,6 @@ public class FsProbe extends AbstractComponent { private final NodeEnvironment nodeEnv; - @Inject public FsProbe(Settings settings, NodeEnvironment nodeEnv) { super(settings); this.nodeEnv = nodeEnv; diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java index c95a7bf8b3a..7019ec48e0b 100644 --- a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java +++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java @@ -20,10 +20,10 @@ package org.elasticsearch.monitor.fs; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.SingleObjectCache; +import org.elasticsearch.env.NodeEnvironment; import java.io.IOException; @@ -35,10 +35,9 @@ public class FsService extends AbstractComponent { private final SingleObjectCache fsStatsCache; - @Inject - public FsService(Settings settings, FsProbe probe) throws IOException { + public FsService(Settings settings, NodeEnvironment nodeEnvironment) throws IOException { super(settings); - this.probe = probe; + this.probe = new FsProbe(settings, nodeEnvironment); TimeValue refreshInterval = settings.getAsTime("monitor.fs.refresh_interval", TimeValue.timeValueSeconds(1)); fsStatsCache = new FsInfoCache(refreshInterval, probe.stats()); logger.debug("Using probe [{}] with refresh_interval [{}]", probe, refreshInterval); diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index 599c86097ff..e224c722d42 100644 --- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; import java.lang.management.*; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -57,7 +58,7 @@ public class JvmInfo implements Streamable, ToXContent { JvmInfo info = new JvmInfo(); info.pid = pid; info.startTime = runtimeMXBean.getStartTime(); - info.version = runtimeMXBean.getSystemProperties().get("java.version"); + info.version = System.getProperty("java.version"); info.vmName = runtimeMXBean.getVmName(); info.vmVendor = runtimeMXBean.getVmVendor(); info.vmVersion = runtimeMXBean.getVmVersion(); @@ -84,7 +85,7 @@ public class JvmInfo implements Streamable, ToXContent { } } info.classPath = runtimeMXBean.getClassPath(); - info.systemProperties = runtimeMXBean.getSystemProperties(); + info.systemProperties = Collections.unmodifiableMap(runtimeMXBean.getSystemProperties()); List gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); info.gcCollectors = new String[gcMxBeans.size()]; @@ -104,6 +105,11 @@ public class JvmInfo implements Streamable, ToXContent { } public static JvmInfo jvmInfo() { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new ManagementPermission("monitor")); + sm.checkPropertyAccess("*"); + } return INSTANCE; } diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java index a11fc2957a4..8d83435bb98 100644 --- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java +++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java @@ -20,7 +20,6 @@ package org.elasticsearch.monitor.jvm; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.FutureUtils; @@ -71,7 +70,6 @@ public class JvmMonitorService extends AbstractLifecycleComponent osStatsCache; - @Inject - public OsService(Settings settings, OsProbe probe) { + public OsService(Settings settings) { super(settings); - this.probe = probe; + this.probe = OsProbe.getInstance(); TimeValue refreshInterval = settings.getAsTime("monitor.os.refresh_interval", TimeValue.timeValueSeconds(1)); this.info = probe.osInfo(); this.info.refreshInterval = refreshInterval.millis(); + this.info.allocatedProcessors = EsExecutors.boundedNumberOfProcessors(settings); + osStatsCache = new OsStatsCache(refreshInterval, probe.osStats()); logger.debug("Using probe [{}] with refresh_interval [{}]", probe, refreshInterval); } diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java index 03f262f883b..ebf7d9fafda 100644 --- a/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java +++ b/core/src/main/java/org/elasticsearch/monitor/os/OsStats.java @@ -36,7 +36,7 @@ public class OsStats implements Streamable, ToXContent { long timestamp; - double loadAverage = -1; + Cpu cpu = null; Mem mem = null; @@ -49,10 +49,7 @@ public class OsStats implements Streamable, ToXContent { return timestamp; } - public double getLoadAverage() { - return loadAverage; - } - + public Cpu getCpu() { return cpu; } public Mem getMem() { return mem; @@ -65,6 +62,8 @@ public class OsStats implements Streamable, ToXContent { static final class Fields { static final XContentBuilderString OS = new XContentBuilderString("os"); static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp"); + static final XContentBuilderString CPU = new XContentBuilderString("cpu"); + static final XContentBuilderString PERCENT = new XContentBuilderString("percent"); static final XContentBuilderString LOAD_AVERAGE = new XContentBuilderString("load_average"); static final XContentBuilderString MEM = new XContentBuilderString("mem"); @@ -85,7 +84,12 @@ public class OsStats implements Streamable, ToXContent { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.OS); builder.field(Fields.TIMESTAMP, getTimestamp()); - builder.field(Fields.LOAD_AVERAGE, getLoadAverage()); + if (cpu != null) { + builder.startObject(Fields.CPU); + builder.field(Fields.PERCENT, cpu.getPercent()); + builder.field(Fields.LOAD_AVERAGE, cpu.getLoadAverage()); + builder.endObject(); + } if (mem != null) { builder.startObject(Fields.MEM); @@ -120,7 +124,7 @@ public class OsStats implements Streamable, ToXContent { @Override public void readFrom(StreamInput in) throws IOException { timestamp = in.readVLong(); - loadAverage = in.readDouble(); + cpu = in.readOptionalStreamable(Cpu::new); if (in.readBoolean()) { mem = Mem.readMem(in); } @@ -132,7 +136,7 @@ public class OsStats implements Streamable, ToXContent { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(timestamp); - out.writeDouble(loadAverage); + out.writeOptionalStreamable(cpu); if (mem == null) { out.writeBoolean(false); } else { @@ -147,6 +151,39 @@ public class OsStats implements Streamable, ToXContent { } } + public static class Cpu implements Streamable { + short percent = -1; + double loadAverage = -1; + + Cpu() {} + + public static Cpu readCpu(StreamInput in) throws IOException { + Cpu cpu = new Cpu(); + cpu.readFrom(in); + return cpu; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + percent = in.readShort(); + loadAverage = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeShort(percent); + out.writeDouble(loadAverage); + } + + public short getPercent() { + return percent; + } + + public double getLoadAverage() { + return loadAverage; + } + } + public static class Swap implements Streamable { long total = -1; @@ -230,5 +267,4 @@ public class OsStats implements Streamable, ToXContent { private static short calculatePercentage(long used, long max) { return max <= 0 ? 0 : (short) (Math.round((100d * used) / max)); } - } diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java index eca3327e779..4cd005982c2 100644 --- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java +++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java @@ -20,6 +20,7 @@ package org.elasticsearch.monitor.process; import org.elasticsearch.bootstrap.BootstrapInfo; +import org.elasticsearch.monitor.Probes; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; @@ -88,17 +89,7 @@ public class ProcessProbe { * Returns the process CPU usage in percent */ public short getProcessCpuPercent() { - if (getProcessCpuLoad != null) { - try { - double load = (double) getProcessCpuLoad.invoke(osMxBean); - if (load >= 0) { - return (short) (load * 100); - } - } catch (Throwable t) { - return -1; - } - } - return -1; + return Probes.getLoadAndScaleToPercent(getProcessCpuLoad, osMxBean); } /** diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java index 08d286dd983..0861dfe5b0c 100644 --- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java +++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java @@ -20,7 +20,6 @@ package org.elasticsearch.monitor.process; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.SingleObjectCache; @@ -34,10 +33,9 @@ public final class ProcessService extends AbstractComponent { private final ProcessInfo info; private final SingleObjectCache processStatsCache; - @Inject - public ProcessService(Settings settings, ProcessProbe probe) { + public ProcessService(Settings settings) { super(settings); - this.probe = probe; + this.probe = ProcessProbe.getInstance(); final TimeValue refreshInterval = settings.getAsTime("monitor.process.refresh_interval", TimeValue.timeValueSeconds(1)); processStatsCache = new ProcessStatsCache(refreshInterval, probe.processStats()); diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index ecd7fddd3be..d3f6367cac0 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -20,8 +20,10 @@ package org.elasticsearch.node; import org.elasticsearch.Build; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionModule; +import org.elasticsearch.bootstrap.Elasticsearch; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClientModule; @@ -31,6 +33,7 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.routing.RoutingService; import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Injector; @@ -40,9 +43,14 @@ import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoveryService; @@ -55,16 +63,18 @@ import org.elasticsearch.gateway.GatewayModule; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.http.HttpServer; import org.elasticsearch.http.HttpServerModule; +import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerModule; import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.memory.IndexingMemoryController; +import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.indices.ttl.IndicesTTLService; -import org.elasticsearch.monitor.MonitorModule; import org.elasticsearch.monitor.MonitorService; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.node.internal.InternalSettingsPreparer; @@ -92,7 +102,16 @@ import org.elasticsearch.tribe.TribeService; import org.elasticsearch.watcher.ResourceWatcherModule; import org.elasticsearch.watcher.ResourceWatcherService; +import java.io.BufferedWriter; import java.io.IOException; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -103,8 +122,6 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; /** * A node represent a node within a cluster (cluster.name). The {@link #client()} can be used * in order to use a {@link Client} to perform actions/operations against the cluster. - *

    In order to create a node, the {@link NodeBuilder} can be used. When done with it, make sure to - * call {@link #close()} on it. */ public class Node implements Releasable { @@ -123,17 +140,16 @@ public class Node implements Releasable { * @param preparedSettings Base settings to configure the node with */ public Node(Settings preparedSettings) { - this(preparedSettings, Version.CURRENT, Collections.>emptyList()); + this(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), Version.CURRENT, Collections.>emptyList()); } - Node(Settings preparedSettings, Version version, Collection> classpathPlugins) { - final Settings pSettings = settingsBuilder().put(preparedSettings) - .put(Client.CLIENT_TYPE_SETTING, CLIENT_TYPE).build(); - Environment tmpEnv = InternalSettingsPreparer.prepareEnvironment(pSettings, null); - Settings tmpSettings = TribeService.processSettings(tmpEnv.settings()); + protected Node(Environment tmpEnv, Version version, Collection> classpathPlugins) { + Settings tmpSettings = settingsBuilder().put(tmpEnv.settings()) + .put(Client.CLIENT_TYPE_SETTING, CLIENT_TYPE).build(); + tmpSettings = TribeService.processSettings(tmpSettings); ESLogger logger = Loggers.getLogger(Node.class, tmpSettings.get("name")); - logger.info("version[{}], pid[{}], build[{}/{}]", version, JvmInfo.jvmInfo().pid(), Build.CURRENT.hashShort(), Build.CURRENT.timestamp()); + logger.info("version[{}], pid[{}], build[{}/{}]", version, JvmInfo.jvmInfo().pid(), Build.CURRENT.shortHash(), Build.CURRENT.date()); logger.info("initializing ..."); @@ -142,7 +158,7 @@ public class Node implements Releasable { tmpEnv.configFile(), Arrays.toString(tmpEnv.dataFiles()), tmpEnv.logsFile(), tmpEnv.pluginsFile()); } - this.pluginsService = new PluginsService(tmpSettings, tmpEnv.pluginsFile(), classpathPlugins); + this.pluginsService = new PluginsService(tmpSettings, tmpEnv.modulesFile(), tmpEnv.pluginsFile(), classpathPlugins); this.settings = pluginsService.updatedSettings(); // create the environment based on the finalized (processed) view of the settings this.environment = new Environment(this.settings()); @@ -153,11 +169,13 @@ public class Node implements Releasable { } catch (IOException ex) { throw new IllegalStateException("Failed to created node environment", ex); } - + final NetworkService networkService = new NetworkService(settings); + final NodeSettingsService nodeSettingsService = new NodeSettingsService(settings); + final SettingsFilter settingsFilter = new SettingsFilter(settings); final ThreadPool threadPool = new ThreadPool(settings); - boolean success = false; try { + final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool); ModulesBuilder modules = new ModulesBuilder(); modules.add(new Version.Module(version)); modules.add(new CircuitBreakerModule(settings)); @@ -166,9 +184,9 @@ public class Node implements Releasable { modules.add(pluginModule); } modules.add(new PluginsModule(pluginsService)); - modules.add(new SettingsModule(this.settings)); - modules.add(new NodeModule(this)); - modules.add(new NetworkModule()); + modules.add(new SettingsModule(this.settings, settingsFilter)); + modules.add(new NodeModule(this, nodeSettingsService, monitorService)); + modules.add(new NetworkModule(networkService)); modules.add(new ScriptModule(this.settings)); modules.add(new EnvironmentModule(environment)); modules.add(new NodeEnvironmentModule(nodeEnvironment)); @@ -181,17 +199,16 @@ public class Node implements Releasable { if (settings.getAsBoolean(HTTP_ENABLED, true)) { modules.add(new HttpServerModule(settings)); } - modules.add(new IndicesModule(settings)); - modules.add(new SearchModule(settings)); + modules.add(new IndicesModule()); + modules.add(new SearchModule()); modules.add(new ActionModule(false)); - modules.add(new MonitorModule(settings)); modules.add(new GatewayModule(settings)); modules.add(new NodeClientModule()); modules.add(new PercolatorModule()); modules.add(new ResourceWatcherModule()); modules.add(new RepositoriesModule()); modules.add(new TribeModule()); - + modules.add(new AnalysisModule(environment)); pluginsService.processModules(modules); @@ -200,6 +217,8 @@ public class Node implements Releasable { client = injector.getInstance(Client.class); threadPool.setNodeSettingsService(injector.getInstance(NodeSettingsService.class)); success = true; + } catch (IOException ex) { + throw new ElasticsearchException("failed to bind service", ex); } finally { if (!success) { nodeEnvironment.close(); @@ -269,6 +288,15 @@ public class Node implements Releasable { injector.getInstance(ResourceWatcherService.class).start(); injector.getInstance(TribeService.class).start(); + if (System.getProperty("es.tests.portsfile", "false").equals("true")) { + if (settings.getAsBoolean("http.enabled", true)) { + HttpServerTransport http = injector.getInstance(HttpServerTransport.class); + writePortsFile("http", http.boundAddress()); + } + TransportService transport = injector.getInstance(TransportService.class); + writePortsFile("transport", transport.boundAddress()); + } + logger.info("started"); return this; @@ -306,6 +334,7 @@ public class Node implements Releasable { for (Class plugin : pluginsService.nodeServices()) { injector.getInstance(plugin).stop(); } + injector.getInstance(RecoverySettings.class).close(); // we should stop this last since it waits for resources to get released // if we had scroll searchers etc or recovery going on we wait for to finish. injector.getInstance(IndicesService.class).stop(); @@ -419,4 +448,27 @@ public class Node implements Releasable { public Injector injector() { return this.injector; } + + /** Writes a file to the logs dir containing the ports for the given transport type */ + private void writePortsFile(String type, BoundTransportAddress boundAddress) { + Path tmpPortsFile = environment.logsFile().resolve(type + ".ports.tmp"); + try (BufferedWriter writer = Files.newBufferedWriter(tmpPortsFile, Charset.forName("UTF-8"))) { + for (TransportAddress address : boundAddress.boundAddresses()) { + InetAddress inetAddress = InetAddress.getByName(address.getAddress()); + if (inetAddress instanceof Inet6Address && inetAddress.isLinkLocalAddress()) { + // no link local, just causes problems + continue; + } + writer.write(NetworkAddress.formatAddress(new InetSocketAddress(inetAddress, address.getPort())) + "\n"); + } + } catch (IOException e) { + throw new RuntimeException("Failed to write ports file", e); + } + Path portsFile = environment.logsFile().resolve(type + ".ports"); + try { + Files.move(tmpPortsFile, portsFile, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new RuntimeException("Failed to rename ports file", e); + } + } } diff --git a/core/src/main/java/org/elasticsearch/node/NodeBuilder.java b/core/src/main/java/org/elasticsearch/node/NodeBuilder.java deleted file mode 100644 index 377c409ccb1..00000000000 --- a/core/src/main/java/org/elasticsearch/node/NodeBuilder.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.node; - -import org.elasticsearch.common.settings.Settings; - -/** - * A node builder is used to construct a {@link Node} instance. - *

    - * Settings will be loaded relative to the ES home (with or without config/ prefix) and if not found, - * within the classpath (with or without config/ prefix). The settings file loaded can either be named - * elasticsearch.yml or elasticsearch.json). - *

    - * Explicit settings can be passed by using the {@link #settings(org.elasticsearch.common.settings.Settings)} method. - *

    - * In any case, settings will be resolved from system properties as well that are either prefixed with es. - * or elasticsearch.. - *

    - * An example for creating a simple node with optional settings loaded from the classpath: - * 

    - * Node node = NodeBuilder.nodeBuilder().node();
- *
    - *

    - * An example for creating a node with explicit settings (in this case, a node in the cluster that does not hold - * data): - * 

    - * Node node = NodeBuilder.nodeBuilder()
- *                      .settings(Settings.settingsBuilder().put("node.data", false)
- *                      .node();
- *
    - *

    - * When done with the node, make sure you call {@link Node#close()} on it. - * - * - */ -public class NodeBuilder { - - private final Settings.Builder settings = Settings.settingsBuilder(); - - /** - * A convenient factory method to create a {@link NodeBuilder}. - */ - public static NodeBuilder nodeBuilder() { - return new NodeBuilder(); - } - - /** - * Set addition settings simply by working directly against the settings builder. - */ - public Settings.Builder settings() { - return settings; - } - - /** - * Set addition settings simply by working directly against the settings builder. - */ - public Settings.Builder getSettings() { - return settings; - } - - /** - * Explicit node settings to set. - */ - public NodeBuilder settings(Settings.Builder settings) { - return settings(settings.build()); - } - - /** - * Explicit node settings to set. - */ - public NodeBuilder settings(Settings settings) { - this.settings.put(settings); - return this; - } - - /** - * Is the node going to be a client node which means it will hold no data (node.data is - * set to false) and other optimizations by different modules. - * - * @param client Should the node be just a client node or not. - */ - public NodeBuilder client(boolean client) { - settings.put("node.client", client); - return this; - } - - /** - * Is the node going to be allowed to allocate data (shards) to it or not. This setting map to - * the node.data setting. Note, when setting {@link #client(boolean)}, the node will - * not hold any data by default. - * - * @param data Should the node be allocated data to or not. - */ - public NodeBuilder data(boolean data) { - settings.put("node.data", data); - return this; - } - - /** - * Is the node a local node. A local node is a node that uses a local (JVM level) discovery and - * transport. Other (local) nodes started within the same JVM (actually, class-loader) will be - * discovered and communicated with. Nodes outside of the JVM will not be discovered. - * - * @param local Should the node be local or not - */ - public NodeBuilder local(boolean local) { - settings.put("node.local", local); - return this; - } - - /** - * The cluster name this node is part of (maps to the cluster.name setting). Defaults - * to elasticsearch. - * - * @param clusterName The cluster name this node is part of. - */ - public NodeBuilder clusterName(String clusterName) { - settings.put("cluster.name", clusterName); - return this; - } - - /** - * Builds the node without starting it. - */ - public Node build() { - return new Node(settings.build()); - } - - /** - * {@link #build()}s and starts the node. - */ - public Node node() { - return build().start(); - } -} diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java index befba85af09..3641c325030 100644 --- a/core/src/main/java/org/elasticsearch/node/NodeModule.java +++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java @@ -22,6 +22,7 @@ package org.elasticsearch.node; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.monitor.MonitorService; import org.elasticsearch.node.Node; import org.elasticsearch.node.service.NodeService; import org.elasticsearch.node.settings.NodeSettingsService; @@ -32,13 +33,17 @@ import org.elasticsearch.node.settings.NodeSettingsService; public class NodeModule extends AbstractModule { private final Node node; + private final NodeSettingsService nodeSettingsService; + private final MonitorService monitorService; // pkg private so tests can mock Class pageCacheRecyclerImpl = PageCacheRecycler.class; Class bigArraysImpl = BigArrays.class; - public NodeModule(Node node) { + public NodeModule(Node node, NodeSettingsService nodeSettingsService, MonitorService monitorService) { this.node = node; + this.nodeSettingsService = nodeSettingsService; + this.monitorService = monitorService; } @Override @@ -55,7 +60,8 @@ public class NodeModule extends AbstractModule { } bind(Node.class).toInstance(node); - bind(NodeSettingsService.class).asEagerSingleton(); + bind(NodeSettingsService.class).toInstance(nodeSettingsService); + bind(MonitorService.class).toInstance(monitorService); bind(NodeService.class).asEagerSingleton(); } } diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java index 3f35ddf033c..7bede53e7ec 100644 --- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java +++ b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java @@ -20,6 +20,8 @@ package org.elasticsearch.node.internal; import java.nio.charset.StandardCharsets; + +import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; @@ -125,11 +127,11 @@ public class InternalSettingsPreparer { if (useSystemProperties(input)) { if (loadDefaults) { for (String prefix : PROPERTY_DEFAULTS_PREFIXES) { - output.putProperties(prefix, System.getProperties()); + output.putProperties(prefix, BootstrapInfo.getSystemProperties()); } } for (String prefix : PROPERTY_PREFIXES) { - output.putProperties(prefix, System.getProperties(), PROPERTY_DEFAULTS_PREFIXES); + output.putProperties(prefix, BootstrapInfo.getSystemProperties(), PROPERTY_DEFAULTS_PREFIXES); } } output.replacePropertyPlaceholders(); diff --git a/core/src/main/java/org/elasticsearch/node/package-info.java b/core/src/main/java/org/elasticsearch/node/package-info.java index fa503a9ce1b..02538cda4f8 100644 --- a/core/src/main/java/org/elasticsearch/node/package-info.java +++ b/core/src/main/java/org/elasticsearch/node/package-info.java @@ -18,7 +18,7 @@ */ /** - * Allow to build a {@link org.elasticsearch.node.Node} using {@link org.elasticsearch.node.NodeBuilder} which is a + * Allow to build a {@link org.elasticsearch.node.Node} which is a * node within the cluster. */ -package org.elasticsearch.node; \ No newline at end of file +package org.elasticsearch.node; diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java index fe57800a466..b4fe59e3473 100644 --- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java +++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java @@ -152,13 +152,14 @@ public class NodeService extends AbstractComponent { transportService.stats(), httpServer == null ? null : httpServer.stats(), circuitBreakerService.stats(), - scriptService.stats() + scriptService.stats(), + discovery.stats() ); } public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, boolean jvm, boolean threadPool, boolean fs, boolean transport, boolean http, boolean circuitBreaker, - boolean script) { + boolean script, boolean discoveryStats) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) return new NodeStats(discovery.localNode(), System.currentTimeMillis(), @@ -171,7 +172,8 @@ public class NodeService extends AbstractComponent { transport ? transportService.stats() : null, http ? (httpServer == null ? null : httpServer.stats()) : null, circuitBreaker ? circuitBreakerService.stats() : null, - script ? scriptService.stats() : null + script ? scriptService.stats() : null, + discoveryStats ? discovery.stats() : null ); } } diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java b/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java index 8cb797cdce0..70abaaaff3d 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; @@ -423,11 +422,6 @@ public class PercolateContext extends SearchContext { return indexService.analysisService(); } - @Override - public IndexQueryParserService queryParserService() { - return indexService.queryParserService(); - } - @Override public SimilarityService similarityService() { return indexService.similarityService(); @@ -450,7 +444,7 @@ public class PercolateContext extends SearchContext { @Override public BitsetFilterCache bitsetFilterCache() { - return indexService.bitsetFilterCache(); + return indexService.cache().bitsetFilterCache(); } @Override @@ -735,7 +729,7 @@ public class PercolateContext extends SearchContext { @Override public Set getHeaders() { - return Collections.EMPTY_SET; + return Collections.emptySet(); } @Override diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java index 86ff604456a..fa7b47766a8 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java @@ -74,6 +74,7 @@ import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.percolator.QueryCollector.Count; @@ -190,7 +191,7 @@ public class PercolatorService extends AbstractComponent { indexShard.shardId().index().name(), request.indices() ); - Query aliasFilter = percolateIndexService.aliasFilter(filteringAliases); + Query aliasFilter = percolateIndexService.aliasFilter(indexShard.getQueryShardContext(), filteringAliases); SearchShardTarget searchShardTarget = new SearchShardTarget(clusterService.localNode().id(), request.shardId().getIndex(), request.shardId().id()); final PercolateContext context = new PercolateContext( @@ -198,7 +199,7 @@ public class PercolatorService extends AbstractComponent { ); SearchContext.setCurrent(context); try { - ParsedDocument parsedDocument = parseRequest(percolateIndexService, request, context, request.shardId().getIndex()); + ParsedDocument parsedDocument = parseRequest(indexShard, request, context, request.shardId().getIndex()); if (context.percolateQueries().isEmpty()) { return new PercolateShardResponse(context, request.shardId()); } @@ -258,7 +259,7 @@ public class PercolatorService extends AbstractComponent { } } - private ParsedDocument parseRequest(IndexService documentIndexService, PercolateShardRequest request, PercolateContext context, String index) { + private ParsedDocument parseRequest(IndexShard shard, PercolateShardRequest request, PercolateContext context, String index) { BytesReference source = request.source(); if (source == null || source.length() == 0) { return null; @@ -276,6 +277,7 @@ public class PercolatorService extends AbstractComponent { // not the in memory percolate doc String[] previousTypes = context.types(); context.types(new String[]{TYPE_NAME}); + QueryShardContext queryShardContext = shard.getQueryShardContext(); try { parser = XContentFactory.xContent(source).createParser(source); String currentFieldName = null; @@ -290,7 +292,7 @@ public class PercolatorService extends AbstractComponent { throw new ElasticsearchParseException("Either specify doc or get, not both"); } - MapperService mapperService = documentIndexService.mapperService(); + MapperService mapperService = shard.mapperService(); DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(request.documentType()); doc = docMapper.getDocumentMapper().parse(source(parser).index(index).type(request.documentType()).flyweight(true)); if (docMapper.getMapping() != null) { @@ -312,12 +314,12 @@ public class PercolatorService extends AbstractComponent { if (context.percolateQuery() != null) { throw new ElasticsearchParseException("Either specify query or filter, not both"); } - context.percolateQuery(documentIndexService.queryParserService().parse(parser).query()); + context.percolateQuery(queryShardContext.parse(parser).query()); } else if ("filter".equals(currentFieldName)) { if (context.percolateQuery() != null) { throw new ElasticsearchParseException("Either specify query or filter, not both"); } - Query filter = documentIndexService.queryParserService().parseInnerFilter(parser).query(); + Query filter = queryShardContext.parseInnerFilter(parser).query(); context.percolateQuery(new ConstantScoreQuery(filter)); } else if ("sort".equals(currentFieldName)) { parseSort(parser, context); diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index 4229c54401a..1db0ac966d7 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -22,10 +22,13 @@ package org.elasticsearch.plugins; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexService; import java.io.Closeable; import java.util.Collection; import java.util.Collections; +import java.util.List; /** * An extension point allowing to plug in custom functionality. @@ -59,20 +62,6 @@ public abstract class Plugin { return Collections.emptyList(); } - /** - * Per index modules. - */ - public Collection indexModules(Settings indexSettings) { - return Collections.emptyList(); - } - - /** - * Per index services that will be automatically closed. - */ - public Collection> indexServices() { - return Collections.emptyList(); - } - /** * Additional node settings loaded by the plugin. Note that settings that are explicit in the nodes settings can't be * overwritten with the additional settings. These settings added if they don't exist. @@ -80,4 +69,18 @@ public abstract class Plugin { public Settings additionalSettings() { return Settings.Builder.EMPTY_SETTINGS; } + + /** + * Called before a new index is created on a node. The given module can be used to register index-level + * extensions. + */ + public void onIndexModule(IndexModule indexModule) {} + + /** + * Old-style guice index level extension point. + * + * @deprecated use #onIndexModule instead + */ + @Deprecated + public final void onModule(IndexModule indexModule) {} } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java index 700c0f7be22..1ebe7813d3c 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -22,6 +22,7 @@ package org.elasticsearch.plugins; import org.apache.lucene.util.IOUtils; import org.elasticsearch.*; import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.cli.Terminal; import org.elasticsearch.common.collect.Tuple; @@ -67,6 +68,10 @@ public class PluginManager { "plugin.bat", "service.bat")); + static final Set MODULES = unmodifiableSet(newHashSet( + "lang-expression", + "lang-groovy")); + static final Set OFFICIAL_PLUGINS = unmodifiableSet(newHashSet( "analysis-icu", "analysis-kuromoji", @@ -78,13 +83,14 @@ public class PluginManager { "discovery-ec2", "discovery-gce", "discovery-multicast", - "lang-expression", - "lang-groovy", "lang-javascript", + "lang-plan-a", "lang-python", + "mapper-attachments", "mapper-murmur3", "mapper-size", "repository-azure", + "repository-hdfs", "repository-s3", "store-smb")); @@ -120,7 +126,7 @@ public class PluginManager { checkForForbiddenName(pluginHandle.name); } else { // if we have no name but url, use temporary name that will be overwritten later - pluginHandle = new PluginHandle("temp_name" + new Random().nextInt(), null, null); + pluginHandle = new PluginHandle("temp_name" + Randomness.get().nextInt(), null, null); } Path pluginFile = download(pluginHandle, terminal); @@ -220,9 +226,10 @@ public class PluginManager { PluginInfo info = PluginInfo.readFromProperties(root); terminal.println(VERBOSE, "%s", info); - // check for jar hell before any copying - if (info.isJvm()) { - jarHellCheck(root, info.isIsolated()); + // don't let luser install plugin as a module... + // they might be unavoidably in maven central and are packaged up the same way) + if (MODULES.contains(info.getName())) { + throw new IOException("plugin '" + info.getName() + "' cannot be installed like this, it is a system module"); } // update name in handle based on 'name' property found in descriptor file @@ -232,6 +239,11 @@ public class PluginManager { throw new IOException("plugin directory " + extractLocation.toAbsolutePath() + " already exists. To update the plugin, uninstall it first using 'remove " + pluginHandle.name + "' command"); } + // check for jar hell before any copying + if (info.isJvm()) { + jarHellCheck(root, info.isIsolated()); + } + // read optional security policy (extra permissions) // if it exists, confirm or warn the user Path policy = root.resolve(PluginInfo.ES_PLUGIN_POLICY); @@ -511,7 +523,7 @@ public class PluginManager { if (removed) { terminal.println("Removed %s", name); } else { - terminal.println("Plugin %s not found. Run plugin --list to get list of installed plugins.", name); + terminal.println("Plugin %s not found. Run \"plugin list\" to get list of installed plugins.", name); } } @@ -574,7 +586,7 @@ public class PluginManager { // Elasticsearch new download service uses groupId org.elasticsearch.plugin from 2.0.0 if (user == null) { if (!Strings.isNullOrEmpty(System.getProperty(PROPERTY_SUPPORT_STAGING_URLS))) { - addUrl(urls, String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip", version, Build.CURRENT.hashShort(), name, version, name, version)); + addUrl(urls, String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip", version, Build.CURRENT.shortHash(), name, version, name, version)); } addUrl(urls, String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%s/%s/%s-%s.zip", name, version, name, version)); } else { diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java b/core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java index 1097752bae5..a8a51db971c 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java @@ -21,6 +21,7 @@ package org.elasticsearch.plugins; import org.apache.commons.cli.CommandLine; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.cli.CliTool; import org.elasticsearch.common.cli.CliToolConfig; import org.elasticsearch.common.cli.Terminal; @@ -65,6 +66,11 @@ public class PluginManagerCliParser extends CliTool { // configure but do not read the logging conf file LogConfigurator.configure(env.settings(), false); int status = new PluginManagerCliParser().execute(args).status(); + exit(status); + } + + @SuppressForbidden(reason = "Allowed to exit explicitly from #main()") + private static void exit(int status) { System.exit(status); } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index 9582d3f1714..5ebd43d5026 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -26,7 +26,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.bootstrap.JarHell; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -37,14 +37,16 @@ import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexModule; -import java.io.Closeable; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.DirectoryStream; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -64,10 +66,10 @@ import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; public class PluginsService extends AbstractComponent { /** - * We keep around a list of plugins + * We keep around a list of plugins and modules */ private final List> plugins; - private final PluginsInfo info; + private final PluginsAndModules info; private final Map> onModuleReferences; @@ -84,13 +86,15 @@ public class PluginsService extends AbstractComponent { /** * Constructs a new PluginService * @param settings The settings of the system + * @param modulesDirectory The directory modules exist in, or null if modules should not be loaded from the filesystem * @param pluginsDirectory The directory plugins exist in, or null if plugins should not be loaded from the filesystem * @param classpathPlugins Plugins that exist in the classpath which should be loaded */ - public PluginsService(Settings settings, Path pluginsDirectory, Collection> classpathPlugins) { + public PluginsService(Settings settings, Path modulesDirectory, Path pluginsDirectory, Collection> classpathPlugins) { super(settings); + info = new PluginsAndModules(); - List> tupleBuilder = new ArrayList<>(); + List> pluginsLoaded = new ArrayList<>(); // first we load plugins that are on the classpath. this is for tests and transport clients for (Class pluginClass : classpathPlugins) { @@ -99,24 +103,39 @@ public class PluginsService extends AbstractComponent { if (logger.isTraceEnabled()) { logger.trace("plugin loaded from classpath [{}]", pluginInfo); } - tupleBuilder.add(new Tuple<>(pluginInfo, plugin)); + pluginsLoaded.add(new Tuple<>(pluginInfo, plugin)); + info.addPlugin(pluginInfo); + } + + // load modules + if (modulesDirectory != null) { + try { + List bundles = getModuleBundles(modulesDirectory); + List> loaded = loadBundles(bundles); + pluginsLoaded.addAll(loaded); + for (Tuple module : loaded) { + info.addModule(module.v1()); + } + } catch (IOException ex) { + throw new IllegalStateException("Unable to initialize modules", ex); + } } // now, find all the ones that are in plugins/ if (pluginsDirectory != null) { try { List bundles = getPluginBundles(pluginsDirectory); - tupleBuilder.addAll(loadBundles(bundles)); + List> loaded = loadBundles(bundles); + pluginsLoaded.addAll(loaded); + for (Tuple plugin : loaded) { + info.addPlugin(plugin.v1()); + } } catch (IOException ex) { throw new IllegalStateException("Unable to initialize plugins", ex); } } - plugins = Collections.unmodifiableList(tupleBuilder); - info = new PluginsInfo(); - for (Tuple tuple : plugins) { - info.add(tuple.v1()); - } + plugins = Collections.unmodifiableList(pluginsLoaded); // We need to build a List of jvm and site plugins for checking mandatory plugins Map jvmPlugins = new HashMap<>(); @@ -146,7 +165,18 @@ public class PluginsService extends AbstractComponent { } } - logger.info("loaded {}, sites {}", jvmPlugins.keySet(), sitePlugins); + // we don't log jars in lib/ we really shouldnt log modules, + // but for now: just be transparent so we can debug any potential issues + Set moduleNames = new HashSet<>(); + Set jvmPluginNames = new HashSet<>(); + for (PluginInfo moduleInfo : info.getModuleInfos()) { + moduleNames.add(moduleInfo.getName()); + } + for (PluginInfo pluginInfo : info.getPluginInfos()) { + jvmPluginNames.add(pluginInfo.getName()); + } + + logger.info("modules {}, plugins {}, sites {}", moduleNames, jvmPluginNames, sitePlugins); Map> onModuleReferences = new HashMap<>(); for (Plugin plugin : jvmPlugins.values()) { @@ -155,6 +185,10 @@ public class PluginsService extends AbstractComponent { if (!method.getName().equals("onModule")) { continue; } + // this is a deprecated final method, so all Plugin subclasses have it + if (method.getParameterTypes().length == 1 && method.getParameterTypes()[0].equals(IndexModule.class)) { + continue; + } if (method.getParameterTypes().length == 0 || method.getParameterTypes().length > 1) { logger.warn("Plugin: {} implementing onModule with no parameters or more than one parameter", plugin.name()); continue; @@ -173,7 +207,7 @@ public class PluginsService extends AbstractComponent { this.onModuleReferences = Collections.unmodifiableMap(onModuleReferences); } - public List> plugins() { + private List> plugins() { return plugins; } @@ -192,8 +226,12 @@ public class PluginsService extends AbstractComponent { if (reference.moduleClass.isAssignableFrom(module.getClass())) { try { reference.onModuleMethod.invoke(plugin.v2(), module); + } catch (IllegalAccessException | InvocationTargetException e) { + logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.v2().name()); + throw new ElasticsearchException("failed to invoke onModule", e); } catch (Exception e) { logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.v2().name()); + throw e; } } } @@ -234,29 +272,18 @@ public class PluginsService extends AbstractComponent { return services; } - public Collection indexModules(Settings indexSettings) { - List modules = new ArrayList<>(); + public void onIndexModule(IndexModule indexModule) { for (Tuple plugin : plugins) { - modules.addAll(plugin.v2().indexModules(indexSettings)); + plugin.v2().onIndexModule(indexModule); } - return modules; } - - public Collection> indexServices() { - List> services = new ArrayList<>(); - for (Tuple plugin : plugins) { - services.addAll(plugin.v2().indexServices()); - } - return services; - } - /** - * Get information about plugins (jvm and site plugins). + * Get information about plugins and modules */ - public PluginsInfo info() { + public PluginsAndModules info() { return info; } - + // a "bundle" is a group of plugins in a single classloader // really should be 1-1, but we are not so fortunate static class Bundle { @@ -264,6 +291,41 @@ public class PluginsService extends AbstractComponent { List urls = new ArrayList<>(); } + // similar in impl to getPluginBundles, but DO NOT try to make them share code. + // we don't need to inherit all the leniency, and things are different enough. + static List getModuleBundles(Path modulesDirectory) throws IOException { + // damn leniency + if (Files.notExists(modulesDirectory)) { + return Collections.emptyList(); + } + List bundles = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(modulesDirectory)) { + for (Path module : stream) { + if (FileSystemUtils.isHidden(module)) { + continue; // skip over .DS_Store etc + } + PluginInfo info = PluginInfo.readFromProperties(module); + if (!info.isJvm()) { + throw new IllegalStateException("modules must be jvm plugins: " + info); + } + if (!info.isIsolated()) { + throw new IllegalStateException("modules must be isolated: " + info); + } + Bundle bundle = new Bundle(); + bundle.plugins.add(info); + // gather urls for jar files + try (DirectoryStream jarStream = Files.newDirectoryStream(module, "*.jar")) { + for (Path jar : jarStream) { + // normalize with toRealPath to get symlinks out of our hair + bundle.urls.add(jar.toRealPath().toUri().toURL()); + } + } + bundles.add(bundle); + } + } + return bundles; + } + static List getPluginBundles(Path pluginsDirectory) throws IOException { ESLogger logger = Loggers.getLogger(PluginsService.class); @@ -271,7 +333,7 @@ public class PluginsService extends AbstractComponent { if (!isAccessibleDirectory(pluginsDirectory, logger)) { return Collections.emptyList(); } - + List bundles = new ArrayList<>(); // a special purgatory for plugins that directly depend on each other bundles.add(new Bundle()); @@ -283,13 +345,21 @@ public class PluginsService extends AbstractComponent { continue; } logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); - PluginInfo info = PluginInfo.readFromProperties(plugin); + final PluginInfo info; + try { + info = PluginInfo.readFromProperties(plugin); + } catch (IOException e) { + throw new IllegalStateException("Could not load plugin descriptor for existing plugin [" + + plugin.getFileName() + "]. Was the plugin built before 2.0?", e); + } + List urls = new ArrayList<>(); if (info.isJvm()) { // a jvm plugin: gather urls for jar files try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) { for (Path jar : jarStream) { - urls.add(jar.toUri().toURL()); + // normalize with toRealPath to get symlinks out of our hair + urls.add(jar.toRealPath().toUri().toURL()); } } } @@ -304,7 +374,7 @@ public class PluginsService extends AbstractComponent { bundle.urls.addAll(urls); } } - + return bundles; } @@ -322,7 +392,7 @@ public class PluginsService extends AbstractComponent { } catch (Exception e) { throw new IllegalStateException("failed to load bundle " + bundle.urls + " due to jar hell", e); } - + // create a child to load the plugins in this bundle ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), getClass().getClassLoader()); for (PluginInfo pluginInfo : bundle.plugins) { diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 8c5088e757b..2648a183362 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -294,7 +294,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent indices = Collections.EMPTY_LIST; + List indices = Collections.emptyList(); Snapshot snapshot = null; try { snapshot = readSnapshot(snapshotId); diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestActionModule.java b/core/src/main/java/org/elasticsearch/rest/action/RestActionModule.java index 02bde5f74d5..f0e4d10d7c4 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/RestActionModule.java +++ b/core/src/main/java/org/elasticsearch/rest/action/RestActionModule.java @@ -57,12 +57,12 @@ import org.elasticsearch.rest.action.admin.indices.exists.indices.RestIndicesExi import org.elasticsearch.rest.action.admin.indices.exists.types.RestTypesExistsAction; import org.elasticsearch.rest.action.admin.indices.flush.RestFlushAction; import org.elasticsearch.rest.action.admin.indices.flush.RestSyncedFlushAction; +import org.elasticsearch.rest.action.admin.indices.forcemerge.RestForceMergeAction; import org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction; import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.mapping.put.RestPutMappingAction; import org.elasticsearch.rest.action.admin.indices.open.RestOpenIndexAction; -import org.elasticsearch.rest.action.admin.indices.optimize.RestOptimizeAction; import org.elasticsearch.rest.action.admin.indices.recovery.RestRecoveryAction; import org.elasticsearch.rest.action.admin.indices.refresh.RestRefreshAction; import org.elasticsearch.rest.action.admin.indices.segments.RestIndicesSegmentsAction; @@ -92,8 +92,10 @@ import org.elasticsearch.rest.action.cat.RestMasterAction; import org.elasticsearch.rest.action.cat.RestNodeAttrsAction; import org.elasticsearch.rest.action.cat.RestNodesAction; import org.elasticsearch.rest.action.cat.RestPluginsAction; +import org.elasticsearch.rest.action.cat.RestRepositoriesAction; import org.elasticsearch.rest.action.cat.RestSegmentsAction; import org.elasticsearch.rest.action.cat.RestShardsAction; +import org.elasticsearch.rest.action.cat.RestSnapshotAction; import org.elasticsearch.rest.action.cat.RestThreadPoolAction; import org.elasticsearch.rest.action.delete.RestDeleteAction; import org.elasticsearch.rest.action.explain.RestExplainAction; @@ -200,7 +202,7 @@ public class RestActionModule extends AbstractModule { bind(RestRefreshAction.class).asEagerSingleton(); bind(RestFlushAction.class).asEagerSingleton(); bind(RestSyncedFlushAction.class).asEagerSingleton(); - bind(RestOptimizeAction.class).asEagerSingleton(); + bind(RestForceMergeAction.class).asEagerSingleton(); bind(RestUpgradeAction.class).asEagerSingleton(); bind(RestClearIndicesCacheAction.class).asEagerSingleton(); @@ -263,6 +265,8 @@ public class RestActionModule extends AbstractModule { catActionMultibinder.addBinding().to(RestPluginsAction.class).asEagerSingleton(); catActionMultibinder.addBinding().to(RestFielddataAction.class).asEagerSingleton(); catActionMultibinder.addBinding().to(RestNodeAttrsAction.class).asEagerSingleton(); + catActionMultibinder.addBinding().to(RestRepositoriesAction.class).asEagerSingleton(); + catActionMultibinder.addBinding().to(RestSnapshotAction.class).asEagerSingleton(); // no abstract cat action bind(RestCatAction.class).asEagerSingleton(); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java index edea49cf20d..98d4feca189 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java @@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.health; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java index 2e3927e665e..910d3dcc833 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java @@ -77,6 +77,7 @@ public class RestNodesStatsAction extends BaseRestHandler { nodesStatsRequest.process(metrics.contains("process")); nodesStatsRequest.breaker(metrics.contains("breaker")); nodesStatsRequest.script(metrics.contains("script")); + nodesStatsRequest.discovery(metrics.contains("discovery")); // check for index specific metrics if (metrics.contains("indices")) { diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java index 200d9dc9825..6e3a889f691 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.cluster.repositories.verify; import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest; -import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -30,9 +29,6 @@ import org.elasticsearch.rest.action.support.RestToXContentListener; import static org.elasticsearch.client.Requests.verifyRepositoryRequest; import static org.elasticsearch.rest.RestRequest.Method.POST; -/** - * Registers repositories - */ public class RestVerifyRepositoryAction extends BaseRestHandler { @Inject @@ -41,12 +37,11 @@ public class RestVerifyRepositoryAction extends BaseRestHandler { controller.registerHandler(POST, "/_snapshot/{repository}/_verify", this); } - @Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { VerifyRepositoryRequest verifyRepositoryRequest = verifyRepositoryRequest(request.param("repository")); verifyRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", verifyRepositoryRequest.masterNodeTimeout())); verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.timeout())); - client.admin().cluster().verifyRepository(verifyRepositoryRequest, new RestToXContentListener(channel)); + client.admin().cluster().verifyRepository(verifyRepositoryRequest, new RestToXContentListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java index 304b3d40a00..21aadd174ec 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java @@ -47,7 +47,10 @@ public class RestGetSnapshotsAction extends BaseRestHandler { public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { String repository = request.param("repository"); String[] snapshots = request.paramAsStringArray("snapshot", Strings.EMPTY_ARRAY); + GetSnapshotsRequest getSnapshotsRequest = getSnapshotsRequest(repository).snapshots(snapshots); + getSnapshotsRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", getSnapshotsRequest.ignoreUnavailable())); + getSnapshotsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSnapshotsRequest.masterNodeTimeout())); client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestToXContentListener(channel)); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java index 4841500cb66..5648abc7f14 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.rest.action.admin.indices.alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.AliasAction; @@ -30,9 +31,10 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.AcknowledgedRestListener; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import static org.elasticsearch.cluster.metadata.AliasAction.newAddAliasAction; import static org.elasticsearch.rest.RestRequest.Method.POST; /** @@ -75,8 +77,8 @@ public class RestIndicesAliasesAction extends BaseRestHandler { } else { throw new IllegalArgumentException("Alias action [" + action + "] not supported"); } - String index = null; - String alias = null; + String[] indices = null; + String[] aliases = null; Map filter = null; String routing = null; boolean routingSet = false; @@ -90,9 +92,9 @@ public class RestIndicesAliasesAction extends BaseRestHandler { currentFieldName = parser.currentName(); } else if (token.isValue()) { if ("index".equals(currentFieldName)) { - index = parser.text(); + indices = new String[] { parser.text() }; } else if ("alias".equals(currentFieldName)) { - alias = parser.text(); + aliases = new String[] { parser.text() }; } else if ("routing".equals(currentFieldName)) { routing = parser.textOrNull(); routingSet = true; @@ -103,6 +105,23 @@ public class RestIndicesAliasesAction extends BaseRestHandler { searchRouting = parser.textOrNull(); searchRoutingSet = true; } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("indices".equals(currentFieldName)) { + List indexNames = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + String index = parser.text(); + indexNames.add(index); + } + indices = indexNames.toArray(new String[indexNames.size()]); + } + if ("aliases".equals(currentFieldName)) { + List aliasNames = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + String alias = parser.text(); + aliasNames.add(alias); + } + aliases = aliasNames.toArray(new String[aliasNames.size()]); + } } else if (token == XContentParser.Token.START_OBJECT) { if ("filter".equals(currentFieldName)) { filter = parser.mapOrdered(); @@ -111,19 +130,19 @@ public class RestIndicesAliasesAction extends BaseRestHandler { } if (type == AliasAction.Type.ADD) { - AliasAction aliasAction = newAddAliasAction(index, alias).filter(filter); + AliasActions aliasActions = new AliasActions(type, indices, aliases); if (routingSet) { - aliasAction.routing(routing); + aliasActions.routing(routing); } if (indexRoutingSet) { - aliasAction.indexRouting(indexRouting); + aliasActions.indexRouting(indexRouting); } if (searchRoutingSet) { - aliasAction.searchRouting(searchRouting); + aliasActions.searchRouting(searchRouting); } - indicesAliasesRequest.addAliasAction(aliasAction); + indicesAliasesRequest.addAliasAction(aliasActions); } else if (type == AliasAction.Type.REMOVE) { - indicesAliasesRequest.removeAlias(index, alias); + indicesAliasesRequest.removeAlias(indices, aliases); } } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetIndicesAliasesAction.java index 34cdcb27962..528799cba28 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetIndicesAliasesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetIndicesAliasesAction.java @@ -70,10 +70,10 @@ public class RestGetIndicesAliasesAction extends BaseRestHandler { final boolean isAllAliasesRequested = isAllOrWildcard(aliases); for (IndexMetaData indexMetaData : metaData) { - builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); + builder.startObject(indexMetaData.getIndex(), XContentBuilder.FieldCaseConversion.NONE); builder.startObject("aliases"); - for (ObjectCursor cursor : indexMetaData.aliases().values()) { + for (ObjectCursor cursor : indexMetaData.getAliases().values()) { if (isAllAliasesRequested || Regex.simpleMatch(aliases, cursor.value.alias())) { AliasMetaData.Builder.toXContent(cursor.value, builder, ToXContent.EMPTY_PARAMS); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java index 57ceb21f41e..3a86911f464 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java @@ -21,7 +21,8 @@ package org.elasticsearch.rest.action.admin.indices.analyze; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -47,6 +48,17 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; */ public class RestAnalyzeAction extends BaseRestHandler { + public static class Fields { + public static final ParseField ANALYZER = new ParseField("analyzer"); + public static final ParseField TEXT = new ParseField("text"); + public static final ParseField FIELD = new ParseField("field"); + public static final ParseField TOKENIZER = new ParseField("tokenizer"); + public static final ParseField TOKEN_FILTERS = new ParseField("token_filters", "filters"); + public static final ParseField CHAR_FILTERS = new ParseField("char_filters"); + public static final ParseField EXPLAIN = new ParseField("explain"); + public static final ParseField ATTRIBUTES = new ParseField("attributes"); + } + @Inject public RestAnalyzeAction(Settings settings, RestController controller, Client client) { super(settings, controller, client); @@ -68,6 +80,8 @@ public class RestAnalyzeAction extends BaseRestHandler { analyzeRequest.tokenizer(request.param("tokenizer")); analyzeRequest.tokenFilters(request.paramAsStringArray("token_filters", request.paramAsStringArray("filters", analyzeRequest.tokenFilters()))); analyzeRequest.charFilters(request.paramAsStringArray("char_filters", analyzeRequest.charFilters())); + analyzeRequest.explain(request.paramAsBoolean("explain", false)); + analyzeRequest.attributes(request.paramAsStringArray("attributes", analyzeRequest.attributes())); if (RestActions.hasBodyContent(request)) { XContentType type = RestActions.guessBodyContentType(request); @@ -78,14 +92,14 @@ public class RestAnalyzeAction extends BaseRestHandler { } } else { // NOTE: if rest request with xcontent body has request parameters, the parameters does not override xcontent values - buildFromContent(RestActions.getRestContent(request), analyzeRequest); + buildFromContent(RestActions.getRestContent(request), analyzeRequest, parseFieldMatcher); } } client.admin().indices().analyze(analyzeRequest, new RestToXContentListener(channel)); } - public static void buildFromContent(BytesReference content, AnalyzeRequest analyzeRequest) { + public static void buildFromContent(BytesReference content, AnalyzeRequest analyzeRequest, ParseFieldMatcher parseFieldMatcher) { try (XContentParser parser = XContentHelper.createParser(content)) { if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("Malforrmed content, must start with an object"); @@ -95,9 +109,9 @@ public class RestAnalyzeAction extends BaseRestHandler { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); - } else if ("text".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { + } else if (parseFieldMatcher.match(currentFieldName, Fields.TEXT) && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.text(parser.text()); - } else if ("text".equals(currentFieldName) && token == XContentParser.Token.START_ARRAY) { + } else if (parseFieldMatcher.match(currentFieldName, Fields.TEXT) && token == XContentParser.Token.START_ARRAY) { List texts = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token.isValue() == false) { @@ -105,14 +119,14 @@ public class RestAnalyzeAction extends BaseRestHandler { } texts.add(parser.text()); } - analyzeRequest.text(texts.toArray(Strings.EMPTY_ARRAY)); - } else if ("analyzer".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { + analyzeRequest.text(texts.toArray(new String[texts.size()])); + } else if (parseFieldMatcher.match(currentFieldName, Fields.ANALYZER) && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.analyzer(parser.text()); - } else if ("field".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { + } else if (parseFieldMatcher.match(currentFieldName, Fields.FIELD) && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.field(parser.text()); - } else if ("tokenizer".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { + } else if (parseFieldMatcher.match(currentFieldName, Fields.TOKENIZER) && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.tokenizer(parser.text()); - } else if (("token_filters".equals(currentFieldName) || "filters".equals(currentFieldName)) && token == XContentParser.Token.START_ARRAY) { + } else if (parseFieldMatcher.match(currentFieldName, Fields.TOKEN_FILTERS) && token == XContentParser.Token.START_ARRAY) { List filters = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token.isValue() == false) { @@ -120,8 +134,8 @@ public class RestAnalyzeAction extends BaseRestHandler { } filters.add(parser.text()); } - analyzeRequest.tokenFilters(filters.toArray(Strings.EMPTY_ARRAY)); - } else if ("char_filters".equals(currentFieldName) && token == XContentParser.Token.START_ARRAY) { + analyzeRequest.tokenFilters(filters.toArray(new String[filters.size()])); + } else if (parseFieldMatcher.match(currentFieldName, Fields.CHAR_FILTERS) && token == XContentParser.Token.START_ARRAY) { List charFilters = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token.isValue() == false) { @@ -129,7 +143,18 @@ public class RestAnalyzeAction extends BaseRestHandler { } charFilters.add(parser.text()); } - analyzeRequest.tokenFilters(charFilters.toArray(Strings.EMPTY_ARRAY)); + analyzeRequest.charFilters(charFilters.toArray(new String[charFilters.size()])); + } else if (parseFieldMatcher.match(currentFieldName, Fields.EXPLAIN) && token == XContentParser.Token.VALUE_BOOLEAN) { + analyzeRequest.explain(parser.booleanValue()); + } else if (parseFieldMatcher.match(currentFieldName, Fields.ATTRIBUTES) && token == XContentParser.Token.START_ARRAY){ + List attributes = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token.isValue() == false) { + throw new IllegalArgumentException(currentFieldName + " array element should only contain attribute name"); + } + attributes.add(parser.text()); + } + analyzeRequest.attributes(attributes.toArray(new String[attributes.size()])); } else { throw new IllegalArgumentException("Unknown parameter [" + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] "); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/optimize/RestOptimizeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java similarity index 57% rename from core/src/main/java/org/elasticsearch/rest/action/admin/indices/optimize/RestOptimizeAction.java rename to core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java index 3ecafae993a..730276c1a2b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/optimize/RestOptimizeAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java @@ -17,10 +17,10 @@ * under the License. */ -package org.elasticsearch.rest.action.admin.indices.optimize; +package org.elasticsearch.rest.action.admin.indices.forcemerge; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; @@ -38,28 +38,25 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh /** * */ -public class RestOptimizeAction extends BaseRestHandler { +public class RestForceMergeAction extends BaseRestHandler { @Inject - public RestOptimizeAction(Settings settings, RestController controller, Client client) { + public RestForceMergeAction(Settings settings, RestController controller, Client client) { super(settings, controller, client); - controller.registerHandler(POST, "/_optimize", this); - controller.registerHandler(POST, "/{index}/_optimize", this); - - controller.registerHandler(GET, "/_optimize", this); - controller.registerHandler(GET, "/{index}/_optimize", this); + controller.registerHandler(POST, "/_forcemerge", this); + controller.registerHandler(POST, "/{index}/_forcemerge", this); } @Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { - OptimizeRequest optimizeRequest = new OptimizeRequest(Strings.splitStringByCommaToArray(request.param("index"))); - optimizeRequest.indicesOptions(IndicesOptions.fromRequest(request, optimizeRequest.indicesOptions())); - optimizeRequest.maxNumSegments(request.paramAsInt("max_num_segments", optimizeRequest.maxNumSegments())); - optimizeRequest.onlyExpungeDeletes(request.paramAsBoolean("only_expunge_deletes", optimizeRequest.onlyExpungeDeletes())); - optimizeRequest.flush(request.paramAsBoolean("flush", optimizeRequest.flush())); - client.admin().indices().optimize(optimizeRequest, new RestBuilderListener(channel) { + ForceMergeRequest mergeRequest = new ForceMergeRequest(Strings.splitStringByCommaToArray(request.param("index"))); + mergeRequest.indicesOptions(IndicesOptions.fromRequest(request, mergeRequest.indicesOptions())); + mergeRequest.maxNumSegments(request.paramAsInt("max_num_segments", mergeRequest.maxNumSegments())); + mergeRequest.onlyExpungeDeletes(request.paramAsBoolean("only_expunge_deletes", mergeRequest.onlyExpungeDeletes())); + mergeRequest.flush(request.paramAsBoolean("flush", mergeRequest.flush())); + client.admin().indices().forceMerge(mergeRequest, new RestBuilderListener(channel) { @Override - public RestResponse buildResponse(OptimizeResponse response, XContentBuilder builder) throws Exception { + public RestResponse buildResponse(ForceMergeResponse response, XContentBuilder builder) throws Exception { builder.startObject(); buildBroadcastShardsHeader(builder, request, response); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java index 67661967d56..0a4592f387f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java @@ -23,16 +23,21 @@ import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import java.io.IOException; + import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestStatus.OK; @@ -43,8 +48,10 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh */ public class RestValidateQueryAction extends BaseRestHandler { + private final IndicesQueriesRegistry indicesQueriesRegistry; + @Inject - public RestValidateQueryAction(Settings settings, RestController controller, Client client) { + public RestValidateQueryAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) { super(settings, controller, client); controller.registerHandler(GET, "/_validate/query", this); controller.registerHandler(POST, "/_validate/query", this); @@ -52,53 +59,52 @@ public class RestValidateQueryAction extends BaseRestHandler { controller.registerHandler(POST, "/{index}/_validate/query", this); controller.registerHandler(GET, "/{index}/{type}/_validate/query", this); controller.registerHandler(POST, "/{index}/{type}/_validate/query", this); + this.indicesQueriesRegistry = indicesQueriesRegistry; } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception { ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest(Strings.splitStringByCommaToArray(request.param("index"))); validateQueryRequest.indicesOptions(IndicesOptions.fromRequest(request, validateQueryRequest.indicesOptions())); + validateQueryRequest.explain(request.paramAsBoolean("explain", false)); if (RestActions.hasBodyContent(request)) { - validateQueryRequest.source(RestActions.getRestContent(request)); + try { + validateQueryRequest.query(RestActions.getQueryContent(RestActions.getRestContent(request), indicesQueriesRegistry, parseFieldMatcher)); + } catch(ParsingException e) { + channel.sendResponse(buildErrorResponse(channel.newBuilder(), e.getDetailedMessage(), validateQueryRequest.explain())); + return; + } catch(Exception e) { + channel.sendResponse(buildErrorResponse(channel.newBuilder(), e.getMessage(), validateQueryRequest.explain())); + return; + } } else { - QuerySourceBuilder querySourceBuilder = RestActions.parseQuerySource(request); - if (querySourceBuilder != null) { - validateQueryRequest.source(querySourceBuilder); + QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request); + if (queryBuilder != null) { + validateQueryRequest.query(queryBuilder); } } validateQueryRequest.types(Strings.splitStringByCommaToArray(request.param("type"))); - if (request.paramAsBoolean("explain", false)) { - validateQueryRequest.explain(true); - } else { - validateQueryRequest.explain(false); - } - if (request.paramAsBoolean("rewrite", false)) { - validateQueryRequest.rewrite(true); - } else { - validateQueryRequest.rewrite(false); - } + validateQueryRequest.rewrite(request.paramAsBoolean("rewrite", false)); client.admin().indices().validateQuery(validateQueryRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(ValidateQueryResponse response, XContentBuilder builder) throws Exception { builder.startObject(); - builder.field("valid", response.isValid()); - + builder.field(VALID_FIELD, response.isValid()); buildBroadcastShardsHeader(builder, request, response); - if (response.getQueryExplanation() != null && !response.getQueryExplanation().isEmpty()) { - builder.startArray("explanations"); + builder.startArray(EXPLANATIONS_FIELD); for (QueryExplanation explanation : response.getQueryExplanation()) { builder.startObject(); if (explanation.getIndex() != null) { - builder.field("index", explanation.getIndex(), XContentBuilder.FieldCaseConversion.NONE); + builder.field(INDEX_FIELD, explanation.getIndex(), XContentBuilder.FieldCaseConversion.NONE); } - builder.field("valid", explanation.isValid()); + builder.field(VALID_FIELD, explanation.isValid()); if (explanation.getError() != null) { - builder.field("error", explanation.getError()); + builder.field(ERROR_FIELD, explanation.getError()); } if (explanation.getExplanation() != null) { - builder.field("explanation", explanation.getExplanation()); + builder.field(EXPLANATION_FIELD, explanation.getExplanation()); } builder.endObject(); } @@ -109,4 +115,20 @@ public class RestValidateQueryAction extends BaseRestHandler { } }); } + + private static BytesRestResponse buildErrorResponse(XContentBuilder builder, String error, boolean explain) throws IOException { + builder.startObject(); + builder.field(VALID_FIELD, false); + if (explain) { + builder.field(ERROR_FIELD, error); + } + builder.endObject(); + return new BytesRestResponse(OK, builder); + } + + private static final XContentBuilderString INDEX_FIELD = new XContentBuilderString("index"); + private static final XContentBuilderString VALID_FIELD = new XContentBuilderString("valid"); + private static final XContentBuilderString EXPLANATIONS_FIELD = new XContentBuilderString("explanations"); + private static final XContentBuilderString ERROR_FIELD = new XContentBuilderString("error"); + private static final XContentBuilderString EXPLANATION_FIELD = new XContentBuilderString("explanation"); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java index a25754d8752..5ebec7130df 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java @@ -41,7 +41,6 @@ import org.elasticsearch.rest.action.support.RestBuilderListener; import org.elasticsearch.script.Script.ScriptField; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.Template; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import java.util.Map; @@ -89,7 +88,7 @@ public class RestRenderSearchTemplateAction extends BaseRestHandler { throw new ElasticsearchParseException("failed to parse request. unknown field [{}] of type [{}]", currentFieldName, token); } } - template = new Template(templateId, ScriptType.INDEXED, MustacheScriptEngineService.NAME, null, params); + template = new Template(templateId, ScriptType.INDEXED, Template.DEFAULT_LANG, null, params); } renderSearchTemplateRequest = new RenderSearchTemplateRequest(); renderSearchTemplateRequest.template(template); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java index 4c421ccc566..b47c2542abf 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java @@ -19,17 +19,24 @@ package org.elasticsearch.rest.action.admin.indices.warmer.put; import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequest; -import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.rest.*; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.support.AcknowledgedRestListener; +import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.search.builder.SearchSourceBuilder; +import java.io.IOException; + import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; @@ -37,9 +44,12 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; */ public class RestPutWarmerAction extends BaseRestHandler { + private final IndicesQueriesRegistry queryRegistry; + @Inject - public RestPutWarmerAction(Settings settings, RestController controller, Client client) { + public RestPutWarmerAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry) { super(settings, controller, client); + this.queryRegistry = queryRegistry; controller.registerHandler(PUT, "/_warmer/{name}", this); controller.registerHandler(PUT, "/{index}/_warmer/{name}", this); controller.registerHandler(PUT, "/{index}/{type}/_warmer/{name}", this); @@ -58,12 +68,14 @@ public class RestPutWarmerAction extends BaseRestHandler { } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { PutWarmerRequest putWarmerRequest = new PutWarmerRequest(request.param("name")); + + BytesReference sourceBytes = RestActions.getRestContent(request); + SearchSourceBuilder source = RestActions.getRestSearchSource(sourceBytes, queryRegistry, parseFieldMatcher); SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index"))) .types(Strings.splitStringByCommaToArray(request.param("type"))) - .requestCache(request.paramAsBoolean("request_cache", null)) - .source(request.content()); + .requestCache(request.paramAsBoolean("request_cache", null)).source(source); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); putWarmerRequest.searchRequest(searchRequest); putWarmerRequest.timeout(request.paramAsTime("timeout", putWarmerRequest.timeout())); diff --git a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java index 90184352714..536b73ba2b5 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java @@ -19,16 +19,11 @@ package org.elasticsearch.rest.action.bulk; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionWriteResponse; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.BulkShardRequest; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Strings; @@ -96,52 +91,7 @@ public class RestBulkAction extends BaseRestHandler { builder.startArray(Fields.ITEMS); for (BulkItemResponse itemResponse : response) { builder.startObject(); - builder.startObject(itemResponse.getOpType()); - builder.field(Fields._INDEX, itemResponse.getIndex()); - builder.field(Fields._TYPE, itemResponse.getType()); - builder.field(Fields._ID, itemResponse.getId()); - long version = itemResponse.getVersion(); - if (version != -1) { - builder.field(Fields._VERSION, itemResponse.getVersion()); - } - if (itemResponse.isFailed()) { - builder.field(Fields.STATUS, itemResponse.getFailure().getStatus().getStatus()); - builder.startObject(Fields.ERROR); - ElasticsearchException.toXContent(builder, request, itemResponse.getFailure().getCause()); - builder.endObject(); - } else { - ActionWriteResponse.ShardInfo shardInfo = itemResponse.getResponse().getShardInfo(); - shardInfo.toXContent(builder, request); - if (itemResponse.getResponse() instanceof DeleteResponse) { - DeleteResponse deleteResponse = itemResponse.getResponse(); - if (deleteResponse.isFound()) { - builder.field(Fields.STATUS, shardInfo.status().getStatus()); - } else { - builder.field(Fields.STATUS, RestStatus.NOT_FOUND.getStatus()); - } - builder.field(Fields.FOUND, deleteResponse.isFound()); - } else if (itemResponse.getResponse() instanceof IndexResponse) { - IndexResponse indexResponse = itemResponse.getResponse(); - if (indexResponse.isCreated()) { - builder.field(Fields.STATUS, RestStatus.CREATED.getStatus()); - } else { - builder.field(Fields.STATUS, shardInfo.status().getStatus()); - } - } else if (itemResponse.getResponse() instanceof UpdateResponse) { - UpdateResponse updateResponse = itemResponse.getResponse(); - if (updateResponse.isCreated()) { - builder.field(Fields.STATUS, RestStatus.CREATED.getStatus()); - } else { - builder.field(Fields.STATUS, shardInfo.status().getStatus()); - } - if (updateResponse.getGetResult() != null) { - builder.startObject(Fields.GET); - updateResponse.getGetResult().toXContentEmbedded(builder, request); - builder.endObject(); - } - } - } - builder.endObject(); + itemResponse.toXContent(builder, request); builder.endObject(); } builder.endArray(); @@ -155,15 +105,7 @@ public class RestBulkAction extends BaseRestHandler { static final class Fields { static final XContentBuilderString ITEMS = new XContentBuilderString("items"); static final XContentBuilderString ERRORS = new XContentBuilderString("errors"); - static final XContentBuilderString _INDEX = new XContentBuilderString("_index"); - static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); - static final XContentBuilderString _ID = new XContentBuilderString("_id"); - static final XContentBuilderString STATUS = new XContentBuilderString("status"); - static final XContentBuilderString ERROR = new XContentBuilderString("error"); static final XContentBuilderString TOOK = new XContentBuilderString("took"); - static final XContentBuilderString _VERSION = new XContentBuilderString("_version"); - static final XContentBuilderString FOUND = new XContentBuilderString("found"); - static final XContentBuilderString GET = new XContentBuilderString("get"); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java index 72057a9cf51..e4d291b5fc8 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java @@ -19,18 +19,24 @@ package org.elasticsearch.rest.action.cat; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.support.QuerySourceBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.rest.*; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestResponseListener; import org.elasticsearch.rest.action.support.RestTable; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -40,11 +46,14 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestCountAction extends AbstractCatAction { + private final IndicesQueriesRegistry indicesQueriesRegistry; + @Inject - public RestCountAction(Settings settings, RestController restController, RestController controller, Client client) { + public RestCountAction(Settings settings, RestController restController, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) { super(settings, controller, client); restController.registerHandler(GET, "/_cat/count", this); restController.registerHandler(GET, "/_cat/count/{index}", this); + this.indicesQueriesRegistry = indicesQueriesRegistry; } @Override @@ -56,20 +65,21 @@ public class RestCountAction extends AbstractCatAction { @Override public void doRequest(final RestRequest request, final RestChannel channel, final Client client) { String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - CountRequest countRequest = new CountRequest(indices); + SearchRequest countRequest = new SearchRequest(indices); String source = request.param("source"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0); + countRequest.source(searchSourceBuilder); if (source != null) { - countRequest.source(source); + searchSourceBuilder.query(RestActions.getQueryContent(new BytesArray(source), indicesQueriesRegistry, parseFieldMatcher)); } else { - QuerySourceBuilder querySourceBuilder = RestActions.parseQuerySource(request); - if (querySourceBuilder != null) { - countRequest.source(querySourceBuilder); + QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request); + if (queryBuilder != null) { + searchSourceBuilder.query(queryBuilder); } } - - client.count(countRequest, new RestResponseListener(channel) { + client.search(countRequest, new RestResponseListener(channel) { @Override - public RestResponse buildResponse(CountResponse countResponse) throws Exception { + public RestResponse buildResponse(SearchResponse countResponse) throws Exception { return RestTable.buildResponse(buildTable(request, countResponse), channel); } }); @@ -88,13 +98,13 @@ public class RestCountAction extends AbstractCatAction { private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("HH:mm:ss"); - private Table buildTable(RestRequest request, CountResponse response) { + private Table buildTable(RestRequest request, SearchResponse response) { Table table = getTableWithHeader(request); long time = System.currentTimeMillis(); table.startRow(); table.addCell(TimeUnit.SECONDS.convert(time, TimeUnit.MILLISECONDS)); table.addCell(dateFormat.print(time)); - table.addCell(response.getCount()); + table.addCell(response.getHits().totalHits()); table.endRow(); return table; diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index 09edd81f323..4dd8033c0bd 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterIndexHealth; +import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; @@ -79,7 +79,8 @@ public class RestIndicesAction extends AbstractCatAction { @Override public void processResponse(final ClusterStateResponse clusterStateResponse) { ClusterState state = clusterStateResponse.getState(); - final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, IndicesOptions.fromOptions(false, true, true, true), indices); + final IndicesOptions concreteIndicesOptions = IndicesOptions.fromOptions(false, true, true, true); + final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, concreteIndicesOptions, indices); final String[] openIndices = indexNameExpressionResolver.concreteIndices(state, IndicesOptions.lenientExpandOpen(), indices); ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(openIndices); clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local())); @@ -87,6 +88,8 @@ public class RestIndicesAction extends AbstractCatAction { @Override public void processResponse(final ClusterHealthResponse clusterHealthResponse) { IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); + indicesStatsRequest.indices(concreteIndices); + indicesStatsRequest.indicesOptions(concreteIndicesOptions); indicesStatsRequest.all(); client.admin().indices().stats(indicesStatsRequest, new RestResponseListener(channel) { @Override @@ -328,8 +331,8 @@ public class RestIndicesAction extends AbstractCatAction { table.addCell(indexStats == null ? null : indexStats.getPrimaries().getDocs().getCount()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getDocs().getDeleted()); - table.addCell(indexMetaData.creationDate()); - table.addCell(new DateTime(indexMetaData.creationDate(), DateTimeZone.UTC)); + table.addCell(indexMetaData.getCreationDate()); + table.addCell(new DateTime(indexMetaData.getCreationDate(), DateTimeZone.UTC)); table.addCell(indexStats == null ? null : indexStats.getTotal().getStore().size()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getStore().size()); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 337dd41b403..208b7e6c374 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -130,6 +130,7 @@ public class RestNodesAction extends AbstractCatAction { table.addCell("file_desc.percent", "default:false;alias:fdp,fileDescriptorPercent;text-align:right;desc:used file descriptor ratio"); table.addCell("file_desc.max", "default:false;alias:fdm,fileDescriptorMax;text-align:right;desc:max file descriptors"); + table.addCell("cpu", "alias:cpu;text-align:right;desc:recent cpu usage"); table.addCell("load", "alias:l;text-align:right;desc:most recent load avg"); table.addCell("uptime", "default:false;alias:u;text-align:right;desc:node uptime"); table.addCell("node.role", "alias:r,role,dc,nodeRole;desc:d:data node, c:client node"); @@ -245,7 +246,7 @@ public class RestNodesAction extends AbstractCatAction { } table.addCell(node.getVersion().number()); - table.addCell(info == null ? null : info.getBuild().hashShort()); + table.addCell(info == null ? null : info.getBuild().shortHash()); table.addCell(jvmInfo == null ? null : jvmInfo.version()); table.addCell(fsInfo == null ? null : fsInfo.getTotal().getAvailable()); table.addCell(jvmStats == null ? null : jvmStats.getMem().getHeapUsed()); @@ -258,7 +259,8 @@ public class RestNodesAction extends AbstractCatAction { table.addCell(processStats == null ? null : calculatePercentage(processStats.getOpenFileDescriptors(), processStats.getMaxFileDescriptors())); table.addCell(processStats == null ? null : processStats.getMaxFileDescriptors()); - table.addCell(osStats == null ? null : String.format(Locale.ROOT, "%.2f", osStats.getLoadAverage())); + table.addCell(osStats == null ? null : Short.toString(osStats.getCpu().getPercent())); + table.addCell(osStats == null ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage())); table.addCell(jvmStats == null ? null : jvmStats.getUptime()); table.addCell(node.clientNode() ? "c" : node.dataNode() ? "d" : "-"); table.addCell(masterId == null ? "x" : masterId.equals(node.id()) ? "*" : node.masterNode() ? "m" : "-"); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java index 058a93bf6b9..b52f8e6fc10 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java @@ -95,7 +95,7 @@ public class RestPluginsAction extends AbstractCatAction { for (DiscoveryNode node : nodes) { NodeInfo info = nodesInfo.getNodesMap().get(node.id()); - for (PluginInfo pluginInfo : info.getPlugins().getInfos()) { + for (PluginInfo pluginInfo : info.getPlugins().getPluginInfos()) { table.startRow(); table.addCell(node.id()); table.addCell(node.name()); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java new file mode 100644 index 00000000000..d19e2b19ca0 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.cat; + +import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.common.Table; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.action.support.RestResponseListener; +import org.elasticsearch.rest.action.support.RestTable; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +/** + * Cat API class to display information about snapshot repositories + */ +public class RestRepositoriesAction extends AbstractCatAction { + @Inject + public RestRepositoriesAction(Settings settings, RestController controller, Client client) { + super(settings, controller, client); + controller.registerHandler(GET, "/_cat/repositories", this); + } + + @Override + protected void doRequest(RestRequest request, RestChannel channel, Client client) { + GetRepositoriesRequest getRepositoriesRequest = new GetRepositoriesRequest(); + getRepositoriesRequest.local(request.paramAsBoolean("local", getRepositoriesRequest.local())); + getRepositoriesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getRepositoriesRequest.masterNodeTimeout())); + + client.admin().cluster().getRepositories(getRepositoriesRequest, new RestResponseListener(channel) { + @Override + public RestResponse buildResponse(GetRepositoriesResponse getRepositoriesResponse) throws Exception { + return RestTable.buildResponse(buildTable(request, getRepositoriesResponse), channel); + } + }); + } + + @Override + protected void documentation(StringBuilder sb) { + sb.append("/_cat/repositories\n"); + } + + @Override + protected Table getTableWithHeader(RestRequest request) { + return new Table() + .startHeaders() + .addCell("id", "alias:id,repoId;desc:unique repository id") + .addCell("type", "alias:t,type;text-align:right;desc:repository type") + .endHeaders(); + } + + private Table buildTable(RestRequest req, GetRepositoriesResponse getRepositoriesResponse) { + Table table = getTableWithHeader(req); + for (RepositoryMetaData repositoryMetaData : getRepositoriesResponse.repositories()) { + table.startRow(); + + table.addCell(repositoryMetaData.name()); + table.addCell(repositoryMetaData.type()); + + table.endRow(); + } + + return table; + } +} diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java index 5ec23c9b287..6574a01ae28 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.ShardRouting; @@ -33,6 +34,8 @@ import org.elasticsearch.common.Table; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.engine.CommitStats; +import org.elasticsearch.index.engine.Engine; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestActionListener; import org.elasticsearch.rest.action.support.RestResponseListener; @@ -91,6 +94,8 @@ public class RestShardsAction extends AbstractCatAction { .addCell("id", "default:false;desc:unique id of node where it lives") .addCell("node", "default:true;alias:n;desc:name of node where it lives"); + table.addCell("sync_id", "alias:sync_id;default:false;desc:sync id"); + table.addCell("unassigned.reason", "alias:ur;default:false;desc:reason shard is unassigned"); table.addCell("unassigned.at", "alias:ua;default:false;desc:time shard became unassigned (UTC)"); table.addCell("unassigned.for", "alias:uf;default:false;text-align:right;desc:time has been unassigned"); @@ -170,7 +175,13 @@ public class RestShardsAction extends AbstractCatAction { Table table = getTableWithHeader(request); for (ShardRouting shard : state.getState().routingTable().allShards()) { - CommonStats shardStats = stats.asMap().get(shard); + ShardStats shardStats = stats.asMap().get(shard); + CommonStats commonStats = null; + CommitStats commitStats = null; + if (shardStats != null) { + commonStats = shardStats.getStats(); + commitStats = shardStats.getCommitStats(); + } table.startRow(); @@ -180,7 +191,7 @@ public class RestShardsAction extends AbstractCatAction { IndexMetaData indexMeta = state.getState().getMetaData().index(shard.index()); boolean usesShadowReplicas = false; if (indexMeta != null) { - usesShadowReplicas = IndexMetaData.isIndexUsingShadowReplicas(indexMeta.settings()); + usesShadowReplicas = IndexMetaData.isIndexUsingShadowReplicas(indexMeta.getSettings()); } if (shard.primary()) { table.addCell("p"); @@ -192,8 +203,8 @@ public class RestShardsAction extends AbstractCatAction { } } table.addCell(shard.state()); - table.addCell(shardStats == null ? null : shardStats.getDocs().getCount()); - table.addCell(shardStats == null ? null : shardStats.getStore().getSize()); + table.addCell(commonStats == null ? null : commonStats.getDocs().getCount()); + table.addCell(commonStats == null ? null : commonStats.getStore().getSize()); if (shard.assignedToNode()) { String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress(); String nodeId = shard.currentNodeId(); @@ -219,10 +230,12 @@ public class RestShardsAction extends AbstractCatAction { table.addCell(null); } + table.addCell(commitStats == null ? null : commitStats.getUserData().get(Engine.SYNC_COMMIT_ID)); + if (shard.unassignedInfo() != null) { table.addCell(shard.unassignedInfo().getReason()); - table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.printer().print(shard.unassignedInfo().getTimestampInMillis())); - table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getTimestampInMillis())); + table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.printer().print(shard.unassignedInfo().getUnassignedTimeInMillis())); + table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis())); table.addCell(shard.unassignedInfo().getDetails()); } else { table.addCell(null); @@ -231,71 +244,71 @@ public class RestShardsAction extends AbstractCatAction { table.addCell(null); } - table.addCell(shardStats == null ? null : shardStats.getCompletion().getSize()); + table.addCell(commonStats == null ? null : commonStats.getCompletion().getSize()); - table.addCell(shardStats == null ? null : shardStats.getFieldData().getMemorySize()); - table.addCell(shardStats == null ? null : shardStats.getFieldData().getEvictions()); + table.addCell(commonStats == null ? null : commonStats.getFieldData().getMemorySize()); + table.addCell(commonStats == null ? null : commonStats.getFieldData().getEvictions()); - table.addCell(shardStats == null ? null : shardStats.getQueryCache().getMemorySize()); - table.addCell(shardStats == null ? null : shardStats.getQueryCache().getEvictions()); + table.addCell(commonStats == null ? null : commonStats.getQueryCache().getMemorySize()); + table.addCell(commonStats == null ? null : commonStats.getQueryCache().getEvictions()); - table.addCell(shardStats == null ? null : shardStats.getFlush().getTotal()); - table.addCell(shardStats == null ? null : shardStats.getFlush().getTotalTime()); + table.addCell(commonStats == null ? null : commonStats.getFlush().getTotal()); + table.addCell(commonStats == null ? null : commonStats.getFlush().getTotalTime()); - table.addCell(shardStats == null ? null : shardStats.getGet().current()); - table.addCell(shardStats == null ? null : shardStats.getGet().getTime()); - table.addCell(shardStats == null ? null : shardStats.getGet().getCount()); - table.addCell(shardStats == null ? null : shardStats.getGet().getExistsTime()); - table.addCell(shardStats == null ? null : shardStats.getGet().getExistsCount()); - table.addCell(shardStats == null ? null : shardStats.getGet().getMissingTime()); - table.addCell(shardStats == null ? null : shardStats.getGet().getMissingCount()); + table.addCell(commonStats == null ? null : commonStats.getGet().current()); + table.addCell(commonStats == null ? null : commonStats.getGet().getTime()); + table.addCell(commonStats == null ? null : commonStats.getGet().getCount()); + table.addCell(commonStats == null ? null : commonStats.getGet().getExistsTime()); + table.addCell(commonStats == null ? null : commonStats.getGet().getExistsCount()); + table.addCell(commonStats == null ? null : commonStats.getGet().getMissingTime()); + table.addCell(commonStats == null ? null : commonStats.getGet().getMissingCount()); - table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexTime()); - table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexCount()); - table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(commonStats == null ? null : commonStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(commonStats == null ? null : commonStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(commonStats == null ? null : commonStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(commonStats == null ? null : commonStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(commonStats == null ? null : commonStats.getIndexing().getTotal().getIndexTime()); + table.addCell(commonStats == null ? null : commonStats.getIndexing().getTotal().getIndexCount()); + table.addCell(commonStats == null ? null : commonStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(shardStats == null ? null : shardStats.getMerge().getCurrent()); - table.addCell(shardStats == null ? null : shardStats.getMerge().getCurrentNumDocs()); - table.addCell(shardStats == null ? null : shardStats.getMerge().getCurrentSize()); - table.addCell(shardStats == null ? null : shardStats.getMerge().getTotal()); - table.addCell(shardStats == null ? null : shardStats.getMerge().getTotalNumDocs()); - table.addCell(shardStats == null ? null : shardStats.getMerge().getTotalSize()); - table.addCell(shardStats == null ? null : shardStats.getMerge().getTotalTime()); + table.addCell(commonStats == null ? null : commonStats.getMerge().getCurrent()); + table.addCell(commonStats == null ? null : commonStats.getMerge().getCurrentNumDocs()); + table.addCell(commonStats == null ? null : commonStats.getMerge().getCurrentSize()); + table.addCell(commonStats == null ? null : commonStats.getMerge().getTotal()); + table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalNumDocs()); + table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalSize()); + table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalTime()); - table.addCell(shardStats == null ? null : shardStats.getPercolate().getCurrent()); - table.addCell(shardStats == null ? null : shardStats.getPercolate().getMemorySize()); - table.addCell(shardStats == null ? null : shardStats.getPercolate().getNumQueries()); - table.addCell(shardStats == null ? null : shardStats.getPercolate().getTime()); - table.addCell(shardStats == null ? null : shardStats.getPercolate().getCount()); + table.addCell(commonStats == null ? null : commonStats.getPercolate().getCurrent()); + table.addCell(commonStats == null ? null : commonStats.getPercolate().getMemorySize()); + table.addCell(commonStats == null ? null : commonStats.getPercolate().getNumQueries()); + table.addCell(commonStats == null ? null : commonStats.getPercolate().getTime()); + table.addCell(commonStats == null ? null : commonStats.getPercolate().getCount()); - table.addCell(shardStats == null ? null : shardStats.getRefresh().getTotal()); - table.addCell(shardStats == null ? null : shardStats.getRefresh().getTotalTime()); + table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotal()); + table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotalTime()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getFetchTime()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getFetchCount()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getOpenContexts()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getQueryTime()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getQueryCount()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getScrollTime()); - table.addCell(shardStats == null ? null : shardStats.getSearch().getTotal().getScrollCount()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getFetchTime()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getFetchCount()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getOpenContexts()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getQueryTime()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getQueryCount()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getScrollTime()); + table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getScrollCount()); - table.addCell(shardStats == null ? null : shardStats.getSegments().getCount()); - table.addCell(shardStats == null ? null : shardStats.getSegments().getMemory()); - table.addCell(shardStats == null ? null : shardStats.getSegments().getIndexWriterMemory()); - table.addCell(shardStats == null ? null : shardStats.getSegments().getIndexWriterMaxMemory()); - table.addCell(shardStats == null ? null : shardStats.getSegments().getVersionMapMemory()); - table.addCell(shardStats == null ? null : shardStats.getSegments().getBitsetMemory()); + table.addCell(commonStats == null ? null : commonStats.getSegments().getCount()); + table.addCell(commonStats == null ? null : commonStats.getSegments().getMemory()); + table.addCell(commonStats == null ? null : commonStats.getSegments().getIndexWriterMemory()); + table.addCell(commonStats == null ? null : commonStats.getSegments().getIndexWriterMaxMemory()); + table.addCell(commonStats == null ? null : commonStats.getSegments().getVersionMapMemory()); + table.addCell(commonStats == null ? null : commonStats.getSegments().getBitsetMemory()); - table.addCell(shardStats == null ? null : shardStats.getWarmer().current()); - table.addCell(shardStats == null ? null : shardStats.getWarmer().total()); - table.addCell(shardStats == null ? null : shardStats.getWarmer().totalTime()); + table.addCell(commonStats == null ? null : commonStats.getWarmer().current()); + table.addCell(commonStats == null ? null : commonStats.getWarmer().total()); + table.addCell(commonStats == null ? null : commonStats.getWarmer().totalTime()); table.endRow(); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java new file mode 100644 index 00000000000..0d98dd20278 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.cat; + + +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Table; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.action.support.RestResponseListener; +import org.elasticsearch.rest.action.support.RestTable; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotState; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +/** + * Cat API class to display information about snapshots + */ +public class RestSnapshotAction extends AbstractCatAction { + @Inject + public RestSnapshotAction(Settings settings, RestController controller, Client client) { + super(settings, controller, client); + controller.registerHandler(GET, "/_cat/snapshots/{repository}", this); + } + + @Override + protected void doRequest(final RestRequest request, RestChannel channel, Client client) { + GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest() + .repository(request.param("repository")) + .snapshots(new String[]{GetSnapshotsRequest.ALL_SNAPSHOTS}); + + getSnapshotsRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", getSnapshotsRequest.ignoreUnavailable())); + + getSnapshotsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSnapshotsRequest.masterNodeTimeout())); + + client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestResponseListener(channel) { + @Override + public RestResponse buildResponse(GetSnapshotsResponse getSnapshotsResponse) throws Exception { + return RestTable.buildResponse(buildTable(request, getSnapshotsResponse), channel); + } + }); + } + + @Override + protected void documentation(StringBuilder sb) { + sb.append("/_cat/snapshots/{repository}\n"); + } + + @Override + protected Table getTableWithHeader(RestRequest request) { + return new Table() + .startHeaders() + .addCell("id", "alias:id,snapshotId;desc:unique snapshot id") + .addCell("status", "alias:s,status;text-align:right;desc:snapshot name") + .addCell("start_epoch", "alias:ste,startEpoch;desc:start time in seconds since 1970-01-01 00:00:00") + .addCell("start_time", "alias:sti,startTime;desc:start time in HH:MM:SS") + .addCell("end_epoch", "alias:ete,endEpoch;desc:end time in seconds since 1970-01-01 00:00:00") + .addCell("end_time", "alias:eti,endTime;desc:end time in HH:MM:SS") + .addCell("duration", "alias:dur,duration;text-align:right;desc:duration") + .addCell("indices", "alias:i,indices;text-align:right;desc:number of indices") + .addCell("successful_shards", "alias:ss,successful_shards;text-align:right;desc:number of successful shards") + .addCell("failed_shards", "alias:fs,failed_shards;text-align:right;desc:number of failed shards") + .addCell("total_shards", "alias:ts,total_shards;text-align:right;desc:number of total shards") + .addCell("reason", "default:false;alias:r,reason;desc:reason for failures") + .endHeaders(); + } + + private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("HH:mm:ss"); + + private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsResponse) { + Table table = getTableWithHeader(req); + for (SnapshotInfo snapshotStatus : getSnapshotsResponse.getSnapshots()) { + table.startRow(); + + table.addCell(snapshotStatus.name()); + table.addCell(snapshotStatus.state()); + table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS)); + table.addCell(dateFormat.print(snapshotStatus.startTime())); + table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS)); + table.addCell(dateFormat.print(snapshotStatus.endTime())); + final long durationMillis; + if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) { + durationMillis = System.currentTimeMillis() - snapshotStatus.startTime(); + } else { + durationMillis = snapshotStatus.endTime() - snapshotStatus.startTime(); + } + table.addCell(TimeValue.timeValueMillis(durationMillis)); + table.addCell(snapshotStatus.indices().size()); + table.addCell(snapshotStatus.successfulShards()); + table.addCell(snapshotStatus.failedShards()); + table.addCell(snapshotStatus.totalShards()); + table.addCell(snapshotStatus.reason()); + + table.endRow(); + } + + return table; + } +} diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java index f1684d05b34..fa2e662c738 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java @@ -62,7 +62,7 @@ public class RestThreadPoolAction extends AbstractCatAction { ThreadPool.Names.GET, ThreadPool.Names.INDEX, ThreadPool.Names.MANAGEMENT, - ThreadPool.Names.OPTIMIZE, + ThreadPool.Names.FORCE_MERGE, ThreadPool.Names.PERCOLATE, ThreadPool.Names.REFRESH, ThreadPool.Names.SEARCH, @@ -78,7 +78,7 @@ public class RestThreadPoolAction extends AbstractCatAction { "g", "i", "ma", - "o", + "fm", "p", "r", "s", @@ -288,7 +288,7 @@ public class RestThreadPoolAction extends AbstractCatAction { } } - table.addCell(poolInfo == null ? null : poolInfo.getType()); + table.addCell(poolInfo == null ? null : poolInfo.getThreadPoolType().getType()); table.addCell(poolStats == null ? null : poolStats.getActive()); table.addCell(poolStats == null ? null : poolStats.getThreads()); table.addCell(poolStats == null ? null : poolStats.getQueue()); diff --git a/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java index 677f3af4508..1ce78e33e3f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java @@ -19,32 +19,36 @@ package org.elasticsearch.rest.action.count; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import org.elasticsearch.search.builder.SearchSourceBuilder; -import static org.elasticsearch.action.count.CountRequest.DEFAULT_MIN_SCORE; -import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastShardsHeader; +import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER; /** * */ public class RestCountAction extends BaseRestHandler { + private final IndicesQueriesRegistry indicesQueriesRegistry; + @Inject - public RestCountAction(Settings settings, RestController controller, Client client) { + public RestCountAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) { super(settings, controller, client); controller.registerHandler(POST, "/_count", this); controller.registerHandler(GET, "/_count", this); @@ -52,22 +56,29 @@ public class RestCountAction extends BaseRestHandler { controller.registerHandler(GET, "/{index}/_count", this); controller.registerHandler(POST, "/{index}/{type}/_count", this); controller.registerHandler(GET, "/{index}/{type}/_count", this); + this.indicesQueriesRegistry = indicesQueriesRegistry; } @Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { - CountRequest countRequest = new CountRequest(Strings.splitStringByCommaToArray(request.param("index"))); + SearchRequest countRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index"))); countRequest.indicesOptions(IndicesOptions.fromRequest(request, countRequest.indicesOptions())); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0); + countRequest.source(searchSourceBuilder); if (RestActions.hasBodyContent(request)) { - countRequest.source(RestActions.getRestContent(request)); + BytesReference restContent = RestActions.getRestContent(request); + searchSourceBuilder.query(RestActions.getQueryContent(restContent, indicesQueriesRegistry, parseFieldMatcher)); } else { - QuerySourceBuilder querySourceBuilder = RestActions.parseQuerySource(request); - if (querySourceBuilder != null) { - countRequest.source(querySourceBuilder); + QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request); + if (queryBuilder != null) { + searchSourceBuilder.query(queryBuilder); } } countRequest.routing(request.param("routing")); - countRequest.minScore(request.paramAsFloat("min_score", DEFAULT_MIN_SCORE)); + float minScore = request.paramAsFloat("min_score", -1f); + if (minScore != -1f) { + searchSourceBuilder.minScore(minScore); + } countRequest.types(Strings.splitStringByCommaToArray(request.param("type"))); countRequest.preference(request.param("preference")); @@ -75,17 +86,18 @@ public class RestCountAction extends BaseRestHandler { if (terminateAfter < 0) { throw new IllegalArgumentException("terminateAfter must be > 0"); } else if (terminateAfter > 0) { - countRequest.terminateAfter(terminateAfter); + searchSourceBuilder.terminateAfter(terminateAfter); } - client.count(countRequest, new RestBuilderListener(channel) { + client.search(countRequest, new RestBuilderListener(channel) { @Override - public RestResponse buildResponse(CountResponse response, XContentBuilder builder) throws Exception { + public RestResponse buildResponse(SearchResponse response, XContentBuilder builder) throws Exception { builder.startObject(); if (terminateAfter != DEFAULT_TERMINATE_AFTER) { - builder.field("terminated_early", response.terminatedEarly()); + builder.field("terminated_early", response.isTerminatedEarly()); } - builder.field("count", response.getCount()); - buildBroadcastShardsHeader(builder, request, response); + builder.field("count", response.getHits().totalHits()); + buildBroadcastShardsHeader(builder, request, response.getTotalShards(), response.getSuccessfulShards(), + response.getFailedShards(), response.getShardFailures()); builder.endObject(); return new BytesRestResponse(response.status(), builder); diff --git a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java index 209ab686ce5..e583ed36274 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.rest.action.delete; -import org.elasticsearch.action.ActionWriteResponse; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; @@ -27,14 +26,13 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import org.elasticsearch.rest.action.support.RestStatusToXContentListener; import static org.elasticsearch.rest.RestRequest.Method.DELETE; -import static org.elasticsearch.rest.RestStatus.NOT_FOUND; /** * @@ -62,31 +60,6 @@ public class RestDeleteAction extends BaseRestHandler { deleteRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel)); } - client.delete(deleteRequest, new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(DeleteResponse result, XContentBuilder builder) throws Exception { - ActionWriteResponse.ShardInfo shardInfo = result.getShardInfo(); - builder.startObject().field(Fields.FOUND, result.isFound()) - .field(Fields._INDEX, result.getIndex()) - .field(Fields._TYPE, result.getType()) - .field(Fields._ID, result.getId()) - .field(Fields._VERSION, result.getVersion()) - .value(shardInfo) - .endObject(); - RestStatus status = shardInfo.status(); - if (!result.isFound()) { - status = NOT_FOUND; - } - return new BytesRestResponse(status, builder); - } - }); - } - - static final class Fields { - static final XContentBuilderString FOUND = new XContentBuilderString("found"); - static final XContentBuilderString _INDEX = new XContentBuilderString("_index"); - static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); - static final XContentBuilderString _ID = new XContentBuilderString("_id"); - static final XContentBuilderString _VERSION = new XContentBuilderString("_version"); + client.delete(deleteRequest, new RestStatusToXContentListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/exists/RestExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/exists/RestExistsAction.java deleted file mode 100644 index 7cfe7caf3fd..00000000000 --- a/core/src/main/java/org/elasticsearch/rest/action/exists/RestExistsAction.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.rest.action.exists; - -import org.elasticsearch.action.exists.ExistsRequest; -import org.elasticsearch.action.exists.ExistsResponse; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.*; -import org.elasticsearch.rest.action.support.RestActions; -import org.elasticsearch.rest.action.support.RestBuilderListener; - -import static org.elasticsearch.action.exists.ExistsRequest.DEFAULT_MIN_SCORE; -import static org.elasticsearch.rest.RestStatus.NOT_FOUND; -import static org.elasticsearch.rest.RestStatus.OK; - -/** - * Action for /_search/exists endpoint - */ -public class RestExistsAction extends BaseRestHandler { - - public RestExistsAction(Settings settings, RestController controller, Client client) { - super(settings, controller, client); - } - - @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { - final ExistsRequest existsRequest = new ExistsRequest(Strings.splitStringByCommaToArray(request.param("index"))); - existsRequest.indicesOptions(IndicesOptions.fromRequest(request, existsRequest.indicesOptions())); - if (RestActions.hasBodyContent(request)) { - existsRequest.source(RestActions.getRestContent(request)); - } else { - QuerySourceBuilder querySourceBuilder = RestActions.parseQuerySource(request); - if (querySourceBuilder != null) { - existsRequest.source(querySourceBuilder); - } - } - existsRequest.routing(request.param("routing")); - existsRequest.minScore(request.paramAsFloat("min_score", DEFAULT_MIN_SCORE)); - existsRequest.types(Strings.splitStringByCommaToArray(request.param("type"))); - existsRequest.preference(request.param("preference")); - - client.exists(existsRequest, new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(ExistsResponse response, XContentBuilder builder) throws Exception { - RestStatus status = response.exists() ? OK : NOT_FOUND; - builder.startObject(); - builder.field("exists", response.exists()); - builder.endObject(); - return new BytesRestResponse(status, builder); - } - }); - } -} diff --git a/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java index 7c01fddf3cc..086446fc53f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java @@ -22,17 +22,16 @@ package org.elasticsearch.rest.action.explain; import org.apache.lucene.search.Explanation; import org.elasticsearch.action.explain.ExplainRequest; import org.elasticsearch.action.explain.ExplainResponse; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.index.get.GetResult; -import org.elasticsearch.index.query.Operator; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; @@ -50,9 +49,12 @@ import static org.elasticsearch.rest.RestStatus.OK; */ public class RestExplainAction extends BaseRestHandler { + private final IndicesQueriesRegistry indicesQueriesRegistry; + @Inject - public RestExplainAction(Settings settings, RestController controller, Client client) { + public RestExplainAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) { super(settings, controller, client); + this.indicesQueriesRegistry = indicesQueriesRegistry; controller.registerHandler(GET, "/{index}/{type}/{id}/_explain", this); controller.registerHandler(POST, "/{index}/{type}/{id}/_explain", this); } @@ -65,22 +67,11 @@ public class RestExplainAction extends BaseRestHandler { explainRequest.preference(request.param("preference")); String queryString = request.param("q"); if (RestActions.hasBodyContent(request)) { - explainRequest.source(RestActions.getRestContent(request)); + BytesReference restContent = RestActions.getRestContent(request); + explainRequest.query(RestActions.getQueryContent(restContent, indicesQueriesRegistry, parseFieldMatcher)); } else if (queryString != null) { - QueryStringQueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery(queryString); - queryStringBuilder.defaultField(request.param("df")); - queryStringBuilder.analyzer(request.param("analyzer")); - queryStringBuilder.analyzeWildcard(request.paramAsBoolean("analyze_wildcard", false)); - queryStringBuilder.lowercaseExpandedTerms(request.paramAsBoolean("lowercase_expanded_terms", true)); - queryStringBuilder.lenient(request.paramAsBoolean("lenient", null)); - String defaultOperator = request.param("default_operator"); - if (defaultOperator != null) { - queryStringBuilder.defaultOperator(Operator.fromString(defaultOperator)); - } - - QuerySourceBuilder querySourceBuilder = new QuerySourceBuilder(); - querySourceBuilder.setQuery(queryStringBuilder); - explainRequest.source(querySourceBuilder); + QueryBuilder query = RestActions.urlParamsToQueryBuilder(request); + explainRequest.query(query); } String sField = request.param("fields"); diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java index 1fe07156d99..c0e45fc6aeb 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java @@ -26,7 +26,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestResponseListener; import org.elasticsearch.search.fetch.source.FetchSourceContext; @@ -77,7 +76,7 @@ public class RestGetSourceAction extends BaseRestHandler { if (!response.isExists()) { return new BytesRestResponse(NOT_FOUND, builder); } else { - XContentHelper.writeDirect(response.getSourceInternal(), builder, request); + builder.rawValue(response.getSourceInternal()); return new BytesRestResponse(OK, builder); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java index d0d0fe68a13..310ce0a1248 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.rest.action.index; -import org.elasticsearch.action.ActionWriteResponse; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; @@ -27,11 +26,11 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import org.elasticsearch.rest.action.support.RestStatusToXContentListener; import java.io.IOException; @@ -74,7 +73,7 @@ public class RestIndexAction extends BaseRestHandler { indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing indexRequest.timestamp(request.param("timestamp")); if (request.hasParam("ttl")) { - indexRequest.ttl(request.paramAsTime("ttl", null).millis()); + indexRequest.ttl(request.param("ttl")); } indexRequest.source(request.content()); indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); @@ -99,33 +98,6 @@ public class RestIndexAction extends BaseRestHandler { if (consistencyLevel != null) { indexRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel)); } - client.index(indexRequest, new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(IndexResponse response, XContentBuilder builder) throws Exception { - builder.startObject(); - ActionWriteResponse.ShardInfo shardInfo = response.getShardInfo(); - builder.field(Fields._INDEX, response.getIndex()) - .field(Fields._TYPE, response.getType()) - .field(Fields._ID, response.getId()) - .field(Fields._VERSION, response.getVersion()); - shardInfo.toXContent(builder, request); - builder.field(Fields.CREATED, response.isCreated()); - builder.endObject(); - RestStatus status = shardInfo.status(); - if (response.isCreated()) { - status = CREATED; - } - return new BytesRestResponse(status, builder); - } - }); + client.index(indexRequest, new RestStatusToXContentListener<>(channel)); } - - static final class Fields { - static final XContentBuilderString _INDEX = new XContentBuilderString("_index"); - static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); - static final XContentBuilderString _ID = new XContentBuilderString("_id"); - static final XContentBuilderString _VERSION = new XContentBuilderString("_version"); - static final XContentBuilderString CREATED = new XContentBuilderString("created"); - } - } diff --git a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java index 5b567b0e32c..df99979eb6b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java @@ -77,8 +77,8 @@ public class RestMainAction extends BaseRestHandler { builder.field("cluster_name", clusterName.value()); builder.startObject("version") .field("number", version.number()) - .field("build_hash", Build.CURRENT.hash()) - .field("build_timestamp", Build.CURRENT.timestamp()) + .field("build_hash", Build.CURRENT.shortHash()) + .field("build_date", Build.CURRENT.date()) .field("build_snapshot", version.snapshot) .field("lucene_version", version.luceneVersion.toString()) .endObject(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index af1f2f464a7..ff51263e08b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -20,16 +20,35 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.action.search.MultiSearchRequest; -import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.rest.*; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.TemplateQueryParser; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestToXContentListener; +import org.elasticsearch.script.Template; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -38,9 +57,11 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestMultiSearchAction extends BaseRestHandler { private final boolean allowExplicitIndex; + private final IndicesQueriesRegistry indicesQueriesRegistry; + @Inject - public RestMultiSearchAction(Settings settings, RestController controller, Client client) { + public RestMultiSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) { super(settings, controller, client); controller.registerHandler(GET, "/_msearch", this); @@ -58,6 +79,7 @@ public class RestMultiSearchAction extends BaseRestHandler { controller.registerHandler(POST, "/{index}/{type}/_msearch/template", this); this.allowExplicitIndex = settings.getAsBoolean("rest.action.multi.allow_explicit_index", true); + this.indicesQueriesRegistry = indicesQueriesRegistry; } @Override @@ -69,12 +91,121 @@ public class RestMultiSearchAction extends BaseRestHandler { String path = request.path(); boolean isTemplateRequest = isTemplateRequest(path); IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, multiSearchRequest.indicesOptions()); - multiSearchRequest.add(RestActions.getRestContent(request), isTemplateRequest, indices, types, request.param("search_type"), request.param("routing"), indicesOptions, allowExplicitIndex); - - client.multiSearch(multiSearchRequest, new RestToXContentListener(channel)); + parseRequest(multiSearchRequest, RestActions.getRestContent(request), isTemplateRequest, indices, types, + request.param("search_type"), request.param("routing"), indicesOptions, allowExplicitIndex, indicesQueriesRegistry, + parseFieldMatcher); + client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel)); } private boolean isTemplateRequest(String path) { return (path != null && path.endsWith("/template")); } + + public static MultiSearchRequest parseRequest(MultiSearchRequest msr, BytesReference data, boolean isTemplateRequest, + @Nullable String[] indices, + @Nullable String[] types, + @Nullable String searchType, + @Nullable String routing, + IndicesOptions indicesOptions, + boolean allowExplicitIndex, IndicesQueriesRegistry indicesQueriesRegistry, + ParseFieldMatcher parseFieldMatcher) throws Exception { + XContent xContent = XContentFactory.xContent(data); + int from = 0; + int length = data.length(); + byte marker = xContent.streamSeparator(); + final QueryParseContext queryParseContext = new QueryParseContext(indicesQueriesRegistry); + while (true) { + int nextMarker = findNextMarker(marker, from, data, length); + if (nextMarker == -1) { + break; + } + // support first line with \n + if (nextMarker == 0) { + from = nextMarker + 1; + continue; + } + + SearchRequest searchRequest = new SearchRequest(); + if (indices != null) { + searchRequest.indices(indices); + } + if (indicesOptions != null) { + searchRequest.indicesOptions(indicesOptions); + } + if (types != null && types.length > 0) { + searchRequest.types(types); + } + if (routing != null) { + searchRequest.routing(routing); + } + searchRequest.searchType(searchType); + + IndicesOptions defaultOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); + + + // now parse the action + if (nextMarker - from > 0) { + try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) { + Map source = parser.map(); + for (Map.Entry entry : source.entrySet()) { + Object value = entry.getValue(); + if ("index".equals(entry.getKey()) || "indices".equals(entry.getKey())) { + if (!allowExplicitIndex) { + throw new IllegalArgumentException("explicit index in multi percolate is not allowed"); + } + searchRequest.indices(nodeStringArrayValue(value)); + } else if ("type".equals(entry.getKey()) || "types".equals(entry.getKey())) { + searchRequest.types(nodeStringArrayValue(value)); + } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) { + searchRequest.searchType(nodeStringValue(value, null)); + } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) { + searchRequest.requestCache(nodeBooleanValue(value)); + } else if ("preference".equals(entry.getKey())) { + searchRequest.preference(nodeStringValue(value, null)); + } else if ("routing".equals(entry.getKey())) { + searchRequest.routing(nodeStringValue(value, null)); + } + } + defaultOptions = IndicesOptions.fromMap(source, defaultOptions); + } + } + searchRequest.indicesOptions(defaultOptions); + + // move pointers + from = nextMarker + 1; + // now for the body + nextMarker = findNextMarker(marker, from, data, length); + if (nextMarker == -1) { + break; + } + final BytesReference slice = data.slice(from, nextMarker - from); + if (isTemplateRequest) { + try (XContentParser parser = XContentFactory.xContent(slice).createParser(slice)) { + queryParseContext.reset(parser); + queryParseContext.parseFieldMatcher(parseFieldMatcher); + Template template = TemplateQueryParser.parse(parser, queryParseContext.parseFieldMatcher(), "params", "template"); + searchRequest.template(template); + } + } else { + try (XContentParser requestParser = XContentFactory.xContent(slice).createParser(slice)) { + queryParseContext.reset(requestParser); + searchRequest.source(SearchSourceBuilder.parseSearchSource(requestParser, queryParseContext)); + } + } + // move pointers + from = nextMarker + 1; + + msr.add(searchRequest); + } + return msr; + } + + private static int findNextMarker(byte marker, int from, BytesReference data, int length) { + for (int i = from; i < length; i++) { + if (data.get(i) == marker) { + return i; + } + } + return -1; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 03a33e0c96b..6db9531af84 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -20,27 +20,36 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.TemplateQueryParser; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.exists.RestExistsAction; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestStatusToXContentListener; +import org.elasticsearch.script.Template; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.suggest.SuggestBuilder; + +import java.io.IOException; +import java.util.Arrays; import static org.elasticsearch.common.unit.TimeValue.parseTimeValue; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -52,9 +61,12 @@ import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion; */ public class RestSearchAction extends BaseRestHandler { + private final IndicesQueriesRegistry queryRegistry; + @Inject - public RestSearchAction(Settings settings, RestController controller, Client client) { + public RestSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry) { super(settings, controller, client); + this.queryRegistry = queryRegistry; controller.registerHandler(GET, "/_search", this); controller.registerHandler(POST, "/_search", this); controller.registerHandler(GET, "/{index}/_search", this); @@ -67,35 +79,38 @@ public class RestSearchAction extends BaseRestHandler { controller.registerHandler(POST, "/{index}/_search/template", this); controller.registerHandler(GET, "/{index}/{type}/_search/template", this); controller.registerHandler(POST, "/{index}/{type}/_search/template", this); - - RestExistsAction restExistsAction = new RestExistsAction(settings, controller, client); - controller.registerHandler(GET, "/_search/exists", restExistsAction); - controller.registerHandler(POST, "/_search/exists", restExistsAction); - controller.registerHandler(GET, "/{index}/_search/exists", restExistsAction); - controller.registerHandler(POST, "/{index}/_search/exists", restExistsAction); - controller.registerHandler(GET, "/{index}/{type}/_search/exists", restExistsAction); - controller.registerHandler(POST, "/{index}/{type}/_search/exists", restExistsAction); } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { SearchRequest searchRequest; - searchRequest = RestSearchAction.parseSearchRequest(request, parseFieldMatcher); - client.search(searchRequest, new RestStatusToXContentListener(channel)); + searchRequest = RestSearchAction.parseSearchRequest(queryRegistry, request, parseFieldMatcher); + client.search(searchRequest, new RestStatusToXContentListener<>(channel)); } - public static SearchRequest parseSearchRequest(RestRequest request, ParseFieldMatcher parseFieldMatcher) { + public static SearchRequest parseSearchRequest(IndicesQueriesRegistry indicesQueriesRegistry, RestRequest request, ParseFieldMatcher parseFieldMatcher) throws IOException { String[] indices = Strings.splitStringByCommaToArray(request.param("index")); SearchRequest searchRequest = new SearchRequest(indices); // get the content, and put it in the body // add content/source as template if template flag is set boolean isTemplateRequest = request.path().endsWith("/template"); + final SearchSourceBuilder builder; if (RestActions.hasBodyContent(request)) { + BytesReference restContent = RestActions.getRestContent(request); + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); if (isTemplateRequest) { - searchRequest.templateSource(RestActions.getRestContent(request)); + try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { + context.reset(parser); + context.parseFieldMatcher(parseFieldMatcher); + Template template = TemplateQueryParser.parse(parser, context.parseFieldMatcher(), "params", "template"); + searchRequest.template(template); + } + builder = null; } else { - searchRequest.source(RestActions.getRestContent(request)); + builder = RestActions.getRestSearchSource(restContent, indicesQueriesRegistry, parseFieldMatcher); } + } else { + builder = null; } // do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types @@ -108,8 +123,15 @@ public class RestSearchAction extends BaseRestHandler { } else { searchRequest.searchType(searchType); } - - searchRequest.extraSource(parseSearchSource(request)); + if (builder == null) { + SearchSourceBuilder extraBuilder = new SearchSourceBuilder(); + if (parseSearchSource(extraBuilder, request)) { + searchRequest.source(extraBuilder); + } + } else { + parseSearchSource(builder, request); + searchRequest.source(builder); + } searchRequest.requestCache(request.paramAsBoolean("request_cache", null)); String scroll = request.param("scroll"); @@ -125,111 +147,89 @@ public class RestSearchAction extends BaseRestHandler { return searchRequest; } - public static SearchSourceBuilder parseSearchSource(RestRequest request) { - SearchSourceBuilder searchSourceBuilder = null; + private static boolean parseSearchSource(final SearchSourceBuilder searchSourceBuilder, RestRequest request) { - QuerySourceBuilder querySourceBuilder = RestActions.parseQuerySource(request); - if (querySourceBuilder != null) { - searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(querySourceBuilder); + boolean modified = false; + QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request); + if (queryBuilder != null) { + searchSourceBuilder.query(queryBuilder); + modified = true; } int from = request.paramAsInt("from", -1); if (from != -1) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } searchSourceBuilder.from(from); + modified = true; } int size = request.paramAsInt("size", -1); if (size != -1) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } searchSourceBuilder.size(size); + modified = true; } if (request.hasParam("explain")) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } searchSourceBuilder.explain(request.paramAsBoolean("explain", null)); + modified = true; } if (request.hasParam("version")) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } searchSourceBuilder.version(request.paramAsBoolean("version", null)); + modified = true; } if (request.hasParam("timeout")) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } searchSourceBuilder.timeout(request.paramAsTime("timeout", null)); + modified = true; } if (request.hasParam("terminate_after")) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } int terminateAfter = request.paramAsInt("terminate_after", SearchContext.DEFAULT_TERMINATE_AFTER); if (terminateAfter < 0) { throw new IllegalArgumentException("terminateAfter must be > 0"); } else if (terminateAfter > 0) { searchSourceBuilder.terminateAfter(terminateAfter); + modified = true; } } String sField = request.param("fields"); if (sField != null) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } if (!Strings.hasText(sField)) { searchSourceBuilder.noFields(); + modified = true; } else { String[] sFields = Strings.splitStringByCommaToArray(sField); if (sFields != null) { for (String field : sFields) { searchSourceBuilder.field(field); + modified = true; } } } } String sFieldDataFields = request.param("fielddata_fields"); if (sFieldDataFields != null) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } if (Strings.hasText(sFieldDataFields)) { String[] sFields = Strings.splitStringByCommaToArray(sFieldDataFields); if (sFields != null) { for (String field : sFields) { searchSourceBuilder.fieldDataField(field); + modified = true; } } } } FetchSourceContext fetchSourceContext = FetchSourceContext.parseFromRestRequest(request); if (fetchSourceContext != null) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } searchSourceBuilder.fetchSource(fetchSourceContext); + modified = true; } if (request.hasParam("track_scores")) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } searchSourceBuilder.trackScores(request.paramAsBoolean("track_scores", false)); + modified = true; } String sSorts = request.param("sort"); if (sSorts != null) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } String[] sorts = Strings.splitStringByCommaToArray(sSorts); for (String sort : sorts) { int delimiter = sort.lastIndexOf(":"); @@ -238,37 +238,33 @@ public class RestSearchAction extends BaseRestHandler { String reverse = sort.substring(delimiter + 1); if ("asc".equals(reverse)) { searchSourceBuilder.sort(sortField, SortOrder.ASC); + modified = true; } else if ("desc".equals(reverse)) { searchSourceBuilder.sort(sortField, SortOrder.DESC); + modified = true; } } else { searchSourceBuilder.sort(sort); + modified = true; } } } String sStats = request.param("stats"); if (sStats != null) { - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } - searchSourceBuilder.stats(Strings.splitStringByCommaToArray(sStats)); + searchSourceBuilder.stats(Arrays.asList(Strings.splitStringByCommaToArray(sStats))); + modified = true; } String suggestField = request.param("suggest_field"); if (suggestField != null) { String suggestText = request.param("suggest_text", request.param("q")); int suggestSize = request.paramAsInt("suggest_size", 5); - if (searchSourceBuilder == null) { - searchSourceBuilder = new SearchSourceBuilder(); - } String suggestMode = request.param("suggest_mode"); - searchSourceBuilder.suggest().addSuggestion( - termSuggestion(suggestField).field(suggestField).text(suggestText).size(suggestSize) - .suggestMode(suggestMode) - ); + searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion( + termSuggestion(suggestField).field(suggestField).text(suggestText).size(suggestSize).suggestMode(suggestMode))); + modified = true; } - - return searchSourceBuilder; + return modified; } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java index 674aa6902b0..14935f5f9a5 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java +++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java @@ -19,18 +19,19 @@ package org.elasticsearch.rest.action.support; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.index.query.Operator; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.index.query.*; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -85,7 +86,7 @@ public class RestActions { builder.endObject(); } - public static QuerySourceBuilder parseQuerySource(RestRequest request) { + public static QueryBuilder urlParamsToQueryBuilder(RestRequest request) { String queryString = request.param("q"); if (queryString == null) { return null; @@ -100,7 +101,17 @@ public class RestActions { if (defaultOperator != null) { queryBuilder.defaultOperator(Operator.fromString(defaultOperator)); } - return new QuerySourceBuilder().setQuery(queryBuilder); + return queryBuilder; + } + + public static SearchSourceBuilder getRestSearchSource(BytesReference sourceBytes, IndicesQueriesRegistry queryRegistry, + ParseFieldMatcher parseFieldMatcher) + throws IOException { + XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes); + QueryParseContext queryParseContext = new QueryParseContext(queryRegistry); + queryParseContext.reset(parser); + queryParseContext.parseFieldMatcher(parseFieldMatcher); + return SearchSourceBuilder.parseSearchSource(parser, queryParseContext); } /** @@ -122,6 +133,19 @@ public class RestActions { return content; } + public static QueryBuilder getQueryContent(BytesReference source, IndicesQueriesRegistry indicesQueriesRegistry, ParseFieldMatcher parseFieldMatcher) { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + try (XContentParser requestParser = XContentFactory.xContent(source).createParser(source)) { + context.reset(requestParser); + context.parseFieldMatcher(parseFieldMatcher); + return context.parseTopLevelQueryBuilder(); + } catch (IOException e) { + throw new ElasticsearchException("failed to parse source", e); + } finally { + context.reset(null); + } + } + /** * guesses the content type from either payload or source parameter * @param request Rest request diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestTable.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestTable.java index e1c62049843..3808e58a527 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/support/RestTable.java +++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestTable.java @@ -75,20 +75,26 @@ public class RestTable { BytesStreamOutput bytesOut = channel.bytesOutput(); UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOut); + int lastHeader = headers.size() - 1; if (verbose) { for (int col = 0; col < headers.size(); col++) { DisplayHeader header = headers.get(col); - pad(new Table.Cell(header.display, table.findHeaderByName(header.name)), width[col], request, out); - out.append(" "); + boolean isLastColumn = col == lastHeader; + pad(new Table.Cell(header.display, table.findHeaderByName(header.name)), width[col], request, out, isLastColumn); + if (!isLastColumn) { + out.append(" "); + } } out.append("\n"); } - for (int row = 0; row < table.getRows().size(); row++) { for (int col = 0; col < headers.size(); col++) { DisplayHeader header = headers.get(col); - pad(table.getAsMap().get(header.name).get(row), width[col], request, out); - out.append(" "); + boolean isLastColumn = col == lastHeader; + pad(table.getAsMap().get(header.name).get(row), width[col], request, out, isLastColumn); + if (!isLastColumn) { + out.append(" "); + } } out.append("\n"); } @@ -236,6 +242,10 @@ public class RestTable { } public static void pad(Table.Cell cell, int width, RestRequest request, UTF8StreamWriter out) throws IOException { + pad(cell, width, request, out, false); + } + + public static void pad(Table.Cell cell, int width, RestRequest request, UTF8StreamWriter out, boolean isLast) throws IOException { String sValue = renderValue(request, cell.value); int length = sValue == null ? 0 : sValue.length(); byte leftOver = (byte) (width - length); @@ -254,8 +264,11 @@ public class RestTable { if (sValue != null) { out.append(sValue); } - for (byte i = 0; i < leftOver; i++) { - out.append(" "); + // Ignores the leftover spaces if the cell is the last of the column. + if (!isLast) { + for (byte i = 0; i < leftOver; i++) { + out.append(" "); + } } } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/template/RestDeleteSearchTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/template/RestDeleteSearchTemplateAction.java index 9b205a8070f..3d0daf37b63 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/template/RestDeleteSearchTemplateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/template/RestDeleteSearchTemplateAction.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.script.RestDeleteIndexedScriptAction; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.script.Template; import static org.elasticsearch.rest.RestRequest.Method.DELETE; @@ -38,6 +38,6 @@ public class RestDeleteSearchTemplateAction extends RestDeleteIndexedScriptActio @Override protected String getScriptLang(RestRequest request) { - return MustacheScriptEngineService.NAME; + return Template.DEFAULT_LANG; } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/rest/action/template/RestGetSearchTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/template/RestGetSearchTemplateAction.java index 39be6a53370..0e8aa357fcd 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/template/RestGetSearchTemplateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/template/RestGetSearchTemplateAction.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.script.RestGetIndexedScriptAction; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.script.Template; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -42,7 +42,7 @@ public class RestGetSearchTemplateAction extends RestGetIndexedScriptAction { @Override protected String getScriptLang(RestRequest request) { - return MustacheScriptEngineService.NAME; + return Template.DEFAULT_LANG; } @Override diff --git a/core/src/main/java/org/elasticsearch/rest/action/template/RestPutSearchTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/template/RestPutSearchTemplateAction.java index a734ce37ca2..0d23645afda 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/template/RestPutSearchTemplateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/template/RestPutSearchTemplateAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.script.RestPutIndexedScriptAction; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.script.Template; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; @@ -59,6 +59,6 @@ public class RestPutSearchTemplateAction extends RestPutIndexedScriptAction { @Override protected String getScriptLang(RestRequest request) { - return MustacheScriptEngineService.NAME; + return Template.DEFAULT_LANG; } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java index a23780db62e..f59c329fbc3 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.rest.action.update; -import org.elasticsearch.action.ActionWriteResponse; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; @@ -29,7 +28,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; @@ -40,6 +38,7 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import org.elasticsearch.rest.action.support.RestStatusToXContentListener; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptParameterParser; import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; @@ -48,7 +47,6 @@ import java.util.HashMap; import java.util.Map; import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.rest.RestStatus.CREATED; /** */ @@ -105,7 +103,7 @@ public class RestUpdateAction extends BaseRestHandler { upsertRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing upsertRequest.timestamp(request.param("timestamp")); if (request.hasParam("ttl")) { - upsertRequest.ttl(request.paramAsTime("ttl", null).millis()); + upsertRequest.ttl(request.param("ttl")); } upsertRequest.version(RestActions.parseVersion(request)); upsertRequest.versionType(VersionType.fromString(request.param("version_type"), upsertRequest.versionType())); @@ -116,45 +114,13 @@ public class RestUpdateAction extends BaseRestHandler { doc.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing doc.timestamp(request.param("timestamp")); if (request.hasParam("ttl")) { - doc.ttl(request.paramAsTime("ttl", null).millis()); + doc.ttl(request.param("ttl")); } doc.version(RestActions.parseVersion(request)); doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType())); } } - client.update(updateRequest, new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(UpdateResponse response, XContentBuilder builder) throws Exception { - builder.startObject(); - ActionWriteResponse.ShardInfo shardInfo = response.getShardInfo(); - builder.field(Fields._INDEX, response.getIndex()) - .field(Fields._TYPE, response.getType()) - .field(Fields._ID, response.getId()) - .field(Fields._VERSION, response.getVersion()); - - shardInfo.toXContent(builder, request); - if (response.getGetResult() != null) { - builder.startObject(Fields.GET); - response.getGetResult().toXContentEmbedded(builder, request); - builder.endObject(); - } - - builder.endObject(); - RestStatus status = shardInfo.status(); - if (response.isCreated()) { - status = CREATED; - } - return new BytesRestResponse(status, builder); - } - }); - } - - static final class Fields { - static final XContentBuilderString _INDEX = new XContentBuilderString("_index"); - static final XContentBuilderString _TYPE = new XContentBuilderString("_type"); - static final XContentBuilderString _ID = new XContentBuilderString("_id"); - static final XContentBuilderString _VERSION = new XContentBuilderString("_version"); - static final XContentBuilderString GET = new XContentBuilderString("get"); + client.update(updateRequest, new RestStatusToXContentListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java index 2d8237e4edb..56bb18d5e6e 100644 --- a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java +++ b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java @@ -19,12 +19,11 @@ package org.elasticsearch.rest.support; -import java.nio.charset.StandardCharsets; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.path.PathTrie; -import org.elasticsearch.common.settings.Settings; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.regex.Pattern; @@ -39,7 +38,6 @@ public class RestUtils { return RestUtils.decodeComponent(value); } }; - public static final String HTTP_CORS_ALLOW_ORIGIN_SETTING = "http.cors.allow-origin"; public static boolean isBrowser(@Nullable String userAgent) { if (userAgent == null) { @@ -224,9 +222,13 @@ public class RestUtils { /** * Determine if CORS setting is a regex + * + * @return a corresponding {@link Pattern} if so and o.w. null. */ - public static Pattern getCorsSettingRegex(Settings settings) { - String corsSetting = settings.get(HTTP_CORS_ALLOW_ORIGIN_SETTING, "*"); + public static Pattern checkCorsSettingForRegex(String corsSetting) { + if (corsSetting == null) { + return null; + } int len = corsSetting.length(); boolean isRegex = len > 2 && corsSetting.startsWith("/") && corsSetting.endsWith("/"); diff --git a/core/src/main/java/org/elasticsearch/script/ClassPermission.java b/core/src/main/java/org/elasticsearch/script/ClassPermission.java new file mode 100644 index 00000000000..eb580bac3ea --- /dev/null +++ b/core/src/main/java/org/elasticsearch/script/ClassPermission.java @@ -0,0 +1,171 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import java.security.BasicPermission; +import java.security.Permission; +import java.security.PermissionCollection; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Set; + +/** + * Checked by scripting engines to allow loading a java class. + *

    + * Examples: + *

    + * Allow permission to {@code java.util.List} + * 

    permission org.elasticsearch.script.ClassPermission "java.util.List";
    + * Allow permission to classes underneath {@code java.util} (and its subpackages such as {@code java.util.zip}) + * 
    permission org.elasticsearch.script.ClassPermission "java.util.*";
    + * Allow permission to standard predefined list of basic classes (see list below) + * 
    permission org.elasticsearch.script.ClassPermission "<<STANDARD>>";
    + * Allow permission to all classes + * 
    permission org.elasticsearch.script.ClassPermission "*";
    + *

    + * Set of classes (allowed by special value <<STANDARD>>): + *

      + *
    • {@link java.lang.Boolean}
      • + *
    • {@link java.lang.Byte}
      • + *
    • {@link java.lang.Character}
      • + *
    • {@link java.lang.Double}
      • + *
    • {@link java.lang.Integer}
      • + *
    • {@link java.lang.Long}
      • + *
    • {@link java.lang.Math}
      • + *
    • {@link java.lang.Object}
      • + *
    • {@link java.lang.Short}
      • + *
    • {@link java.lang.String}
      • + *
    • {@link java.math.BigDecimal}
      • + *
    • {@link java.util.ArrayList}
      • + *
    • {@link java.util.Arrays}
      • + *
    • {@link java.util.Date}
      • + *
    • {@link java.util.HashMap}
      • + *
    • {@link java.util.HashSet}
      • + *
    • {@link java.util.Iterator}
      • + *
    • {@link java.util.List}
      • + *
    • {@link java.util.Map}
      • + *
    • {@link java.util.Set}
      • + *
    • {@link java.util.UUID}
      • + *
    • {@link org.joda.time.DateTime}
      • + *
    • {@link org.joda.time.DateTimeUtils}
      • + *
    • {@link org.joda.time.DateTimeZone}
      • + *
    • {@link org.joda.time.Instant}
      • + *
    + */ +public final class ClassPermission extends BasicPermission { + private static final long serialVersionUID = 3530711429252193884L; + + public static final String STANDARD = "<>"; + /** Typical set of classes for scripting: basic data types, math, dates, and simple collections */ + // this is the list from the old grovy sandbox impl (+ some things like String, Iterator, etc that were missing) + public static final Set STANDARD_CLASSES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + // jdk classes + java.lang.Boolean.class.getName(), + java.lang.Byte.class.getName(), + java.lang.Character.class.getName(), + java.lang.Double.class.getName(), + java.lang.Integer.class.getName(), + java.lang.Long.class.getName(), + java.lang.Math.class.getName(), + java.lang.Object.class.getName(), + java.lang.Short.class.getName(), + java.lang.String.class.getName(), + java.math.BigDecimal.class.getName(), + java.util.ArrayList.class.getName(), + java.util.Arrays.class.getName(), + java.util.Date.class.getName(), + java.util.HashMap.class.getName(), + java.util.HashSet.class.getName(), + java.util.Iterator.class.getName(), + java.util.List.class.getName(), + java.util.Map.class.getName(), + java.util.Set.class.getName(), + java.util.UUID.class.getName(), + // joda-time + org.joda.time.DateTime.class.getName(), + org.joda.time.DateTimeUtils.class.getName(), + org.joda.time.DateTimeZone.class.getName(), + org.joda.time.Instant.class.getName() + ))); + + /** + * Creates a new ClassPermission object. + * + * @param name class to grant permission to + */ + public ClassPermission(String name) { + super(name); + } + + /** + * Creates a new ClassPermission object. + * This constructor exists for use by the {@code Policy} object to instantiate new Permission objects. + * + * @param name class to grant permission to + * @param actions ignored + */ + public ClassPermission(String name, String actions) { + this(name); + } + + @Override + public boolean implies(Permission p) { + // check for a special value of STANDARD to imply the basic set + if (p != null && p.getClass() == getClass()) { + ClassPermission other = (ClassPermission) p; + if (STANDARD.equals(getName()) && STANDARD_CLASSES.contains(other.getName())) { + return true; + } + } + return super.implies(p); + } + + @Override + public PermissionCollection newPermissionCollection() { + // BasicPermissionCollection only handles wildcards, we expand <> here + PermissionCollection impl = super.newPermissionCollection(); + return new PermissionCollection() { + private static final long serialVersionUID = 6792220143549780002L; + + @Override + public void add(Permission permission) { + if (permission instanceof ClassPermission && STANDARD.equals(permission.getName())) { + for (String clazz : STANDARD_CLASSES) { + impl.add(new ClassPermission(clazz)); + } + } else { + impl.add(permission); + } + } + + @Override + public boolean implies(Permission permission) { + return impl.implies(permission); + } + + @Override + public Enumeration elements() { + return impl.elements(); + } + }; + } +} diff --git a/core/src/main/java/org/elasticsearch/script/ScriptContext.java b/core/src/main/java/org/elasticsearch/script/ScriptContext.java index a12fc85a53c..4b1b6de63f2 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptContext.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptContext.java @@ -37,7 +37,7 @@ public interface ScriptContext { */ enum Standard implements ScriptContext { - AGGS("aggs"), MAPPING("mapping"), SEARCH("search"), UPDATE("update"); + AGGS("aggs"), SEARCH("search"), UPDATE("update"); private final String key; diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java index 3c19826a190..f3bdad64b66 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -22,9 +22,7 @@ package org.elasticsearch.script; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.multibindings.MapBinder; import org.elasticsearch.common.inject.multibindings.Multibinder; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import java.util.ArrayList; import java.util.HashMap; @@ -75,13 +73,6 @@ public class ScriptModule extends AbstractModule { Multibinder multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class); multibinder.addBinding().to(NativeScriptEngineService.class); - - try { - Class.forName("com.github.mustachejava.Mustache"); - multibinder.addBinding().to(MustacheScriptEngineService.class).asEagerSingleton(); - } catch (Throwable t) { - Loggers.getLogger(ScriptService.class, settings).debug("failed to load mustache", t); - } for (Class scriptEngine : scriptEngines) { multibinder.addBinding().to(scriptEngine).asEagerSingleton(); diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index 87a5a9a506d..3b91f2d3110 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -246,8 +246,7 @@ public class ScriptService extends AbstractComponent implements Closeable { // TODO: fix this through some API or something, thats wrong // special exception to prevent expressions from compiling as update or mapping scripts boolean expression = "expression".equals(script.getLang()); - boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()) || - scriptContext.getKey().equals(ScriptContext.Standard.MAPPING.getKey()); + boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()); if (expression && notSupported) { throw new ScriptException("scripts of type [" + script.getType() + "]," + " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported"); diff --git a/core/src/main/java/org/elasticsearch/script/Template.java b/core/src/main/java/org/elasticsearch/script/Template.java index 293a8b3d5ea..c9bb9085051 100644 --- a/core/src/main/java/org/elasticsearch/script/Template.java +++ b/core/src/main/java/org/elasticsearch/script/Template.java @@ -29,13 +29,15 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import java.io.IOException; import java.util.Collections; import java.util.Map; public class Template extends Script { + + /** Default templating language */ + public static final String DEFAULT_LANG = "mustache"; private XContentType contentType; @@ -46,17 +48,17 @@ public class Template extends Script { /** * Constructor for simple inline template. The template will have no lang, * content type or params set. - * + * * @param template * The inline template. */ public Template(String template) { - super(template, MustacheScriptEngineService.NAME); + super(template, DEFAULT_LANG); } /** * Constructor for Template. - * + * * @param template * The cache key of the template to be compiled/executed. For * inline templates this is the actual templates source code. For @@ -73,13 +75,13 @@ public class Template extends Script { */ public Template(String template, ScriptType type, @Nullable String lang, @Nullable XContentType xContentType, @Nullable Map params) { - super(template, type, lang, params); + super(template, type, lang == null ? DEFAULT_LANG : lang, params); this.contentType = xContentType; } /** * Method for getting the {@link XContentType} of the template. - * + * * @return The {@link XContentType} of the template. */ public XContentType getContentType() { @@ -119,19 +121,17 @@ public class Template extends Script { return template; } - @SuppressWarnings("unchecked") public static Script parse(Map config, boolean removeMatchedEntries, ParseFieldMatcher parseFieldMatcher) { - return new TemplateParser(Collections.EMPTY_MAP, MustacheScriptEngineService.NAME).parse(config, removeMatchedEntries, parseFieldMatcher); + return new TemplateParser(Collections.emptyMap(), DEFAULT_LANG).parse(config, removeMatchedEntries, parseFieldMatcher); } - @SuppressWarnings("unchecked") public static Template parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException { - return new TemplateParser(Collections.EMPTY_MAP, MustacheScriptEngineService.NAME).parse(parser, parseFieldMatcher); + return new TemplateParser(Collections.emptyMap(), DEFAULT_LANG).parse(parser, parseFieldMatcher); } @Deprecated public static Template parse(XContentParser parser, Map additionalTemplateFieldNames, ParseFieldMatcher parseFieldMatcher) throws IOException { - return new TemplateParser(additionalTemplateFieldNames, MustacheScriptEngineService.NAME).parse(parser, parseFieldMatcher); + return new TemplateParser(additionalTemplateFieldNames, DEFAULT_LANG).parse(parser, parseFieldMatcher); } @Deprecated @@ -174,7 +174,7 @@ public class Template extends Script { @Override protected Template createSimpleScript(XContentParser parser) throws IOException { - return new Template(String.valueOf(parser.objectText()), ScriptType.INLINE, MustacheScriptEngineService.NAME, contentType, null); + return new Template(String.valueOf(parser.objectText()), ScriptType.INLINE, DEFAULT_LANG, contentType, null); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index aec5cd18206..8096213b8fa 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -21,7 +21,6 @@ package org.elasticsearch.search; import org.apache.lucene.index.*; -import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; @@ -571,12 +570,8 @@ public enum MultiValueMode implements Writeable { * NOTE: Calling the returned instance on docs that are not root docs is illegal * The returned instance can only be evaluate the current and upcoming docs */ - public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet rootDocs, final DocIdSet innerDocSet, int maxDoc) throws IOException { - if (rootDocs == null || innerDocSet == null) { - return select(DocValues.emptySortedNumeric(maxDoc), missingValue); - } - final DocIdSetIterator innerDocs = innerDocSet.iterator(); - if (innerDocs == null) { + public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException { + if (rootDocs == null || innerDocs == null) { return select(DocValues.emptySortedNumeric(maxDoc), missingValue); } @@ -666,12 +661,8 @@ public enum MultiValueMode implements Writeable { * NOTE: Calling the returned instance on docs that are not root docs is illegal * The returned instance can only be evaluate the current and upcoming docs */ - public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet rootDocs, final DocIdSet innerDocSet, int maxDoc) throws IOException { - if (rootDocs == null || innerDocSet == null) { - return select(FieldData.emptySortedNumericDoubles(maxDoc), missingValue); - } - final DocIdSetIterator innerDocs = innerDocSet.iterator(); - if (innerDocs == null) { + public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException { + if (rootDocs == null || innerDocs == null) { return select(FieldData.emptySortedNumericDoubles(maxDoc), missingValue); } @@ -761,12 +752,8 @@ public enum MultiValueMode implements Writeable { * NOTE: Calling the returned instance on docs that are not root docs is illegal * The returned instance can only be evaluate the current and upcoming docs */ - public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet rootDocs, final DocIdSet innerDocSet, int maxDoc) throws IOException { - if (rootDocs == null || innerDocSet == null) { - return select(FieldData.emptySortedBinary(maxDoc), missingValue); - } - final DocIdSetIterator innerDocs = innerDocSet.iterator(); - if (innerDocs == null) { + public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException { + if (rootDocs == null || innerDocs == null) { return select(FieldData.emptySortedBinary(maxDoc), missingValue); } final BinaryDocValues selectedValues = select(values, null); @@ -861,12 +848,8 @@ public enum MultiValueMode implements Writeable { * NOTE: Calling the returned instance on docs that are not root docs is illegal * The returned instance can only be evaluate the current and upcoming docs */ - public SortedDocValues select(final RandomAccessOrds values, final BitSet rootDocs, final DocIdSet innerDocSet) throws IOException { - if (rootDocs == null || innerDocSet == null) { - return select(DocValues.emptySortedSet()); - } - final DocIdSetIterator innerDocs = innerDocSet.iterator(); - if (innerDocs == null) { + public SortedDocValues select(final RandomAccessOrds values, final BitSet rootDocs, final DocIdSetIterator innerDocs) throws IOException { + if (rootDocs == null || innerDocs == null) { return select(DocValues.emptySortedSet()); } final SortedDocValues selectedValues = select(values); diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index 7959eaa0a9a..b84a5804c05 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -73,6 +73,8 @@ import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityPars import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinality; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsParser; import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds; +import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidParser; +import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroid; import org.elasticsearch.search.aggregations.metrics.max.InternalMax; import org.elasticsearch.search.aggregations.metrics.max.MaxParser; import org.elasticsearch.search.aggregations.metrics.min.InternalMin; @@ -154,7 +156,6 @@ import java.util.Set; */ public class SearchModule extends AbstractModule { - private final Settings settings; private final Set> aggParsers = new HashSet<>(); private final Set> pipelineAggParsers = new HashSet<>(); private final Highlighters highlighters = new Highlighters(); @@ -167,19 +168,6 @@ public class SearchModule extends AbstractModule { // pkg private so tests can mock Class searchServiceImpl = SearchService.class; - public SearchModule(Settings settings) { - this.settings = settings; - } - - // TODO document public API - public void registerStream(SignificanceHeuristicStreams.Stream stream) { - SignificanceHeuristicStreams.registerStream(stream); - } - - public void registerStream(MovAvgModelStreams.Stream stream) { - MovAvgModelStreams.registerStream(stream); - } - public void registerHighlighter(String key, Class clazz) { highlighters.registerExtension(key, clazz); } @@ -288,6 +276,7 @@ public class SearchModule extends AbstractModule { multibinderAggParser.addBinding().to(ReverseNestedParser.class); multibinderAggParser.addBinding().to(TopHitsParser.class); multibinderAggParser.addBinding().to(GeoBoundsParser.class); + multibinderAggParser.addBinding().to(GeoCentroidParser.class); multibinderAggParser.addBinding().to(ScriptedMetricParser.class); multibinderAggParser.addBinding().to(ChildrenParser.class); for (Class parser : aggParsers) { @@ -357,6 +346,7 @@ public class SearchModule extends AbstractModule { InternalHDRPercentileRanks.registerStreams(); InternalCardinality.registerStreams(); InternalScriptedMetric.registerStreams(); + InternalGeoCentroid.registerStreams(); // buckets InternalGlobal.registerStreams(); diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index e7973494666..eb8414bb32e 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -19,15 +19,14 @@ package org.elasticsearch.search; +import com.carrotsearch.hppc.ObjectFloatHashMap; import com.carrotsearch.hppc.ObjectHashSet; import com.carrotsearch.hppc.ObjectSet; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.search.TopDocs; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cache.recycler.PageCacheRecycler; @@ -38,7 +37,6 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -46,8 +44,8 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; @@ -62,43 +60,33 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Loading; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; -import org.elasticsearch.index.query.TemplateQueryParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.search.stats.ShardSearchStats; import org.elasticsearch.index.search.stats.StatsGroupsParseElement; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.IndicesWarmer.TerminationHandle; -import org.elasticsearch.indices.IndicesWarmer.WarmerContext; import org.elasticsearch.indices.cache.request.IndicesRequestCache; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.Script.ScriptParseException; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.Template; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.dfs.DfsPhase; import org.elasticsearch.search.dfs.DfsSearchResult; -import org.elasticsearch.search.fetch.FetchPhase; -import org.elasticsearch.search.fetch.FetchSearchResult; -import org.elasticsearch.search.fetch.QueryFetchSearchResult; -import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult; -import org.elasticsearch.search.fetch.ShardFetchRequest; -import org.elasticsearch.search.internal.DefaultSearchContext; -import org.elasticsearch.search.internal.InternalScrollSearchRequest; -import org.elasticsearch.search.internal.ScrollContext; -import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.fetch.*; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.script.ScriptFieldsContext.ScriptField; +import org.elasticsearch.search.highlight.HighlightBuilder; +import org.elasticsearch.search.internal.*; import org.elasticsearch.search.internal.SearchContext.Lifetime; -import org.elasticsearch.search.internal.ShardSearchLocalRequest; -import org.elasticsearch.search.internal.ShardSearchRequest; -import org.elasticsearch.search.query.QueryPhase; -import org.elasticsearch.search.query.QuerySearchRequest; -import org.elasticsearch.search.query.QuerySearchResult; -import org.elasticsearch.search.query.QuerySearchResultProvider; -import org.elasticsearch.search.query.ScrollQuerySearchResult; +import org.elasticsearch.search.query.*; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.threadpool.ThreadPool; @@ -112,14 +100,13 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicLong; import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.common.Strings.hasLength; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes; /** * */ -public class SearchService extends AbstractLifecycleComponent { +public class SearchService extends AbstractLifecycleComponent implements IndexEventListener { public static final String NORMS_LOADING_KEY = "index.norms.loading"; public static final String DEFAULT_KEEPALIVE_KEY = "search.default_keep_alive"; @@ -173,27 +160,6 @@ public class SearchService extends AbstractLifecycleComponent { this.threadPool = threadPool; this.clusterService = clusterService; this.indicesService = indicesService; - indicesService.indicesLifecycle().addListener(new IndicesLifecycle.Listener() { - @Override - public void afterIndexClosed(Index index, @IndexSettings Settings indexSettings) { - // once an index is closed we can just clean up all the pending search context information - // to release memory and let references to the filesystem go etc. - IndexMetaData idxMeta = SearchService.this.clusterService.state().metaData().index(index.getName()); - if (idxMeta != null && idxMeta.state() == IndexMetaData.State.CLOSE) { - // we need to check if it's really closed - // since sometimes due to a relocation we already closed the shard and that causes the index to be closed - // if we then close all the contexts we can get some search failures along the way which are not expected. - // it's fine to keep the contexts open if the index is still "alive" - // unfortunately we don't have a clear way to signal today why an index is closed. - afterIndexDeleted(index, indexSettings); - } - } - - @Override - public void afterIndexDeleted(Index index, @IndexSettings Settings indexSettings) { - freeAllContextForIndex(index); - } - }); this.indicesWarmer = indicesWarmer; this.scriptService = scriptService; this.pageCacheRecycler = pageCacheRecycler; @@ -216,8 +182,8 @@ public class SearchService extends AbstractLifecycleComponent { this.keepAliveReaper = threadPool.scheduleWithFixedDelay(new Reaper(), keepAliveInterval); - this.indicesWarmer.addListener(new NormsWarmer()); - this.indicesWarmer.addListener(new FieldDataWarmer()); + this.indicesWarmer.addListener(new NormsWarmer(indicesWarmer)); + this.indicesWarmer.addListener(new FieldDataWarmer(indicesWarmer)); this.indicesWarmer.addListener(new SearchWarmer()); defaultSearchTimeout = settings.getAsTime(DEFAULT_SEARCH_TIMEOUT, NO_TIMEOUT); @@ -235,6 +201,26 @@ public class SearchService extends AbstractLifecycleComponent { } } + @Override + public void afterIndexClosed(Index index, Settings indexSettings) { + // once an index is closed we can just clean up all the pending search context information + // to release memory and let references to the filesystem go etc. + IndexMetaData idxMeta = SearchService.this.clusterService.state().metaData().index(index.getName()); + if (idxMeta != null && idxMeta.getState() == IndexMetaData.State.CLOSE) { + // we need to check if it's really closed + // since sometimes due to a relocation we already closed the shard and that causes the index to be closed + // if we then close all the contexts we can get some search failures along the way which are not expected. + // it's fine to keep the contexts open if the index is still "alive" + // unfortunately we don't have a clear way to signal today why an index is closed. + afterIndexDeleted(index, indexSettings); + } + } + + @Override + public void afterIndexDeleted(Index index, Settings indexSettings) { + freeAllContextForIndex(index); + } + protected void putContext(SearchContext context) { final SearchContext previous = activeContexts.put(context.id(), context); assert previous == null; @@ -567,15 +553,23 @@ public class SearchService extends AbstractLifecycleComponent { SearchContext context = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget, engineSearcher, indexService, indexShard, scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher, defaultSearchTimeout); SearchContext.setCurrent(context); + try { if (request.scroll() != null) { context.scrollContext(new ScrollContext()); context.scrollContext().scroll = request.scroll(); } - - parseTemplate(request, context); + if (request.template() != null) { + ExecutableScript executable = this.scriptService.executable(request.template(), ScriptContext.Standard.SEARCH, context); + BytesReference run = (BytesReference) executable.run(); + try (XContentParser parser = XContentFactory.xContent(run).createParser(run)) { + QueryParseContext queryParseContext = new QueryParseContext(indicesService.getIndicesQueryRegistry()); + queryParseContext.reset(parser); + queryParseContext.parseFieldMatcher(parseFieldMatcher); + parseSource(context, SearchSourceBuilder.parseSearchSource(parser, queryParseContext)); + } + } parseSource(context, request.source()); - parseSource(context, request.extraSource()); // if the from and size are still not set, default them if (context.from() == -1) { @@ -664,113 +658,222 @@ public class SearchService extends AbstractLifecycleComponent { } } - private void parseTemplate(ShardSearchRequest request, SearchContext searchContext) { - - BytesReference processedQuery; - if (request.template() != null) { - ExecutableScript executable = this.scriptService.executable(request.template(), ScriptContext.Standard.SEARCH, searchContext); - processedQuery = (BytesReference) executable.run(); - } else { - if (!hasLength(request.templateSource())) { - return; - } - XContentParser parser = null; - Template template = null; - - try { - parser = XContentFactory.xContent(request.templateSource()).createParser(request.templateSource()); - template = TemplateQueryParser.parse(parser, searchContext.parseFieldMatcher(), "params", "template"); - - if (template.getType() == ScriptService.ScriptType.INLINE) { - //Try to double parse for nested template id/file - parser = null; - try { - ExecutableScript executable = this.scriptService.executable(template, ScriptContext.Standard.SEARCH, searchContext); - processedQuery = (BytesReference) executable.run(); - parser = XContentFactory.xContent(processedQuery).createParser(processedQuery); - } catch (ElasticsearchParseException epe) { - //This was an non-nested template, the parse failure was due to this, it is safe to assume this refers to a file - //for backwards compatibility and keep going - template = new Template(template.getScript(), ScriptService.ScriptType.FILE, MustacheScriptEngineService.NAME, - null, template.getParams()); - ExecutableScript executable = this.scriptService.executable(template, ScriptContext.Standard.SEARCH, searchContext); - processedQuery = (BytesReference) executable.run(); - } - if (parser != null) { - try { - Template innerTemplate = TemplateQueryParser.parse(parser, searchContext.parseFieldMatcher()); - if (hasLength(innerTemplate.getScript()) && !innerTemplate.getType().equals(ScriptService.ScriptType.INLINE)) { - //An inner template referring to a filename or id - template = new Template(innerTemplate.getScript(), innerTemplate.getType(), - MustacheScriptEngineService.NAME, null, template.getParams()); - ExecutableScript executable = this.scriptService.executable(template, ScriptContext.Standard.SEARCH, - searchContext); - processedQuery = (BytesReference) executable.run(); - } - } catch (ScriptParseException e) { - // No inner template found, use original template from above - } - } - } else { - ExecutableScript executable = this.scriptService.executable(template, ScriptContext.Standard.SEARCH, searchContext); - processedQuery = (BytesReference) executable.run(); - } - } catch (IOException e) { - throw new ElasticsearchParseException("Failed to parse template", e); - } finally { - Releasables.closeWhileHandlingException(parser); - } - - if (!hasLength(template.getScript())) { - throw new ElasticsearchParseException("Template must have [template] field configured"); - } - } - request.source(processedQuery); - } - - private void parseSource(SearchContext context, BytesReference source) throws SearchParseException { + private void parseSource(SearchContext context, SearchSourceBuilder source) throws SearchContextException { // nothing to parse... - if (source == null || source.length() == 0) { + if (source == null) { return; } - XContentParser parser = null; - try { - parser = XContentFactory.xContent(source).createParser(source); - XContentParser.Token token; - token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("failed to parse search source. source must be an object, but found [{}] instead", token.name()); + final IndexShard indexShard = context.indexShard(); + QueryShardContext queryShardContext = indexShard.getQueryShardContext(); + context.from(source.from()); + context.size(source.size()); + ObjectFloatHashMap indexBoostMap = source.indexBoost(); + if (indexBoostMap != null) { + Float indexBoost = indexBoostMap.get(context.shardTarget().index()); + if (indexBoost != null) { + context.queryBoost(indexBoost); } - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - String fieldName = parser.currentName(); + } + if (source.query() != null) { + context.parsedQuery(queryShardContext.toQuery(source.query())); + } + if (source.postFilter() != null) { + context.parsedPostFilter(queryShardContext.toQuery(source.postFilter())); + } + if (source.sorts() != null) { + XContentParser completeSortParser = null; + try { + XContentBuilder completeSortBuilder = XContentFactory.jsonBuilder(); + completeSortBuilder.startObject(); + completeSortBuilder.startArray("sort"); + for (BytesReference sort : source.sorts()) { + XContentParser parser = XContentFactory.xContent(sort).createParser(sort); parser.nextToken(); - SearchParseElement element = elementParsers.get(fieldName); - if (element == null) { - throw new SearchParseException(context, "failed to parse search source. unknown search element [" + fieldName + "]", parser.getTokenLocation()); - } - element.parse(parser, context); - } else { - if (token == null) { - throw new ElasticsearchParseException("failed to parse search source. end of query source reached but query is not complete."); + completeSortBuilder.copyCurrentStructure(parser); + } + completeSortBuilder.endArray(); + completeSortBuilder.endObject(); + BytesReference completeSortBytes = completeSortBuilder.bytes(); + completeSortParser = XContentFactory.xContent(completeSortBytes).createParser(completeSortBytes); + completeSortParser.nextToken(); + completeSortParser.nextToken(); + completeSortParser.nextToken(); + this.elementParsers.get("sort").parse(completeSortParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = completeSortParser != null ? completeSortParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse sort source [" + sSource + "]", location, e); + } + } + context.trackScores(source.trackScores()); + if (source.minScore() != null) { + context.minimumScore(source.minScore()); + } + context.timeoutInMillis(source.timeoutInMillis()); + context.terminateAfter(source.terminateAfter()); + if (source.aggregations() != null) { + XContentParser completeAggregationsParser = null; + try { + XContentBuilder completeAggregationsBuilder = XContentFactory.jsonBuilder(); + completeAggregationsBuilder.startObject(); + for (BytesReference agg : source.aggregations()) { + XContentParser parser = XContentFactory.xContent(agg).createParser(agg); + parser.nextToken(); + parser.nextToken(); + completeAggregationsBuilder.field(parser.currentName()); + parser.nextToken(); + completeAggregationsBuilder.copyCurrentStructure(parser); + } + completeAggregationsBuilder.endObject(); + BytesReference completeAggregationsBytes = completeAggregationsBuilder.bytes(); + completeAggregationsParser = XContentFactory.xContent(completeAggregationsBytes).createParser(completeAggregationsBytes); + completeAggregationsParser.nextToken(); + this.elementParsers.get("aggregations").parse(completeAggregationsParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = completeAggregationsParser != null ? completeAggregationsParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse rescore source [" + sSource + "]", location, e); + } + } + if (source.suggest() != null) { + XContentParser suggestParser = null; + try { + suggestParser = XContentFactory.xContent(source.suggest()).createParser(source.suggest()); + suggestParser.nextToken(); + this.elementParsers.get("suggest").parse(suggestParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = suggestParser != null ? suggestParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e); + } + } + if (source.rescores() != null) { + XContentParser completeRescoreParser = null; + try { + XContentBuilder completeRescoreBuilder = XContentFactory.jsonBuilder(); + completeRescoreBuilder.startObject(); + completeRescoreBuilder.startArray("rescore"); + for (BytesReference rescore : source.rescores()) { + XContentParser parser = XContentFactory.xContent(rescore).createParser(rescore); + parser.nextToken(); + completeRescoreBuilder.copyCurrentStructure(parser); + } + completeRescoreBuilder.endArray(); + completeRescoreBuilder.endObject(); + BytesReference completeRescoreBytes = completeRescoreBuilder.bytes(); + completeRescoreParser = XContentFactory.xContent(completeRescoreBytes).createParser(completeRescoreBytes); + completeRescoreParser.nextToken(); + completeRescoreParser.nextToken(); + completeRescoreParser.nextToken(); + this.elementParsers.get("rescore").parse(completeRescoreParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = completeRescoreParser != null ? completeRescoreParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse rescore source [" + sSource + "]", location, e); + } + } + if (source.fields() != null) { + context.fieldNames().addAll(source.fields()); + } + if (source.explain() != null) { + context.explain(source.explain()); + } + if (source.fetchSource() != null) { + context.fetchSourceContext(source.fetchSource()); + } + if (source.fieldDataFields() != null) { + FieldDataFieldsContext fieldDataFieldsContext = context.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY); + for (String field : source.fieldDataFields()) { + fieldDataFieldsContext.add(new FieldDataField(field)); + } + fieldDataFieldsContext.setHitExecutionNeeded(true); + } + if (source.highlighter() != null) { + HighlightBuilder highlightBuilder = source.highlighter(); + try { + context.highlight(highlightBuilder.build(context.indexShard().getQueryShardContext())); + } catch (IOException e) { + throw new SearchContextException(context, "failed to create SearchContextHighlighter", e); + } + } + if (source.innerHits() != null) { + XContentParser innerHitsParser = null; + try { + innerHitsParser = XContentFactory.xContent(source.innerHits()).createParser(source.innerHits()); + innerHitsParser.nextToken(); + this.elementParsers.get("inner_hits").parse(innerHitsParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = innerHitsParser != null ? innerHitsParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e); + } + } + if (source.scriptFields() != null) { + for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) { + SearchScript searchScript = context.scriptService().search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH); + context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, field.ignoreFailure())); + } + } + if (source.ext() != null) { + XContentParser extParser = null; + try { + extParser = XContentFactory.xContent(source.ext()).createParser(source.ext()); + XContentParser.Token token = extParser.nextToken(); + String currentFieldName = null; + while ((token = extParser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = extParser.currentName(); } else { - throw new ElasticsearchParseException("failed to parse search source. expected field name but got [{}]", token); + SearchParseElement parseElement = this.elementParsers.get(currentFieldName); + if (parseElement == null) { + throw new SearchParseException(context, "Unknown element [" + currentFieldName + "] in [ext]", + extParser.getTokenLocation()); + } else { + parseElement.parse(extParser, context); + } } } + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = extParser != null ? extParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse ext source [" + sSource + "]", location, e); } - } catch (Throwable e) { - String sSource = "_na_"; - try { - sSource = XContentHelper.convertToJson(source, false); - } catch (Throwable e1) { - // ignore - } - XContentLocation location = parser != null ? parser.getTokenLocation() : null; - throw new SearchParseException(context, "failed to parse search source [" + sSource + "]", location, e); - } finally { - if (parser != null) { - parser.close(); - } + } + if (source.version() != null) { + context.version(source.version()); + } + if (source.stats() != null) { + context.groupStats(source.stats()); } } @@ -842,11 +945,15 @@ public class SearchService extends AbstractLifecycleComponent { return this.activeContexts.size(); } - static class NormsWarmer extends IndicesWarmer.Listener { + static class NormsWarmer implements IndicesWarmer.Listener { + private final IndicesWarmer indicesWarmer; + public NormsWarmer(IndicesWarmer indicesWarmer) { + this.indicesWarmer = indicesWarmer; + } @Override - public TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { - final Loading defaultLoading = Loading.parse(indexMetaData.settings().get(NORMS_LOADING_KEY), Loading.LAZY); + public TerminationHandle warmNewReaders(final IndexShard indexShard, final Engine.Searcher searcher) { + final Loading defaultLoading = Loading.parse(indexShard.getIndexSettings().getSettings().get(NORMS_LOADING_KEY), Loading.LAZY); final MapperService mapperService = indexShard.mapperService(); final ObjectSet warmUp = new ObjectHashSet<>(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { @@ -864,14 +971,14 @@ public class SearchService extends AbstractLifecycleComponent { final CountDownLatch latch = new CountDownLatch(1); // Norms loading may be I/O intensive but is not CPU intensive, so we execute it in a single task - threadPool.executor(executor()).execute(new Runnable() { + indicesWarmer.getExecutor().execute(new Runnable() { @Override public void run() { try { for (ObjectCursor stringObjectCursor : warmUp) { final String indexName = stringObjectCursor.value; final long start = System.nanoTime(); - for (final LeafReaderContext ctx : context.searcher().reader().leaves()) { + for (final LeafReaderContext ctx : searcher.reader().leaves()) { final NumericDocValues values = ctx.reader().getNormValues(indexName); if (values != null) { values.get(0); @@ -898,15 +1005,21 @@ public class SearchService extends AbstractLifecycleComponent { } @Override - public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) { + public TerminationHandle warmTopReader(IndexShard indexShard, final Engine.Searcher searcher) { return TerminationHandle.NO_WAIT; } } - static class FieldDataWarmer extends IndicesWarmer.Listener { + static class FieldDataWarmer implements IndicesWarmer.Listener { + + private final IndicesWarmer indicesWarmer; + + public FieldDataWarmer(IndicesWarmer indicesWarmer) { + this.indicesWarmer = indicesWarmer; + } @Override - public TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { + public TerminationHandle warmNewReaders(final IndexShard indexShard, final Engine.Searcher searcher) { final MapperService mapperService = indexShard.mapperService(); final Map warmUp = new HashMap<>(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { @@ -941,9 +1054,9 @@ public class SearchService extends AbstractLifecycleComponent { } } final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService(); - final Executor executor = threadPool.executor(executor()); - final CountDownLatch latch = new CountDownLatch(context.searcher().reader().leaves().size() * warmUp.size()); - for (final LeafReaderContext ctx : context.searcher().reader().leaves()) { + final Executor executor = indicesWarmer.getExecutor(); + final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size() * warmUp.size()); + for (final LeafReaderContext ctx : searcher.reader().leaves()) { for (final MappedFieldType fieldType : warmUp.values()) { executor.execute(new Runnable() { @@ -974,7 +1087,7 @@ public class SearchService extends AbstractLifecycleComponent { } @Override - public TerminationHandle warmTopReader(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { + public TerminationHandle warmTopReader(final IndexShard indexShard, final Engine.Searcher searcher) { final MapperService mapperService = indexShard.mapperService(); final Map warmUpGlobalOrdinals = new HashMap<>(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { @@ -1007,7 +1120,7 @@ public class SearchService extends AbstractLifecycleComponent { } } final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService(); - final Executor executor = threadPool.executor(executor()); + final Executor executor = indicesWarmer.getExecutor(); final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size()); for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) { executor.execute(new Runnable() { @@ -1016,7 +1129,7 @@ public class SearchService extends AbstractLifecycleComponent { try { final long start = System.nanoTime(); IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType); - ifd.loadGlobal(context.getDirectoryReader()); + ifd.loadGlobal(searcher.getDirectoryReader()); if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard.warmerService().logger().trace("warmed global ordinals for [{}], took [{}]", fieldType.names().fullName(), TimeValue.timeValueNanos(System.nanoTime() - start)); } @@ -1037,74 +1150,73 @@ public class SearchService extends AbstractLifecycleComponent { } } - class SearchWarmer extends IndicesWarmer.Listener { + class SearchWarmer implements IndicesWarmer.Listener { @Override - public TerminationHandle warmNewReaders(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) { - return internalWarm(indexShard, indexMetaData, context, threadPool, false); + public TerminationHandle warmNewReaders(IndexShard indexShard, final Engine.Searcher searcher) { + return internalWarm(indexShard, searcher, false); } @Override - public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) { - return internalWarm(indexShard, indexMetaData, context, threadPool, true); + public TerminationHandle warmTopReader(IndexShard indexShard, final Engine.Searcher searcher) { + return internalWarm(indexShard, searcher, true); } - public TerminationHandle internalWarm(final IndexShard indexShard, final IndexMetaData indexMetaData, final IndicesWarmer.WarmerContext warmerContext, ThreadPool threadPool, final boolean top) { - IndexWarmersMetaData custom = indexMetaData.custom(IndexWarmersMetaData.TYPE); + public TerminationHandle internalWarm(final IndexShard indexShard, final Engine.Searcher searcher, final boolean top) { + IndexWarmersMetaData custom = indexShard.getIndexSettings().getIndexMetaData().custom(IndexWarmersMetaData.TYPE); if (custom == null) { return TerminationHandle.NO_WAIT; } - final Executor executor = threadPool.executor(executor()); + final Executor executor = indicesWarmer.getExecutor(); final CountDownLatch latch = new CountDownLatch(custom.entries().size()); for (final IndexWarmersMetaData.Entry entry : custom.entries()) { - executor.execute(new Runnable() { - - @Override - public void run() { - SearchContext context = null; + executor.execute(() -> { + SearchContext context = null; + try { + long now = System.nanoTime(); + final IndexService indexService = indicesService.indexServiceSafe(indexShard.shardId().index().name()); + QueryParseContext queryParseContext = new QueryParseContext(indicesService.getIndicesQueryRegistry()); + queryParseContext.parseFieldMatcher(indexService.getIndexSettings().getParseFieldMatcher()); + ShardSearchRequest request = new ShardSearchLocalRequest(indexShard.shardId(), indexShard.getIndexSettings() + .getNumberOfShards(), + SearchType.QUERY_THEN_FETCH, entry.source().build(queryParseContext), entry.types(), entry.requestCache()); + context = createContext(request, searcher); + // if we use sort, we need to do query to sort on + // it and load relevant field data + // if not, we might as well set size=0 (and cache + // if needed) + if (context.sort() == null) { + context.size(0); + } + boolean canCache = indicesQueryCache.canCache(request, context); + // early terminate when we can cache, since we + // can only do proper caching on top level searcher + // also, if we can't cache, and its top, we don't + // need to execute it, since we already did when its + // not top + if (canCache != top) { + return; + } + loadOrExecuteQueryPhase(request, context, queryPhase); + long took = System.nanoTime() - now; + if (indexShard.warmerService().logger().isTraceEnabled()) { + indexShard.warmerService().logger().trace("warmed [{}], took [{}]", entry.name(), TimeValue.timeValueNanos(took)); + } + } catch (Throwable t) { + indexShard.warmerService().logger().warn("warmer [{}] failed", t, entry.name()); + } finally { try { - long now = System.nanoTime(); - ShardSearchRequest request = new ShardSearchLocalRequest(indexShard.shardId(), indexMetaData.numberOfShards(), - SearchType.QUERY_THEN_FETCH, entry.source(), entry.types(), entry.requestCache()); - context = createContext(request, warmerContext.searcher()); - // if we use sort, we need to do query to sort on it and load relevant field data - // if not, we might as well set size=0 (and cache if needed) - if (context.sort() == null) { - context.size(0); + if (context != null) { + freeContext(context.id()); + cleanContext(context); } - boolean canCache = indicesQueryCache.canCache(request, context); - // early terminate when we can cache, since we can only do proper caching on top level searcher - // also, if we can't cache, and its top, we don't need to execute it, since we already did when its not top - if (canCache != top) { - return; - } - loadOrExecuteQueryPhase(request, context, queryPhase); - long took = System.nanoTime() - now; - if (indexShard.warmerService().logger().isTraceEnabled()) { - indexShard.warmerService().logger().trace("warmed [{}], took [{}]", entry.name(), TimeValue.timeValueNanos(took)); - } - } catch (Throwable t) { - indexShard.warmerService().logger().warn("warmer [{}] failed", t, entry.name()); } finally { - try { - if (context != null) { - freeContext(context.id()); - cleanContext(context); - } - } finally { - latch.countDown(); - } + latch.countDown(); } } - }); } - return new TerminationHandle() { - @Override - public void awaitTermination() throws InterruptedException { - latch.await(); - } - }; + return () -> latch.await(); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index 13a162df7fc..dd9c5a30a9b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -55,6 +55,8 @@ import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityBuilder; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBounds; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsBuilder; +import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroid; +import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidBuilder; import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.search.aggregations.metrics.max.MaxBuilder; import org.elasticsearch.search.aggregations.metrics.min.Min; @@ -287,6 +289,13 @@ public class AggregationBuilders { return new GeoBoundsBuilder(name); } + /** + * Create a new {@link GeoCentroid} aggregation with the given name. + */ + public static GeoCentroidBuilder geoCentroid(String name) { + return new GeoCentroidBuilder(name); + } + /** * Create a new {@link ScriptedMetric} aggregation with the given name. */ diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java index 3841030240e..9b23997b6d4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java @@ -194,7 +194,7 @@ public class InternalAggregations implements Aggregations, ToXContent, Streamabl } public static InternalAggregations readOptionalAggregations(StreamInput in) throws IOException { - return in.readOptionalStreamable(new InternalAggregations()); + return in.readOptionalStreamable(InternalAggregations::new); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java index a7f01e4414c..ab655497c4c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java @@ -40,7 +40,7 @@ import java.util.Map; */ public abstract class BucketsAggregator extends AggregatorBase { - protected final BigArrays bigArrays; + private final BigArrays bigArrays; private IntArray docCounts; public BucketsAggregator(String name, AggregatorFactories factories, AggregationContext context, Aggregator parent, @@ -67,7 +67,7 @@ public abstract class BucketsAggregator extends AggregatorBase { /** * Utility method to collect the given doc in the given bucket (identified by the bucket ordinal) */ - public void collectBucket(LeafBucketCollector subCollector, int doc, long bucketOrd) throws IOException { + public final void collectBucket(LeafBucketCollector subCollector, int doc, long bucketOrd) throws IOException { grow(bucketOrd + 1); collectExistingBucket(subCollector, doc, bucketOrd); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenParser.java index 14545a10f32..2c597bdcee9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenParser.java @@ -18,8 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket.children; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Query; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.index.mapper.DocumentMapper; @@ -75,8 +74,8 @@ public class ChildrenParser implements Aggregator.Parser { DocumentMapper childDocMapper = context.mapperService().documentMapper(childType); String parentType = null; - Filter parentFilter = null; - Filter childFilter = null; + Query parentFilter = null; + Query childFilter = null; if (childDocMapper != null) { ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper(); if (!parentFieldMapper.active()) { @@ -86,8 +85,8 @@ public class ChildrenParser implements Aggregator.Parser { DocumentMapper parentDocMapper = context.mapperService().documentMapper(parentType); if (parentDocMapper != null) { // TODO: use the query API - parentFilter = new QueryWrapperFilter(parentDocMapper.typeFilter()); - childFilter = new QueryWrapperFilter(childDocMapper.typeFilter()); + parentFilter = parentDocMapper.typeFilter(); + childFilter = childDocMapper.typeFilter(); ParentChildIndexFieldData parentChildIndexFieldData = context.fieldData().getForField(parentFieldMapper.fieldType()); config.fieldContext(new FieldContext(parentFieldMapper.fieldType().names().indexName(), parentChildIndexFieldData, parentFieldMapper.fieldType())); } else { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java index dc98416ab13..6d9a1edc712 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java @@ -68,7 +68,7 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { private Set replay = new LinkedHashSet<>(); public ParentToChildrenAggregator(String name, AggregatorFactories factories, AggregationContext aggregationContext, - Aggregator parent, String parentType, Filter childFilter, Filter parentFilter, + Aggregator parent, String parentType, Query childFilter, Query parentFilter, ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, long maxOrd, List pipelineAggregators, Map metaData) throws IOException { super(name, factories, aggregationContext, parent, pipelineAggregators, metaData); @@ -185,10 +185,10 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { public static class Factory extends ValuesSourceAggregatorFactory { private final String parentType; - private final Filter parentFilter; - private final Filter childFilter; + private final Query parentFilter; + private final Query childFilter; - public Factory(String name, ValuesSourceConfig config, String parentType, Filter parentFilter, Filter childFilter) { + public Factory(String name, ValuesSourceConfig config, String parentType, Query parentFilter, Query childFilter) { super(name, InternalChildren.TYPE.name(), config); this.parentType = parentType; this.parentFilter = parentFilter; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java index b0a31695ff9..192d624b5e0 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java @@ -40,7 +40,7 @@ public class FilterParser implements Aggregator.Parser { @Override public AggregatorFactory parse(String aggregationName, XContentParser parser, SearchContext context) throws IOException { - ParsedQuery filter = context.queryParserService().parseInnerFilter(parser); + ParsedQuery filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); return new FilterAggregator.Factory(aggregationName, filter == null ? new MatchAllDocsQuery() : filter.query()); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java index a80feac5842..8ed37078012 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java @@ -82,7 +82,7 @@ public class FiltersParser implements Aggregator.Parser { if (token == XContentParser.Token.FIELD_NAME) { key = parser.currentName(); } else { - ParsedQuery filter = context.queryParserService().parseInnerFilter(parser); + ParsedQuery filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); filters.add(new FiltersAggregator.KeyedFilter(key, filter == null ? Queries.newMatchAllQuery() : filter.query())); } } @@ -95,7 +95,7 @@ public class FiltersParser implements Aggregator.Parser { keyed = false; int idx = 0; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - ParsedQuery filter = context.queryParserService().parseInnerFilter(parser); + ParsedQuery filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); filters.add(new FiltersAggregator.KeyedFilter(String.valueOf(idx), filter == null ? Queries.newMatchAllQuery() : filter.query())); idx++; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java index 2f9856ad594..6456fba8640 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGrid.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import java.util.List; @@ -30,10 +29,9 @@ import java.util.List; public interface GeoHashGrid extends MultiBucketsAggregation { /** - * A bucket that is associated with a {@code geohash_grid} cell. The key of the bucket is the {@cod geohash} of the cell + * A bucket that is associated with a {@code geohash_grid} cell. The key of the bucket is the {@code geohash} of the cell */ - public static interface Bucket extends MultiBucketsAggregation.Bucket { - public GeoPoint getCentroid(); + interface Bucket extends MultiBucketsAggregation.Bucket { } /** diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java index 41af33ddd42..343d335cfa2 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java @@ -20,7 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.LongArray; @@ -51,7 +51,6 @@ public class GeoHashGridAggregator extends BucketsAggregator { private final int shardSize; private final GeoHashGridParser.GeoGridFactory.CellIdSource valuesSource; private final LongHash bucketOrds; - private LongArray bucketCentroids; public GeoHashGridAggregator(String name, AggregatorFactories factories, GeoHashGridParser.GeoGridFactory.CellIdSource valuesSource, int requiredSize, int shardSize, AggregationContext aggregationContext, Aggregator parent, List pipelineAggregators, @@ -61,7 +60,6 @@ public class GeoHashGridAggregator extends BucketsAggregator { this.requiredSize = requiredSize; this.shardSize = shardSize; bucketOrds = new LongHash(1, aggregationContext.bigArrays()); - bucketCentroids = aggregationContext.bigArrays().newLongArray(1, true); } @Override @@ -69,28 +67,6 @@ public class GeoHashGridAggregator extends BucketsAggregator { return (valuesSource != null && valuesSource.needsScores()) || super.needsScores(); } - @Override - public void collectBucket(LeafBucketCollector subCollector, int doc, long bucketOrd) throws IOException { - bucketCentroids = bigArrays.grow(bucketCentroids, bucketOrd + 1); - super.collectBucket(subCollector, doc, bucketOrd); - } - - protected final void adjustCentroid(long bucketOrd, long geohash) { - final int numDocs = getDocCounts().get(bucketOrd); - final GeoPoint oldCentroid = new GeoPoint(); - final GeoPoint nextLoc = new GeoPoint(); - - if (numDocs > 1) { - final long curCentroid = bucketCentroids.get(bucketOrd); - oldCentroid.resetFromGeoHash(curCentroid); - nextLoc.resetFromGeoHash(geohash); - bucketCentroids.set(bucketOrd, XGeoHashUtils.longEncode(oldCentroid.lon() + (nextLoc.lon() - oldCentroid.lon()) / numDocs, - oldCentroid.lat() + (nextLoc.lat() - oldCentroid.lat()) / numDocs, XGeoHashUtils.PRECISION)); - } else { - bucketCentroids.set(bucketOrd, geohash); - } - } - @Override public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { @@ -104,8 +80,7 @@ public class GeoHashGridAggregator extends BucketsAggregator { long previous = Long.MAX_VALUE; for (int i = 0; i < valuesCount; ++i) { - final long valFullRes = values.valueAt(i); - final long val = XGeoHashUtils.longEncode(valFullRes, valuesSource.precision()); + final long val = values.valueAt(i); if (previous != val || i == 0) { long bucketOrdinal = bucketOrds.add(val); if (bucketOrdinal < 0) { // already seen @@ -114,7 +89,6 @@ public class GeoHashGridAggregator extends BucketsAggregator { } else { collectBucket(sub, doc, bucketOrdinal); } - adjustCentroid(bucketOrdinal, valFullRes); previous = val; } } @@ -128,7 +102,7 @@ public class GeoHashGridAggregator extends BucketsAggregator { long bucketOrd; public OrdinalBucket() { - super(0, 0, new GeoPoint(), (InternalAggregations) null); + super(0, 0, (InternalAggregations) null); } } @@ -146,7 +120,6 @@ public class GeoHashGridAggregator extends BucketsAggregator { } spare.geohashAsLong = bucketOrds.get(i); - spare.centroid.resetFromGeoHash(bucketCentroids.get(i)); spare.docCount = bucketDocCount(i); spare.bucketOrd = i; spare = (OrdinalBucket) ordered.insertWithOverflow(spare); @@ -170,7 +143,6 @@ public class GeoHashGridAggregator extends BucketsAggregator { @Override public void doClose() { Releasables.close(bucketOrds); - Releasables.close(bucketCentroids); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridBuilder.java index 9f382d86906..a1f12f465ca 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridBuilder.java @@ -30,8 +30,8 @@ public class GeoHashGridBuilder extends AggregationBuilder { private String field; - private int precision = GeoHashGridParser.DEFAULT_PRECISION; - private int requiredSize = GeoHashGridParser.DEFAULT_MAX_NUM_CELLS; + private int precision = GeoHashGridParams.DEFAULT_PRECISION; + private int requiredSize = GeoHashGridParams.DEFAULT_MAX_NUM_CELLS; private int shardSize = 0; /** @@ -54,11 +54,7 @@ public class GeoHashGridBuilder extends AggregationBuilder { * precision, the more fine-grained this aggregation will be. */ public GeoHashGridBuilder precision(int precision) { - if ((precision < 1) || (precision > 12)) { - throw new IllegalArgumentException("Invalid geohash aggregation precision of " + precision - + "must be between 1 and 12"); - } - this.precision = precision; + this.precision = GeoHashGridParams.checkPrecision(precision); return this; } @@ -85,14 +81,14 @@ public class GeoHashGridBuilder extends AggregationBuilder { if (field != null) { builder.field("field", field); } - if (precision != GeoHashGridParser.DEFAULT_PRECISION) { - builder.field("precision", precision); + if (precision != GeoHashGridParams.DEFAULT_PRECISION) { + builder.field(GeoHashGridParams.FIELD_PRECISION.getPreferredName(), precision); } - if (requiredSize != GeoHashGridParser.DEFAULT_MAX_NUM_CELLS) { - builder.field("size", requiredSize); + if (requiredSize != GeoHashGridParams.DEFAULT_MAX_NUM_CELLS) { + builder.field(GeoHashGridParams.FIELD_SIZE.getPreferredName(), requiredSize); } if (shardSize != 0) { - builder.field("shard_size", shardSize); + builder.field(GeoHashGridParams.FIELD_SHARD_SIZE.getPreferredName(), shardSize); } return builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParams.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParams.java new file mode 100644 index 00000000000..640418920d2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParams.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.ParseField; + +/** + * Encapsulates relevant parameter defaults and validations for the geo hash grid aggregation. + */ +final class GeoHashGridParams { + /* default values */ + public static final int DEFAULT_PRECISION = 5; + public static final int DEFAULT_MAX_NUM_CELLS = 10000; + + /* recognized field names in JSON */ + public static final ParseField FIELD_PRECISION = new ParseField("precision"); + public static final ParseField FIELD_SIZE = new ParseField("size"); + public static final ParseField FIELD_SHARD_SIZE = new ParseField("shard_size"); + + + public static int checkPrecision(int precision) { + if ((precision < 1) || (precision > 12)) { + throw new IllegalArgumentException("Invalid geohash aggregation precision of " + precision + + ". Must be between 1 and 12."); + } + return precision; + } + + private GeoHashGridParams() { + throw new AssertionError("No instances intended"); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java index 0025880a580..109301fdbff 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java @@ -20,7 +20,8 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.MultiGeoPointValues; @@ -28,6 +29,7 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingNumericDocValues; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.InternalAggregation; @@ -58,16 +60,13 @@ public class GeoHashGridParser implements Aggregator.Parser { return InternalGeoHashGrid.TYPE.name(); } - public static final int DEFAULT_PRECISION = 5; - public static final int DEFAULT_MAX_NUM_CELLS = 10000; - @Override public AggregatorFactory parse(String aggregationName, XContentParser parser, SearchContext context) throws IOException { ValuesSourceParser vsParser = ValuesSourceParser.geoPoint(aggregationName, InternalGeoHashGrid.TYPE, context).build(); - int precision = DEFAULT_PRECISION; - int requiredSize = DEFAULT_MAX_NUM_CELLS; + int precision = GeoHashGridParams.DEFAULT_PRECISION; + int requiredSize = GeoHashGridParams.DEFAULT_MAX_NUM_CELLS; int shardSize = -1; XContentParser.Token token; @@ -77,14 +76,18 @@ public class GeoHashGridParser implements Aggregator.Parser { currentFieldName = parser.currentName(); } else if (vsParser.token(currentFieldName, token, parser)) { continue; - } else if (token == XContentParser.Token.VALUE_NUMBER) { - if ("precision".equals(currentFieldName)) { - precision = parser.intValue(); - } else if ("size".equals(currentFieldName)) { + } else if (token == XContentParser.Token.VALUE_NUMBER || + token == XContentParser.Token.VALUE_STRING) { //Be lenient and also allow numbers enclosed in quotes + if (context.parseFieldMatcher().match(currentFieldName, GeoHashGridParams.FIELD_PRECISION)) { + precision = GeoHashGridParams.checkPrecision(parser.intValue()); + } else if (context.parseFieldMatcher().match(currentFieldName, GeoHashGridParams.FIELD_SIZE)) { requiredSize = parser.intValue(); - } else if ("shard_size".equals(currentFieldName) || "shardSize".equals(currentFieldName)) { + } else if (context.parseFieldMatcher().match(currentFieldName, GeoHashGridParams.FIELD_SHARD_SIZE)) { shardSize = parser.intValue(); } + } else if (token != XContentParser.Token.START_OBJECT) { + throw new SearchParseException(context, "Unexpected token " + token + " in [" + aggregationName + "].", + parser.getTokenLocation()); } } @@ -112,9 +115,9 @@ public class GeoHashGridParser implements Aggregator.Parser { static class GeoGridFactory extends ValuesSourceAggregatorFactory { - private int precision; - private int requiredSize; - private int shardSize; + private final int precision; + private final int requiredSize; + private final int shardSize; public GeoGridFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize) { super(name, InternalGeoHashGrid.TYPE.name(), config); @@ -150,9 +153,11 @@ public class GeoHashGridParser implements Aggregator.Parser { private static class CellValues extends SortingNumericDocValues { private MultiGeoPointValues geoValues; + private int precision; - protected CellValues(MultiGeoPointValues geoValues) { + protected CellValues(MultiGeoPointValues geoValues, int precision) { this.geoValues = geoValues; + this.precision = precision; } @Override @@ -161,7 +166,7 @@ public class GeoHashGridParser implements Aggregator.Parser { resize(geoValues.count()); for (int i = 0; i < count(); ++i) { GeoPoint target = geoValues.valueAt(i); - values[i] = XGeoHashUtils.longEncode(target.getLon(), target.getLat(), XGeoHashUtils.PRECISION); + values[i] = GeoHashUtils.longEncode(target.getLon(), target.getLat(), precision); } sort(); } @@ -188,7 +193,7 @@ public class GeoHashGridParser implements Aggregator.Parser { @Override public SortedNumericDocValues longValues(LeafReaderContext ctx) { - return new CellValues(valuesSource.geoPointValues(ctx)); + return new CellValues(valuesSource.geoPointValues(ctx), precision); } @Override @@ -203,5 +208,4 @@ public class GeoHashGridParser implements Aggregator.Parser { } } - } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java index 50c1d733b4e..75d089ebbc8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -87,23 +87,21 @@ public class InternalGeoHashGrid extends InternalMultiBucketAggregation buckets, ReduceContext context) { List aggregationsList = new ArrayList<>(buckets.size()); long docCount = 0; - double cLon = 0; - double cLat = 0; for (Bucket bucket : buckets) { docCount += bucket.docCount; - cLon += (bucket.docCount * bucket.centroid.lon()); - cLat += (bucket.docCount * bucket.centroid.lat()); aggregationsList.add(bucket.aggregations); } final InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context); - return new Bucket(geohashAsLong, docCount, new GeoPoint(cLat/docCount, cLon/docCount), aggs); + return new Bucket(geohashAsLong, docCount, aggs); } @Override public void readFrom(StreamInput in) throws IOException { geohashAsLong = in.readLong(); docCount = in.readVLong(); - centroid = GeoPoint.fromGeohash(in.readLong()); aggregations = InternalAggregations.readAggregations(in); } @@ -164,7 +152,6 @@ public class InternalGeoHashGrid extends InternalMultiBucketAggregation 0; } } - - public static final class GeoFields { - public static final XContentBuilderString CENTROID = new XContentBuilderString("centroid"); - } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java index c47c519d503..d2ca0a9121a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java @@ -173,7 +173,7 @@ public class HistogramAggregator extends BucketsAggregator { @Override protected Aggregator createUnmapped(AggregationContext aggregationContext, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { - return new HistogramAggregator(name, factories, rounding, order, keyed, minDocCount, null, null, config.formatter(), + return new HistogramAggregator(name, factories, rounding, order, keyed, minDocCount, extendedBounds, null, config.formatter(), histogramFactory, aggregationContext, parent, pipelineAggregators, metaData); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeBuilder.java index acb55f68ea5..c7723972b07 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeBuilder.java @@ -41,7 +41,7 @@ public class RangeBuilder extends AbstractRangeBuilder { * * @param key the key to use for this range in the response * @param from the lower bound on the distances, inclusive - * @parap to the upper bound on the distances, exclusive + * @param to the upper bound on the distances, exclusive */ public RangeBuilder addRange(String key, double from, double to) { ranges.add(new Range(key, from, to)); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeBuilder.java index 35c8a3011d1..4bd57580fae 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeBuilder.java @@ -42,7 +42,7 @@ public class DateRangeBuilder extends AbstractRangeBuilder { * * @param key the key to use for this range in the response * @param from the lower bound on the distances, inclusive - * @parap to the upper bound on the distances, exclusive + * @param to the upper bound on the distances, exclusive */ public DateRangeBuilder addRange(String key, Object from, Object to) { ranges.add(new Range(key, from, to)); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceBuilder.java index e6649fa6a90..ae8fc22d769 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceBuilder.java @@ -168,7 +168,7 @@ public class GeoDistanceBuilder extends AggregationBuilder { * * @param key the key to use for this range in the response * @param from the lower bound on the distances, inclusive - * @parap to the upper bound on the distances, exclusive + * @param to the upper bound on the distances, exclusive */ public GeoDistanceBuilder addRange(String key, double from, double to) { ranges.add(new Range(key, from, to)); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java index 218f0dcbc90..5ac3f2a6d4e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java @@ -19,18 +19,15 @@ package org.elasticsearch.search.aggregations.bucket.range.ipv4; +import org.elasticsearch.common.network.Cidrs; import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilderException; -import java.util.regex.Pattern; - /** * Builder for the {@code IPv4Range} aggregation. */ public class IPv4RangeBuilder extends AbstractRangeBuilder { - private static final Pattern MASK_PATTERN = Pattern.compile("[\\.|/]"); - /** * Sole constructor. */ @@ -43,7 +40,7 @@ public class IPv4RangeBuilder extends AbstractRangeBuilder { * * @param key the key to use for this range in the response * @param from the lower bound on the distances, inclusive - * @parap to the upper bound on the distances, exclusive + * @param to the upper bound on the distances, exclusive */ public IPv4RangeBuilder addRange(String key, String from, String to) { ranges.add(new Range(key, from, to)); @@ -61,11 +58,13 @@ public class IPv4RangeBuilder extends AbstractRangeBuilder { * Add a range based on a CIDR mask. */ public IPv4RangeBuilder addMaskRange(String key, String mask) { - long[] fromTo = cidrMaskToMinMax(mask); - if (fromTo == null) { - throw new SearchSourceBuilderException("invalid CIDR mask [" + mask + "] in ip_range aggregation [" + getName() + "]"); + long[] fromTo; + try { + fromTo = Cidrs.cidrMaskToMinMax(mask); + } catch (IllegalArgumentException e) { + throw new SearchSourceBuilderException("invalid CIDR mask [" + mask + "] in ip_range aggregation [" + getName() + "]", e); } - ranges.add(new Range(key, fromTo[0] < 0 ? null : fromTo[0], fromTo[1] < 0 ? null : fromTo[1])); + ranges.add(new Range(key, fromTo[0] == 0 ? null : fromTo[0], fromTo[1] == InternalIPv4Range.MAX_IP ? null : fromTo[1])); return this; } @@ -108,59 +107,4 @@ public class IPv4RangeBuilder extends AbstractRangeBuilder { public IPv4RangeBuilder addUnboundedFrom(String from) { return addUnboundedFrom(null, from); } - - /** - * Computes the min & max ip addresses (represented as long values - same way as stored in index) represented by the given CIDR mask - * expression. The returned array has the length of 2, where the first entry represents the {@code min} address and the second the {@code max}. - * A {@code -1} value for either the {@code min} or the {@code max}, represents an unbounded end. In other words: - * - *

    - * {@code min == -1 == "0.0.0.0" } - *

    - * - * and - * - *

    - * {@code max == -1 == "255.255.255.255" } - *

    - */ - static long[] cidrMaskToMinMax(String cidr) { - String[] parts = MASK_PATTERN.split(cidr); - if (parts.length != 5) { - return null; - } - int addr = (( Integer.parseInt(parts[0]) << 24 ) & 0xFF000000) - | (( Integer.parseInt(parts[1]) << 16 ) & 0xFF0000) - | (( Integer.parseInt(parts[2]) << 8 ) & 0xFF00) - | ( Integer.parseInt(parts[3]) & 0xFF); - - int mask = (-1) << (32 - Integer.parseInt(parts[4])); - - if (Integer.parseInt(parts[4]) == 0) { - mask = 0 << 32; - } - - int from = addr & mask; - long longFrom = intIpToLongIp(from); - if (longFrom == 0) { - longFrom = -1; - } - - int to = from + (~mask); - long longTo = intIpToLongIp(to) + 1; // we have to +1 here as the range is non-inclusive on the "to" side - - if (longTo == InternalIPv4Range.MAX_IP) { - longTo = -1; - } - - return new long[] { longFrom, longTo }; - } - - private static long intIpToLongIp(int i) { - long p1 = ((long) ((i >> 24 ) & 0xFF)) << 24; - int p2 = ((i >> 16 ) & 0xFF) << 16; - int p3 = ((i >> 8 ) & 0xFF) << 8; - int p4 = i & 0xFF; - return p1 + p2 + p3 + p4; - } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java index 65be35d1350..a50c1c109f3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java @@ -32,12 +32,13 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import static org.elasticsearch.index.mapper.ip.IpFieldMapper.MAX_IP; + /** * */ public class InternalIPv4Range extends InternalRange { - - public static final long MAX_IP = 4294967296l; + public static final long MAX_IP = 1L << 32; public final static Type TYPE = new Type("ip_range", "iprange"); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java index 37891f6f239..8b0862fed29 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java @@ -18,7 +18,9 @@ */ package org.elasticsearch.search.aggregations.bucket.range.ipv4; +import org.elasticsearch.common.network.Cidrs; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.ip.IpFieldMapper; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -124,13 +126,15 @@ public class IpRangeParser implements Aggregator.Parser { } private static void parseMaskRange(String cidr, RangeAggregator.Range range, String aggregationName, SearchContext ctx) { - long[] fromTo = IPv4RangeBuilder.cidrMaskToMinMax(cidr); - if (fromTo == null) { + long[] fromTo; + try { + fromTo = Cidrs.cidrMaskToMinMax(cidr); + } catch (IllegalArgumentException e) { throw new SearchParseException(ctx, "invalid CIDR mask [" + cidr + "] in aggregation [" + aggregationName + "]", - null); + null, e); } - range.from = fromTo[0] < 0 ? Double.NEGATIVE_INFINITY : fromTo[0]; - range.to = fromTo[1] < 0 ? Double.POSITIVE_INFINITY : fromTo[1]; + range.from = fromTo[0] == 0 ? Double.NEGATIVE_INFINITY : fromTo[0]; + range.to = fromTo[1] == InternalIPv4Range.MAX_IP ? Double.POSITIVE_INFINITY : fromTo[1]; if (range.key == null) { range.key = cidr; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java index 27d9f58a7ff..21b92e83fff 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java @@ -173,7 +173,7 @@ public abstract class InternalSignificantTerms terms = (InternalSignificantTerms) aggregation; for (Bucket bucket : terms.buckets) { - List existingBuckets = buckets.get(bucket.getKey()); + List existingBuckets = buckets.get(bucket.getKeyAsString()); if (existingBuckets == null) { existingBuckets = new ArrayList<>(aggregations.size()); buckets.put(bucket.getKeyAsString(), existingBuckets); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java index 9768a8617da..020229867f6 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java @@ -66,7 +66,7 @@ public class SignificantTermsParametersParser extends AbstractTermsParametersPar if (significanceHeuristicParser != null) { significanceHeuristic = significanceHeuristicParser.parse(parser, context.parseFieldMatcher(), context); } else if (context.parseFieldMatcher().match(currentFieldName, BACKGROUND_FILTER)) { - filter = context.queryParserService().parseInnerFilter(parser).query(); + filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser).query(); } else { throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].", parser.getTokenLocation()); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicStreams.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicStreams.java index 18a6f6b93dd..64d2ae659e0 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicStreams.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/SignificanceHeuristicStreams.java @@ -31,7 +31,7 @@ import java.util.Map; */ public class SignificanceHeuristicStreams { - private static Map STREAMS = Collections.EMPTY_MAP; + private static Map STREAMS = Collections.emptyMap(); static { HashMap map = new HashMap<>(); @@ -79,7 +79,7 @@ public class SignificanceHeuristicStreams { * @param name The given name * @return The associated stream */ - public static synchronized Stream stream(String name) { + private static synchronized Stream stream(String name) { return STREAMS.get(name); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroid.java new file mode 100644 index 00000000000..2cdf462f042 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroid.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.geocentroid; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.search.aggregations.Aggregation; + +/** + * Interface for {@link org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregator} + */ +public interface GeoCentroid extends Aggregation { + GeoPoint centroid(); + long count(); +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java new file mode 100644 index 00000000000..3834a65a3ad --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.geocentroid; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.GeoHashUtils; +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.metrics.MetricsAggregator; +import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * A geo metric aggregator that computes a geo-centroid from a {@code geo_point} type field + */ +public final class GeoCentroidAggregator extends MetricsAggregator { + private final ValuesSource.GeoPoint valuesSource; + LongArray centroids; + LongArray counts; + + protected GeoCentroidAggregator(String name, AggregationContext aggregationContext, Aggregator parent, + ValuesSource.GeoPoint valuesSource, List pipelineAggregators, + Map metaData) throws IOException { + super(name, aggregationContext, parent, pipelineAggregators, metaData); + this.valuesSource = valuesSource; + if (valuesSource != null) { + final BigArrays bigArrays = context.bigArrays(); + centroids = bigArrays.newLongArray(1, true); + counts = bigArrays.newLongArray(1, true); + } + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final BigArrays bigArrays = context.bigArrays(); + final MultiGeoPointValues values = valuesSource.geoPointValues(ctx); + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + centroids = bigArrays.grow(centroids, bucket + 1); + counts = bigArrays.grow(counts, bucket + 1); + + values.setDocument(doc); + final int valueCount = values.count(); + if (valueCount > 0) { + double[] pt = new double[2]; + // get the previously accumulated number of counts + long prevCounts = counts.get(bucket); + // increment by the number of points for this document + counts.increment(bucket, valueCount); + // get the previous GeoPoint if a moving avg was computed + if (prevCounts > 0) { + final GeoPoint centroid = GeoPoint.fromIndexLong(centroids.get(bucket)); + pt[0] = centroid.lon(); + pt[1] = centroid.lat(); + } + // update the moving average + for (int i = 0; i < valueCount; ++i) { + GeoPoint value = values.valueAt(i); + pt[0] = pt[0] + (value.getLon() - pt[0]) / ++prevCounts; + pt[1] = pt[1] + (value.getLat() - pt[1]) / prevCounts; + } + centroids.set(bucket, GeoUtils.mortonHash(pt[0], pt[1])); + } + } + }; + } + + @Override + public InternalAggregation buildAggregation(long bucket) { + if (valuesSource == null || bucket >= centroids.size()) { + return buildEmptyAggregation(); + } + final long bucketCount = counts.get(bucket); + final GeoPoint bucketCentroid = (bucketCount > 0) ? GeoPoint.fromIndexLong(centroids.get(bucket)) : + new GeoPoint(Double.NaN, Double.NaN); + return new InternalGeoCentroid(name, bucketCentroid , bucketCount, pipelineAggregators(), metaData()); + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalGeoCentroid(name, null, 0l, pipelineAggregators(), metaData()); + } + + @Override + public void doClose() { + Releasables.close(centroids, counts); + } + + public static class Factory extends ValuesSourceAggregatorFactory.LeafOnly { + protected Factory(String name, ValuesSourceConfig config) { + super(name, InternalGeoBounds.TYPE.name(), config); + } + + @Override + protected Aggregator createUnmapped(AggregationContext aggregationContext, Aggregator parent, + List pipelineAggregators, Map metaData) throws IOException { + return new GeoCentroidAggregator(name, aggregationContext, parent, null, pipelineAggregators, metaData); + } + + @Override + protected Aggregator doCreateInternal(ValuesSource.GeoPoint valuesSource, AggregationContext aggregationContext, Aggregator parent, + boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) + throws IOException { + return new GeoCentroidAggregator(name, aggregationContext, parent, valuesSource, pipelineAggregators, metaData); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidBuilder.java new file mode 100644 index 00000000000..9d6823c6752 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidBuilder.java @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.geocentroid; + +import org.elasticsearch.search.aggregations.metrics.ValuesSourceMetricsAggregationBuilder; + +/** + * Builder class for {@link org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregator} + */ +public class GeoCentroidBuilder extends ValuesSourceMetricsAggregationBuilder { + + public GeoCentroidBuilder(String name) { + super(name, InternalGeoCentroid.TYPE.name()); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java new file mode 100644 index 00000000000..49a7bc8e969 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.geocentroid; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchParseException; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceParser; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; + +/** + * Parser class for {@link org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregator} + */ +public class GeoCentroidParser implements Aggregator.Parser { + + @Override + public String type() { + return InternalGeoCentroid.TYPE.name(); + } + + @Override + public AggregatorFactory parse(String aggregationName, XContentParser parser, SearchContext context) throws IOException { + ValuesSourceParser vsParser = ValuesSourceParser.geoPoint(aggregationName, InternalGeoCentroid.TYPE, context) + .targetValueType(ValueType.GEOPOINT) + .formattable(true) + .build(); + XContentParser.Token token; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (vsParser.token(currentFieldName, token, parser)) { + continue; + } else { + throw new SearchParseException(context, "Unknown key for a " + token + " in aggregation [" + aggregationName + "]: [" + + currentFieldName + "].", parser.getTokenLocation()); + } + } + return new GeoCentroidAggregator.Factory(aggregationName, vsParser.config()); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java new file mode 100644 index 00000000000..b9eeb19354c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java @@ -0,0 +1,160 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.geocentroid; + +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.search.aggregations.AggregationStreams; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Serialization and merge logic for {@link org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregator} + */ +public class InternalGeoCentroid extends InternalMetricsAggregation implements GeoCentroid { + + public final static Type TYPE = new Type("geo_centroid"); + public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() { + @Override + public InternalGeoCentroid readResult(StreamInput in) throws IOException { + InternalGeoCentroid result = new InternalGeoCentroid(); + result.readFrom(in); + return result; + } + }; + + public static void registerStreams() { + AggregationStreams.registerStream(STREAM, TYPE.stream()); + } + + protected GeoPoint centroid; + protected long count; + + protected InternalGeoCentroid() { + } + + public InternalGeoCentroid(String name, GeoPoint centroid, long count, List + pipelineAggregators, Map metaData) { + super(name, pipelineAggregators, metaData); + this.centroid = centroid; + assert count >= 0; + this.count = count; + } + + @Override + public GeoPoint centroid() { + return (centroid == null || Double.isNaN(centroid.lon()) ? null : centroid); + } + + @Override + public long count() { + return count; + } + + @Override + public Type type() { + return TYPE; + } + + @Override + public InternalGeoCentroid doReduce(List aggregations, ReduceContext reduceContext) { + double lonSum = Double.NaN; + double latSum = Double.NaN; + int totalCount = 0; + for (InternalAggregation aggregation : aggregations) { + InternalGeoCentroid centroidAgg = (InternalGeoCentroid) aggregation; + if (centroidAgg.count > 0) { + totalCount += centroidAgg.count; + if (Double.isNaN(lonSum)) { + lonSum = centroidAgg.count * centroidAgg.centroid.getLon(); + latSum = centroidAgg.count * centroidAgg.centroid.getLat(); + } else { + lonSum += (centroidAgg.count * centroidAgg.centroid.getLon()); + latSum += (centroidAgg.count * centroidAgg.centroid.getLat()); + } + } + } + final GeoPoint result = (Double.isNaN(lonSum)) ? null : new GeoPoint(latSum/totalCount, lonSum/totalCount); + return new InternalGeoCentroid(name, result, totalCount, pipelineAggregators(), getMetaData()); + } + + @Override + public Object getProperty(List path) { + if (path.isEmpty()) { + return this; + } else if (path.size() == 1) { + String coordinate = path.get(0); + switch (coordinate) { + case "value": + return centroid; + case "lat": + return centroid.lat(); + case "lon": + return centroid.lon(); + default: + throw new IllegalArgumentException("Found unknown path element [" + coordinate + "] in [" + getName() + "]"); + } + } else { + throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path); + } + } + + @Override + protected void doReadFrom(StreamInput in) throws IOException { + count = in.readVLong(); + if (in.readBoolean()) { + centroid = GeoPoint.fromIndexLong(in.readLong()); + } else { + centroid = null; + } + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeVLong(count); + if (centroid != null) { + out.writeBoolean(true); + out.writeLong(GeoUtils.mortonHash(centroid.lon(), centroid.lat())); + } else { + out.writeBoolean(false); + } + } + + static class Fields { + public static final XContentBuilderString CENTROID = new XContentBuilderString("location"); + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + if (centroid != null) { + builder.startObject(Fields.CENTROID).field("lat", centroid.lat()).field("lon", centroid.lon()).endObject(); + } + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggegator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java similarity index 91% rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggegator.java rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java index 5cc7ddb5dda..6e648cb50e2 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggegator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java @@ -43,7 +43,7 @@ import java.util.Map; /** * */ -public class StatsAggegator extends NumericMetricsAggregator.MultiValue { +public class StatsAggregator extends NumericMetricsAggregator.MultiValue { final ValuesSource.Numeric valuesSource; final ValueFormatter formatter; @@ -54,10 +54,10 @@ public class StatsAggegator extends NumericMetricsAggregator.MultiValue { DoubleArray maxes; - public StatsAggegator(String name, ValuesSource.Numeric valuesSource, ValueFormatter formatter, - AggregationContext context, - Aggregator parent, List pipelineAggregators, - Map metaData) throws IOException { + public StatsAggregator(String name, ValuesSource.Numeric valuesSource, ValueFormatter formatter, + AggregationContext context, + Aggregator parent, List pipelineAggregators, + Map metaData) throws IOException { super(name, context, parent, pipelineAggregators, metaData); this.valuesSource = valuesSource; if (valuesSource != null) { @@ -164,14 +164,14 @@ public class StatsAggegator extends NumericMetricsAggregator.MultiValue { @Override protected Aggregator createUnmapped(AggregationContext aggregationContext, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { - return new StatsAggegator(name, null, config.formatter(), aggregationContext, parent, pipelineAggregators, metaData); + return new StatsAggregator(name, null, config.formatter(), aggregationContext, parent, pipelineAggregators, metaData); } @Override protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, AggregationContext aggregationContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new StatsAggegator(name, valuesSource, config.formatter(), aggregationContext, parent, pipelineAggregators, metaData); + return new StatsAggregator(name, valuesSource, config.formatter(), aggregationContext, parent, pipelineAggregators, metaData); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java index 5ec9b2a59a7..86c85e40ce5 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java @@ -34,6 +34,6 @@ public class StatsParser extends NumericValuesSourceMetricsAggregatorParser config) { - return new StatsAggegator.Factory(aggregationName, config); + return new StatsAggregator.Factory(aggregationName, config); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsBuilder.java index 62bd22ab082..1efd4a7cd24 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsBuilder.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.tophits; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -29,7 +28,6 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; import java.io.IOException; -import java.util.Map; /** * Builder for the {@link TopHits} aggregation. @@ -89,6 +87,14 @@ public class TopHitsBuilder extends AbstractAggregationBuilder { return this; } + /** + * Adds a field to loaded and returned. + */ + public TopHitsBuilder addField(String field) { + sourceBuilder().field(field); + return this; + } + /** * Sets no fields to be loaded, resulting in only id and type to be returned per field. */ @@ -173,185 +179,6 @@ public class TopHitsBuilder extends AbstractAggregationBuilder { return this; } - /** - * Adds a field to be highlighted with default fragment size of 100 characters, and - * default number of fragments of 5. - * - * @param name The field to highlight - */ - public TopHitsBuilder addHighlightedField(String name) { - highlightBuilder().field(name); - return this; - } - - - /** - * Adds a field to be highlighted with a provided fragment size (in characters), and - * default number of fragments of 5. - * - * @param name The field to highlight - * @param fragmentSize The size of a fragment in characters - */ - public TopHitsBuilder addHighlightedField(String name, int fragmentSize) { - highlightBuilder().field(name, fragmentSize); - return this; - } - - /** - * Adds a field to be highlighted with a provided fragment size (in characters), and - * a provided (maximum) number of fragments. - * - * @param name The field to highlight - * @param fragmentSize The size of a fragment in characters - * @param numberOfFragments The (maximum) number of fragments - */ - public TopHitsBuilder addHighlightedField(String name, int fragmentSize, int numberOfFragments) { - highlightBuilder().field(name, fragmentSize, numberOfFragments); - return this; - } - - /** - * Adds a field to be highlighted with a provided fragment size (in characters), - * a provided (maximum) number of fragments and an offset for the highlight. - * - * @param name The field to highlight - * @param fragmentSize The size of a fragment in characters - * @param numberOfFragments The (maximum) number of fragments - */ - public TopHitsBuilder addHighlightedField(String name, int fragmentSize, int numberOfFragments, - int fragmentOffset) { - highlightBuilder().field(name, fragmentSize, numberOfFragments, fragmentOffset); - return this; - } - - /** - * Adds a highlighted field. - */ - public TopHitsBuilder addHighlightedField(HighlightBuilder.Field field) { - highlightBuilder().field(field); - return this; - } - - /** - * Set a tag scheme that encapsulates a built in pre and post tags. The allows schemes - * are styled and default. - * - * @param schemaName The tag scheme name - */ - public TopHitsBuilder setHighlighterTagsSchema(String schemaName) { - highlightBuilder().tagsSchema(schemaName); - return this; - } - - public TopHitsBuilder setHighlighterFragmentSize(Integer fragmentSize) { - highlightBuilder().fragmentSize(fragmentSize); - return this; - } - - public TopHitsBuilder setHighlighterNumOfFragments(Integer numOfFragments) { - highlightBuilder().numOfFragments(numOfFragments); - return this; - } - - public TopHitsBuilder setHighlighterFilter(Boolean highlightFilter) { - highlightBuilder().highlightFilter(highlightFilter); - return this; - } - - /** - * The encoder to set for highlighting - */ - public TopHitsBuilder setHighlighterEncoder(String encoder) { - highlightBuilder().encoder(encoder); - return this; - } - - /** - * Explicitly set the pre tags that will be used for highlighting. - */ - public TopHitsBuilder setHighlighterPreTags(String... preTags) { - highlightBuilder().preTags(preTags); - return this; - } - - /** - * Explicitly set the post tags that will be used for highlighting. - */ - public TopHitsBuilder setHighlighterPostTags(String... postTags) { - highlightBuilder().postTags(postTags); - return this; - } - - /** - * The order of fragments per field. By default, ordered by the order in the - * highlighted text. Can be score, which then it will be ordered - * by score of the fragments. - */ - public TopHitsBuilder setHighlighterOrder(String order) { - highlightBuilder().order(order); - return this; - } - - public TopHitsBuilder setHighlighterRequireFieldMatch(boolean requireFieldMatch) { - highlightBuilder().requireFieldMatch(requireFieldMatch); - return this; - } - - public TopHitsBuilder setHighlighterBoundaryMaxScan(Integer boundaryMaxScan) { - highlightBuilder().boundaryMaxScan(boundaryMaxScan); - return this; - } - - public TopHitsBuilder setHighlighterBoundaryChars(char[] boundaryChars) { - highlightBuilder().boundaryChars(boundaryChars); - return this; - } - - /** - * The highlighter type to use. - */ - public TopHitsBuilder setHighlighterType(String type) { - highlightBuilder().highlighterType(type); - return this; - } - - public TopHitsBuilder setHighlighterFragmenter(String fragmenter) { - highlightBuilder().fragmenter(fragmenter); - return this; - } - - /** - * Sets a query to be used for highlighting all fields instead of the search query. - */ - public TopHitsBuilder setHighlighterQuery(QueryBuilder highlightQuery) { - highlightBuilder().highlightQuery(highlightQuery); - return this; - } - - /** - * Sets the size of the fragment to return from the beginning of the field if there are no matches to - * highlight and the field doesn't also define noMatchSize. - * @param noMatchSize integer to set or null to leave out of request. default is null. - * @return this builder for chaining - */ - public TopHitsBuilder setHighlighterNoMatchSize(Integer noMatchSize) { - highlightBuilder().noMatchSize(noMatchSize); - return this; - } - - /** - * Sets the maximum number of phrases the fvh will consider if the field doesn't also define phraseLimit. - */ - public TopHitsBuilder setHighlighterPhraseLimit(Integer phraseLimit) { - highlightBuilder().phraseLimit(phraseLimit); - return this; - } - - public TopHitsBuilder setHighlighterOptions(Map options) { - highlightBuilder().options(options); - return this; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(getName()).field(type); @@ -366,7 +193,12 @@ public class TopHitsBuilder extends AbstractAggregationBuilder { return sourceBuilder; } - public HighlightBuilder highlightBuilder() { + public HighlightBuilder highlighter() { return sourceBuilder().highlighter(); } + + public TopHitsBuilder highlighter(HighlightBuilder highlightBuilder) { + sourceBuilder().highlighter(highlightBuilder); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java index 206587ac6a4..50b3482f568 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java @@ -25,6 +25,7 @@ import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.fetch.FetchPhase; +import org.elasticsearch.search.fetch.FieldsParseElement; import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsParseElement; import org.elasticsearch.search.fetch.script.ScriptFieldsParseElement; import org.elasticsearch.search.fetch.source.FetchSourceParseElement; @@ -46,15 +47,19 @@ public class TopHitsParser implements Aggregator.Parser { private final HighlighterParseElement highlighterParseElement; private final FieldDataFieldsParseElement fieldDataFieldsParseElement; private final ScriptFieldsParseElement scriptFieldsParseElement; + private final FieldsParseElement fieldsParseElement; @Inject - public TopHitsParser(FetchPhase fetchPhase, SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { + public TopHitsParser(FetchPhase fetchPhase, SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, + HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, + ScriptFieldsParseElement scriptFieldsParseElement, FieldsParseElement fieldsParseElement) { this.fetchPhase = fetchPhase; this.sortParseElement = sortParseElement; this.sourceParseElement = sourceParseElement; this.highlighterParseElement = highlighterParseElement; this.fieldDataFieldsParseElement = fieldDataFieldsParseElement; this.scriptFieldsParseElement = scriptFieldsParseElement; + this.fieldsParseElement = fieldsParseElement; } @Override @@ -75,6 +80,8 @@ public class TopHitsParser implements Aggregator.Parser { sortParseElement.parse(parser, subSearchContext); } else if ("_source".equals(currentFieldName)) { sourceParseElement.parse(parser, subSearchContext); + } else if ("fields".equals(currentFieldName)) { + fieldsParseElement.parse(parser, subSearchContext); } else if (token.isValue()) { switch (currentFieldName) { case "from": diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java index 89955ef0278..0f2ffea9a75 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java @@ -78,7 +78,7 @@ public abstract class BucketMetricsPipelineAggregator extends SiblingPipelineAgg } } } - return buildAggregation(Collections.EMPTY_LIST, metaData()); + return buildAggregation(Collections.emptyList(), metaData()); } /** @@ -123,4 +123,4 @@ public abstract class BucketMetricsPipelineAggregator extends SiblingPipelineAgg gapPolicy.writeTo(out); } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java index 8101c3caae2..95a70af7934 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java @@ -90,7 +90,7 @@ public class MaxBucketPipelineAggregator extends BucketMetricsPipelineAggregator @Override protected InternalAggregation buildAggregation(List pipelineAggregators, Map metadata) { String[] keys = maxBucketKeys.toArray(new String[maxBucketKeys.size()]); - return new InternalBucketMetricValue(name(), keys, maxValue, formatter, Collections.EMPTY_LIST, metaData()); + return new InternalBucketMetricValue(name(), keys, maxValue, formatter, Collections.emptyList(), metaData()); } public static class Factory extends PipelineAggregatorFactory { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java index 5da74281290..755b2060ae6 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java @@ -91,7 +91,7 @@ public class MinBucketPipelineAggregator extends BucketMetricsPipelineAggregator protected InternalAggregation buildAggregation(java.util.List pipelineAggregators, java.util.Map metadata) { String[] keys = minBucketKeys.toArray(new String[minBucketKeys.size()]); - return new InternalBucketMetricValue(name(), keys, minValue, formatter, Collections.EMPTY_LIST, metaData()); + return new InternalBucketMetricValue(name(), keys, minValue, formatter, Collections.emptyList(), metaData()); }; public static class Factory extends PipelineAggregatorFactory { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelStreams.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelStreams.java index faee8a9f75b..f1238fb9fbd 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelStreams.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelStreams.java @@ -32,7 +32,7 @@ import java.util.Map; */ public class MovAvgModelStreams { - private static Map STREAMS = Collections.EMPTY_MAP; + private static Map STREAMS = Collections.emptyMap(); static { HashMap map = new HashMap<>(); @@ -79,7 +79,7 @@ public class MovAvgModelStreams { * @param name The given name * @return The associated stream */ - public static synchronized Stream stream(String name) { + private static synchronized Stream stream(String name) { return STREAMS.get(name); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/format/ValueFormatter.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/format/ValueFormatter.java index 4e0060d6162..a4b6c2cf282 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/format/ValueFormatter.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/format/ValueFormatter.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.search.aggregations.support.format; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -250,7 +250,7 @@ public interface ValueFormatter extends Streamable { @Override public String format(long value) { - return XGeoHashUtils.stringEncode(value); + return GeoHashUtils.stringEncode(value); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 3b87030fec8..465729ca714 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -20,19 +20,25 @@ package org.elasticsearch.search.builder; import com.carrotsearch.hppc.ObjectFloatHashMap; -import java.nio.charset.StandardCharsets; -import org.elasticsearch.ElasticsearchGenerationException; -import org.elasticsearch.action.support.QuerySourceBuilder; +import com.carrotsearch.hppc.cursors.ObjectCursor; + import org.elasticsearch.action.support.ToXContentToBytes; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.fetch.innerhits.InnerHitsBuilder; @@ -48,9 +54,8 @@ import org.elasticsearch.search.suggest.SuggestBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.Map; +import java.util.Objects; /** * A search source builder allowing to easily build search source. Simple @@ -59,7 +64,43 @@ import java.util.Map; * * @see org.elasticsearch.action.search.SearchRequest#source(SearchSourceBuilder) */ -public class SearchSourceBuilder extends ToXContentToBytes { +public final class SearchSourceBuilder extends ToXContentToBytes implements Writeable { + + public static final ParseField FROM_FIELD = new ParseField("from"); + public static final ParseField SIZE_FIELD = new ParseField("size"); + public static final ParseField TIMEOUT_FIELD = new ParseField("timeout"); + public static final ParseField TERMINATE_AFTER_FIELD = new ParseField("terminate_after"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField POST_FILTER_FIELD = new ParseField("post_filter"); + public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); + public static final ParseField VERSION_FIELD = new ParseField("version"); + public static final ParseField EXPLAIN_FIELD = new ParseField("explain"); + public static final ParseField _SOURCE_FIELD = new ParseField("_source"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + public static final ParseField FIELDDATA_FIELDS_FIELD = new ParseField("fielddata_fields"); + public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields"); + public static final ParseField SCRIPT_FIELD = new ParseField("script"); + public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure"); + public static final ParseField SORT_FIELD = new ParseField("sort"); + public static final ParseField TRACK_SCORES_FIELD = new ParseField("track_scores"); + public static final ParseField INDICES_BOOST_FIELD = new ParseField("indices_boost"); + public static final ParseField AGGREGATIONS_FIELD = new ParseField("aggregations", "aggs"); + public static final ParseField HIGHLIGHT_FIELD = new ParseField("highlight"); + public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); + public static final ParseField SUGGEST_FIELD = new ParseField("suggest"); + public static final ParseField RESCORE_FIELD = new ParseField("rescore"); + public static final ParseField STATS_FIELD = new ParseField("stats"); + public static final ParseField EXT_FIELD = new ParseField("ext"); + + private static final SearchSourceBuilder PROTOTYPE = new SearchSourceBuilder(); + + public static SearchSourceBuilder readSearchSourceFrom(StreamInput in) throws IOException { + return PROTOTYPE.readFrom(in); + } + + public static SearchSourceBuilder parseSearchSource(XContentParser parser, QueryParseContext context) throws IOException { + return PROTOTYPE.fromXContent(parser, context); + } /** * A static factory method to construct a new search source. @@ -75,11 +116,9 @@ public class SearchSourceBuilder extends ToXContentToBytes { return new HighlightBuilder(); } - private QuerySourceBuilder querySourceBuilder; + private QueryBuilder queryBuilder; - private QueryBuilder postQueryBuilder; - - private BytesReference filterBinary; + private QueryBuilder postQueryBuilder; private int from = -1; @@ -89,7 +128,7 @@ public class SearchSourceBuilder extends ToXContentToBytes { private Boolean version; - private List sorts; + private List sorts; private boolean trackScores = false; @@ -103,21 +142,21 @@ public class SearchSourceBuilder extends ToXContentToBytes { private List scriptFields; private FetchSourceContext fetchSourceContext; - private List aggregations; - private BytesReference aggregationsBinary; + private List aggregations; private HighlightBuilder highlightBuilder; - private SuggestBuilder suggestBuilder; + private BytesReference suggestBuilder; - private InnerHitsBuilder innerHitsBuilder; + private BytesReference innerHitsBuilder; - private List rescoreBuilders; - private Integer defaultRescoreWindowSize; + private List rescoreBuilders; private ObjectFloatHashMap indexBoost = null; - private String[] stats; + private List stats; + + private BytesReference ext = null; /** * Constructs a new search source builder. @@ -126,77 +165,20 @@ public class SearchSourceBuilder extends ToXContentToBytes { } /** - * Sets the query provided as a {@link QuerySourceBuilder} - */ - public SearchSourceBuilder query(QuerySourceBuilder querySourceBuilder) { - this.querySourceBuilder = querySourceBuilder; - return this; - } - - /** - * Constructs a new search source builder with a search query. + * Sets the search query for this request. * * @see org.elasticsearch.index.query.QueryBuilders */ - public SearchSourceBuilder query(QueryBuilder query) { - if (this.querySourceBuilder == null) { - this.querySourceBuilder = new QuerySourceBuilder(); - } - this.querySourceBuilder.setQuery(query); + public SearchSourceBuilder query(QueryBuilder query) { + this.queryBuilder = query; return this; } /** - * Constructs a new search source builder with a raw search query. + * Gets the query for this request */ - public SearchSourceBuilder query(byte[] queryBinary) { - return query(queryBinary, 0, queryBinary.length); - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchSourceBuilder query(byte[] queryBinary, int queryBinaryOffset, int queryBinaryLength) { - return query(new BytesArray(queryBinary, queryBinaryOffset, queryBinaryLength)); - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchSourceBuilder query(BytesReference queryBinary) { - if (this.querySourceBuilder == null) { - this.querySourceBuilder = new QuerySourceBuilder(); - } - this.querySourceBuilder.setQuery(queryBinary); - return this; - } - - /** - * Constructs a new search source builder with a raw search query. - */ - public SearchSourceBuilder query(String queryString) { - return query(queryString.getBytes(StandardCharsets.UTF_8)); - } - - /** - * Constructs a new search source builder with a query from a builder. - */ - public SearchSourceBuilder query(XContentBuilder query) { - return query(query.bytes()); - } - - /** - * Constructs a new search source builder with a query from a map. - */ - @SuppressWarnings("unchecked") - public SearchSourceBuilder query(Map query) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE); - builder.map(query); - return query(builder); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + query + "]", e); - } + public QueryBuilder query() { + return queryBuilder; } /** @@ -204,63 +186,16 @@ public class SearchSourceBuilder extends ToXContentToBytes { * only has affect on the search hits (not aggregations). This filter is * always executed as last filtering mechanism. */ - public SearchSourceBuilder postFilter(QueryBuilder postFilter) { + public SearchSourceBuilder postFilter(QueryBuilder postFilter) { this.postQueryBuilder = postFilter; return this; } /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). + * Gets the post filter for this request */ - public SearchSourceBuilder postFilter(String postFilterString) { - return postFilter(postFilterString.getBytes(StandardCharsets.UTF_8)); - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchSourceBuilder postFilter(byte[] postFilter) { - return postFilter(postFilter, 0, postFilter.length); - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchSourceBuilder postFilter(byte[] postFilterBinary, int postFilterBinaryOffset, int postFilterBinaryLength) { - return postFilter(new BytesArray(postFilterBinary, postFilterBinaryOffset, postFilterBinaryLength)); - } - - /** - * Sets a filter on the query executed that only applies to the search query - * (and not aggs for example). - */ - public SearchSourceBuilder postFilter(BytesReference postFilterBinary) { - this.filterBinary = postFilterBinary; - return this; - } - - /** - * Constructs a new search source builder with a query from a builder. - */ - public SearchSourceBuilder postFilter(XContentBuilder postFilter) { - return postFilter(postFilter.bytes()); - } - - /** - * Constructs a new search source builder with a query from a map. - */ - @SuppressWarnings("unchecked") - public SearchSourceBuilder postFilter(Map postFilter) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE); - builder.map(postFilter); - return postFilter(builder); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + postFilter + "]", e); - } + public QueryBuilder postFilter() { + return postQueryBuilder; } /** @@ -271,6 +206,13 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Gets the from index to start the search from. + **/ + public int from() { + return from; + } + /** * The number of search hits to return. Defaults to 10. */ @@ -279,6 +221,13 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Gets the number of search hits to return. + */ + public int size() { + return size; + } + /** * Sets the minimum score below which docs will be filtered out. */ @@ -287,6 +236,13 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Gets the minimum score below which docs will be filtered out. + */ + public Float minScore() { + return minScore; + } + /** * Should each {@link org.elasticsearch.search.SearchHit} be returned with * an explanation of the hit (ranking). @@ -296,6 +252,14 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Indicates whether each search hit will be returned with an explanation of + * the hit (ranking) + */ + public Boolean explain() { + return explain; + } + /** * Should each {@link org.elasticsearch.search.SearchHit} be returned with a * version associated with it. @@ -305,6 +269,14 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Indicates whether the document's version will be included in the search + * hits. + */ + public Boolean version() { + return version; + } + /** * An optional timeout to control how long search is allowed to take. */ @@ -314,11 +286,10 @@ public class SearchSourceBuilder extends ToXContentToBytes { } /** - * An optional timeout to control how long search is allowed to take. + * Gets the timeout to control how long search is allowed to take. */ - public SearchSourceBuilder timeout(String timeout) { - this.timeoutInMillis = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout").millis(); - return this; + public long timeoutInMillis() { + return timeoutInMillis; } /** @@ -326,13 +297,20 @@ public class SearchSourceBuilder extends ToXContentToBytes { * terminateAfter documents */ public SearchSourceBuilder terminateAfter(int terminateAfter) { - if (terminateAfter <= 0) { + if (terminateAfter < 0) { throw new IllegalArgumentException("terminateAfter must be > 0"); } this.terminateAfter = terminateAfter; return this; } + /** + * Gets the number of documents to terminate after collecting. + */ + public int terminateAfter() { + return terminateAfter; + } + /** * Adds a sort against the given field name and the sort ordering. * @@ -359,11 +337,26 @@ public class SearchSourceBuilder extends ToXContentToBytes { * Adds a sort builder. */ public SearchSourceBuilder sort(SortBuilder sort) { - if (sorts == null) { - sorts = new ArrayList<>(); + try { + if (sorts == null) { + sorts = new ArrayList<>(); + } + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + sort.toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + sorts.add(builder.bytes()); + return this; + } catch (IOException e) { + throw new RuntimeException(e); } - sorts.add(sort); - return this; + } + + /** + * Gets the bytes representing the sort builders for this request. + */ + public List sorts() { + return sorts; } /** @@ -376,102 +369,105 @@ public class SearchSourceBuilder extends ToXContentToBytes { } /** - * Add an get to perform as part of the search. + * Indicates whether scores will be tracked for this request. + */ + public boolean trackScores() { + return trackScores; + } + + /** + * Add an aggregation to perform as part of the search. */ public SearchSourceBuilder aggregation(AbstractAggregationBuilder aggregation) { - if (aggregations == null) { - aggregations = new ArrayList<>(); - } - aggregations.add(aggregation); - return this; - } - - /** - * Sets a raw (xcontent / json) addAggregation. - */ - public SearchSourceBuilder aggregations(byte[] aggregationsBinary) { - return aggregations(aggregationsBinary, 0, aggregationsBinary.length); - } - - /** - * Sets a raw (xcontent / json) addAggregation. - */ - public SearchSourceBuilder aggregations(byte[] aggregationsBinary, int aggregationsBinaryOffset, int aggregationsBinaryLength) { - return aggregations(new BytesArray(aggregationsBinary, aggregationsBinaryOffset, aggregationsBinaryLength)); - } - - /** - * Sets a raw (xcontent / json) addAggregation. - */ - public SearchSourceBuilder aggregations(BytesReference aggregationsBinary) { - this.aggregationsBinary = aggregationsBinary; - return this; - } - - /** - * Sets a raw (xcontent / json) addAggregation. - */ - public SearchSourceBuilder aggregations(XContentBuilder aggs) { - return aggregations(aggs.bytes()); - } - - /** - * Set the rescore window size for rescores that don't specify their window. - */ - public SearchSourceBuilder defaultRescoreWindowSize(int defaultRescoreWindowSize) { - this.defaultRescoreWindowSize = defaultRescoreWindowSize; - return this; - } - - /** - * Sets a raw (xcontent / json) addAggregation. - */ - @SuppressWarnings("unchecked") - public SearchSourceBuilder aggregations(Map aggregations) { try { - XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE); - builder.map(aggregations); - return aggregations(builder); + if (aggregations == null) { + aggregations = new ArrayList<>(); + } + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + aggregation.toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + aggregations.add(builder.bytes()); + return this; } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + aggregations + "]", e); + throw new RuntimeException(e); } } - public HighlightBuilder highlighter() { - if (highlightBuilder == null) { - highlightBuilder = new HighlightBuilder(); - } - return highlightBuilder; + /** + * Gets the bytes representing the aggregation builders for this request. + */ + public List aggregations() { + return aggregations; } /** * Adds highlight to perform as part of the search. */ - public SearchSourceBuilder highlight(HighlightBuilder highlightBuilder) { + public SearchSourceBuilder highlighter(HighlightBuilder highlightBuilder) { this.highlightBuilder = highlightBuilder; return this; } - public InnerHitsBuilder innerHitsBuilder() { - if (innerHitsBuilder == null) { - innerHitsBuilder = new InnerHitsBuilder(); + /** + * Gets the hightlighter builder for this request. + */ + public HighlightBuilder highlighter() { + return highlightBuilder; + } + + public SearchSourceBuilder innerHits(InnerHitsBuilder innerHitsBuilder) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + innerHitsBuilder.innerXContent(builder, EMPTY_PARAMS); + builder.endObject(); + this.innerHitsBuilder = builder.bytes(); + return this; + } catch (IOException e) { + throw new RuntimeException(e); } + } + + /** + * Gets the bytes representing the inner hits builder for this request. + */ + public BytesReference innerHits() { return innerHitsBuilder; } - public SuggestBuilder suggest() { - if (suggestBuilder == null) { - suggestBuilder = new SuggestBuilder("suggest"); + public SearchSourceBuilder suggest(SuggestBuilder suggestBuilder) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + suggestBuilder.toXContent(builder, EMPTY_PARAMS); + this.suggestBuilder = builder.bytes(); + return this; + } catch (IOException e) { + throw new RuntimeException(e); } + } + + /** + * Gets the bytes representing the suggester builder for this request. + */ + public BytesReference suggest() { return suggestBuilder; } public SearchSourceBuilder addRescorer(RescoreBuilder rescoreBuilder) { - if (rescoreBuilders == null) { - rescoreBuilders = new ArrayList<>(); + try { + if (rescoreBuilders == null) { + rescoreBuilders = new ArrayList<>(); + } + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + rescoreBuilder.toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + rescoreBuilders.add(builder.bytes()); + return this; + } catch (IOException e) { + throw new RuntimeException(e); } - rescoreBuilders.add(rescoreBuilder); - return this; } public SearchSourceBuilder clearRescorers() { @@ -479,6 +475,13 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Gets the bytes representing the rescore builders for this request. + */ + public List rescores() { + return rescoreBuilders; + } + /** * Indicates whether the response should contain the stored _source for * every hit @@ -535,33 +538,11 @@ public class SearchSourceBuilder extends ToXContentToBytes { } /** - * Sets no fields to be loaded, resulting in only id and type to be returned - * per field. + * Gets the {@link FetchSourceContext} which defines how the _source should + * be fetched. */ - public SearchSourceBuilder noFields() { - this.fieldNames = Collections.emptyList(); - return this; - } - - /** - * Sets the fields to load and return as part of the search request. If none - * are specified, the source of the document will be returned. - */ - public SearchSourceBuilder fields(List fields) { - this.fieldNames = fields; - return this; - } - - /** - * Adds the fields to load and return as part of the search request. If none - * are specified, the source of the document will be returned. - */ - public SearchSourceBuilder fields(String... fields) { - if (fieldNames == null) { - fieldNames = new ArrayList<>(); - } - Collections.addAll(fieldNames, fields); - return this; + public FetchSourceContext fetchSource() { + return fetchSourceContext; } /** @@ -577,6 +558,31 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Sets the fields to load and return as part of the search request. If none + * are specified, the source of the document will be returned. + */ + public SearchSourceBuilder fields(List fields) { + this.fieldNames = fields; + return this; + } + + /** + * Sets no fields to be loaded, resulting in only id and type to be returned + * per field. + */ + public SearchSourceBuilder noFields() { + this.fieldNames = Collections.emptyList(); + return this; + } + + /** + * Gets the fields to load and return as part of the search request. + */ + public List fields() { + return fieldNames; + } + /** * Adds a field to load from the field data cache and return as part of the * search request. @@ -589,6 +595,13 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Gets the field-data fields. + */ + public List fieldDataFields() { + return fieldDataFields; + } + /** * Adds a script field under the given name with the provided script. * @@ -598,13 +611,33 @@ public class SearchSourceBuilder extends ToXContentToBytes { * The script */ public SearchSourceBuilder scriptField(String name, Script script) { + scriptField(name, script, false); + return this; + } + + /** + * Adds a script field under the given name with the provided script. + * + * @param name + * The name of the field + * @param script + * The script + */ + public SearchSourceBuilder scriptField(String name, Script script, boolean ignoreFailure) { if (scriptFields == null) { scriptFields = new ArrayList<>(); } - scriptFields.add(new ScriptField(name, script)); + scriptFields.add(new ScriptField(name, script, ignoreFailure)); return this; } + /** + * Gets the script fields. + */ + public List scriptFields() { + return scriptFields; + } + /** * Sets the boost a specific index will receive when the query is executeed * against it. @@ -622,14 +655,238 @@ public class SearchSourceBuilder extends ToXContentToBytes { return this; } + /** + * Gets the boost a specific indices will receive when the query is + * executeed against them. + */ + public ObjectFloatHashMap indexBoost() { + return indexBoost; + } + /** * The stats groups this request will be aggregated under. */ - public SearchSourceBuilder stats(String... statsGroups) { + public SearchSourceBuilder stats(List statsGroups) { this.stats = statsGroups; return this; } + /** + * The stats groups this request will be aggregated under. + */ + public List stats() { + return stats; + } + + public SearchSourceBuilder ext(XContentBuilder ext) { + this.ext = ext.bytes(); + return this; + } + + public BytesReference ext() { + return ext; + } + + public SearchSourceBuilder fromXContent(XContentParser parser, QueryParseContext context) throws IOException { + SearchSourceBuilder builder = new SearchSourceBuilder(); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = null; + if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) { + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] but found [" + token + "]", + parser.getTokenLocation()); + } + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (context.parseFieldMatcher().match(currentFieldName, FROM_FIELD)) { + builder.from = parser.intValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, SIZE_FIELD)) { + builder.size = parser.intValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, TIMEOUT_FIELD)) { + builder.timeoutInMillis = parser.longValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, TERMINATE_AFTER_FIELD)) { + builder.terminateAfter = parser.intValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, MIN_SCORE_FIELD)) { + builder.minScore = parser.floatValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, VERSION_FIELD)) { + builder.version = parser.booleanValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, EXPLAIN_FIELD)) { + builder.explain = parser.booleanValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, TRACK_SCORES_FIELD)) { + builder.trackScores = parser.booleanValue(); + } else if (context.parseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) { + builder.fetchSourceContext = FetchSourceContext.parse(parser, context); + } else if (context.parseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) { + List fieldNames = new ArrayList<>(); + fieldNames.add(parser.text()); + builder.fieldNames = fieldNames; + } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { + builder.sort(parser.text()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (context.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { + builder.queryBuilder = context.parseInnerQueryBuilder(); + } else if (context.parseFieldMatcher().match(currentFieldName, POST_FILTER_FIELD)) { + builder.postQueryBuilder = context.parseInnerQueryBuilder(); + } else if (context.parseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) { + builder.fetchSourceContext = FetchSourceContext.parse(parser, context); + } else if (context.parseFieldMatcher().match(currentFieldName, SCRIPT_FIELDS_FIELD)) { + List scriptFields = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + String scriptFieldName = parser.currentName(); + token = parser.nextToken(); + if (token == XContentParser.Token.START_OBJECT) { + Script script = null; + boolean ignoreFailure = false; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (context.parseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) { + script = Script.parse(parser, context.parseFieldMatcher()); + } else if (context.parseFieldMatcher().match(currentFieldName, IGNORE_FAILURE_FIELD)) { + ignoreFailure = parser.booleanValue(); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + + "].", parser.getTokenLocation()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (context.parseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) { + script = Script.parse(parser, context.parseFieldMatcher()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + + "].", parser.getTokenLocation()); + } + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + + "].", parser.getTokenLocation()); + } + } + scriptFields.add(new ScriptField(scriptFieldName, script, ignoreFailure)); + } else { + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] in [" + + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); + } + } + builder.scriptFields = scriptFields; + } else if (context.parseFieldMatcher().match(currentFieldName, INDICES_BOOST_FIELD)) { + ObjectFloatHashMap indexBoost = new ObjectFloatHashMap(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + indexBoost.put(currentFieldName, parser.floatValue()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } + builder.indexBoost = indexBoost; + } else if (context.parseFieldMatcher().match(currentFieldName, AGGREGATIONS_FIELD)) { + List aggregations = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token == XContentParser.Token.START_OBJECT) { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + xContentBuilder.field(currentFieldName); + xContentBuilder.copyCurrentStructure(parser); + xContentBuilder.endObject(); + aggregations.add(xContentBuilder.bytes()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } + builder.aggregations = aggregations; + } else if (context.parseFieldMatcher().match(currentFieldName, HIGHLIGHT_FIELD)) { + builder.highlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context); + } else if (context.parseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); + builder.innerHitsBuilder = xContentBuilder.bytes(); + } else if (context.parseFieldMatcher().match(currentFieldName, SUGGEST_FIELD)) { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); + builder.suggestBuilder = xContentBuilder.bytes(); + } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { + List sorts = new ArrayList<>(); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); + sorts.add(xContentBuilder.bytes()); + builder.sorts = sorts; + } else if (context.parseFieldMatcher().match(currentFieldName, EXT_FIELD)) { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); + builder.ext = xContentBuilder.bytes(); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } else if (token == XContentParser.Token.START_ARRAY) { + + if (context.parseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) { + List fieldNames = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + fieldNames.add(parser.text()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in [" + + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); + } + } + builder.fieldNames = fieldNames; + } else if (context.parseFieldMatcher().match(currentFieldName, FIELDDATA_FIELDS_FIELD)) { + List fieldDataFields = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + fieldDataFields.add(parser.text()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in [" + + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); + } + } + builder.fieldDataFields = fieldDataFields; + } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { + List sorts = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); + sorts.add(xContentBuilder.bytes()); + } + builder.sorts = sorts; + } else if (context.parseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) { + List rescoreBuilders = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); + rescoreBuilders.add(xContentBuilder.bytes()); + } + builder.rescoreBuilders = rescoreBuilders; + } else if (context.parseFieldMatcher().match(currentFieldName, STATS_FIELD)) { + List stats = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + stats.add(parser.text()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in [" + + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); + } + } + builder.stats = stats; + } else if (context.parseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) { + builder.fetchSourceContext = FetchSourceContext.parse(parser, context); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } + return builder; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -640,65 +897,49 @@ public class SearchSourceBuilder extends ToXContentToBytes { public void innerToXContent(XContentBuilder builder, Params params) throws IOException { if (from != -1) { - builder.field("from", from); + builder.field(FROM_FIELD.getPreferredName(), from); } if (size != -1) { - builder.field("size", size); + builder.field(SIZE_FIELD.getPreferredName(), size); } if (timeoutInMillis != -1) { - builder.field("timeout", timeoutInMillis); + builder.field(TIMEOUT_FIELD.getPreferredName(), timeoutInMillis); } if (terminateAfter != SearchContext.DEFAULT_TERMINATE_AFTER) { - builder.field("terminate_after", terminateAfter); + builder.field(TERMINATE_AFTER_FIELD.getPreferredName(), terminateAfter); } - if (querySourceBuilder != null) { - querySourceBuilder.innerToXContent(builder, params); + if (queryBuilder != null) { + builder.field(QUERY_FIELD.getPreferredName(), queryBuilder); } if (postQueryBuilder != null) { - builder.field("post_filter"); - postQueryBuilder.toXContent(builder, params); - } - - if (filterBinary != null) { - if (XContentFactory.xContentType(filterBinary) == builder.contentType()) { - builder.rawField("filter", filterBinary); - } else { - builder.field("filter_binary", filterBinary); - } + builder.field(POST_FILTER_FIELD.getPreferredName(), postQueryBuilder); } if (minScore != null) { - builder.field("min_score", minScore); + builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); } if (version != null) { - builder.field("version", version); + builder.field(VERSION_FIELD.getPreferredName(), version); } if (explain != null) { - builder.field("explain", explain); + builder.field(EXPLAIN_FIELD.getPreferredName(), explain); } if (fetchSourceContext != null) { - if (!fetchSourceContext.fetchSource()) { - builder.field("_source", false); - } else { - builder.startObject("_source"); - builder.array("includes", fetchSourceContext.includes()); - builder.array("excludes", fetchSourceContext.excludes()); - builder.endObject(); - } + builder.field(_SOURCE_FIELD.getPreferredName(), fetchSourceContext); } if (fieldNames != null) { if (fieldNames.size() == 1) { - builder.field("fields", fieldNames.get(0)); + builder.field(FIELDS_FIELD.getPreferredName(), fieldNames.get(0)); } else { - builder.startArray("fields"); + builder.startArray(FIELDS_FIELD.getPreferredName()); for (String fieldName : fieldNames) { builder.value(fieldName); } @@ -707,39 +948,37 @@ public class SearchSourceBuilder extends ToXContentToBytes { } if (fieldDataFields != null) { - builder.startArray("fielddata_fields"); - for (String fieldName : fieldDataFields) { - builder.value(fieldName); + builder.startArray(FIELDDATA_FIELDS_FIELD.getPreferredName()); + for (String fieldDataField : fieldDataFields) { + builder.value(fieldDataField); } builder.endArray(); } if (scriptFields != null) { - builder.startObject("script_fields"); + builder.startObject(SCRIPT_FIELDS_FIELD.getPreferredName()); for (ScriptField scriptField : scriptFields) { - builder.startObject(scriptField.fieldName()); - builder.field("script", scriptField.script()); - builder.endObject(); + scriptField.toXContent(builder, params); } builder.endObject(); } if (sorts != null) { - builder.startArray("sort"); - for (SortBuilder sort : sorts) { - builder.startObject(); - sort.toXContent(builder, params); - builder.endObject(); + builder.startArray(SORT_FIELD.getPreferredName()); + for (BytesReference sort : sorts) { + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(sort); + parser.nextToken(); + builder.copyCurrentStructure(parser); } builder.endArray(); } if (trackScores) { - builder.field("track_scores", true); + builder.field(TRACK_SCORES_FIELD.getPreferredName(), true); } if (indexBoost != null) { - builder.startObject("indices_boost"); + builder.startObject(INDICES_BOOST_FIELD.getPreferredName()); assert !indexBoost.containsKey(null); final Object[] keys = indexBoost.keys; final float[] values = indexBoost.values; @@ -752,82 +991,73 @@ public class SearchSourceBuilder extends ToXContentToBytes { } if (aggregations != null) { - builder.field("aggregations"); + builder.field(AGGREGATIONS_FIELD.getPreferredName()); builder.startObject(); - for (AbstractAggregationBuilder aggregation : aggregations) { - aggregation.toXContent(builder, params); + for (BytesReference aggregation : aggregations) { + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(aggregation); + parser.nextToken(); + parser.nextToken(); + builder.copyCurrentStructure(parser); } builder.endObject(); } - if (aggregationsBinary != null) { - if (XContentFactory.xContentType(aggregationsBinary) == builder.contentType()) { - builder.rawField("aggregations", aggregationsBinary); - } else { - builder.field("aggregations_binary", aggregationsBinary); - } - } - if (highlightBuilder != null) { - highlightBuilder.toXContent(builder, params); + this.highlightBuilder.toXContent(builder, params); } if (innerHitsBuilder != null) { - innerHitsBuilder.toXContent(builder, params); + builder.field(INNER_HITS_FIELD.getPreferredName()); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(innerHitsBuilder); + parser.nextToken(); + builder.copyCurrentStructure(parser); } if (suggestBuilder != null) { - suggestBuilder.toXContent(builder, params); + builder.field(SUGGEST_FIELD.getPreferredName()); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(suggestBuilder); + parser.nextToken(); + builder.copyCurrentStructure(parser); } if (rescoreBuilders != null) { - // Strip empty rescoreBuilders from the request - Iterator itr = rescoreBuilders.iterator(); - while (itr.hasNext()) { - if (itr.next().isEmpty()) { - itr.remove(); - } - } - - // Now build the request taking care to skip empty lists and only send the object form - // if there is just one builder. - if (rescoreBuilders.size() == 1) { - builder.startObject("rescore"); - rescoreBuilders.get(0).toXContent(builder, params); - if (rescoreBuilders.get(0).windowSize() == null && defaultRescoreWindowSize != null) { - builder.field("window_size", defaultRescoreWindowSize); - } - builder.endObject(); - } else if (!rescoreBuilders.isEmpty()) { - builder.startArray("rescore"); - for (RescoreBuilder rescoreBuilder : rescoreBuilders) { - builder.startObject(); - rescoreBuilder.toXContent(builder, params); - if (rescoreBuilder.windowSize() == null && defaultRescoreWindowSize != null) { - builder.field("window_size", defaultRescoreWindowSize); - } - builder.endObject(); - } - builder.endArray(); - } - } - - if (stats != null) { - builder.startArray("stats"); - for (String stat : stats) { - builder.value(stat); + builder.startArray(RESCORE_FIELD.getPreferredName()); + for (BytesReference rescoreBuilder : rescoreBuilders) { + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(rescoreBuilder); + parser.nextToken(); + builder.copyCurrentStructure(parser); } builder.endArray(); } + + if (stats != null) { + builder.field(STATS_FIELD.getPreferredName(), stats); + } + + if (ext != null) { + builder.field(EXT_FIELD.getPreferredName()); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(ext); + parser.nextToken(); + builder.copyCurrentStructure(parser); + } } - private static class ScriptField { + public static class ScriptField implements Writeable, ToXContent { + + public static final ScriptField PROTOTYPE = new ScriptField(null, null); + + private final boolean ignoreFailure; private final String fieldName; private final Script script; private ScriptField(String fieldName, Script script) { + this(fieldName, script, false); + } + + private ScriptField(String fieldName, Script script, boolean ignoreFailure) { this.fieldName = fieldName; this.script = script; + this.ignoreFailure = ignoreFailure; } public String fieldName() { @@ -837,5 +1067,303 @@ public class SearchSourceBuilder extends ToXContentToBytes { public Script script() { return script; } + + public boolean ignoreFailure() { + return ignoreFailure; + } + + @Override + public ScriptField readFrom(StreamInput in) throws IOException { + return new ScriptField(in.readString(), Script.readScript(in), in.readBoolean()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(fieldName); + script.writeTo(out); + out.writeBoolean(ignoreFailure); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(fieldName); + builder.field(SCRIPT_FIELD.getPreferredName(), script); + builder.field(IGNORE_FAILURE_FIELD.getPreferredName(), ignoreFailure); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, script, ignoreFailure); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ScriptField other = (ScriptField) obj; + return Objects.equals(fieldName, other.fieldName) + && Objects.equals(script, other.script) + && Objects.equals(ignoreFailure, other.ignoreFailure); + } + } + + @Override + public SearchSourceBuilder readFrom(StreamInput in) throws IOException { + SearchSourceBuilder builder = new SearchSourceBuilder(); + if (in.readBoolean()) { + int size = in.readVInt(); + List aggregations = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + aggregations.add(in.readBytesReference()); + } + builder.aggregations = aggregations; + } + builder.explain = in.readOptionalBoolean(); + builder.fetchSourceContext = FetchSourceContext.optionalReadFromStream(in); + boolean hasFieldDataFields = in.readBoolean(); + if (hasFieldDataFields) { + int size = in.readVInt(); + List fieldDataFields = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + fieldDataFields.add(in.readString()); + } + builder.fieldDataFields = fieldDataFields; + } + boolean hasFieldNames = in.readBoolean(); + if (hasFieldNames) { + int size = in.readVInt(); + List fieldNames = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + fieldNames.add(in.readString()); + } + builder.fieldNames = fieldNames; + } + builder.from = in.readVInt(); + if (in.readBoolean()) { + builder.highlightBuilder = HighlightBuilder.PROTOTYPE.readFrom(in); + } + boolean hasIndexBoost = in.readBoolean(); + if (hasIndexBoost) { + int size = in.readVInt(); + ObjectFloatHashMap indexBoost = new ObjectFloatHashMap(size); + for (int i = 0; i < size; i++) { + indexBoost.put(in.readString(), in.readFloat()); + } + builder.indexBoost = indexBoost; + } + if (in.readBoolean()) { + builder.innerHitsBuilder = in.readBytesReference(); + } + if (in.readBoolean()) { + builder.minScore = in.readFloat(); + } + if (in.readBoolean()) { + builder.postQueryBuilder = in.readQuery(); + } + if (in.readBoolean()) { + builder.queryBuilder = in.readQuery(); + } + if (in.readBoolean()) { + int size = in.readVInt(); + List rescoreBuilders = new ArrayList<>(); + for (int i = 0; i < size; i++) { + rescoreBuilders.add(in.readBytesReference()); + } + builder.rescoreBuilders = rescoreBuilders; + } + if (in.readBoolean()) { + int size = in.readVInt(); + List scriptFields = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + scriptFields.add(ScriptField.PROTOTYPE.readFrom(in)); + } + builder.scriptFields = scriptFields; + } + builder.size = in.readVInt(); + if (in.readBoolean()) { + int size = in.readVInt(); + List sorts = new ArrayList<>(); + for (int i = 0; i < size; i++) { + sorts.add(in.readBytesReference()); + } + builder.sorts = sorts; + } + if (in.readBoolean()) { + int size = in.readVInt(); + List stats = new ArrayList<>(); + for (int i = 0; i < size; i++) { + stats.add(in.readString()); + } + builder.stats = stats; + } + if (in.readBoolean()) { + builder.suggestBuilder = in.readBytesReference(); + } + builder.terminateAfter = in.readVInt(); + builder.timeoutInMillis = in.readLong(); + builder.trackScores = in.readBoolean(); + builder.version = in.readOptionalBoolean(); + if (in.readBoolean()) { + builder.ext = in.readBytesReference(); + } + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + boolean hasAggregations = aggregations != null; + out.writeBoolean(hasAggregations); + if (hasAggregations) { + out.writeVInt(aggregations.size()); + for (BytesReference aggregation : aggregations) { + out.writeBytesReference(aggregation); + } + } + out.writeOptionalBoolean(explain); + FetchSourceContext.optionalWriteToStream(fetchSourceContext, out); + boolean hasFieldDataFields = fieldDataFields != null; + out.writeBoolean(hasFieldDataFields); + if (hasFieldDataFields) { + out.writeVInt(fieldDataFields.size()); + for (String field : fieldDataFields) { + out.writeString(field); + } + } + boolean hasFieldNames = fieldNames != null; + out.writeBoolean(hasFieldNames); + if (hasFieldNames) { + out.writeVInt(fieldNames.size()); + for (String field : fieldNames) { + out.writeString(field); + } + } + out.writeVInt(from); + boolean hasHighlightBuilder = highlightBuilder != null; + out.writeBoolean(hasHighlightBuilder); + if (hasHighlightBuilder) { + highlightBuilder.writeTo(out); + } + boolean hasIndexBoost = indexBoost != null; + out.writeBoolean(hasIndexBoost); + if (hasIndexBoost) { + out.writeVInt(indexBoost.size()); + for (ObjectCursor key : indexBoost.keys()) { + out.writeString(key.value); + out.writeFloat(indexBoost.get(key.value)); + } + } + boolean hasInnerHitsBuilder = innerHitsBuilder != null; + out.writeBoolean(hasInnerHitsBuilder); + if (hasInnerHitsBuilder) { + out.writeBytesReference(innerHitsBuilder); + } + boolean hasMinScore = minScore != null; + out.writeBoolean(hasMinScore); + if (hasMinScore) { + out.writeFloat(minScore); + } + boolean hasPostQuery = postQueryBuilder != null; + out.writeBoolean(hasPostQuery); + if (hasPostQuery) { + out.writeQuery(postQueryBuilder); + } + boolean hasQuery = queryBuilder != null; + out.writeBoolean(hasQuery); + if (hasQuery) { + out.writeQuery(queryBuilder); + } + boolean hasRescoreBuilders = rescoreBuilders != null; + out.writeBoolean(hasRescoreBuilders); + if (hasRescoreBuilders) { + out.writeVInt(rescoreBuilders.size()); + for (BytesReference rescoreBuilder : rescoreBuilders) { + out.writeBytesReference(rescoreBuilder); + } + } + boolean hasScriptFields = scriptFields != null; + out.writeBoolean(hasScriptFields); + if (hasScriptFields) { + out.writeVInt(scriptFields.size()); + for (ScriptField scriptField : scriptFields) { + scriptField.writeTo(out); + } + } + out.writeVInt(size); + boolean hasSorts = sorts != null; + out.writeBoolean(hasSorts); + if (hasSorts) { + out.writeVInt(sorts.size()); + for (BytesReference sort : sorts) { + out.writeBytesReference(sort); + } + } + boolean hasStats = stats != null; + out.writeBoolean(hasStats); + if (hasStats) { + out.writeVInt(stats.size()); + for (String stat : stats) { + out.writeString(stat); + } + } + boolean hasSuggestBuilder = suggestBuilder != null; + out.writeBoolean(hasSuggestBuilder); + if (hasSuggestBuilder) { + out.writeBytesReference(suggestBuilder); + } + out.writeVInt(terminateAfter); + out.writeLong(timeoutInMillis); + out.writeBoolean(trackScores); + out.writeOptionalBoolean(version); + boolean hasExt = ext != null; + out.writeBoolean(hasExt); + if (hasExt) { + out.writeBytesReference(ext); + } + } + + @Override + public int hashCode() { + return Objects.hash(aggregations, explain, fetchSourceContext, fieldDataFields, fieldNames, from, + highlightBuilder, indexBoost, innerHitsBuilder, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, + size, sorts, stats, suggestBuilder, terminateAfter, timeoutInMillis, trackScores, version); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + SearchSourceBuilder other = (SearchSourceBuilder) obj; + return Objects.equals(aggregations, other.aggregations) + && Objects.equals(explain, other.explain) + && Objects.equals(fetchSourceContext, other.fetchSourceContext) + && Objects.equals(fieldDataFields, other.fieldDataFields) + && Objects.equals(fieldNames, other.fieldNames) + && Objects.equals(from, other.from) + && Objects.equals(highlightBuilder, other.highlightBuilder) + && Objects.equals(indexBoost, other.indexBoost) + && Objects.equals(innerHitsBuilder, other.innerHitsBuilder) + && Objects.equals(minScore, other.minScore) + && Objects.equals(postQueryBuilder, other.postQueryBuilder) + && Objects.equals(queryBuilder, other.queryBuilder) + && Objects.equals(rescoreBuilders, other.rescoreBuilders) + && Objects.equals(scriptFields, other.scriptFields) + && Objects.equals(size, other.size) + && Objects.equals(sorts, other.sorts) + && Objects.equals(stats, other.stats) + && Objects.equals(suggestBuilder, other.suggestBuilder) + && Objects.equals(terminateAfter, other.terminateAfter) + && Objects.equals(timeoutInMillis, other.timeoutInMillis) + && Objects.equals(trackScores, other.trackScores) + && Objects.equals(version, other.version); } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index be3798e37d9..227141e4ddf 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -21,21 +21,21 @@ package org.elasticsearch.search.fetch; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; -import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.BitSet; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.text.StringAndBytesText; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.fieldvisitor.AllFieldsVisitor; import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; import org.elasticsearch.index.fieldvisitor.FieldsVisitor; import org.elasticsearch.index.mapper.DocumentMapper; @@ -55,13 +55,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder; @@ -98,9 +92,7 @@ public class FetchPhase implements SearchPhase { public void execute(SearchContext context) { FieldsVisitor fieldsVisitor; Set fieldNames = null; - List extractFieldNames = null; - - boolean loadAllStored = false; + List fieldNamePatterns = null; if (!context.hasFieldNames()) { // no fields specified, default to return source if no explicit indication if (!context.hasScriptFields() && !context.hasFetchSourceContext()) { @@ -111,10 +103,6 @@ public class FetchPhase implements SearchPhase { fieldsVisitor = new FieldsVisitor(context.sourceRequested()); } else { for (String fieldName : context.fieldNames()) { - if (fieldName.equals("*")) { - loadAllStored = true; - continue; - } if (fieldName.equals(SourceFieldMapper.NAME)) { if (context.hasFetchSourceContext()) { context.fetchSourceContext().fetchSource(true); @@ -123,32 +111,28 @@ public class FetchPhase implements SearchPhase { } continue; } - MappedFieldType fieldType = context.smartNameFieldType(fieldName); - if (fieldType == null) { - // Only fail if we know it is a object field, missing paths / fields shouldn't fail. - if (context.getObjectMapper(fieldName) != null) { - throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field"); + if (Regex.isSimpleMatchPattern(fieldName)) { + if (fieldNamePatterns == null) { + fieldNamePatterns = new ArrayList<>(); + } + fieldNamePatterns.add(fieldName); + } else { + MappedFieldType fieldType = context.smartNameFieldType(fieldName); + if (fieldType == null) { + // Only fail if we know it is a object field, missing paths / fields shouldn't fail. + if (context.getObjectMapper(fieldName) != null) { + throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field"); + } } - } else if (fieldType.stored()) { if (fieldNames == null) { fieldNames = new HashSet<>(); } - fieldNames.add(fieldType.names().indexName()); - } else { - if (extractFieldNames == null) { - extractFieldNames = new ArrayList<>(); - } - extractFieldNames.add(fieldName); + fieldNames.add(fieldName); } } - if (loadAllStored) { - fieldsVisitor = new AllFieldsVisitor(); // load everything, including _source - } else if (fieldNames != null) { - boolean loadSource = extractFieldNames != null || context.sourceRequested(); - fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource); - } else { - fieldsVisitor = new FieldsVisitor(extractFieldNames != null || context.sourceRequested()); - } + boolean loadSource = context.sourceRequested(); + fieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames, + fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource); } InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()]; @@ -163,9 +147,9 @@ public class FetchPhase implements SearchPhase { try { int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId); if (rootDocId != -1) { - searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, extractFieldNames, loadAllStored, fieldNames, subReaderContext); + searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns, subReaderContext); } else { - searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, extractFieldNames, subReaderContext); + searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext); } } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); @@ -199,7 +183,7 @@ public class FetchPhase implements SearchPhase { return -1; } - private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId, List extractFieldNames, LeafReaderContext subReaderContext) { + private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId, LeafReaderContext subReaderContext) { loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId); fieldsVisitor.postProcess(context.mapperService()); @@ -219,45 +203,24 @@ public class FetchPhase implements SearchPhase { typeText = documentMapper.typeText(); } InternalSearchHit searchHit = new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields); - - // go over and extract fields that are not mapped / stored + // Set _source if requested. SourceLookup sourceLookup = context.lookup().source(); sourceLookup.setSegmentAndDocument(subReaderContext, subDocId); if (fieldsVisitor.source() != null) { sourceLookup.setSource(fieldsVisitor.source()); } - if (extractFieldNames != null) { - for (String extractFieldName : extractFieldNames) { - List values = context.lookup().source().extractRawValues(extractFieldName); - if (!values.isEmpty()) { - if (searchHit.fieldsOrNull() == null) { - searchHit.fields(new HashMap(2)); - } - - SearchHitField hitField = searchHit.fields().get(extractFieldName); - if (hitField == null) { - hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2)); - searchHit.fields().put(extractFieldName, hitField); - } - for (Object value : values) { - hitField.values().add(value); - } - } - } - } - return searchHit; } - private InternalSearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId, int rootSubDocId, List extractFieldNames, boolean loadAllStored, Set fieldNames, LeafReaderContext subReaderContext) throws IOException { + private InternalSearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId, int rootSubDocId, Set fieldNames, List fieldNamePatterns, LeafReaderContext subReaderContext) throws IOException { // Also if highlighting is requested on nested documents we need to fetch the _source from the root document, // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail, // because the entire _source is only stored with the root document. - final FieldsVisitor rootFieldsVisitor = new FieldsVisitor(context.sourceRequested() || extractFieldNames != null || context.highlight() != null); + final FieldsVisitor rootFieldsVisitor = new FieldsVisitor(context.sourceRequested() || context.highlight() != null); loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId); rootFieldsVisitor.postProcess(context.mapperService()); - Map searchFields = getSearchFields(context, nestedSubDocId, loadAllStored, fieldNames, subReaderContext); + Map searchFields = getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext); DocumentMapper documentMapper = context.mapperService().documentMapper(rootFieldsVisitor.uid().type()); SourceLookup sourceLookup = context.lookup().source(); sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId); @@ -299,39 +262,14 @@ public class FetchPhase implements SearchPhase { } InternalSearchHit searchHit = new InternalSearchHit(nestedTopDocId, rootFieldsVisitor.uid().id(), documentMapper.typeText(), nestedIdentity, searchFields); - if (extractFieldNames != null) { - for (String extractFieldName : extractFieldNames) { - List values = context.lookup().source().extractRawValues(extractFieldName); - if (!values.isEmpty()) { - if (searchHit.fieldsOrNull() == null) { - searchHit.fields(new HashMap(2)); - } - - SearchHitField hitField = searchHit.fields().get(extractFieldName); - if (hitField == null) { - hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2)); - searchHit.fields().put(extractFieldName, hitField); - } - for (Object value : values) { - hitField.values().add(value); - } - } - } - } - return searchHit; } - private Map getSearchFields(SearchContext context, int nestedSubDocId, boolean loadAllStored, Set fieldNames, LeafReaderContext subReaderContext) { + private Map getSearchFields(SearchContext context, int nestedSubDocId, Set fieldNames, List fieldNamePatterns, LeafReaderContext subReaderContext) { Map searchFields = null; if (context.hasFieldNames() && !context.fieldNames().isEmpty()) { - FieldsVisitor nestedFieldsVisitor = null; - if (loadAllStored) { - nestedFieldsVisitor = new AllFieldsVisitor(); - } else if (fieldNames != null) { - nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames, false); - } - + FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames, + fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, false); if (nestedFieldsVisitor != null) { loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId); nestedFieldsVisitor.postProcess(context.mapperService()); @@ -353,7 +291,7 @@ public class FetchPhase implements SearchPhase { String originalName = nestedObjectMapper.name(); InternalSearchHit.InternalNestedIdentity nestedIdentity = null; do { - Filter parentFilter; + Query parentFilter; nestedParentObjectMapper = documentMapper.findParentObjectMapper(current); if (nestedParentObjectMapper != null) { if (nestedParentObjectMapper.nested().isNested() == false) { @@ -365,18 +303,13 @@ public class FetchPhase implements SearchPhase { parentFilter = Queries.newNonNestedFilter(); } - Filter childFilter = nestedObjectMapper.nestedTypeFilter(); + Query childFilter = nestedObjectMapper.nestedTypeFilter(); if (childFilter == null) { current = nestedParentObjectMapper; continue; } - // We can pass down 'null' as acceptedDocs, because we're fetching matched docId that matched in the query phase. - DocIdSet childDocSet = childFilter.getDocIdSet(subReaderContext, null); - if (childDocSet == null) { - current = nestedParentObjectMapper; - continue; - } - DocIdSetIterator childIter = childDocSet.iterator(); + final Weight childWeight = context.searcher().createNormalizedWeight(childFilter, false); + DocIdSetIterator childIter = childWeight.scorer(subReaderContext); if (childIter == null) { current = nestedParentObjectMapper; continue; diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java index 237f43519fa..981408b1bdd 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java @@ -33,7 +33,7 @@ public class FetchSubPhaseContext { /** * Set if this phase should be executed at all. */ - void setHitExecutionNeeded(boolean hitExecutionNeeded) { + public void setHitExecutionNeeded(boolean hitExecutionNeeded) { this.hitExecutionNeeded = hitExecutionNeeded; } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsBuilder.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsBuilder.java index a14fdfe9f8f..2e76a4c3703 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsBuilder.java @@ -42,12 +42,16 @@ public class InnerHitsBuilder implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("inner_hits"); + innerXContent(builder, params); + return builder.endObject(); + } + + public void innerXContent(XContentBuilder builder, Params params) throws IOException { for (Map.Entry entry : innerHits.entrySet()) { builder.startObject(entry.getKey()); entry.getValue().toXContent(builder, params); builder.endObject(); } - return builder.endObject(); } /** @@ -261,187 +265,12 @@ public class InnerHitsBuilder implements ToXContent { return this; } - public HighlightBuilder highlightBuilder() { + public HighlightBuilder highlighter() { return sourceBuilder().highlighter(); } - /** - * Adds a field to be highlighted with default fragment size of 100 characters, and - * default number of fragments of 5. - * - * @param name The field to highlight - */ - public InnerHit addHighlightedField(String name) { - highlightBuilder().field(name); - return this; - } - - - /** - * Adds a field to be highlighted with a provided fragment size (in characters), and - * default number of fragments of 5. - * - * @param name The field to highlight - * @param fragmentSize The size of a fragment in characters - */ - public InnerHit addHighlightedField(String name, int fragmentSize) { - highlightBuilder().field(name, fragmentSize); - return this; - } - - /** - * Adds a field to be highlighted with a provided fragment size (in characters), and - * a provided (maximum) number of fragments. - * - * @param name The field to highlight - * @param fragmentSize The size of a fragment in characters - * @param numberOfFragments The (maximum) number of fragments - */ - public InnerHit addHighlightedField(String name, int fragmentSize, int numberOfFragments) { - highlightBuilder().field(name, fragmentSize, numberOfFragments); - return this; - } - - /** - * Adds a field to be highlighted with a provided fragment size (in characters), - * a provided (maximum) number of fragments and an offset for the highlight. - * - * @param name The field to highlight - * @param fragmentSize The size of a fragment in characters - * @param numberOfFragments The (maximum) number of fragments - */ - public InnerHit addHighlightedField(String name, int fragmentSize, int numberOfFragments, - int fragmentOffset) { - highlightBuilder().field(name, fragmentSize, numberOfFragments, fragmentOffset); - return this; - } - - /** - * Adds a highlighted field. - */ - public InnerHit addHighlightedField(HighlightBuilder.Field field) { - highlightBuilder().field(field); - return this; - } - - /** - * Set a tag scheme that encapsulates a built in pre and post tags. The allows schemes - * are styled and default. - * - * @param schemaName The tag scheme name - */ - public InnerHit setHighlighterTagsSchema(String schemaName) { - highlightBuilder().tagsSchema(schemaName); - return this; - } - - public InnerHit setHighlighterFragmentSize(Integer fragmentSize) { - highlightBuilder().fragmentSize(fragmentSize); - return this; - } - - public InnerHit setHighlighterNumOfFragments(Integer numOfFragments) { - highlightBuilder().numOfFragments(numOfFragments); - return this; - } - - public InnerHit setHighlighterFilter(Boolean highlightFilter) { - highlightBuilder().highlightFilter(highlightFilter); - return this; - } - - /** - * The encoder to set for highlighting - */ - public InnerHit setHighlighterEncoder(String encoder) { - highlightBuilder().encoder(encoder); - return this; - } - - /** - * Explicitly set the pre tags that will be used for highlighting. - */ - public InnerHit setHighlighterPreTags(String... preTags) { - highlightBuilder().preTags(preTags); - return this; - } - - /** - * Explicitly set the post tags that will be used for highlighting. - */ - public InnerHit setHighlighterPostTags(String... postTags) { - highlightBuilder().postTags(postTags); - return this; - } - - /** - * The order of fragments per field. By default, ordered by the order in the - * highlighted text. Can be score, which then it will be ordered - * by score of the fragments. - */ - public InnerHit setHighlighterOrder(String order) { - highlightBuilder().order(order); - return this; - } - - public InnerHit setHighlighterRequireFieldMatch(boolean requireFieldMatch) { - highlightBuilder().requireFieldMatch(requireFieldMatch); - return this; - } - - public InnerHit setHighlighterBoundaryMaxScan(Integer boundaryMaxScan) { - highlightBuilder().boundaryMaxScan(boundaryMaxScan); - return this; - } - - public InnerHit setHighlighterBoundaryChars(char[] boundaryChars) { - highlightBuilder().boundaryChars(boundaryChars); - return this; - } - - /** - * The highlighter type to use. - */ - public InnerHit setHighlighterType(String type) { - highlightBuilder().highlighterType(type); - return this; - } - - public InnerHit setHighlighterFragmenter(String fragmenter) { - highlightBuilder().fragmenter(fragmenter); - return this; - } - - /** - * Sets a query to be used for highlighting all fields instead of the search query. - */ - public InnerHit setHighlighterQuery(QueryBuilder highlightQuery) { - highlightBuilder().highlightQuery(highlightQuery); - return this; - } - - /** - * Sets the size of the fragment to return from the beginning of the field if there are no matches to - * highlight and the field doesn't also define noMatchSize. - * - * @param noMatchSize integer to set or null to leave out of request. default is null. - * @return this builder for chaining - */ - public InnerHit setHighlighterNoMatchSize(Integer noMatchSize) { - highlightBuilder().noMatchSize(noMatchSize); - return this; - } - - /** - * Sets the maximum number of phrases the fvh will consider if the field doesn't also define phraseLimit. - */ - public InnerHit setHighlighterPhraseLimit(Integer phraseLimit) { - highlightBuilder().phraseLimit(phraseLimit); - return this; - } - - public InnerHit setHighlighterOptions(Map options) { - highlightBuilder().options(options); + public InnerHit highlighter(HighlightBuilder highlightBuilder) { + sourceBuilder().highlighter(highlightBuilder); return this; } @@ -460,24 +289,8 @@ public class InnerHitsBuilder implements ToXContent { return this; } - - - - /** - * Adds a nested inner hit definition that collects inner hits for hits - * on this inner hit level. - */ - public InnerHit addNestedInnerHits(String name, String path, InnerHit innerHit) { - sourceBuilder().innerHitsBuilder().addNestedInnerHits(name, path, innerHit); - return this; - } - - /** - * Adds a nested inner hit definition that collects inner hits for hits - * on this inner hit level. - */ - public InnerHit addParentChildInnerHits(String name, String type, InnerHit innerHit) { - sourceBuilder().innerHitsBuilder().addParentChildInnerHits(name, type, innerHit); + public InnerHit innerHits(InnerHitsBuilder innerHitsBuilder) { + sourceBuilder().innerHits(innerHitsBuilder); return this; } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java index 2887a83ad8f..e1884e36609 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java @@ -110,14 +110,14 @@ public final class InnerHitsContext { @Override public TopDocs topDocs(SearchContext context, FetchSubPhase.HitContext hitContext) throws IOException { - Filter rawParentFilter; + Query rawParentFilter; if (parentObjectMapper == null) { rawParentFilter = Queries.newNonNestedFilter(); } else { rawParentFilter = parentObjectMapper.nestedTypeFilter(); } BitSetProducer parentFilter = context.bitsetFilterCache().getBitSetProducer(rawParentFilter); - Filter childFilter = childObjectMapper.nestedTypeFilter(); + Query childFilter = childObjectMapper.nestedTypeFilter(); Query q = Queries.filtered(query.query(), new NestedChildrenQuery(parentFilter, childFilter, hitContext)); if (size() == 0) { @@ -147,11 +147,11 @@ public final class InnerHitsContext { static class NestedChildrenQuery extends Query { private final BitSetProducer parentFilter; - private final Filter childFilter; + private final Query childFilter; private final int docId; private final LeafReader leafReader; - NestedChildrenQuery(BitSetProducer parentFilter, Filter childFilter, FetchSubPhase.HitContext hitContext) { + NestedChildrenQuery(BitSetProducer parentFilter, Query childFilter, FetchSubPhase.HitContext hitContext) { this.parentFilter = parentFilter; this.childFilter = childFilter; this.docId = hitContext.docId(); @@ -187,6 +187,7 @@ public final class InnerHitsContext { @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + final Weight childWeight = childFilter.createWeight(searcher, false); return new ConstantScoreWeight(this) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { @@ -208,11 +209,7 @@ public final class InnerHitsContext { return null; } - final DocIdSet children = childFilter.getDocIdSet(context, null); - if (children == null) { - return null; - } - final DocIdSetIterator childrenIterator = children.iterator(); + final DocIdSetIterator childrenIterator = childWeight.scorer(context); if (childrenIterator == null) { return null; } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java index ac6dc18d7d4..6ed635e8c81 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java @@ -59,7 +59,7 @@ public class InnerHitsParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext searchContext) throws Exception { - QueryShardContext context = searchContext.queryParserService().getShardContext(); + QueryShardContext context = searchContext.indexShard().getQueryShardContext(); context.reset(parser); Map innerHitsMap = parseInnerHits(parser, context, searchContext); if (innerHitsMap != null) { @@ -149,7 +149,7 @@ public class InnerHitsParseElement implements SearchParseElement { if (documentMapper == null) { throw new IllegalArgumentException("type [" + type + "] doesn't exist"); } - return new InnerHitsContext.ParentChildInnerHits(parseResult.context(), parseResult.query(), parseResult.childInnerHits(), context.mapperService(), documentMapper); + return new InnerHitsContext.ParentChildInnerHits(parseResult.context(), parseResult.query(), parseResult.childInnerHits(), context.getMapperService(), documentMapper); } private InnerHitsContext.NestedInnerHits parseNested(XContentParser parser, QueryShardContext context, SearchContext searchContext, String nestedPath) throws Exception { @@ -178,7 +178,7 @@ public class InnerHitsParseElement implements SearchParseElement { fieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { if ("query".equals(fieldName)) { - Query q = searchContext.queryParserService().parseInnerQuery(context); + Query q = context.parseInnerQuery(); query = new ParsedQuery(q, context.copyNamedQueries()); } else if ("inner_hits".equals(fieldName)) { childInnerHits = parseInnerHits(parser, context, searchContext); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceContext.java b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceContext.java index 9db7aea6b95..c14e2882d39 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceContext.java @@ -19,35 +19,49 @@ package org.elasticsearch.search.fetch.source; -import org.elasticsearch.Version; import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.RestRequest; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** */ -public class FetchSourceContext implements Streamable { +public class FetchSourceContext implements Streamable, ToXContent { + + public static final ParseField INCLUDES_FIELD = new ParseField("includes", "include"); + public static final ParseField EXCLUDES_FIELD = new ParseField("excludes", "exclude"); public static final FetchSourceContext FETCH_SOURCE = new FetchSourceContext(true); public static final FetchSourceContext DO_NOT_FETCH_SOURCE = new FetchSourceContext(false); private boolean fetchSource; - private boolean transformSource; private String[] includes; private String[] excludes; + public static FetchSourceContext parse(XContentParser parser, QueryParseContext context) throws IOException { + FetchSourceContext fetchSourceContext = new FetchSourceContext(); + fetchSourceContext.fromXContent(parser, context); + return fetchSourceContext; + } FetchSourceContext() { } public FetchSourceContext(boolean fetchSource) { - this(fetchSource, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, false); + this(fetchSource, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY); } public FetchSourceContext(String include) { @@ -57,23 +71,21 @@ public class FetchSourceContext implements Streamable { public FetchSourceContext(String include, String exclude) { this(true, include == null ? Strings.EMPTY_ARRAY : new String[]{include}, - exclude == null ? Strings.EMPTY_ARRAY : new String[]{exclude}, - false); + exclude == null ? Strings.EMPTY_ARRAY : new String[]{exclude}); } public FetchSourceContext(String[] includes) { - this(true, includes, Strings.EMPTY_ARRAY, false); + this(true, includes, Strings.EMPTY_ARRAY); } public FetchSourceContext(String[] includes, String[] excludes) { - this(true, includes, excludes, false); + this(true, includes, excludes); } - public FetchSourceContext(boolean fetchSource, String[] includes, String[] excludes, boolean transform) { + public FetchSourceContext(boolean fetchSource, String[] includes, String[] excludes) { this.fetchSource = fetchSource; this.includes = includes == null ? Strings.EMPTY_ARRAY : includes; this.excludes = excludes == null ? Strings.EMPTY_ARRAY : excludes; - this.transformSource = transform; } public boolean fetchSource() { @@ -85,22 +97,6 @@ public class FetchSourceContext implements Streamable { return this; } - /** - * Should the document be transformed after the source is loaded? - */ - public boolean transformSource() { - return this.transformSource; - } - - /** - * Should the document be transformed after the source is loaded? - * @return this for chaining - */ - public FetchSourceContext transformSource(boolean transformSource) { - this.transformSource = transformSource; - return this; - } - public String[] includes() { return this.includes; } @@ -164,20 +160,98 @@ public class FetchSourceContext implements Streamable { source_excludes = Strings.splitStringByCommaToArray(sExcludes); } - boolean transform = request.paramAsBoolean("_source_transform", false); - - if (fetchSource != null || source_includes != null || source_excludes != null || transform) { - return new FetchSourceContext(fetchSource == null ? true : fetchSource, source_includes, source_excludes, transform); + if (fetchSource != null || source_includes != null || source_excludes != null) { + return new FetchSourceContext(fetchSource == null ? true : fetchSource, source_includes, source_excludes); } return null; } + public void fromXContent(XContentParser parser, QueryParseContext context) throws IOException { + XContentParser.Token token = parser.currentToken(); + boolean fetchSource = true; + String[] includes = Strings.EMPTY_ARRAY; + String[] excludes = Strings.EMPTY_ARRAY; + if (token == XContentParser.Token.VALUE_BOOLEAN) { + fetchSource = parser.booleanValue(); + } else if (token == XContentParser.Token.VALUE_STRING) { + includes = new String[]{parser.text()}; + } else if (token == XContentParser.Token.START_ARRAY) { + ArrayList list = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + list.add(parser.text()); + } + includes = list.toArray(new String[list.size()]); + } else if (token == XContentParser.Token.START_OBJECT) { + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_ARRAY) { + if (context.parseFieldMatcher().match(currentFieldName, INCLUDES_FIELD)) { + List includesList = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + includesList.add(parser.text()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } + includes = includesList.toArray(new String[includesList.size()]); + } else if (context.parseFieldMatcher().match(currentFieldName, EXCLUDES_FIELD)) { + List excludesList = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + excludesList.add(parser.text()); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } + excludes = excludesList.toArray(new String[excludesList.size()]); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } else if (token == XContentParser.Token.VALUE_STRING) { + if (context.parseFieldMatcher().match(currentFieldName, INCLUDES_FIELD)) { + includes = new String[] {parser.text()}; + } else if (context.parseFieldMatcher().match(currentFieldName, EXCLUDES_FIELD)) { + excludes = new String[] {parser.text()}; + } + } else { + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation()); + } + } + } else { + throw new ParsingException(parser.getTokenLocation(), "Expected one of [" + XContentParser.Token.VALUE_BOOLEAN + ", " + + XContentParser.Token.START_OBJECT + "] but found [" + token + "]", parser.getTokenLocation()); + } + this.fetchSource = fetchSource; + this.includes = includes; + this.excludes = excludes; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (fetchSource) { + builder.startObject(); + builder.array(INCLUDES_FIELD.getPreferredName(), includes); + builder.array(EXCLUDES_FIELD.getPreferredName(), excludes); + builder.endObject(); + } else { + builder.value(false); + } + return builder; + } + @Override public void readFrom(StreamInput in) throws IOException { fetchSource = in.readBoolean(); includes = in.readStringArray(); excludes = in.readStringArray(); - transformSource = in.readBoolean(); + in.readBoolean(); // Used to be transformSource but that was dropped in 2.1 } @Override @@ -185,7 +259,7 @@ public class FetchSourceContext implements Streamable { out.writeBoolean(fetchSource); out.writeStringArray(includes); out.writeStringArray(excludes); - out.writeBoolean(transformSource); + out.writeBoolean(false); // Used to be transformSource but that was dropped in 2.1 } @Override diff --git a/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java new file mode 100644 index 00000000000..b4de465cc74 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java @@ -0,0 +1,646 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.highlight; + +import org.apache.lucene.search.highlight.SimpleFragmenter; +import org.apache.lucene.search.highlight.SimpleSpanFragmenter; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.highlight.HighlightBuilder.Order; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * This abstract class holds parameters shared by {@link HighlightBuilder} and {@link HighlightBuilder.Field} + * and provides the common setters, equality, hashCode calculation and common serialization + */ +public abstract class AbstractHighlighterBuilder> { + + public static final ParseField PRE_TAGS_FIELD = new ParseField("pre_tags"); + public static final ParseField POST_TAGS_FIELD = new ParseField("post_tags"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + public static final ParseField ORDER_FIELD = new ParseField("order"); + public static final ParseField TAGS_SCHEMA_FIELD = new ParseField("tags_schema"); + public static final ParseField HIGHLIGHT_FILTER_FIELD = new ParseField("highlight_filter"); + public static final ParseField FRAGMENT_SIZE_FIELD = new ParseField("fragment_size"); + public static final ParseField FRAGMENT_OFFSET_FIELD = new ParseField("fragment_offset"); + public static final ParseField NUMBER_OF_FRAGMENTS_FIELD = new ParseField("number_of_fragments"); + public static final ParseField ENCODER_FIELD = new ParseField("encoder"); + public static final ParseField REQUIRE_FIELD_MATCH_FIELD = new ParseField("require_field_match"); + public static final ParseField BOUNDARY_MAX_SCAN_FIELD = new ParseField("boundary_max_scan"); + public static final ParseField BOUNDARY_CHARS_FIELD = new ParseField("boundary_chars"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField FRAGMENTER_FIELD = new ParseField("fragmenter"); + public static final ParseField NO_MATCH_SIZE_FIELD = new ParseField("no_match_size"); + public static final ParseField FORCE_SOURCE_FIELD = new ParseField("force_source"); + public static final ParseField PHRASE_LIMIT_FIELD = new ParseField("phrase_limit"); + public static final ParseField OPTIONS_FIELD = new ParseField("options"); + public static final ParseField HIGHLIGHT_QUERY_FIELD = new ParseField("highlight_query"); + public static final ParseField MATCHED_FIELDS_FIELD = new ParseField("matched_fields"); + + protected String[] preTags; + + protected String[] postTags; + + protected Integer fragmentSize; + + protected Integer numOfFragments; + + protected String highlighterType; + + protected String fragmenter; + + protected QueryBuilder highlightQuery; + + protected Order order; + + protected Boolean highlightFilter; + + protected Boolean forceSource; + + protected Integer boundaryMaxScan; + + protected char[] boundaryChars; + + protected Integer noMatchSize; + + protected Integer phraseLimit; + + protected Map options; + + protected Boolean requireFieldMatch; + + /** + * Set the pre tags that will be used for highlighting. + */ + @SuppressWarnings("unchecked") + public HB preTags(String... preTags) { + this.preTags = preTags; + return (HB) this; + } + + /** + * @return the value set by {@link #preTags(String...)} + */ + public String[] preTags() { + return this.preTags; + } + + /** + * Set the post tags that will be used for highlighting. + */ + @SuppressWarnings("unchecked") + public HB postTags(String... postTags) { + this.postTags = postTags; + return (HB) this; + } + + /** + * @return the value set by {@link #postTags(String...)} + */ + public String[] postTags() { + return this.postTags; + } + + /** + * Set the fragment size in characters, defaults to {@link HighlightBuilder#DEFAULT_FRAGMENT_CHAR_SIZE} + */ + @SuppressWarnings("unchecked") + public HB fragmentSize(Integer fragmentSize) { + this.fragmentSize = fragmentSize; + return (HB) this; + } + + /** + * @return the value set by {@link #fragmentSize(Integer)} + */ + public Integer fragmentSize() { + return this.fragmentSize; + } + + /** + * Set the number of fragments, defaults to {@link HighlightBuilder#DEFAULT_NUMBER_OF_FRAGMENTS} + */ + @SuppressWarnings("unchecked") + public HB numOfFragments(Integer numOfFragments) { + this.numOfFragments = numOfFragments; + return (HB) this; + } + + /** + * @return the value set by {@link #numOfFragments(Integer)} + */ + public Integer numOfFragments() { + return this.numOfFragments; + } + + /** + * Set type of highlighter to use. Out of the box supported types + * are plain, fvh and postings. + * The default option selected is dependent on the mappings defined for your index. + * Details of the different highlighter types are covered in the reference guide. + */ + @SuppressWarnings("unchecked") + public HB highlighterType(String highlighterType) { + this.highlighterType = highlighterType; + return (HB) this; + } + + /** + * @return the value set by {@link #highlighterType(String)} + */ + public String highlighterType() { + return this.highlighterType; + } + + /** + * Sets what fragmenter to use to break up text that is eligible for highlighting. + * This option is only applicable when using the plain highlighterType highlighter. + * Permitted values are "simple" or "span" relating to {@link SimpleFragmenter} and + * {@link SimpleSpanFragmenter} implementations respectively with the default being "span" + */ + @SuppressWarnings("unchecked") + public HB fragmenter(String fragmenter) { + this.fragmenter = fragmenter; + return (HB) this; + } + + /** + * @return the value set by {@link #fragmenter(String)} + */ + public String fragmenter() { + return this.fragmenter; + } + + /** + * Sets a query to be used for highlighting instead of the search query. + */ + @SuppressWarnings("unchecked") + public HB highlightQuery(QueryBuilder highlightQuery) { + this.highlightQuery = highlightQuery; + return (HB) this; + } + + /** + * @return the value set by {@link #highlightQuery(QueryBuilder)} + */ + public QueryBuilder highlightQuery() { + return this.highlightQuery; + } + + /** + * The order of fragments per field. By default, ordered by the order in the + * highlighted text. Can be score, which then it will be ordered + * by score of the fragments, or none. + */ + public HB order(String order) { + return order(Order.fromString(order)); + } + + /** + * By default, fragments of a field are ordered by the order in the highlighted text. + * If set to {@link Order#SCORE}, this changes order to score of the fragments. + */ + @SuppressWarnings("unchecked") + public HB order(Order scoreOrdered) { + this.order = scoreOrdered; + return (HB) this; + } + + /** + * @return the value set by {@link #order(Order)} + */ + public Order order() { + return this.order; + } + + /** + * Set this to true when using the highlighterType fvh + * and you want to provide highlighting on filter clauses in your + * query. Default is false. + */ + @SuppressWarnings("unchecked") + public HB highlightFilter(Boolean highlightFilter) { + this.highlightFilter = highlightFilter; + return (HB) this; + } + + /** + * @return the value set by {@link #highlightFilter(Boolean)} + */ + public Boolean highlightFilter() { + return this.highlightFilter; + } + + /** + * When using the highlighterType fvh this setting + * controls how far to look for boundary characters, and defaults to 20. + */ + @SuppressWarnings("unchecked") + public HB boundaryMaxScan(Integer boundaryMaxScan) { + this.boundaryMaxScan = boundaryMaxScan; + return (HB) this; + } + + /** + * @return the value set by {@link #boundaryMaxScan(Integer)} + */ + public Integer boundaryMaxScan() { + return this.boundaryMaxScan; + } + + /** + * When using the highlighterType fvh this setting + * defines what constitutes a boundary for highlighting. It’s a single string with + * each boundary character defined in it. It defaults to .,!? \t\n + */ + @SuppressWarnings("unchecked") + public HB boundaryChars(char[] boundaryChars) { + this.boundaryChars = boundaryChars; + return (HB) this; + } + + /** + * @return the value set by {@link #boundaryChars(char[])} + */ + public char[] boundaryChars() { + return this.boundaryChars; + } + + /** + * Allows to set custom options for custom highlighters. + */ + @SuppressWarnings("unchecked") + public HB options(Map options) { + this.options = options; + return (HB) this; + } + + /** + * @return the value set by {@link #options(Map)} + */ + public Map options() { + return this.options; + } + + /** + * Set to true to cause a field to be highlighted only if a query matches that field. + * Default is false meaning that terms are highlighted on all requested fields regardless + * if the query matches specifically on them. + */ + @SuppressWarnings("unchecked") + public HB requireFieldMatch(Boolean requireFieldMatch) { + this.requireFieldMatch = requireFieldMatch; + return (HB) this; + } + + /** + * @return the value set by {@link #requireFieldMatch(Boolean)} + */ + public Boolean requireFieldMatch() { + return this.requireFieldMatch; + } + + /** + * Sets the size of the fragment to return from the beginning of the field if there are no matches to + * highlight and the field doesn't also define noMatchSize. + * @param noMatchSize integer to set or null to leave out of request. default is null. + * @return this for chaining + */ + @SuppressWarnings("unchecked") + public HB noMatchSize(Integer noMatchSize) { + this.noMatchSize = noMatchSize; + return (HB) this; + } + + /** + * @return the value set by {@link #noMatchSize(Integer)} + */ + public Integer noMatchSize() { + return this.noMatchSize; + } + + /** + * Sets the maximum number of phrases the fvh will consider if the field doesn't also define phraseLimit. + * @param phraseLimit maximum number of phrases the fvh will consider + * @return this for chaining + */ + @SuppressWarnings("unchecked") + public HB phraseLimit(Integer phraseLimit) { + this.phraseLimit = phraseLimit; + return (HB) this; + } + + /** + * @return the value set by {@link #phraseLimit(Integer)} + */ + public Integer phraseLimit() { + return this.noMatchSize; + } + + /** + * Forces the highlighting to highlight fields based on the source even if fields are stored separately. + */ + @SuppressWarnings("unchecked") + public HB forceSource(Boolean forceSource) { + this.forceSource = forceSource; + return (HB) this; + } + + /** + * @return the value set by {@link #forceSource(Boolean)} + */ + public Boolean forceSource() { + return this.forceSource; + } + + void commonOptionsToXContent(XContentBuilder builder) throws IOException { + if (preTags != null) { + builder.array(PRE_TAGS_FIELD.getPreferredName(), preTags); + } + if (postTags != null) { + builder.array(POST_TAGS_FIELD.getPreferredName(), postTags); + } + if (fragmentSize != null) { + builder.field(FRAGMENT_SIZE_FIELD.getPreferredName(), fragmentSize); + } + if (numOfFragments != null) { + builder.field(NUMBER_OF_FRAGMENTS_FIELD.getPreferredName(), numOfFragments); + } + if (highlighterType != null) { + builder.field(TYPE_FIELD.getPreferredName(), highlighterType); + } + if (fragmenter != null) { + builder.field(FRAGMENTER_FIELD.getPreferredName(), fragmenter); + } + if (highlightQuery != null) { + builder.field(HIGHLIGHT_QUERY_FIELD.getPreferredName(), highlightQuery); + } + if (order != null) { + builder.field(ORDER_FIELD.getPreferredName(), order.toString()); + } + if (highlightFilter != null) { + builder.field(HIGHLIGHT_FILTER_FIELD.getPreferredName(), highlightFilter); + } + if (boundaryMaxScan != null) { + builder.field(BOUNDARY_MAX_SCAN_FIELD.getPreferredName(), boundaryMaxScan); + } + if (boundaryChars != null) { + builder.field(BOUNDARY_CHARS_FIELD.getPreferredName(), new String(boundaryChars)); + } + if (options != null && options.size() > 0) { + builder.field(OPTIONS_FIELD.getPreferredName(), options); + } + if (forceSource != null) { + builder.field(FORCE_SOURCE_FIELD.getPreferredName(), forceSource); + } + if (requireFieldMatch != null) { + builder.field(REQUIRE_FIELD_MATCH_FIELD.getPreferredName(), requireFieldMatch); + } + if (noMatchSize != null) { + builder.field(NO_MATCH_SIZE_FIELD.getPreferredName(), noMatchSize); + } + if (phraseLimit != null) { + builder.field(PHRASE_LIMIT_FIELD.getPreferredName(), phraseLimit); + } + } + + /** + * Creates a new {@link HighlightBuilder} from the highlighter held by the {@link QueryParseContext} + * in {@link org.elasticsearch.common.xcontent.XContent} format + * + * @param parseContext containing the parser positioned at the structure to be parsed from. + * the state on the parser contained in this context will be changed as a side effect of this + * method call + * @return the new {@link AbstractHighlighterBuilder} + */ + public HB fromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = null; + HB highlightBuilder = createInstance(parser); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_ARRAY) { + if (parseContext.parseFieldMatcher().match(currentFieldName, PRE_TAGS_FIELD)) { + List preTagsList = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + preTagsList.add(parser.text()); + } + highlightBuilder.preTags(preTagsList.toArray(new String[preTagsList.size()])); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, POST_TAGS_FIELD)) { + List postTagsList = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + postTagsList.add(parser.text()); + } + highlightBuilder.postTags(postTagsList.toArray(new String[postTagsList.size()])); + } else if (false == highlightBuilder.doFromXContent(parseContext, currentFieldName, token)) { + throw new ParsingException(parser.getTokenLocation(), "cannot parse array with name [{}]", currentFieldName); + } + } else if (token.isValue()) { + if (parseContext.parseFieldMatcher().match(currentFieldName, ORDER_FIELD)) { + highlightBuilder.order(Order.fromString(parser.text())); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, HIGHLIGHT_FILTER_FIELD)) { + highlightBuilder.highlightFilter(parser.booleanValue()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FRAGMENT_SIZE_FIELD)) { + highlightBuilder.fragmentSize(parser.intValue()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, NUMBER_OF_FRAGMENTS_FIELD)) { + highlightBuilder.numOfFragments(parser.intValue()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REQUIRE_FIELD_MATCH_FIELD)) { + highlightBuilder.requireFieldMatch(parser.booleanValue()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, BOUNDARY_MAX_SCAN_FIELD)) { + highlightBuilder.boundaryMaxScan(parser.intValue()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, BOUNDARY_CHARS_FIELD)) { + highlightBuilder.boundaryChars(parser.text().toCharArray()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { + highlightBuilder.highlighterType(parser.text()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FRAGMENTER_FIELD)) { + highlightBuilder.fragmenter(parser.text()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, NO_MATCH_SIZE_FIELD)) { + highlightBuilder.noMatchSize(parser.intValue()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FORCE_SOURCE_FIELD)) { + highlightBuilder.forceSource(parser.booleanValue()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PHRASE_LIMIT_FIELD)) { + highlightBuilder.phraseLimit(parser.intValue()); + } else if (false == highlightBuilder.doFromXContent(parseContext, currentFieldName, token)) { + throw new ParsingException(parser.getTokenLocation(), "unexpected fieldname [{}]", currentFieldName); + } + } else if (token == XContentParser.Token.START_OBJECT && currentFieldName != null) { + if (parseContext.parseFieldMatcher().match(currentFieldName, OPTIONS_FIELD)) { + highlightBuilder.options(parser.map()); + } else if (parseContext.parseFieldMatcher().match(currentFieldName, HIGHLIGHT_QUERY_FIELD)) { + highlightBuilder.highlightQuery(parseContext.parseInnerQueryBuilder()); + } else if (false == highlightBuilder.doFromXContent(parseContext, currentFieldName, token)) { + throw new ParsingException(parser.getTokenLocation(), "cannot parse object with name [{}]", currentFieldName); + } + } else if (currentFieldName != null) { + throw new ParsingException(parser.getTokenLocation(), "unexpected token [{}] after [{}]", token, currentFieldName); + } + } + + if (highlightBuilder.preTags() != null && highlightBuilder.postTags() == null) { + throw new ParsingException(parser.getTokenLocation(), "Highlighter global preTags are set, but global postTags are not set"); + } + return highlightBuilder; + } + + /** + * @param parser the input parser. Implementing classes might advance the parser depending on the + * information they need to instantiate a new instance + * @return a new instance + */ + protected abstract HB createInstance(XContentParser parser) throws IOException; + + /** + * Implementing subclasses can handle parsing special options depending on the + * current token, field name and the parse context. + * @return true if an option was found and successfully parsed, otherwise false + */ + protected abstract boolean doFromXContent(QueryParseContext parseContext, String currentFieldName, XContentParser.Token endMarkerToken) throws IOException; + + @Override + public final int hashCode() { + return Objects.hash(getClass(), Arrays.hashCode(preTags), Arrays.hashCode(postTags), fragmentSize, + numOfFragments, highlighterType, fragmenter, highlightQuery, order, highlightFilter, + forceSource, boundaryMaxScan, Arrays.hashCode(boundaryChars), noMatchSize, + phraseLimit, options, requireFieldMatch, doHashCode()); + } + + /** + * fields only present in subclass should contribute to hashCode in the implementation + */ + protected abstract int doHashCode(); + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + @SuppressWarnings("unchecked") + HB other = (HB) obj; + return Arrays.equals(preTags, other.preTags) && + Arrays.equals(postTags, other.postTags) && + Objects.equals(fragmentSize, other.fragmentSize) && + Objects.equals(numOfFragments, other.numOfFragments) && + Objects.equals(highlighterType, other.highlighterType) && + Objects.equals(fragmenter, other.fragmenter) && + Objects.equals(highlightQuery, other.highlightQuery) && + Objects.equals(order, other.order) && + Objects.equals(highlightFilter, other.highlightFilter) && + Objects.equals(forceSource, other.forceSource) && + Objects.equals(boundaryMaxScan, other.boundaryMaxScan) && + Arrays.equals(boundaryChars, other.boundaryChars) && + Objects.equals(noMatchSize, other.noMatchSize) && + Objects.equals(phraseLimit, other.phraseLimit) && + Objects.equals(options, other.options) && + Objects.equals(requireFieldMatch, other.requireFieldMatch) && + doEquals(other); + } + + /** + * fields only present in subclass should be checked for equality in the implementation + */ + protected abstract boolean doEquals(HB other); + + /** + * read common parameters from {@link StreamInput} + */ + @SuppressWarnings("unchecked") + protected HB readOptionsFrom(StreamInput in) throws IOException { + preTags(in.readOptionalStringArray()); + postTags(in.readOptionalStringArray()); + fragmentSize(in.readOptionalVInt()); + numOfFragments(in.readOptionalVInt()); + highlighterType(in.readOptionalString()); + fragmenter(in.readOptionalString()); + if (in.readBoolean()) { + highlightQuery(in.readQuery()); + } + if (in.readBoolean()) { + order(Order.PROTOTYPE.readFrom(in)); + } + highlightFilter(in.readOptionalBoolean()); + forceSource(in.readOptionalBoolean()); + boundaryMaxScan(in.readOptionalVInt()); + if (in.readBoolean()) { + boundaryChars(in.readString().toCharArray()); + } + noMatchSize(in.readOptionalVInt()); + phraseLimit(in.readOptionalVInt()); + if (in.readBoolean()) { + options(in.readMap()); + } + requireFieldMatch(in.readOptionalBoolean()); + return (HB) this; + } + + /** + * write common parameters to {@link StreamOutput} + */ + protected void writeOptionsTo(StreamOutput out) throws IOException { + out.writeOptionalStringArray(preTags); + out.writeOptionalStringArray(postTags); + out.writeOptionalVInt(fragmentSize); + out.writeOptionalVInt(numOfFragments); + out.writeOptionalString(highlighterType); + out.writeOptionalString(fragmenter); + boolean hasQuery = highlightQuery != null; + out.writeBoolean(hasQuery); + if (hasQuery) { + out.writeQuery(highlightQuery); + } + boolean hasSetOrder = order != null; + out.writeBoolean(hasSetOrder); + if (hasSetOrder) { + order.writeTo(out); + } + out.writeOptionalBoolean(highlightFilter); + out.writeOptionalBoolean(forceSource); + out.writeOptionalVInt(boundaryMaxScan); + boolean hasBounaryChars = boundaryChars != null; + out.writeBoolean(hasBounaryChars); + if (hasBounaryChars) { + out.writeString(String.valueOf(boundaryChars)); + } + out.writeOptionalVInt(noMatchSize); + out.writeOptionalVInt(phraseLimit); + boolean hasOptions = options != null; + out.writeBoolean(hasOptions); + if (hasOptions) { + out.writeMap(options); + } + out.writeOptionalBoolean(requireFieldMatch); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java b/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java index 6914d854100..9e86edef47d 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.highlight; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.QueryScorer; import org.apache.lucene.search.highlight.WeightedSpanTerm; @@ -83,9 +82,6 @@ public final class CustomQueryScorer extends QueryScorer { } else if (query instanceof FiltersFunctionScoreQuery) { query = ((FiltersFunctionScoreQuery) query).getSubQuery(); extract(query, query.getBoost(), terms); - } else if (query instanceof FilteredQuery) { - query = ((FilteredQuery) query).getQuery(); - extract(query, 1F, terms); } else { extractWeightedTerms(terms, query, query.getBoost()); } diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java index 695598e4fec..c0b1aeea3be 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java @@ -19,16 +19,34 @@ package org.elasticsearch.search.highlight; -import org.apache.lucene.search.highlight.SimpleFragmenter; -import org.apache.lucene.search.highlight.SimpleSpanFragmenter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.vectorhighlight.SimpleBoundaryScanner; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.highlight.SearchContextHighlight.FieldOptions; +import org.elasticsearch.search.highlight.SearchContextHighlight.FieldOptions.Builder; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; /** * A builder for search highlighting. Settings can control how large fields @@ -36,46 +54,61 @@ import java.util.Map; * * @see org.elasticsearch.search.builder.SearchSourceBuilder#highlight() */ -public class HighlightBuilder implements ToXContent { +public class HighlightBuilder extends AbstractHighlighterBuilder implements Writeable, ToXContent { - private List fields; + public static final HighlightBuilder PROTOTYPE = new HighlightBuilder(); - private String tagsSchema; + public static final String HIGHLIGHT_ELEMENT_NAME = "highlight"; - private Boolean highlightFilter; + /** default for whether to highlight fields based on the source even if stored separately */ + public static final boolean DEFAULT_FORCE_SOURCE = false; + /** default for whether a field should be highlighted only if a query matches that field */ + public static final boolean DEFAULT_REQUIRE_FIELD_MATCH = true; + /** default for whether fvh should provide highlighting on filter clauses */ + public static final boolean DEFAULT_HIGHLIGHT_FILTER = false; + /** default for highlight fragments being ordered by score */ + public static final boolean DEFAULT_SCORE_ORDERED = false; + /** the default encoder setting */ + public static final String DEFAULT_ENCODER = "default"; + /** default for the maximum number of phrases the fvh will consider */ + public static final int DEFAULT_PHRASE_LIMIT = 256; + /** default for fragment size when there are no matches */ + public static final int DEFAULT_NO_MATCH_SIZE = 0; + /** the default number of fragments for highlighting */ + public static final int DEFAULT_NUMBER_OF_FRAGMENTS = 5; + /** the default number of fragments size in characters */ + public static final int DEFAULT_FRAGMENT_CHAR_SIZE = 100; + /** the default opening tag */ + public static final String[] DEFAULT_PRE_TAGS = new String[]{""}; + /** the default closing tag */ + public static final String[] DEFAULT_POST_TAGS = new String[]{""}; - private Integer fragmentSize; + /** the default opening tags when tag_schema = "styled" */ + public static final String[] DEFAULT_STYLED_PRE_TAG = { + "", "", "", + "", "", "", + "", "", "", + "" + }; + /** the default closing tags when tag_schema = "styled" */ + public static final String[] DEFAULT_STYLED_POST_TAGS = {""}; - private Integer numOfFragments; + /** + * a {@link FieldOptions.Builder} with default settings + */ + public final static Builder defaultFieldOptions() { + return new SearchContextHighlight.FieldOptions.Builder() + .preTags(DEFAULT_PRE_TAGS).postTags(DEFAULT_POST_TAGS).scoreOrdered(DEFAULT_SCORE_ORDERED).highlightFilter(DEFAULT_HIGHLIGHT_FILTER) + .requireFieldMatch(DEFAULT_REQUIRE_FIELD_MATCH).forceSource(DEFAULT_FORCE_SOURCE).fragmentCharSize(DEFAULT_FRAGMENT_CHAR_SIZE).numberOfFragments(DEFAULT_NUMBER_OF_FRAGMENTS) + .encoder(DEFAULT_ENCODER).boundaryMaxScan(SimpleBoundaryScanner.DEFAULT_MAX_SCAN) + .boundaryChars(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) + .noMatchSize(DEFAULT_NO_MATCH_SIZE).phraseLimit(DEFAULT_PHRASE_LIMIT); + } - private String[] preTags; - - private String[] postTags; - - private String order; + private final List fields = new ArrayList<>(); private String encoder; - private Boolean requireFieldMatch; - - private Integer boundaryMaxScan; - - private char[] boundaryChars; - - private String highlighterType; - - private String fragmenter; - - private QueryBuilder highlightQuery; - - private Integer noMatchSize; - - private Integer phraseLimit; - - private Map options; - - private Boolean forceSource; - private boolean useExplicitFieldOrder = false; /** @@ -85,14 +118,9 @@ public class HighlightBuilder implements ToXContent { * @param name The field to highlight */ public HighlightBuilder field(String name) { - if (fields == null) { - fields = new ArrayList<>(); - } - fields.add(new Field(name)); - return this; + return field(new Field(name)); } - /** * Adds a field to be highlighted with a provided fragment size (in characters), and * default number of fragments of 5. @@ -101,11 +129,7 @@ public class HighlightBuilder implements ToXContent { * @param fragmentSize The size of a fragment in characters */ public HighlightBuilder field(String name, int fragmentSize) { - if (fields == null) { - fields = new ArrayList<>(); - } - fields.add(new Field(name).fragmentSize(fragmentSize)); - return this; + return field(new Field(name).fragmentSize(fragmentSize)); } @@ -118,14 +142,9 @@ public class HighlightBuilder implements ToXContent { * @param numberOfFragments The (maximum) number of fragments */ public HighlightBuilder field(String name, int fragmentSize, int numberOfFragments) { - if (fields == null) { - fields = new ArrayList<>(); - } - fields.add(new Field(name).fragmentSize(fragmentSize).numOfFragments(numberOfFragments)); - return this; + return field(new Field(name).fragmentSize(fragmentSize).numOfFragments(numberOfFragments)); } - /** * Adds a field to be highlighted with a provided fragment size (in characters), and * a provided (maximum) number of fragments. @@ -136,56 +155,38 @@ public class HighlightBuilder implements ToXContent { * @param fragmentOffset The offset from the start of the fragment to the start of the highlight */ public HighlightBuilder field(String name, int fragmentSize, int numberOfFragments, int fragmentOffset) { - if (fields == null) { - fields = new ArrayList<>(); - } - fields.add(new Field(name).fragmentSize(fragmentSize).numOfFragments(numberOfFragments) + return field(new Field(name).fragmentSize(fragmentSize).numOfFragments(numberOfFragments) .fragmentOffset(fragmentOffset)); - return this; } public HighlightBuilder field(Field field) { - if (fields == null) { - fields = new ArrayList<>(); - } fields.add(field); return this; } + public List fields() { + return this.fields; + } + /** - * Set a tag scheme that encapsulates a built in pre and post tags. The allows schemes + * Set a tag scheme that encapsulates a built in pre and post tags. The allowed schemes * are styled and default. * * @param schemaName The tag scheme name */ public HighlightBuilder tagsSchema(String schemaName) { - this.tagsSchema = schemaName; - return this; - } - - /** - * Set this to true when using the highlighterType fast-vector-highlighter - * and you want to provide highlighting on filter clauses in your - * query. Default is false. - */ - public HighlightBuilder highlightFilter(boolean highlightFilter) { - this.highlightFilter = highlightFilter; - return this; - } - - /** - * Sets the size of a fragment in characters (defaults to 100) - */ - public HighlightBuilder fragmentSize(Integer fragmentSize) { - this.fragmentSize = fragmentSize; - return this; - } - - /** - * Sets the maximum number of fragments returned - */ - public HighlightBuilder numOfFragments(Integer numOfFragments) { - this.numOfFragments = numOfFragments; + switch (schemaName) { + case "default": + preTags(DEFAULT_PRE_TAGS); + postTags(DEFAULT_POST_TAGS); + break; + case "styled": + preTags(DEFAULT_STYLED_PRE_TAG); + postTags(DEFAULT_STYLED_POST_TAGS); + break; + default: + throw new IllegalArgumentException("Unknown tag schema ["+ schemaName +"]"); + } return this; } @@ -201,125 +202,10 @@ public class HighlightBuilder implements ToXContent { } /** - * Explicitly set the pre tags that will be used for highlighting. + * Getter for {@link #encoder(String)} */ - public HighlightBuilder preTags(String... preTags) { - this.preTags = preTags; - return this; - } - - /** - * Explicitly set the post tags that will be used for highlighting. - */ - public HighlightBuilder postTags(String... postTags) { - this.postTags = postTags; - return this; - } - - /** - * The order of fragments per field. By default, ordered by the order in the - * highlighted text. Can be score, which then it will be ordered - * by score of the fragments. - */ - public HighlightBuilder order(String order) { - this.order = order; - return this; - } - - /** - * Set to true to cause a field to be highlighted only if a query matches that field. - * Default is false meaning that terms are highlighted on all requested fields regardless - * if the query matches specifically on them. - */ - public HighlightBuilder requireFieldMatch(boolean requireFieldMatch) { - this.requireFieldMatch = requireFieldMatch; - return this; - } - - /** - * When using the highlighterType fast-vector-highlighter this setting - * controls how far to look for boundary characters, and defaults to 20. - */ - public HighlightBuilder boundaryMaxScan(Integer boundaryMaxScan) { - this.boundaryMaxScan = boundaryMaxScan; - return this; - } - - /** - * When using the highlighterType fast-vector-highlighter this setting - * defines what constitutes a boundary for highlighting. It’s a single string with - * each boundary character defined in it. It defaults to .,!? \t\n - */ - public HighlightBuilder boundaryChars(char[] boundaryChars) { - this.boundaryChars = boundaryChars; - return this; - } - - /** - * Set type of highlighter to use. Supported types - * are highlighter, fast-vector-highlighter and postings-highlighter. - * The default option selected is dependent on the mappings defined for your index. - * Details of the different highlighter types are covered in the reference guide. - */ - public HighlightBuilder highlighterType(String highlighterType) { - this.highlighterType = highlighterType; - return this; - } - - /** - * Sets what fragmenter to use to break up text that is eligible for highlighting. - * This option is only applicable when using the plain highlighterType highlighter. - * Permitted values are "simple" or "span" relating to {@link SimpleFragmenter} and - * {@link SimpleSpanFragmenter} implementations respectively with the default being "span" - */ - public HighlightBuilder fragmenter(String fragmenter) { - this.fragmenter = fragmenter; - return this; - } - - /** - * Sets a query to be used for highlighting all fields instead of the search query. - */ - public HighlightBuilder highlightQuery(QueryBuilder highlightQuery) { - this.highlightQuery = highlightQuery; - return this; - } - - /** - * Sets the size of the fragment to return from the beginning of the field if there are no matches to - * highlight and the field doesn't also define noMatchSize. - * @param noMatchSize integer to set or null to leave out of request. default is null. - * @return this for chaining - */ - public HighlightBuilder noMatchSize(Integer noMatchSize) { - this.noMatchSize = noMatchSize; - return this; - } - - /** - * Sets the maximum number of phrases the fvh will consider if the field doesn't also define phraseLimit. - * @param phraseLimit maximum number of phrases the fvh will consider - * @return this for chaining - */ - public HighlightBuilder phraseLimit(Integer phraseLimit) { - this.phraseLimit = phraseLimit; - return this; - } - - /** - * Allows to set custom options for custom highlighters. - */ - public HighlightBuilder options(Map options) { - this.options = options; - return this; - } - - /** - * Forces the highlighting to highlight fields based on the source even if fields are stored separately. - */ - public HighlightBuilder forceSource(boolean forceSource) { - this.forceSource = forceSource; - return this; + public String encoder() { + return this.encoder; } /** @@ -331,130 +217,177 @@ public class HighlightBuilder implements ToXContent { return this; } + /** + * Gets value set with {@link #useExplicitFieldOrder(boolean)} + */ + public Boolean useExplicitFieldOrder() { + return this.useExplicitFieldOrder; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("highlight"); - if (tagsSchema != null) { - builder.field("tags_schema", tagsSchema); + builder.startObject(HIGHLIGHT_ELEMENT_NAME); + innerXContent(builder); + builder.endObject(); + return builder; + } + + /** + * parse options only present in top level highlight builder (`tags_schema`, `encoder` and nested `fields`) + */ + @Override + protected boolean doFromXContent(QueryParseContext parseContext, String currentFieldName, Token currentToken) throws IOException { + XContentParser parser = parseContext.parser(); + XContentParser.Token token; + boolean foundCurrentFieldMatch = false; + if (currentToken.isValue()) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TAGS_SCHEMA_FIELD)) { + tagsSchema(parser.text()); + foundCurrentFieldMatch = true; + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ENCODER_FIELD)) { + encoder(parser.text()); + foundCurrentFieldMatch = true; + } + } else if (currentToken == Token.START_ARRAY && parseContext.parseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) { + useExplicitFieldOrder(true); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + field(HighlightBuilder.Field.PROTOTYPE.fromXContent(parseContext)); + } + } + foundCurrentFieldMatch = true; + } else { + throw new ParsingException(parser.getTokenLocation(), + "If highlighter fields is an array it must contain objects containing a single field"); + } + } + } else if (currentToken == Token.START_OBJECT && parseContext.parseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + field(HighlightBuilder.Field.PROTOTYPE.fromXContent(parseContext)); + } + } + foundCurrentFieldMatch = true; } - if (preTags != null) { - builder.array("pre_tags", preTags); + return foundCurrentFieldMatch; + } + + public SearchContextHighlight build(QueryShardContext context) throws IOException { + // create template global options that are later merged with any partial field options + final SearchContextHighlight.FieldOptions.Builder globalOptionsBuilder = new SearchContextHighlight.FieldOptions.Builder(); + globalOptionsBuilder.encoder(this.encoder); + transferOptions(this, globalOptionsBuilder, context); + + // overwrite unset global options by default values + globalOptionsBuilder.merge(defaultFieldOptions().build()); + + // create field options + Collection fieldOptions = new ArrayList<>(); + for (Field field : this.fields) { + final SearchContextHighlight.FieldOptions.Builder fieldOptionsBuilder = new SearchContextHighlight.FieldOptions.Builder(); + fieldOptionsBuilder.fragmentOffset(field.fragmentOffset); + if (field.matchedFields != null) { + Set matchedFields = new HashSet(field.matchedFields.length); + Collections.addAll(matchedFields, field.matchedFields); + fieldOptionsBuilder.matchedFields(matchedFields); + } + transferOptions(field, fieldOptionsBuilder, context); + fieldOptions.add(new SearchContextHighlight.Field(field.name(), fieldOptionsBuilder.merge(globalOptionsBuilder.build()).build())); } - if (postTags != null) { - builder.array("post_tags", postTags); + return new SearchContextHighlight(fieldOptions); + } + + /** + * Transfers field options present in the input {@link AbstractHighlighterBuilder} to the receiving + * {@link FieldOptions.Builder}, effectively overwriting existing settings + * @param targetOptionsBuilder the receiving options builder + * @param highlighterBuilder highlight builder with the input options + * @param context needed to convert {@link QueryBuilder} to {@link Query} + * @throws IOException on errors parsing any optional nested highlight query + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static void transferOptions(AbstractHighlighterBuilder highlighterBuilder, SearchContextHighlight.FieldOptions.Builder targetOptionsBuilder, QueryShardContext context) throws IOException { + if (highlighterBuilder.preTags != null) { + targetOptionsBuilder.preTags(highlighterBuilder.preTags); } - if (order != null) { - builder.field("order", order); + if (highlighterBuilder.postTags != null) { + targetOptionsBuilder.postTags(highlighterBuilder.postTags); } - if (highlightFilter != null) { - builder.field("highlight_filter", highlightFilter); + if (highlighterBuilder.order != null) { + targetOptionsBuilder.scoreOrdered(highlighterBuilder.order == Order.SCORE); } - if (fragmentSize != null) { - builder.field("fragment_size", fragmentSize); + if (highlighterBuilder.highlightFilter != null) { + targetOptionsBuilder.highlightFilter(highlighterBuilder.highlightFilter); } - if (numOfFragments != null) { - builder.field("number_of_fragments", numOfFragments); + if (highlighterBuilder.fragmentSize != null) { + targetOptionsBuilder.fragmentCharSize(highlighterBuilder.fragmentSize); } + if (highlighterBuilder.numOfFragments != null) { + targetOptionsBuilder.numberOfFragments(highlighterBuilder.numOfFragments); + } + if (highlighterBuilder.requireFieldMatch != null) { + targetOptionsBuilder.requireFieldMatch(highlighterBuilder.requireFieldMatch); + } + if (highlighterBuilder.boundaryMaxScan != null) { + targetOptionsBuilder.boundaryMaxScan(highlighterBuilder.boundaryMaxScan); + } + if (highlighterBuilder.boundaryChars != null) { + targetOptionsBuilder.boundaryChars(convertCharArray(highlighterBuilder.boundaryChars)); + } + if (highlighterBuilder.highlighterType != null) { + targetOptionsBuilder.highlighterType(highlighterBuilder.highlighterType); + } + if (highlighterBuilder.fragmenter != null) { + targetOptionsBuilder.fragmenter(highlighterBuilder.fragmenter); + } + if (highlighterBuilder.noMatchSize != null) { + targetOptionsBuilder.noMatchSize(highlighterBuilder.noMatchSize); + } + if (highlighterBuilder.forceSource != null) { + targetOptionsBuilder.forceSource(highlighterBuilder.forceSource); + } + if (highlighterBuilder.phraseLimit != null) { + targetOptionsBuilder.phraseLimit(highlighterBuilder.phraseLimit); + } + if (highlighterBuilder.options != null) { + targetOptionsBuilder.options(highlighterBuilder.options); + } + if (highlighterBuilder.highlightQuery != null) { + targetOptionsBuilder.highlightQuery(highlighterBuilder.highlightQuery.toQuery(context)); + } + } + + private static Character[] convertCharArray(char[] array) { + if (array == null) { + return null; + } + Character[] charArray = new Character[array.length]; + for (int i = 0; i < array.length; i++) { + charArray[i] = array[i]; + } + return charArray; + } + + public void innerXContent(XContentBuilder builder) throws IOException { + // first write common options + commonOptionsToXContent(builder); + // special options for top-level highlighter if (encoder != null) { - builder.field("encoder", encoder); + builder.field(ENCODER_FIELD.getPreferredName(), encoder); } - if (requireFieldMatch != null) { - builder.field("require_field_match", requireFieldMatch); - } - if (boundaryMaxScan != null) { - builder.field("boundary_max_scan", boundaryMaxScan); - } - if (boundaryChars != null) { - builder.field("boundary_chars", boundaryChars); - } - if (highlighterType != null) { - builder.field("type", highlighterType); - } - if (fragmenter != null) { - builder.field("fragmenter", fragmenter); - } - if (highlightQuery != null) { - builder.field("highlight_query", highlightQuery); - } - if (noMatchSize != null) { - builder.field("no_match_size", noMatchSize); - } - if (phraseLimit != null) { - builder.field("phrase_limit", phraseLimit); - } - if (options != null && options.size() > 0) { - builder.field("options", options); - } - if (forceSource != null) { - builder.field("force_source", forceSource); - } - if (fields != null) { + if (fields.size() > 0) { if (useExplicitFieldOrder) { - builder.startArray("fields"); + builder.startArray(FIELDS_FIELD.getPreferredName()); } else { - builder.startObject("fields"); + builder.startObject(FIELDS_FIELD.getPreferredName()); } for (Field field : fields) { if (useExplicitFieldOrder) { builder.startObject(); } - builder.startObject(field.name()); - if (field.preTags != null) { - builder.field("pre_tags", field.preTags); - } - if (field.postTags != null) { - builder.field("post_tags", field.postTags); - } - if (field.fragmentSize != -1) { - builder.field("fragment_size", field.fragmentSize); - } - if (field.numOfFragments != -1) { - builder.field("number_of_fragments", field.numOfFragments); - } - if (field.fragmentOffset != -1) { - builder.field("fragment_offset", field.fragmentOffset); - } - if (field.highlightFilter != null) { - builder.field("highlight_filter", field.highlightFilter); - } - if (field.order != null) { - builder.field("order", field.order); - } - if (field.requireFieldMatch != null) { - builder.field("require_field_match", field.requireFieldMatch); - } - if (field.boundaryMaxScan != -1) { - builder.field("boundary_max_scan", field.boundaryMaxScan); - } - if (field.boundaryChars != null) { - builder.field("boundary_chars", field.boundaryChars); - } - if (field.highlighterType != null) { - builder.field("type", field.highlighterType); - } - if (field.fragmenter != null) { - builder.field("fragmenter", field.fragmenter); - } - if (field.highlightQuery != null) { - builder.field("highlight_query", field.highlightQuery); - } - if (field.noMatchSize != null) { - builder.field("no_match_size", field.noMatchSize); - } - if (field.matchedFields != null) { - builder.field("matched_fields", field.matchedFields); - } - if (field.phraseLimit != null) { - builder.field("phrase_limit", field.phraseLimit); - } - if (field.options != null && field.options.size() > 0) { - builder.field("options", field.options); - } - if (field.forceSource != null) { - builder.field("force_source", field.forceSource); - } - - builder.endObject(); + field.innerXContent(builder); if (useExplicitFieldOrder) { builder.endObject(); } @@ -465,30 +398,69 @@ public class HighlightBuilder implements ToXContent { builder.endObject(); } } - builder.endObject(); - return builder; } - public static class Field { - final String name; - String[] preTags; - String[] postTags; - int fragmentSize = -1; + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + toXContent(builder, EMPTY_PARAMS); + return builder.string(); + } catch (Exception e) { + return "{ \"error\" : \"" + ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + + @Override + protected HighlightBuilder createInstance(XContentParser parser) { + return new HighlightBuilder(); + } + + @Override + protected int doHashCode() { + return Objects.hash(encoder, useExplicitFieldOrder, fields); + } + + @Override + protected boolean doEquals(HighlightBuilder other) { + return Objects.equals(encoder, other.encoder) && + Objects.equals(useExplicitFieldOrder, other.useExplicitFieldOrder) && + Objects.equals(fields, other.fields); + } + + @Override + public HighlightBuilder readFrom(StreamInput in) throws IOException { + HighlightBuilder highlightBuilder = new HighlightBuilder(); + highlightBuilder.readOptionsFrom(in) + .encoder(in.readOptionalString()) + .useExplicitFieldOrder(in.readBoolean()); + int fields = in.readVInt(); + for (int i = 0; i < fields; i++) { + highlightBuilder.field(Field.PROTOTYPE.readFrom(in)); + } + return highlightBuilder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeOptionsTo(out); + out.writeOptionalString(encoder); + out.writeBoolean(useExplicitFieldOrder); + out.writeVInt(fields.size()); + for (int i = 0; i < fields.size(); i++) { + fields.get(i).writeTo(out); + } + } + + public static class Field extends AbstractHighlighterBuilder implements Writeable { + static final Field PROTOTYPE = new Field("_na_"); + + private final String name; + int fragmentOffset = -1; - int numOfFragments = -1; - Boolean highlightFilter; - String order; - Boolean requireFieldMatch; - int boundaryMaxScan = -1; - char[] boundaryChars; - String highlighterType; - String fragmenter; - QueryBuilder highlightQuery; - Integer noMatchSize; + String[] matchedFields; - Integer phraseLimit; - Map options; - Boolean forceSource; public Field(String name) { this.name = name; @@ -498,118 +470,11 @@ public class HighlightBuilder implements ToXContent { return name; } - /** - * Explicitly set the pre tags for this field that will be used for highlighting. - * This overrides global settings set by {@link HighlightBuilder#preTags(String...)}. - */ - public Field preTags(String... preTags) { - this.preTags = preTags; - return this; - } - - /** - * Explicitly set the post tags for this field that will be used for highlighting. - * This overrides global settings set by {@link HighlightBuilder#postTags(String...)}. - */ - public Field postTags(String... postTags) { - this.postTags = postTags; - return this; - } - - public Field fragmentSize(int fragmentSize) { - this.fragmentSize = fragmentSize; - return this; - } - public Field fragmentOffset(int fragmentOffset) { this.fragmentOffset = fragmentOffset; return this; } - public Field numOfFragments(int numOfFragments) { - this.numOfFragments = numOfFragments; - return this; - } - - public Field highlightFilter(boolean highlightFilter) { - this.highlightFilter = highlightFilter; - return this; - } - - /** - * The order of fragments per field. By default, ordered by the order in the - * highlighted text. Can be score, which then it will be ordered - * by score of the fragments. - * This overrides global settings set by {@link HighlightBuilder#order(String)}. - */ - public Field order(String order) { - this.order = order; - return this; - } - - public Field requireFieldMatch(boolean requireFieldMatch) { - this.requireFieldMatch = requireFieldMatch; - return this; - } - - public Field boundaryMaxScan(int boundaryMaxScan) { - this.boundaryMaxScan = boundaryMaxScan; - return this; - } - - public Field boundaryChars(char[] boundaryChars) { - this.boundaryChars = boundaryChars; - return this; - } - - /** - * Set type of highlighter to use. Supported types - * are highlighter, fast-vector-highlighter nad postings-highlighter. - * This overrides global settings set by {@link HighlightBuilder#highlighterType(String)}. - */ - public Field highlighterType(String highlighterType) { - this.highlighterType = highlighterType; - return this; - } - - /** - * Sets what fragmenter to use to break up text that is eligible for highlighting. - * This option is only applicable when using plain / normal highlighter. - * This overrides global settings set by {@link HighlightBuilder#fragmenter(String)}. - */ - public Field fragmenter(String fragmenter) { - this.fragmenter = fragmenter; - return this; - } - - /** - * Sets a query to use for highlighting this field instead of the search query. - */ - public Field highlightQuery(QueryBuilder highlightQuery) { - this.highlightQuery = highlightQuery; - return this; - } - - /** - * Sets the size of the fragment to return from the beginning of the field if there are no matches to - * highlight. - * @param noMatchSize integer to set or null to leave out of request. default is null. - * @return this for chaining - */ - public Field noMatchSize(Integer noMatchSize) { - this.noMatchSize = noMatchSize; - return this; - } - - /** - * Allows to set custom options for custom highlighters. - * This overrides global settings set by {@link HighlightBuilder#options(Map)}. - */ - public Field options(Map options) { - this.options = options; - return this; - } - /** * Set the matched fields to highlight against this field data. Default to null, meaning just * the named field. If you provide a list of fields here then don't forget to include name as @@ -620,24 +485,111 @@ public class HighlightBuilder implements ToXContent { return this; } - /** - * Sets the maximum number of phrases the fvh will consider. - * @param phraseLimit maximum number of phrases the fvh will consider - * @return this for chaining - */ - public Field phraseLimit(Integer phraseLimit) { - this.phraseLimit = phraseLimit; - return this; + public void innerXContent(XContentBuilder builder) throws IOException { + builder.startObject(name); + // write common options + commonOptionsToXContent(builder); + // write special field-highlighter options + if (fragmentOffset != -1) { + builder.field(FRAGMENT_OFFSET_FIELD.getPreferredName(), fragmentOffset); + } + if (matchedFields != null) { + builder.field(MATCHED_FIELDS_FIELD.getPreferredName(), matchedFields); + } + builder.endObject(); } - /** - * Forces the highlighting to highlight this field based on the source even if this field is stored separately. + * parse options only present in field highlight builder (`fragment_offset`, `matched_fields`) */ - public Field forceSource(boolean forceSource) { - this.forceSource = forceSource; - return this; + @Override + protected boolean doFromXContent(QueryParseContext parseContext, String currentFieldName, Token currentToken) throws IOException { + XContentParser parser = parseContext.parser(); + boolean foundCurrentFieldMatch = false; + if (parseContext.parseFieldMatcher().match(currentFieldName, FRAGMENT_OFFSET_FIELD) && currentToken.isValue()) { + fragmentOffset(parser.intValue()); + foundCurrentFieldMatch = true; + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MATCHED_FIELDS_FIELD) + && currentToken == XContentParser.Token.START_ARRAY) { + List matchedFields = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + matchedFields.add(parser.text()); + } + matchedFields(matchedFields.toArray(new String[matchedFields.size()])); + foundCurrentFieldMatch = true; + } + return foundCurrentFieldMatch; } + @Override + protected Field createInstance(XContentParser parser) throws IOException { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + String fieldname = parser.currentName(); + return new Field(fieldname); + } else { + throw new ParsingException(parser.getTokenLocation(), "unknown token type [{}], expected field name", parser.currentToken()); + } + } + + @Override + protected int doHashCode() { + return Objects.hash(name, fragmentOffset, Arrays.hashCode(matchedFields)); + } + + @Override + protected boolean doEquals(Field other) { + return Objects.equals(name, other.name) && + Objects.equals(fragmentOffset, other.fragmentOffset) && + Arrays.equals(matchedFields, other.matchedFields); + } + + @Override + public Field readFrom(StreamInput in) throws IOException { + Field field = new Field(in.readString()); + field.fragmentOffset(in.readVInt()); + field.matchedFields(in.readOptionalStringArray()); + field.readOptionsFrom(in); + return field; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeVInt(fragmentOffset); + out.writeOptionalStringArray(matchedFields); + writeOptionsTo(out); + } + } + + public enum Order implements Writeable { + NONE, SCORE; + + static Order PROTOTYPE = NONE; + + @Override + public Order readFrom(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Order ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(this.ordinal()); + } + + public static Order fromString(String order) { + if (order.toUpperCase(Locale.ROOT).equals(SCORE.name())) { + return Order.SCORE; + } + return NONE; + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java index 12f376f3878..38534ba0ff3 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java @@ -19,10 +19,9 @@ package org.elasticsearch.search.highlight; -import org.apache.lucene.search.vectorhighlight.SimpleBoundaryScanner; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.internal.SearchContext; @@ -52,37 +51,21 @@ import java.util.Set; */ public class HighlighterParseElement implements SearchParseElement { - private static final String[] DEFAULT_PRE_TAGS = new String[]{""}; - private static final String[] DEFAULT_POST_TAGS = new String[]{""}; - - private static final String[] STYLED_PRE_TAG = { - "", "", "", - "", "", "", - "", "", "", - "" - }; - private static final String[] STYLED_POST_TAGS = {""}; - @Override public void parse(XContentParser parser, SearchContext context) throws Exception { try { - context.highlight(parse(parser, context.queryParserService())); + context.highlight(parse(parser, context.indexShard().getQueryShardContext())); } catch (IllegalArgumentException ex) { throw new SearchParseException(context, "Error while trying to parse Highlighter element in request", parser.getTokenLocation()); } } - public SearchContextHighlight parse(XContentParser parser, IndexQueryParserService queryParserService) throws IOException { + public SearchContextHighlight parse(XContentParser parser, QueryShardContext queryShardContext) throws IOException { XContentParser.Token token; String topLevelFieldName = null; final List> fieldsOptions = new ArrayList<>(); - final SearchContextHighlight.FieldOptions.Builder globalOptionsBuilder = new SearchContextHighlight.FieldOptions.Builder() - .preTags(DEFAULT_PRE_TAGS).postTags(DEFAULT_POST_TAGS).scoreOrdered(false).highlightFilter(false) - .requireFieldMatch(true).forceSource(false).fragmentCharSize(100).numberOfFragments(5) - .encoder("default").boundaryMaxScan(SimpleBoundaryScanner.DEFAULT_MAX_SCAN) - .boundaryChars(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) - .noMatchSize(0).phraseLimit(256); + final SearchContextHighlight.FieldOptions.Builder globalOptionsBuilder = HighlightBuilder.defaultFieldOptions(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -111,7 +94,7 @@ public class HighlighterParseElement implements SearchParseElement { } highlightFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - fieldsOptions.add(Tuple.tuple(highlightFieldName, parseFields(parser, queryParserService))); + fieldsOptions.add(Tuple.tuple(highlightFieldName, parseFields(parser, queryShardContext))); } } } else { @@ -125,8 +108,8 @@ public class HighlighterParseElement implements SearchParseElement { } else if ("tags_schema".equals(topLevelFieldName) || "tagsSchema".equals(topLevelFieldName)) { String schema = parser.text(); if ("styled".equals(schema)) { - globalOptionsBuilder.preTags(STYLED_PRE_TAG); - globalOptionsBuilder.postTags(STYLED_POST_TAGS); + globalOptionsBuilder.preTags(HighlightBuilder.DEFAULT_STYLED_PRE_TAG); + globalOptionsBuilder.postTags(HighlightBuilder.DEFAULT_STYLED_POST_TAGS); } } else if ("highlight_filter".equals(topLevelFieldName) || "highlightFilter".equals(topLevelFieldName)) { globalOptionsBuilder.highlightFilter(parser.booleanValue()); @@ -167,11 +150,11 @@ public class HighlighterParseElement implements SearchParseElement { if (token == XContentParser.Token.FIELD_NAME) { highlightFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - fieldsOptions.add(Tuple.tuple(highlightFieldName, parseFields(parser, queryParserService))); + fieldsOptions.add(Tuple.tuple(highlightFieldName, parseFields(parser, queryShardContext))); } } } else if ("highlight_query".equals(topLevelFieldName) || "highlightQuery".equals(topLevelFieldName)) { - globalOptionsBuilder.highlightQuery(queryParserService.parse(parser).query()); + globalOptionsBuilder.highlightQuery(queryShardContext.parse(parser).query()); } } } @@ -189,7 +172,7 @@ public class HighlighterParseElement implements SearchParseElement { return new SearchContextHighlight(fields); } - protected SearchContextHighlight.FieldOptions.Builder parseFields(XContentParser parser, IndexQueryParserService queryParserService) throws IOException { + private static SearchContextHighlight.FieldOptions.Builder parseFields(XContentParser parser, QueryShardContext queryShardContext) throws IOException { XContentParser.Token token; final SearchContextHighlight.FieldOptions.Builder fieldOptionsBuilder = new SearchContextHighlight.FieldOptions.Builder(); @@ -252,7 +235,7 @@ public class HighlighterParseElement implements SearchParseElement { } } else if (token == XContentParser.Token.START_OBJECT) { if ("highlight_query".equals(fieldName) || "highlightQuery".equals(fieldName)) { - fieldOptionsBuilder.highlightQuery(queryParserService.parse(parser).query()); + fieldOptionsBuilder.highlightQuery(queryShardContext.parse(parser).query()); } else if ("options".equals(fieldName)) { fieldOptionsBuilder.options(parser.map()); } diff --git a/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java b/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java index 1e519957aac..54366bee8c9 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java @@ -19,8 +19,6 @@ package org.elasticsearch.search.highlight; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.ExtensionPoint; @@ -31,26 +29,18 @@ import java.util.*; */ public class Highlighters extends ExtensionPoint.ClassMap { - @Deprecated // remove in 3.0 - private static final String FAST_VECTOR_HIGHLIGHTER = "fast-vector-highlighter"; private static final String FVH = "fvh"; - @Deprecated // remove in 3.0 - private static final String HIGHLIGHTER = "highlighter"; private static final String PLAIN = "plain"; - @Deprecated // remove in 3.0 - private static final String POSTINGS_HIGHLIGHTER = "postings-highlighter"; private static final String POSTINGS = "postings"; - private final Map parsers; - private final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Highlighters.class.getName())); public Highlighters(){ - this(Collections.EMPTY_MAP); + this(Collections.emptyMap()); } private Highlighters(Map parsers) { - super("highlighter", Highlighter.class, new HashSet<>(Arrays.asList(FVH, FAST_VECTOR_HIGHLIGHTER, PLAIN, HIGHLIGHTER, POSTINGS, POSTINGS_HIGHLIGHTER)), + super("highlighter", Highlighter.class, new HashSet<>(Arrays.asList(FVH, PLAIN, POSTINGS)), Highlighters.class); this.parsers = Collections.unmodifiableMap(parsers); } @@ -61,31 +51,15 @@ public class Highlighters extends ExtensionPoint.ClassMap { } private static Map addBuiltIns(Settings settings, Map parsers) { - // build in highlighers Map map = new HashMap<>(); map.put(FVH, new FastVectorHighlighter(settings)); - map.put(FAST_VECTOR_HIGHLIGHTER, map.get(FVH)); map.put(PLAIN, new PlainHighlighter()); - map.put(HIGHLIGHTER, map.get(PLAIN)); map.put(POSTINGS, new PostingsHighlighter()); - map.put(POSTINGS_HIGHLIGHTER, map.get(POSTINGS)); map.putAll(parsers); return map; } public Highlighter get(String type) { - switch (type) { - case FAST_VECTOR_HIGHLIGHTER: - deprecationLogger.deprecated("highlighter key [{}] is deprecated and will be removed in 3.x use [{}] instead", FAST_VECTOR_HIGHLIGHTER, FVH); - break; - case HIGHLIGHTER: - deprecationLogger.deprecated("highlighter key [{}] is deprecated and will be removed in 3.x use [{}] instead", HIGHLIGHTER, PLAIN); - break; - case POSTINGS_HIGHLIGHTER: - deprecationLogger.deprecated("highlighter key [{}] is deprecated and will be removed in 3.x use [{}] instead", POSTINGS_HIGHLIGHTER, POSTINGS); - break; - } return parsers.get(type); } - } diff --git a/core/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java b/core/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java index 38a8147b105..293143fb1db 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java @@ -53,6 +53,10 @@ public class SearchContextHighlight { this.globalForceSource = globalForceSource; } + boolean globalForceSource() { + return this.globalForceSource; + } + public boolean forceSource(Field field) { if (globalForceSource) { return true; diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 64d765985cb..1174fcdd8a9 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -35,14 +35,13 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.cache.query.*; import org.elasticsearch.index.cache.query.QueryCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; @@ -181,7 +180,7 @@ public class DefaultSearchContext extends SearchContext { long resultWindow = from + size; // We need settingsService's view of the settings because its dynamic. // indexService's isn't. - int maxResultWindow = indexService.settingsService().getSettings().getAsInt(MAX_RESULT_WINDOW, Defaults.MAX_RESULT_WINDOW); + int maxResultWindow = indexService.getIndexSettings().getSettings().getAsInt(MAX_RESULT_WINDOW, Defaults.MAX_RESULT_WINDOW); if (resultWindow > maxResultWindow) { throw new QueryPhaseExecutionException(this, @@ -193,19 +192,21 @@ public class DefaultSearchContext extends SearchContext { } // initialize the filtering alias based on the provided filters - aliasFilter = indexService.aliasFilter(request.filteringAliases()); + aliasFilter = indexService.aliasFilter(indexShard.getQueryShardContext(), request.filteringAliases()); if (query() == null) { parsedQuery(ParsedQuery.parsedMatchAllQuery()); } - if (queryBoost() != 1.0f) { + if (queryBoost() != AbstractQueryBuilder.DEFAULT_BOOST) { parsedQuery(new ParsedQuery(new FunctionScoreQuery(query(), new WeightFactorFunction(queryBoost)), parsedQuery())); } Query searchFilter = searchFilter(types()); if (searchFilter != null) { if (Queries.isConstantMatchAllQuery(query())) { Query q = new ConstantScoreQuery(searchFilter); - q.setBoost(query().getBoost()); + if (query().getBoost() != AbstractQueryBuilder.DEFAULT_BOOST) { + q = new BoostQuery(q, query().getBoost()); + } parsedQuery(new ParsedQuery(q, parsedQuery())); } else { BooleanQuery filtered = new BooleanQuery.Builder() @@ -430,11 +431,6 @@ public class DefaultSearchContext extends SearchContext { return indexService.analysisService(); } - @Override - public IndexQueryParserService queryParserService() { - return indexService.queryParserService(); - } - @Override public SimilarityService similarityService() { return indexService.similarityService(); @@ -457,7 +453,7 @@ public class DefaultSearchContext extends SearchContext { @Override public BitsetFilterCache bitsetFilterCache() { - return indexService.bitsetFilterCache(); + return indexService.cache().bitsetFilterCache(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 5687e9eeb10..7225c7b32bd 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -34,7 +34,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; @@ -257,11 +256,6 @@ public abstract class FilteredSearchContext extends SearchContext { return in.analysisService(); } - @Override - public IndexQueryParserService queryParserService() { - return in.queryParserService(); - } - @Override public SimilarityService similarityService() { return in.similarityService(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java index c1194f1f401..96fd103fa6f 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java +++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java @@ -559,7 +559,7 @@ public class InternalSearchHit implements SearchHit { score = in.readFloat(); id = in.readText(); type = in.readText(); - nestedIdentity = in.readOptionalStreamable(new InternalNestedIdentity()); + nestedIdentity = in.readOptionalStreamable(InternalNestedIdentity::new); version = in.readLong(); source = in.readBytesReference(); if (source.length() == 0) { @@ -810,7 +810,7 @@ public class InternalSearchHit implements SearchHit { public void readFrom(StreamInput in) throws IOException { field = in.readOptionalText(); offset = in.readInt(); - child = in.readOptionalStreamable(new InternalNestedIdentity()); + child = in.readOptionalStreamable(InternalNestedIdentity::new); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index e632241d8df..0f61b2bc6a3 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -40,7 +40,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; @@ -206,8 +205,6 @@ public abstract class SearchContext extends DelegatingHasContextAndHeaders imple public abstract AnalysisService analysisService(); - public abstract IndexQueryParserService queryParserService(); - public abstract SimilarityService similarityService(); public abstract ScriptService scriptService(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java index ca8c074627b..47791aeddfa 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.script.Template; import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -65,9 +66,7 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S private Scroll scroll; private String[] types = Strings.EMPTY_ARRAY; private String[] filteringAliases; - private BytesReference source; - private BytesReference extraSource; - private BytesReference templateSource; + private SearchSourceBuilder source; private Template template; private Boolean requestCache; private long nowInMillis; @@ -79,8 +78,6 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S String[] filteringAliases, long nowInMillis) { this(shardRouting.shardId(), numberOfShards, searchRequest.searchType(), searchRequest.source(), searchRequest.types(), searchRequest.requestCache()); - this.extraSource = searchRequest.extraSource(); - this.templateSource = searchRequest.templateSource(); this.template = searchRequest.template(); this.scroll = searchRequest.scroll(); this.filteringAliases = filteringAliases; @@ -98,8 +95,8 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S this.filteringAliases = filteringAliases; } - public ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType searchType, - BytesReference source, String[] types, Boolean requestCache) { + public ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType searchType, SearchSourceBuilder source, String[] types, + Boolean requestCache) { this.index = shardId.getIndex(); this.shardId = shardId.id(); this.numberOfShards = numberOfShards; @@ -125,20 +122,15 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S } @Override - public BytesReference source() { + public SearchSourceBuilder source() { return source; } @Override - public void source(BytesReference source) { + public void source(SearchSourceBuilder source) { this.source = source; } - @Override - public BytesReference extraSource() { - return extraSource; - } - @Override public int numberOfShards() { return numberOfShards; @@ -158,17 +150,11 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S public long nowInMillis() { return nowInMillis; } - @Override public Template template() { return template; } - @Override - public BytesReference templateSource() { - return templateSource; - } - @Override public Boolean requestCache() { return requestCache; @@ -188,18 +174,13 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S if (in.readBoolean()) { scroll = readScroll(in); } - - source = in.readBytesReference(); - extraSource = in.readBytesReference(); - + if (in.readBoolean()) { + source = SearchSourceBuilder.readSearchSourceFrom(in); + } types = in.readStringArray(); filteringAliases = in.readStringArray(); nowInMillis = in.readVLong(); - - templateSource = in.readBytesReference(); - if (in.readBoolean()) { - template = Template.readTemplate(in); - } + template = in.readOptionalStreamable(Template::new); requestCache = in.readOptionalBoolean(); } @@ -216,20 +197,20 @@ public class ShardSearchLocalRequest extends ContextAndHeaderHolder implements S out.writeBoolean(true); scroll.writeTo(out); } - out.writeBytesReference(source); - out.writeBytesReference(extraSource); + if (source == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + source.writeTo(out); + + } out.writeStringArray(types); out.writeStringArrayNullable(filteringAliases); if (!asKey) { out.writeVLong(nowInMillis); } - out.writeBytesReference(templateSource); - boolean hasTemplate = template != null; - out.writeBoolean(hasTemplate); - if (hasTemplate) { - template.writeTo(out); - } + out.writeOptionalStreamable(template); out.writeOptionalBoolean(requestCache); } diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 6d9734f8994..fb631b08270 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -20,12 +20,11 @@ package org.elasticsearch.search.internal; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.HasContext; import org.elasticsearch.common.HasContextAndHeaders; -import org.elasticsearch.common.HasHeaders; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.script.Template; import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -42,11 +41,9 @@ public interface ShardSearchRequest extends HasContextAndHeaders { String[] types(); - BytesReference source(); + SearchSourceBuilder source(); - void source(BytesReference source); - - BytesReference extraSource(); + void source(SearchSourceBuilder source); int numberOfShards(); @@ -58,8 +55,6 @@ public interface ShardSearchRequest extends HasContextAndHeaders { Template template(); - BytesReference templateSource(); - Boolean requestCache(); Scroll scroll(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java index e7b1e2f9e62..279d9d6bd20 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.script.Template; import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.transport.TransportRequest; import java.io.IOException; @@ -87,20 +88,15 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha } @Override - public BytesReference source() { + public SearchSourceBuilder source() { return shardSearchLocalRequest.source(); } @Override - public void source(BytesReference source) { + public void source(SearchSourceBuilder source) { shardSearchLocalRequest.source(source); } - @Override - public BytesReference extraSource() { - return shardSearchLocalRequest.extraSource(); - } - @Override public int numberOfShards() { return shardSearchLocalRequest.numberOfShards(); @@ -120,17 +116,11 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha public long nowInMillis() { return shardSearchLocalRequest.nowInMillis(); } - @Override public Template template() { return shardSearchLocalRequest.template(); } - @Override - public BytesReference templateSource() { - return shardSearchLocalRequest.templateSource(); - } - @Override public Boolean requestCache() { return shardSearchLocalRequest.requestCache(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index 670f6788d4e..5feabad7920 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.search.internal; -import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.util.Counter; import org.elasticsearch.action.search.SearchType; @@ -83,7 +83,7 @@ public class SubSearchContext extends FilteredSearchContext { } @Override - public Filter searchFilter(String[] types) { + public Query searchFilter(String[] types) { throw new UnsupportedOperationException("this context should be read only"); } diff --git a/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java index 5e6a6df5fc0..a7b965dbfb1 100644 --- a/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java +++ b/core/src/main/java/org/elasticsearch/search/lookup/LeafIndexLookup.java @@ -134,7 +134,7 @@ public class LeafIndexLookup extends MinimalMap { indexField = new IndexField(stringField, this); indexFields.put(stringField, indexField); } catch (IOException e) { - throw new ElasticsearchException(e.getMessage()); + throw new ElasticsearchException(e); } } return indexField; diff --git a/core/src/main/java/org/elasticsearch/search/query/FilterBinaryParseElement.java b/core/src/main/java/org/elasticsearch/search/query/FilterBinaryParseElement.java deleted file mode 100644 index facb17e61f0..00000000000 --- a/core/src/main/java/org/elasticsearch/search/query/FilterBinaryParseElement.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.query; - -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.ParsedQuery; -import org.elasticsearch.search.SearchParseElement; -import org.elasticsearch.search.internal.SearchContext; - -/** - * - */ -public class FilterBinaryParseElement implements SearchParseElement { - - @Override - public void parse(XContentParser parser, SearchContext context) throws Exception { - byte[] filterSource = parser.binaryValue(); - try (XContentParser fSourceParser = XContentFactory.xContent(filterSource).createParser(filterSource)) { - ParsedQuery filter = context.queryParserService().parseInnerFilter(fSourceParser); - if (filter != null) { - context.parsedPostFilter(filter); - } - } - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java b/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java index bbe6b0a55df..3f3093b6996 100644 --- a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java @@ -30,7 +30,7 @@ public class PostFilterParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - ParsedQuery postFilter = context.queryParserService().parseInnerFilter(parser); + ParsedQuery postFilter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); if (postFilter != null) { context.parsedPostFilter(postFilter); } diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryBinaryParseElement.java b/core/src/main/java/org/elasticsearch/search/query/QueryBinaryParseElement.java deleted file mode 100644 index 1d6119bddac..00000000000 --- a/core/src/main/java/org/elasticsearch/search/query/QueryBinaryParseElement.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.query; - -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.SearchParseElement; -import org.elasticsearch.search.internal.SearchContext; - -/** - * - */ -public class QueryBinaryParseElement implements SearchParseElement { - - @Override - public void parse(XContentParser parser, SearchContext context) throws Exception { - byte[] querySource = parser.binaryValue(); - try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) { - context.parsedQuery(context.queryParserService().parse(qSourceParser)); - } - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java b/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java index 8db4cb3ef55..12a080acdea 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java @@ -30,6 +30,6 @@ public class QueryParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - context.parsedQuery(context.queryParserService().parse(parser)); + context.parsedQuery(context.indexShard().getQueryShardContext().parse(parser)); } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index c4aa23fc2a6..ce8836cd336 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -90,13 +90,8 @@ public class QueryPhase implements SearchPhase { parseElements.put("indices_boost", new IndicesBoostParseElement()); parseElements.put("indicesBoost", new IndicesBoostParseElement()); parseElements.put("query", new QueryParseElement()); - parseElements.put("queryBinary", new QueryBinaryParseElement()); - parseElements.put("query_binary", new QueryBinaryParseElement()); - parseElements.put("filter", new PostFilterParseElement()); // For bw comp reason, should be removed in version 1.1 parseElements.put("post_filter", new PostFilterParseElement()); parseElements.put("postFilter", new PostFilterParseElement()); - parseElements.put("filterBinary", new FilterBinaryParseElement()); - parseElements.put("filter_binary", new FilterBinaryParseElement()); parseElements.put("sort", new SortParseElement()); parseElements.put("trackScores", new TrackScoresParseElement()); parseElements.put("track_scores", new TrackScoresParseElement()); diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java index b4c39252574..afbd034840e 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java @@ -183,7 +183,7 @@ public final class QueryRescorer implements Rescorer { private static final ObjectParser RESCORE_PARSER = new ObjectParser<>("query", null); static { - RESCORE_PARSER.declareObject(QueryRescoreContext::setParsedQuery, (p, c) -> c.queryParserService().parse(p), new ParseField("rescore_query")); + RESCORE_PARSER.declareObject(QueryRescoreContext::setParsedQuery, (p, c) -> c.indexShard().getQueryShardContext().parse(p), new ParseField("rescore_query")); RESCORE_PARSER.declareFloat(QueryRescoreContext::setQueryWeight, new ParseField("query_weight")); RESCORE_PARSER.declareFloat(QueryRescoreContext::setRescoreQueryWeight, new ParseField("rescore_query_weight")); RESCORE_PARSER.declareString(QueryRescoreContext::setScoreMode, new ParseField("score_mode")); diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java index 405e2cc7e23..2a272f742e3 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java +++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java @@ -21,10 +21,9 @@ package org.elasticsearch.search.sort; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.util.BitSet; @@ -73,7 +72,7 @@ public class GeoDistanceSortParser implements SortParser { MultiValueMode sortMode = null; NestedInnerQueryParseSupport nestedHelper = null; - final boolean indexCreatedBeforeV2_0 = context.queryParserService().getIndexCreatedVersion().before(Version.V_2_0_0); + final boolean indexCreatedBeforeV2_0 = context.indexShard().getIndexSettings().getIndexVersionCreated().before(Version.V_2_0_0); boolean coerce = false; boolean ignoreMalformed = false; @@ -171,14 +170,14 @@ public class GeoDistanceSortParser implements SortParser { final Nested nested; if (nestedHelper != null && nestedHelper.getPath() != null) { BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter()); - Filter innerDocumentsFilter; + Query innerDocumentsFilter; if (nestedHelper.filterFound()) { // TODO: use queries instead - innerDocumentsFilter = new QueryWrapperFilter(nestedHelper.getInnerFilter()); + innerDocumentsFilter = nestedHelper.getInnerFilter(); } else { innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); } - nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); + nested = new Nested(rootDocumentsFilter, context.searcher().createNormalizedWeight(innerDocumentsFilter, false)); } else { nested = null; } @@ -202,7 +201,7 @@ public class GeoDistanceSortParser implements SortParser { selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE); } else { final BitSet rootDocs = nested.rootDocs(context); - final DocIdSet innerDocs = nested.innerDocs(context); + final DocIdSetIterator innerDocs = nested.innerDocs(context); selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, context.reader().maxDoc()); } return selectedValues.getRawDoubleValues(); diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java index e772c8f6d1b..c465eaf6efb 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java @@ -21,8 +21,7 @@ package org.elasticsearch.search.sort; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; import org.apache.lucene.search.join.BitSetProducer; @@ -145,14 +144,14 @@ public class ScriptSortParser implements SortParser { final Nested nested; if (nestedHelper != null && nestedHelper.getPath() != null) { BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter()); - Filter innerDocumentsFilter; + Query innerDocumentsFilter; if (nestedHelper.filterFound()) { // TODO: use queries instead - innerDocumentsFilter = new QueryWrapperFilter(nestedHelper.getInnerFilter()); + innerDocumentsFilter = nestedHelper.getInnerFilter(); } else { innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); } - nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); + nested = new Nested(rootDocumentsFilter, context.searcher().createNormalizedWeight(innerDocumentsFilter, false)); } else { nested = null; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java index 8454537be75..80a9daa53f0 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java @@ -19,8 +19,7 @@ package org.elasticsearch.search.sort; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.join.BitSetProducer; @@ -240,14 +239,14 @@ public class SortParseElement implements SearchParseElement { final Nested nested; if (nestedHelper != null && nestedHelper.getPath() != null) { BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter()); - Filter innerDocumentsFilter; + Query innerDocumentsFilter; if (nestedHelper.filterFound()) { // TODO: use queries instead - innerDocumentsFilter = new QueryWrapperFilter(nestedHelper.getInnerFilter()); + innerDocumentsFilter = nestedHelper.getInnerFilter(); } else { innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); } - nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); + nested = new Nested(rootDocumentsFilter, context.searcher().createNormalizedWeight(innerDocumentsFilter, false)); } else { nested = null; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java index 95ce7ce70a0..db60d58953a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java @@ -126,7 +126,7 @@ public class Suggest implements Iterable>(); + suggestion = new Suggestion(); break; } suggestion.readFrom(in); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java index ea45c1033e9..5621e03e7de 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java @@ -20,9 +20,6 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.search.suggest.context.CategoryContextMapping; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; -import org.elasticsearch.search.suggest.context.GeolocationContextMapping; import java.io.IOException; import java.util.ArrayList; @@ -101,90 +98,18 @@ public class SuggestBuilder extends ToXContentToBytes { private String name; private String suggester; private String text; + private String prefix; + private String regex; private String field; private String analyzer; private Integer size; private Integer shardSize; - - private List contextQueries = new ArrayList<>(); public SuggestionBuilder(String name, String suggester) { this.name = name; this.suggester = suggester; } - @SuppressWarnings("unchecked") - private T addContextQuery(ContextQuery ctx) { - this.contextQueries.add(ctx); - return (T) this; - } - - /** - * Setup a Geolocation for suggestions. See {@link GeolocationContextMapping}. - * @param lat Latitude of the location - * @param lon Longitude of the Location - * @return this - */ - public T addGeoLocation(String name, double lat, double lon, int ... precisions) { - return addContextQuery(GeolocationContextMapping.query(name, lat, lon, precisions)); - } - - /** - * Setup a Geolocation for suggestions. See {@link GeolocationContextMapping}. - * @param lat Latitude of the location - * @param lon Longitude of the Location - * @param precisions precisions as string var-args - * @return this - */ - public T addGeoLocationWithPrecision(String name, double lat, double lon, String ... precisions) { - return addContextQuery(GeolocationContextMapping.query(name, lat, lon, precisions)); - } - - /** - * Setup a Geolocation for suggestions. See {@link GeolocationContextMapping}. - * @param geohash Geohash of the location - * @return this - */ - public T addGeoLocation(String name, String geohash) { - return addContextQuery(GeolocationContextMapping.query(name, geohash)); - } - - /** - * Setup a Category for suggestions. See {@link CategoryContextMapping}. - * @param categories name of the category - * @return this - */ - public T addCategory(String name, CharSequence...categories) { - return addContextQuery(CategoryContextMapping.query(name, categories)); - } - - /** - * Setup a Category for suggestions. See {@link CategoryContextMapping}. - * @param categories name of the category - * @return this - */ - public T addCategory(String name, Iterable categories) { - return addContextQuery(CategoryContextMapping.query(name, categories)); - } - - /** - * Setup a Context Field for suggestions. See {@link CategoryContextMapping}. - * @param fieldvalues name of the category - * @return this - */ - public T addContextField(String name, CharSequence...fieldvalues) { - return addContextQuery(CategoryContextMapping.query(name, fieldvalues)); - } - - /** - * Setup a Context Field for suggestions. See {@link CategoryContextMapping}. - * @param fieldvalues name of the category - * @return this - */ - public T addContextField(String name, Iterable fieldvalues) { - return addContextQuery(CategoryContextMapping.query(name, fieldvalues)); - } - /** * Same as in {@link SuggestBuilder#setText(String)}, but in the suggestion scope. */ @@ -194,12 +119,26 @@ public class SuggestBuilder extends ToXContentToBytes { return (T) this; } + protected void setPrefix(String prefix) { + this.prefix = prefix; + } + + protected void setRegex(String regex) { + this.regex = regex; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(name); if (text != null) { builder.field("text", text); } + if (prefix != null) { + builder.field("prefix", prefix); + } + if (regex != null) { + builder.field("regex", regex); + } builder.startObject(suggester); if (analyzer != null) { builder.field("analyzer", analyzer); @@ -214,13 +153,6 @@ public class SuggestBuilder extends ToXContentToBytes { builder.field("shard_size", shardSize); } - if (!contextQueries.isEmpty()) { - builder.startObject("context"); - for (ContextQuery query : contextQueries) { - query.toXContent(builder, params); - } - builder.endObject(); - } builder = innerToXContent(builder, params); builder.endObject(); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java index 16957986e27..66b917394ff 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; @@ -61,15 +60,4 @@ public abstract class SuggestBuilders { public static CompletionSuggestionBuilder completionSuggestion(String name) { return new CompletionSuggestionBuilder(name); } - - /** - * Creates a fuzzy completion suggestion lookup query with the provided name - * - * @param name The suggestion name - * @return a {@link org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder} - * instance - */ - public static CompletionSuggestionFuzzyBuilder fuzzyCompletionSuggestion(String name) { - return new CompletionSuggestionFuzzyBuilder(name); - } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java index 040ee4e85ea..a8050d1acaf 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java @@ -20,13 +20,12 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.common.HasContextAndHeaders; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; import java.io.IOException; public interface SuggestContextParser { - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, - IndexQueryParserService queryParserService, HasContextAndHeaders headersContext) throws IOException; + SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService indexFieldDataService, HasContextAndHeaders headersContext) throws IOException; } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java index 3dcca4a8086..650eb76b1c4 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java @@ -22,8 +22,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.HasContextAndHeaders; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; @@ -45,13 +45,13 @@ public final class SuggestParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.queryParserService(), + SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.fieldData(), context.shardTarget().index(), context.shardTarget().shardId(), context); context.suggest(suggestionSearchContext); } - public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, - IndexQueryParserService queryParserService, String index, int shardId, HasContextAndHeaders headersContext) throws IOException { + public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService, + String index, int shardId, HasContextAndHeaders headersContext) throws IOException { SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); BytesRef globalText = null; @@ -71,6 +71,8 @@ public final class SuggestParseElement implements SearchParseElement { } else if (token == XContentParser.Token.START_OBJECT) { String suggestionName = fieldName; BytesRef suggestText = null; + BytesRef prefix = null; + BytesRef regex = null; SuggestionContext suggestionContext = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -79,6 +81,10 @@ public final class SuggestParseElement implements SearchParseElement { } else if (token.isValue()) { if ("text".equals(fieldName)) { suggestText = parser.utf8Bytes(); + } else if ("prefix".equals(fieldName)) { + prefix = parser.utf8Bytes(); + } else if ("regex".equals(fieldName)) { + regex = parser.utf8Bytes(); } else { throw new IllegalArgumentException("[suggest] does not support [" + fieldName + "]"); } @@ -90,14 +96,22 @@ public final class SuggestParseElement implements SearchParseElement { throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported"); } final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser(); - suggestionContext = contextParser.parse(parser, mapperService, queryParserService, headersContext); + suggestionContext = contextParser.parse(parser, mapperService, fieldDataService, headersContext); } } if (suggestionContext != null) { - suggestionContext.setText(suggestText); + if (suggestText != null && prefix == null) { + suggestionContext.setPrefix(suggestText); + suggestionContext.setText(suggestText); + } else if (suggestText == null && prefix != null) { + suggestionContext.setPrefix(prefix); + suggestionContext.setText(prefix); + } else if (regex != null) { + suggestionContext.setRegex(regex); + suggestionContext.setText(regex); + } suggestionContexts.put(suggestionName, suggestionContext); } - } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java index 25703e80b6a..9eba50f478a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.ExtensionPoint; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.phrase.PhraseSuggester; @@ -34,7 +35,7 @@ public final class Suggesters extends ExtensionPoint.ClassMap { private final Map parsers; public Suggesters() { - this(Collections.EMPTY_MAP); + this(Collections.emptyMap()); } public Suggesters(Map suggesters) { @@ -43,13 +44,13 @@ public final class Suggesters extends ExtensionPoint.ClassMap { } @Inject - public Suggesters(Map suggesters, ScriptService scriptService) { - this(addBuildIns(suggesters, scriptService)); + public Suggesters(Map suggesters, ScriptService scriptService, IndicesService indexServices) { + this(addBuildIns(suggesters, scriptService, indexServices)); } - private static Map addBuildIns(Map suggesters, ScriptService scriptService) { + private static Map addBuildIns(Map suggesters, ScriptService scriptService, IndicesService indexServices) { final Map map = new HashMap<>(); - map.put("phrase", new PhraseSuggester(scriptService)); + map.put("phrase", new PhraseSuggester(scriptService, indexServices)); map.put("term", new TermSuggester()); map.put("completion", new CompletionSuggester()); map.putAll(suggesters); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java index 2cb36f53914..1d3339e0578 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java @@ -40,6 +40,8 @@ public class SuggestionSearchContext { public static class SuggestionContext { private BytesRef text; + private BytesRef prefix; + private BytesRef regex; private final Suggester suggester; private String field; private Analyzer analyzer; @@ -55,7 +57,23 @@ public class SuggestionSearchContext { public void setText(BytesRef text) { this.text = text; } - + + public BytesRef getPrefix() { + return prefix; + } + + public void setPrefix(BytesRef prefix) { + this.prefix = prefix; + } + + public BytesRef getRegex() { + return regex; + } + + public void setRegex(BytesRef regex) { + this.regex = regex; + } + public SuggestionContext(Suggester suggester) { this.suggester = suggester; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java deleted file mode 100644 index c5b1b5931e9..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java +++ /dev/null @@ -1,407 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.suggest.completion; - -import com.carrotsearch.hppc.ObjectLongHashMap; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.TokenStreamToAutomaton; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; -import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Accountables; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.IntsRef; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.LimitedFiniteStringsIterator; -import org.apache.lucene.util.fst.ByteSequenceOutputs; -import org.apache.lucene.util.fst.FST; -import org.apache.lucene.util.fst.PairOutputs; -import org.apache.lucene.util.fst.PairOutputs.Pair; -import org.apache.lucene.util.fst.PositiveIntOutputs; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.LookupFactory; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider { - - // for serialization - public static final int SERIALIZE_PRESERVE_SEPARATORS = 1; - public static final int SERIALIZE_HAS_PAYLOADS = 2; - public static final int SERIALIZE_PRESERVE_POSITION_INCREMENTS = 4; - - private static final int MAX_SURFACE_FORMS_PER_ANALYZED_FORM = 256; - private static final int MAX_GRAPH_EXPANSIONS = -1; - - public static final String CODEC_NAME = "analyzing"; - public static final int CODEC_VERSION_START = 1; - public static final int CODEC_VERSION_SERIALIZED_LABELS = 2; - public static final int CODEC_VERSION_CHECKSUMS = 3; - public static final int CODEC_VERSION_LATEST = CODEC_VERSION_CHECKSUMS; - - private final boolean preserveSep; - private final boolean preservePositionIncrements; - private final int maxSurfaceFormsPerAnalyzedForm; - private final int maxGraphExpansions; - private final boolean hasPayloads; - private final XAnalyzingSuggester prototype; - - public AnalyzingCompletionLookupProvider(boolean preserveSep, boolean exactFirst, boolean preservePositionIncrements, boolean hasPayloads) { - this.preserveSep = preserveSep; - this.preservePositionIncrements = preservePositionIncrements; - this.hasPayloads = hasPayloads; - this.maxSurfaceFormsPerAnalyzedForm = MAX_SURFACE_FORMS_PER_ANALYZED_FORM; - this.maxGraphExpansions = MAX_GRAPH_EXPANSIONS; - int options = preserveSep ? XAnalyzingSuggester.PRESERVE_SEP : 0; - // needs to fixed in the suggester first before it can be supported - //options |= exactFirst ? XAnalyzingSuggester.EXACT_FIRST : 0; - prototype = new XAnalyzingSuggester(null, null, null, options, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, preservePositionIncrements, null, false, 1, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); - } - - @Override - public String getName() { - return "analyzing"; - } - - public boolean getPreserveSep() { - return preserveSep; - } - - public boolean getPreservePositionsIncrements() { - return preservePositionIncrements; - } - - public boolean hasPayloads() { - return hasPayloads; - } - - @Override - public FieldsConsumer consumer(final IndexOutput output) throws IOException { - CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION_LATEST); - return new FieldsConsumer() { - private Map fieldOffsets = new HashMap<>(); - - @Override - public void close() throws IOException { - try { - /* - * write the offsets per field such that we know where - * we need to load the FSTs from - */ - long pointer = output.getFilePointer(); - output.writeVInt(fieldOffsets.size()); - for (Map.Entry entry : fieldOffsets.entrySet()) { - output.writeString(entry.getKey()); - output.writeVLong(entry.getValue()); - } - output.writeLong(pointer); - CodecUtil.writeFooter(output); - } finally { - IOUtils.close(output); - } - } - - @Override - public void write(Fields fields) throws IOException { - for(String field : fields) { - Terms terms = fields.terms(field); - if (terms == null) { - continue; - } - TermsEnum termsEnum = terms.iterator(); - PostingsEnum docsEnum = null; - final SuggestPayload spare = new SuggestPayload(); - int maxAnalyzedPathsForOneInput = 0; - final XAnalyzingSuggester.XBuilder builder = new XAnalyzingSuggester.XBuilder(maxSurfaceFormsPerAnalyzedForm, hasPayloads, XAnalyzingSuggester.PAYLOAD_SEP); - int docCount = 0; - while (true) { - BytesRef term = termsEnum.next(); - if (term == null) { - break; - } - docsEnum = termsEnum.postings(docsEnum, PostingsEnum.PAYLOADS); - builder.startTerm(term); - int docFreq = 0; - while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { - for (int i = 0; i < docsEnum.freq(); i++) { - final int position = docsEnum.nextPosition(); - AnalyzingCompletionLookupProvider.this.parsePayload(docsEnum.getPayload(), spare); - builder.addSurface(spare.surfaceForm.get(), spare.payload.get(), spare.weight); - // multi fields have the same surface form so we sum up here - maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position + 1); - } - docFreq++; - docCount = Math.max(docCount, docsEnum.docID()+1); - } - builder.finishTerm(docFreq); - } - /* - * Here we are done processing the field and we can - * buid the FST and write it to disk. - */ - FST> build = builder.build(); - assert build != null || docCount == 0: "the FST is null but docCount is != 0 actual value: [" + docCount + "]"; - /* - * it's possible that the FST is null if we have 2 segments that get merged - * and all docs that have a value in this field are deleted. This will cause - * a consumer to be created but it doesn't consume any values causing the FSTBuilder - * to return null. - */ - if (build != null) { - fieldOffsets.put(field, output.getFilePointer()); - build.save(output); - /* write some more meta-info */ - output.writeVInt(maxAnalyzedPathsForOneInput); - output.writeVInt(maxSurfaceFormsPerAnalyzedForm); - output.writeInt(maxGraphExpansions); // can be negative - int options = 0; - options |= preserveSep ? SERIALIZE_PRESERVE_SEPARATORS : 0; - options |= hasPayloads ? SERIALIZE_HAS_PAYLOADS : 0; - options |= preservePositionIncrements ? SERIALIZE_PRESERVE_POSITION_INCREMENTS : 0; - output.writeVInt(options); - output.writeVInt(XAnalyzingSuggester.SEP_LABEL); - output.writeVInt(XAnalyzingSuggester.END_BYTE); - output.writeVInt(XAnalyzingSuggester.PAYLOAD_SEP); - output.writeVInt(XAnalyzingSuggester.HOLE_CHARACTER); - } - } - } - }; - } - - - @Override - public LookupFactory load(IndexInput input) throws IOException { - long sizeInBytes = 0; - int version = CodecUtil.checkHeader(input, CODEC_NAME, CODEC_VERSION_START, CODEC_VERSION_LATEST); - if (version >= CODEC_VERSION_CHECKSUMS) { - CodecUtil.checksumEntireFile(input); - } - final long metaPointerPosition = input.length() - (version >= CODEC_VERSION_CHECKSUMS? 8 + CodecUtil.footerLength() : 8); - final Map lookupMap = new HashMap<>(); - input.seek(metaPointerPosition); - long metaPointer = input.readLong(); - input.seek(metaPointer); - int numFields = input.readVInt(); - - Map meta = new TreeMap<>(); - for (int i = 0; i < numFields; i++) { - String name = input.readString(); - long offset = input.readVLong(); - meta.put(offset, name); - } - - for (Map.Entry entry : meta.entrySet()) { - input.seek(entry.getKey()); - FST> fst = new FST<>(input, new PairOutputs<>( - PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton())); - int maxAnalyzedPathsForOneInput = input.readVInt(); - int maxSurfaceFormsPerAnalyzedForm = input.readVInt(); - int maxGraphExpansions = input.readInt(); - int options = input.readVInt(); - boolean preserveSep = (options & SERIALIZE_PRESERVE_SEPARATORS) != 0; - boolean hasPayloads = (options & SERIALIZE_HAS_PAYLOADS) != 0; - boolean preservePositionIncrements = (options & SERIALIZE_PRESERVE_POSITION_INCREMENTS) != 0; - - // first version did not include these three fields, so fall back to old default (before the analyzingsuggester - // was updated in Lucene, so we cannot use the suggester defaults) - int sepLabel, payloadSep, endByte, holeCharacter; - switch (version) { - case CODEC_VERSION_START: - sepLabel = 0xFF; - payloadSep = '\u001f'; - endByte = 0x0; - holeCharacter = '\u001E'; - break; - default: - sepLabel = input.readVInt(); - endByte = input.readVInt(); - payloadSep = input.readVInt(); - holeCharacter = input.readVInt(); - } - - AnalyzingSuggestHolder holder = new AnalyzingSuggestHolder(preserveSep, preservePositionIncrements, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, - hasPayloads, maxAnalyzedPathsForOneInput, fst, sepLabel, payloadSep, endByte, holeCharacter); - sizeInBytes += fst.ramBytesUsed(); - lookupMap.put(entry.getValue(), holder); - } - final long ramBytesUsed = sizeInBytes; - return new LookupFactory() { - @Override - public Lookup getLookup(CompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext) { - AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(fieldType.names().indexName()); - if (analyzingSuggestHolder == null) { - return null; - } - int flags = analyzingSuggestHolder.getPreserveSeparator() ? XAnalyzingSuggester.PRESERVE_SEP : 0; - - final XAnalyzingSuggester suggester; - final Automaton queryPrefix = fieldType.requiresContext() ? ContextQuery.toAutomaton(analyzingSuggestHolder.getPreserveSeparator(), suggestionContext.getContextQueries()) : null; - - if (suggestionContext.isFuzzy()) { - suggester = new XFuzzySuggester(fieldType.indexAnalyzer(), queryPrefix, fieldType.searchAnalyzer(), flags, - analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions, - suggestionContext.getFuzzyEditDistance(), suggestionContext.isFuzzyTranspositions(), - suggestionContext.getFuzzyPrefixLength(), suggestionContext.getFuzzyMinLength(), suggestionContext.isFuzzyUnicodeAware(), - analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads, - analyzingSuggestHolder.maxAnalyzedPathsForOneInput, analyzingSuggestHolder.sepLabel, analyzingSuggestHolder.payloadSep, analyzingSuggestHolder.endByte, - analyzingSuggestHolder.holeCharacter); - } else { - suggester = new XAnalyzingSuggester(fieldType.indexAnalyzer(), queryPrefix, fieldType.searchAnalyzer(), flags, - analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions, - analyzingSuggestHolder.preservePositionIncrements, analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads, - analyzingSuggestHolder.maxAnalyzedPathsForOneInput, analyzingSuggestHolder.sepLabel, analyzingSuggestHolder.payloadSep, analyzingSuggestHolder.endByte, - analyzingSuggestHolder.holeCharacter); - } - return suggester; - } - - @Override - public CompletionStats stats(String... fields) { - long sizeInBytes = 0; - ObjectLongHashMap completionFields = null; - if (fields != null && fields.length > 0) { - completionFields = new ObjectLongHashMap<>(fields.length); - } - - for (Map.Entry entry : lookupMap.entrySet()) { - sizeInBytes += entry.getValue().fst.ramBytesUsed(); - if (fields == null || fields.length == 0) { - continue; - } - if (Regex.simpleMatch(fields, entry.getKey())) { - long fstSize = entry.getValue().fst.ramBytesUsed(); - completionFields.addTo(entry.getKey(), fstSize); - } - } - - return new CompletionStats(sizeInBytes, completionFields); - } - - @Override - AnalyzingSuggestHolder getAnalyzingSuggestHolder(MappedFieldType fieldType) { - return lookupMap.get(fieldType.names().indexName()); - } - - @Override - public long ramBytesUsed() { - return ramBytesUsed; - } - - @Override - public Collection getChildResources() { - return Accountables.namedAccountables("field", lookupMap); - } - }; - } - - static class AnalyzingSuggestHolder implements Accountable { - final boolean preserveSep; - final boolean preservePositionIncrements; - final int maxSurfaceFormsPerAnalyzedForm; - final int maxGraphExpansions; - final boolean hasPayloads; - final int maxAnalyzedPathsForOneInput; - final FST> fst; - final int sepLabel; - final int payloadSep; - final int endByte; - final int holeCharacter; - - public AnalyzingSuggestHolder(boolean preserveSep, boolean preservePositionIncrements, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions, - boolean hasPayloads, int maxAnalyzedPathsForOneInput, FST> fst) { - this(preserveSep, preservePositionIncrements, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, hasPayloads, maxAnalyzedPathsForOneInput, fst, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); - } - - public AnalyzingSuggestHolder(boolean preserveSep, boolean preservePositionIncrements, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions, boolean hasPayloads, int maxAnalyzedPathsForOneInput, FST> fst, int sepLabel, int payloadSep, int endByte, int holeCharacter) { - this.preserveSep = preserveSep; - this.preservePositionIncrements = preservePositionIncrements; - this.maxSurfaceFormsPerAnalyzedForm = maxSurfaceFormsPerAnalyzedForm; - this.maxGraphExpansions = maxGraphExpansions; - this.hasPayloads = hasPayloads; - this.maxAnalyzedPathsForOneInput = maxAnalyzedPathsForOneInput; - this.fst = fst; - this.sepLabel = sepLabel; - this.payloadSep = payloadSep; - this.endByte = endByte; - this.holeCharacter = holeCharacter; - } - - public boolean getPreserveSeparator() { - return preserveSep; - } - - public boolean getPreservePositionIncrements() { - return preservePositionIncrements; - } - - public boolean hasPayloads() { - return hasPayloads; - } - - @Override - public long ramBytesUsed() { - if (fst != null) { - return fst.ramBytesUsed(); - } else { - return 0; - } - } - - @Override - public Collection getChildResources() { - if (fst != null) { - return Collections.singleton(Accountables.namedAccountable("fst", fst)); - } else { - return Collections.emptyList(); - } - } - } - - @Override - public Set toFiniteStrings(TokenStream stream) throws IOException { - return prototype.toFiniteStrings(stream); - } - - -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/Completion090PostingsFormat.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/Completion090PostingsFormat.java deleted file mode 100644 index 447b3fd7198..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/Completion090PostingsFormat.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.suggest.completion; - -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.FilterLeafReader.FilterTerms; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.Terms; -import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.store.IOContext.Context; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.InputStreamDataInput; -import org.apache.lucene.store.OutputStreamDataOutput; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Accountables; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream.ToFiniteStrings; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.singletonMap; - -/** - * This {@link PostingsFormat} is basically a T-Sink for a default postings - * format that is used to store postings on disk fitting the lucene APIs and - * builds a suggest FST as an auxiliary data structure next to the actual - * postings format. It uses the delegate postings format for simplicity to - * handle all the merge operations. The auxiliary suggest FST data structure is - * only loaded if a FieldsProducer is requested for reading, for merging it uses - * the low memory delegate postings format. - */ -public class Completion090PostingsFormat extends PostingsFormat { - - public static final String CODEC_NAME = "completion090"; - public static final int SUGGEST_CODEC_VERSION = 1; - public static final int SUGGEST_VERSION_CURRENT = SUGGEST_CODEC_VERSION; - public static final String EXTENSION = "cmp"; - - private static final ESLogger logger = Loggers.getLogger(Completion090PostingsFormat.class); - private static final CompletionLookupProvider LOOKUP_PROVIDER = new AnalyzingCompletionLookupProvider(true, false, true, false); - private static final Map PROVIDERS = singletonMap(LOOKUP_PROVIDER.getName(), LOOKUP_PROVIDER); - private PostingsFormat delegatePostingsFormat; - private CompletionLookupProvider writeProvider; - - public Completion090PostingsFormat(PostingsFormat delegatePostingsFormat, CompletionLookupProvider provider) { - super(CODEC_NAME); - this.delegatePostingsFormat = delegatePostingsFormat; - this.writeProvider = provider; - assert delegatePostingsFormat != null && writeProvider != null; - } - - /* - * Used only by core Lucene at read-time via Service Provider instantiation - * do not use at Write-time in application code. - */ - public Completion090PostingsFormat() { - super(CODEC_NAME); - } - - @Override - public CompletionFieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - if (delegatePostingsFormat == null) { - throw new UnsupportedOperationException("Error - " + getClass().getName() - + " has been constructed without a choice of PostingsFormat"); - } - assert writeProvider != null; - return new CompletionFieldsConsumer(state); - } - - @Override - public CompletionFieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new CompletionFieldsProducer(state); - } - - private class CompletionFieldsConsumer extends FieldsConsumer { - - private FieldsConsumer delegatesFieldsConsumer; - private FieldsConsumer suggestFieldsConsumer; - - public CompletionFieldsConsumer(SegmentWriteState state) throws IOException { - this.delegatesFieldsConsumer = delegatePostingsFormat.fieldsConsumer(state); - String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); - IndexOutput output = null; - boolean success = false; - try { - output = state.directory.createOutput(suggestFSTFile, state.context); - CodecUtil.writeHeader(output, CODEC_NAME, SUGGEST_VERSION_CURRENT); - /* - * we write the delegate postings format name so we can load it - * without getting an instance in the ctor - */ - output.writeString(delegatePostingsFormat.getName()); - output.writeString(writeProvider.getName()); - this.suggestFieldsConsumer = writeProvider.consumer(output); - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(output); - } - } - } - - @Override - public void write(Fields fields) throws IOException { - delegatesFieldsConsumer.write(fields); - suggestFieldsConsumer.write(fields); - } - - @Override - public void close() throws IOException { - IOUtils.close(delegatesFieldsConsumer, suggestFieldsConsumer); - } - } - - private static class CompletionFieldsProducer extends FieldsProducer { - // TODO make this class lazyload all the things in order to take advantage of the new merge instance API - // today we just load everything up-front - private final FieldsProducer delegateProducer; - private final LookupFactory lookupFactory; - private final int version; - - public CompletionFieldsProducer(SegmentReadState state) throws IOException { - String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); - IndexInput input = state.directory.openInput(suggestFSTFile, state.context); - version = CodecUtil.checkHeader(input, CODEC_NAME, SUGGEST_CODEC_VERSION, SUGGEST_VERSION_CURRENT); - FieldsProducer delegateProducer = null; - boolean success = false; - try { - PostingsFormat delegatePostingsFormat = PostingsFormat.forName(input.readString()); - String providerName = input.readString(); - CompletionLookupProvider completionLookupProvider = PROVIDERS.get(providerName); - if (completionLookupProvider == null) { - throw new IllegalStateException("no provider with name [" + providerName + "] registered"); - } - // TODO: we could clone the ReadState and make it always forward IOContext.MERGE to prevent unecessary heap usage? - delegateProducer = delegatePostingsFormat.fieldsProducer(state); - /* - * If we are merging we don't load the FSTs at all such that we - * don't consume so much memory during merge - */ - if (state.context.context != Context.MERGE) { - // TODO: maybe we can do this in a fully lazy fashion based on some configuration - // eventually we should have some kind of curciut breaker that prevents us from going OOM here - // with some configuration - this.lookupFactory = completionLookupProvider.load(input); - } else { - this.lookupFactory = null; - } - this.delegateProducer = delegateProducer; - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(delegateProducer, input); - } else { - IOUtils.close(input); - } - } - } - - @Override - public void close() throws IOException { - IOUtils.close(delegateProducer); - } - - @Override - public Iterator iterator() { - return delegateProducer.iterator(); - } - - @Override - public Terms terms(String field) throws IOException { - final Terms terms = delegateProducer.terms(field); - if (terms == null || lookupFactory == null) { - return terms; - } - return new CompletionTerms(terms, lookupFactory); - } - - @Override - public int size() { - return delegateProducer.size(); - } - - @Override - public long ramBytesUsed() { - return (lookupFactory == null ? 0 : lookupFactory.ramBytesUsed()) + delegateProducer.ramBytesUsed(); - } - - @Override - public Collection getChildResources() { - List resources = new ArrayList<>(); - if (lookupFactory != null) { - resources.add(Accountables.namedAccountable("lookup", lookupFactory)); - } - resources.add(Accountables.namedAccountable("delegate", delegateProducer)); - return Collections.unmodifiableList(resources); - } - - @Override - public void checkIntegrity() throws IOException { - delegateProducer.checkIntegrity(); - } - - @Override - public FieldsProducer getMergeInstance() throws IOException { - return delegateProducer.getMergeInstance(); - } - } - - public static final class CompletionTerms extends FilterTerms { - private final LookupFactory lookup; - - public CompletionTerms(Terms delegate, LookupFactory lookup) { - super(delegate); - this.lookup = lookup; - } - - public Lookup getLookup(CompletionFieldMapper.CompletionFieldType mapper, CompletionSuggestionContext suggestionContext) { - return lookup.getLookup(mapper, suggestionContext); - } - - public CompletionStats stats(String ... fields) { - return lookup.stats(fields); - } - } - - public static abstract class CompletionLookupProvider implements PayloadProcessor, ToFiniteStrings { - - public static final char UNIT_SEPARATOR = '\u001f'; - - public abstract FieldsConsumer consumer(IndexOutput output) throws IOException; - - public abstract String getName(); - - public abstract LookupFactory load(IndexInput input) throws IOException; - - @Override - public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { - if (weight < -1 || weight > Integer.MAX_VALUE) { - throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE"); - } - for (int i = 0; i < surfaceForm.length; i++) { - if (surfaceForm.bytes[i] == UNIT_SEPARATOR) { - throw new IllegalArgumentException( - "surface form cannot contain unit separator character U+001F; this character is reserved"); - } - } - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream); - output.writeVLong(weight + 1); - output.writeVInt(surfaceForm.length); - output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); - output.writeVInt(payload.length); - output.writeBytes(payload.bytes, 0, payload.length); - - output.close(); - return new BytesRef(byteArrayOutputStream.toByteArray()); - } - - @Override - public void parsePayload(BytesRef payload, SuggestPayload ref) throws IOException { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(payload.bytes, payload.offset, payload.length); - InputStreamDataInput input = new InputStreamDataInput(byteArrayInputStream); - ref.weight = input.readVLong() - 1; - int len = input.readVInt(); - ref.surfaceForm.grow(len); - ref.surfaceForm.setLength(len); - input.readBytes(ref.surfaceForm.bytes(), 0, ref.surfaceForm.length()); - len = input.readVInt(); - ref.payload.grow(len); - ref.payload.setLength(len); - input.readBytes(ref.payload.bytes(), 0, ref.payload.length()); - input.close(); - } - } - - public CompletionStats completionStats(IndexReader indexReader, String ... fields) { - CompletionStats completionStats = new CompletionStats(); - for (LeafReaderContext atomicReaderContext : indexReader.leaves()) { - LeafReader atomicReader = atomicReaderContext.reader(); - try { - for (String fieldName : atomicReader.fields()) { - Terms terms = atomicReader.fields().terms(fieldName); - if (terms instanceof CompletionTerms) { - CompletionTerms completionTerms = (CompletionTerms) terms; - completionStats.add(completionTerms.stats(fields)); - } - } - } catch (IOException e) { - logger.error("Could not get completion stats: {}", e, e.getMessage()); - } - } - - return completionStats; - } - - public static abstract class LookupFactory implements Accountable { - public abstract Lookup getLookup(CompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext); - public abstract CompletionStats stats(String ... fields); - abstract AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder getAnalyzingSuggestHolder(MappedFieldType fieldType); - } -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java new file mode 100644 index 00000000000..08c0302f81e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion; + +import com.carrotsearch.hppc.ObjectLongHashMap; + +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.suggest.document.CompletionTerms; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.regex.Regex; + +import java.io.IOException; + +public class CompletionFieldStats { + + /** + * Returns total in-heap bytes used by all suggesters. This method has CPU cost O(numIndexedFields). + * + * @param fieldNamePatterns if non-null, any completion field name matching any of these patterns will break out its in-heap bytes + * separately in the returned {@link CompletionStats} + */ + public static CompletionStats completionStats(IndexReader indexReader, String ... fieldNamePatterns) { + long sizeInBytes = 0; + ObjectLongHashMap completionFields = null; + if (fieldNamePatterns != null && fieldNamePatterns.length > 0) { + completionFields = new ObjectLongHashMap<>(fieldNamePatterns.length); + } + for (LeafReaderContext atomicReaderContext : indexReader.leaves()) { + LeafReader atomicReader = atomicReaderContext.reader(); + try { + Fields fields = atomicReader.fields(); + for (String fieldName : fields) { + Terms terms = fields.terms(fieldName); + if (terms instanceof CompletionTerms) { + // TODO: currently we load up the suggester for reporting its size + long fstSize = ((CompletionTerms) terms).suggester().ramBytesUsed(); + if (fieldNamePatterns != null && fieldNamePatterns.length > 0 && Regex.simpleMatch(fieldNamePatterns, fieldName)) { + completionFields.addTo(fieldName, fstSize); + } + sizeInBytes += fstSize; + } + } + } catch (IOException ioe) { + throw new ElasticsearchException(ioe); + } + } + return new CompletionStats(sizeInBytes, completionFields); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java index 8470633fdc5..928a1342ec1 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java @@ -18,107 +18,158 @@ */ package org.elasticsearch.search.suggest.completion; +import org.apache.lucene.analysis.Analyzer; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.HasContextAndHeaders; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestionSearchContext; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; +import org.elasticsearch.search.suggest.completion.context.ContextMapping; +import org.elasticsearch.search.suggest.completion.context.ContextMappings; import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.search.suggest.SuggestUtils.parseSuggestContext; +import java.util.*; /** + * Parses query options for {@link CompletionSuggester} * + * Acceptable input: + * { + * "field" : STRING + * "size" : INT + * "fuzzy" : BOOLEAN | FUZZY_OBJECT + * "contexts" : QUERY_CONTEXTS + * "regex" : REGEX_OBJECT + * } + * + * FUZZY_OBJECT : { + * "edit_distance" : STRING | INT + * "transpositions" : BOOLEAN + * "min_length" : INT + * "prefix_length" : INT + * "unicode_aware" : BOOLEAN + * "max_determinized_states" : INT + * } + * + * REGEX_OBJECT: { + * "flags" : REGEX_FLAGS + * "max_determinized_states" : INT + * } + * + * see {@link RegexpFlag} for REGEX_FLAGS */ public class CompletionSuggestParser implements SuggestContextParser { - private CompletionSuggester completionSuggester; - private static final ParseField FUZZINESS = Fuzziness.FIELD.withDeprecation("edit_distance"); + private static ObjectParser TLP_PARSER = new ObjectParser<>("completion", null); + private static ObjectParser REGEXP_PARSER = new ObjectParser<>("regexp", CompletionSuggestionBuilder.RegexOptionsBuilder::new); + private static ObjectParser FUZZY_PARSER = new ObjectParser<>("fuzzy", CompletionSuggestionBuilder.FuzzyOptionsBuilder::new); + static { + FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyMinLength, new ParseField("min_length")); + FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setMaxDeterminizedStates, new ParseField("max_determinized_states")); + FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setUnicodeAware, new ParseField("unicode_aware")); + FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyPrefixLength, new ParseField("prefix_length")); + FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setTranspositions, new ParseField("transpositions")); + FUZZY_PARSER.declareValue((a, b) -> { + try { + a.setFuzziness(Fuzziness.parse(b).asDistance()); + } catch (IOException e) { + throw new ElasticsearchException(e); + } + }, new ParseField("fuzziness")); + REGEXP_PARSER.declareInt(CompletionSuggestionBuilder.RegexOptionsBuilder::setMaxDeterminizedStates, new ParseField("max_determinized_states")); + REGEXP_PARSER.declareStringOrNull(CompletionSuggestionBuilder.RegexOptionsBuilder::setFlags, new ParseField("flags")); + + TLP_PARSER.declareStringArray(CompletionSuggestionContext::setPayloadFields, new ParseField("payload")); + TLP_PARSER.declareObjectOrDefault(CompletionSuggestionContext::setFuzzyOptionsBuilder, FUZZY_PARSER, CompletionSuggestionBuilder.FuzzyOptionsBuilder::new, new ParseField("fuzzy")); + TLP_PARSER.declareObject(CompletionSuggestionContext::setRegexOptionsBuilder, REGEXP_PARSER, new ParseField("regexp")); + TLP_PARSER.declareString(SuggestionSearchContext.SuggestionContext::setField, new ParseField("field")); + TLP_PARSER.declareField((p, v, c) -> { + String analyzerName = p.text(); + Analyzer analyzer = c.mapperService.analysisService().analyzer(analyzerName); + if (analyzer == null) { + throw new IllegalArgumentException("Analyzer [" + analyzerName + "] doesn't exists"); + } + v.setAnalyzer(analyzer); + }, new ParseField("analyzer"), ObjectParser.ValueType.STRING); + TLP_PARSER.declareString(SuggestionSearchContext.SuggestionContext::setField, new ParseField("analyzer")); + TLP_PARSER.declareInt(SuggestionSearchContext.SuggestionContext::setSize, new ParseField("size")); + TLP_PARSER.declareInt(SuggestionSearchContext.SuggestionContext::setShardSize, new ParseField("size")); + TLP_PARSER.declareField((p, v, c) -> { + // Copy the current structure. We will parse, once the mapping is provided + XContentBuilder builder = XContentFactory.contentBuilder(p.contentType()); + builder.copyCurrentStructure(p); + BytesReference bytes = builder.bytes(); + c.contextParser = XContentFactory.xContent(bytes).createParser(bytes); + p.skipChildren(); + }, new ParseField("contexts", "context"), ObjectParser.ValueType.OBJECT); // context is deprecated + } + + private static class ContextAndSuggest { + XContentParser contextParser; + final MapperService mapperService; + + ContextAndSuggest(MapperService mapperService) { + this.mapperService = mapperService; + } + } + + private final CompletionSuggester completionSuggester; public CompletionSuggestParser(CompletionSuggester completionSuggester) { this.completionSuggester = completionSuggester; } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, - IndexQueryParserService queryParserService, HasContextAndHeaders headersContext) throws IOException { - XContentParser.Token token; - String fieldName = null; - CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester); - - XContentParser contextParser = null; - - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } else if (token.isValue()) { - if (!parseSuggestContext(parser, mapperService, fieldName, suggestion, queryParserService.parseFieldMatcher())) { - if (token == XContentParser.Token.VALUE_BOOLEAN && "fuzzy".equals(fieldName)) { - suggestion.setFuzzy(parser.booleanValue()); + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService, + HasContextAndHeaders headersContext) throws IOException { + final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService); + final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService); + TLP_PARSER.parse(parser, suggestion, contextAndSuggest); + final XContentParser contextParser = contextAndSuggest.contextParser; + MappedFieldType mappedFieldType = mapperService.smartNameFieldType(suggestion.getField()); + if (mappedFieldType == null) { + throw new ElasticsearchException("Field [" + suggestion.getField() + "] is not a completion suggest field"); + } else if (mappedFieldType instanceof CompletionFieldMapper.CompletionFieldType) { + CompletionFieldMapper.CompletionFieldType type = (CompletionFieldMapper.CompletionFieldType) mappedFieldType; + if (type.hasContextMappings() == false && contextParser != null) { + throw new IllegalArgumentException("suggester [" + type.names().fullName() + "] doesn't expect any context"); + } + Map> queryContexts = Collections.emptyMap(); + if (type.hasContextMappings() && contextParser != null) { + ContextMappings contextMappings = type.getContextMappings(); + contextParser.nextToken(); + queryContexts = new HashMap<>(contextMappings.size()); + assert contextParser.currentToken() == XContentParser.Token.START_OBJECT; + XContentParser.Token currentToken; + String currentFieldName; + while ((currentToken = contextParser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (currentToken == XContentParser.Token.FIELD_NAME) { + currentFieldName = contextParser.currentName(); + final ContextMapping mapping = contextMappings.get(currentFieldName); + queryContexts.put(currentFieldName, mapping.parseQueryContext(contextParser)); } } - } else if (token == XContentParser.Token.START_OBJECT) { - if("fuzzy".equals(fieldName)) { - suggestion.setFuzzy(true); - String fuzzyConfigName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fuzzyConfigName = parser.currentName(); - } else if (token.isValue()) { - if (queryParserService.parseFieldMatcher().match(fuzzyConfigName, FUZZINESS)) { - suggestion.setFuzzyEditDistance(Fuzziness.parse(parser).asDistance()); - } else if ("transpositions".equals(fuzzyConfigName)) { - suggestion.setFuzzyTranspositions(parser.booleanValue()); - } else if ("min_length".equals(fuzzyConfigName) || "minLength".equals(fuzzyConfigName)) { - suggestion.setFuzzyMinLength(parser.intValue()); - } else if ("prefix_length".equals(fuzzyConfigName) || "prefixLength".equals(fuzzyConfigName)) { - suggestion.setFuzzyPrefixLength(parser.intValue()); - } else if ("unicode_aware".equals(fuzzyConfigName) || "unicodeAware".equals(fuzzyConfigName)) { - suggestion.setFuzzyUnicodeAware(parser.booleanValue()); - } - } - } - } else if("context".equals(fieldName)) { - // Copy the current structure. We will parse, once the mapping is provided - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()); - builder.copyCurrentStructure(parser); - BytesReference bytes = builder.bytes(); - contextParser = parser.contentType().xContent().createParser(bytes); - } else { - throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]"); - } - } else { - throw new IllegalArgumentException("suggester[completion] doesn't support field [" + fieldName + "]"); + contextParser.close(); } + suggestion.setFieldType(type); + suggestion.setQueryContexts(queryContexts); + return suggestion; + } else { + throw new IllegalArgumentException("Field [" + suggestion.getField() + "] is not a completion suggest field"); } - - suggestion.fieldType((CompletionFieldMapper.CompletionFieldType) mapperService.smartNameFieldType(suggestion.getField())); - - CompletionFieldMapper.CompletionFieldType fieldType = suggestion.fieldType(); - if (fieldType != null) { - if (fieldType.requiresContext()) { - if (contextParser == null) { - throw new IllegalArgumentException("suggester [completion] requires context to be setup"); - } else { - contextParser.nextToken(); - List contextQueries = ContextQuery.parseQueries(fieldType.getContextMapping(), contextParser); - suggestion.setContextQuery(contextQueries); - } - } else if (contextParser != null) { - throw new IllegalArgumentException("suggester [completion] doesn't expect any context"); - } - } - return suggestion; } + + } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java index b3e5e2dc2a5..106672ae7ae 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java @@ -18,97 +18,242 @@ */ package org.elasticsearch.search.suggest.completion; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Terms; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.CollectionUtil; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.bytes.BytesArray; +import org.apache.lucene.search.suggest.document.CompletionQuery; +import org.apache.lucene.search.suggest.document.TopSuggestDocs; +import org.apache.lucene.search.suggest.document.TopSuggestDocsCollector; +import org.apache.lucene.util.*; +import org.apache.lucene.util.PriorityQueue; import org.elasticsearch.common.text.StringText; +import org.elasticsearch.index.fielddata.AtomicFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.Suggester; -import org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option; import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class CompletionSuggester extends Suggester { - private static final ScoreComparator scoreComparator = new ScoreComparator(); - - - @Override - protected Suggest.Suggestion> innerExecute(String name, - CompletionSuggestionContext suggestionContext, IndexSearcher searcher, CharsRefBuilder spare) throws IOException { - if (suggestionContext.fieldType() == null) { - throw new ElasticsearchException("Field [" + suggestionContext.getField() + "] is not a completion suggest field"); - } - final IndexReader indexReader = searcher.getIndexReader(); - CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize()); - spare.copyUTF8Bytes(suggestionContext.getText()); - - CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new StringText(spare.toString()), 0, spare.length()); - completionSuggestion.addTerm(completionSuggestEntry); - - String fieldName = suggestionContext.getField(); - Map results = new HashMap<>(indexReader.leaves().size() * suggestionContext.getSize()); - for (LeafReaderContext atomicReaderContext : indexReader.leaves()) { - LeafReader atomicReader = atomicReaderContext.reader(); - Terms terms = atomicReader.fields().terms(fieldName); - if (terms instanceof Completion090PostingsFormat.CompletionTerms) { - final Completion090PostingsFormat.CompletionTerms lookupTerms = (Completion090PostingsFormat.CompletionTerms) terms; - final Lookup lookup = lookupTerms.getLookup(suggestionContext.fieldType(), suggestionContext); - if (lookup == null) { - // we don't have a lookup for this segment.. this might be possible if a merge dropped all - // docs from the segment that had a value in this segment. - continue; - } - List lookupResults = lookup.lookup(spare.get(), false, suggestionContext.getSize()); - for (Lookup.LookupResult res : lookupResults) { - - final String key = res.key.toString(); - final float score = res.value; - final Option value = results.get(key); - if (value == null) { - final Option option = new CompletionSuggestion.Entry.Option(new StringText(key), score, res.payload == null ? null - : new BytesArray(res.payload)); - results.put(key, option); - } else if (value.getScore() < score) { - value.setScore(score); - value.setPayload(res.payload == null ? null : new BytesArray(res.payload)); - } - } - } - } - final List options = new ArrayList<>(results.values()); - CollectionUtil.introSort(options, scoreComparator); - - int optionCount = Math.min(suggestionContext.getSize(), options.size()); - for (int i = 0 ; i < optionCount ; i++) { - completionSuggestEntry.addOption(options.get(i)); - } - - return completionSuggestion; - } - - @Override public SuggestContextParser getContextParser() { return new CompletionSuggestParser(this); } - public static class ScoreComparator implements Comparator { + @Override + protected Suggest.Suggestion> innerExecute(String name, + final CompletionSuggestionContext suggestionContext, final IndexSearcher searcher, CharsRefBuilder spare) throws IOException { + final CompletionFieldMapper.CompletionFieldType fieldType = suggestionContext.getFieldType(); + if (fieldType == null) { + throw new IllegalArgumentException("field [" + suggestionContext.getField() + "] is not a completion field"); + } + CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize()); + spare.copyUTF8Bytes(suggestionContext.getText()); + CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new StringText(spare.toString()), 0, spare.length()); + completionSuggestion.addTerm(completionSuggestEntry); + TopSuggestDocsCollector collector = new TopDocumentsCollector(suggestionContext.getSize()); + suggest(searcher, suggestionContext.toQuery(), collector); + int numResult = 0; + List leaves = searcher.getIndexReader().leaves(); + for (TopSuggestDocs.SuggestScoreDoc suggestScoreDoc : collector.get().scoreLookupDocs()) { + TopDocumentsCollector.SuggestDoc suggestDoc = (TopDocumentsCollector.SuggestDoc) suggestScoreDoc; + // collect contexts + Map> contexts = Collections.emptyMap(); + if (fieldType.hasContextMappings() && suggestDoc.getContexts().isEmpty() == false) { + contexts = fieldType.getContextMappings().getNamedContexts(suggestDoc.getContexts()); + } + // collect payloads + final Map> payload = new HashMap<>(0); + Set payloadFields = suggestionContext.getPayloadFields(); + if (payloadFields.isEmpty() == false) { + final int readerIndex = ReaderUtil.subIndex(suggestDoc.doc, leaves); + final LeafReaderContext subReaderContext = leaves.get(readerIndex); + final int subDocId = suggestDoc.doc - subReaderContext.docBase; + for (String field : payloadFields) { + MappedFieldType payloadFieldType = suggestionContext.getMapperService().smartNameFieldType(field); + if (payloadFieldType != null) { + final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType).load(subReaderContext); + final ScriptDocValues scriptValues = data.getScriptValues(); + scriptValues.setNextDocId(subDocId); + payload.put(field, new ArrayList<>(scriptValues.getValues())); + } else { + throw new IllegalArgumentException("payload field [" + field + "] does not exist"); + } + } + } + if (numResult++ < suggestionContext.getSize()) { + CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option( + new StringText(suggestDoc.key.toString()), suggestDoc.score, contexts, payload); + completionSuggestEntry.addOption(option); + } else { + break; + } + } + return completionSuggestion; + } + + private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSuggestDocsCollector collector) throws IOException { + query = (CompletionQuery) query.rewrite(searcher.getIndexReader()); + Weight weight = query.createWeight(searcher, collector.needsScores()); + for (LeafReaderContext context : searcher.getIndexReader().leaves()) { + BulkScorer scorer = weight.bulkScorer(context); + if (scorer != null) { + try { + scorer.score(collector.getLeafCollector(context), context.reader().getLiveDocs()); + } catch (CollectionTerminatedException e) { + // collection was terminated prematurely + // continue with the following leaf + } + } + } + } + + // TODO: this should be refactored and moved to lucene + // see https://issues.apache.org/jira/browse/LUCENE-6880 + private final static class TopDocumentsCollector extends TopSuggestDocsCollector { + + /** + * Holds a list of suggest meta data for a doc + */ + private final static class SuggestDoc extends TopSuggestDocs.SuggestScoreDoc { + + private List suggestScoreDocs; + + public SuggestDoc(int doc, CharSequence key, CharSequence context, float score) { + super(doc, key, context, score); + } + + void add(CharSequence key, CharSequence context, float score) { + if (suggestScoreDocs == null) { + suggestScoreDocs = new ArrayList<>(1); + } + suggestScoreDocs.add(new TopSuggestDocs.SuggestScoreDoc(doc, key, context, score)); + } + + public List getKeys() { + if (suggestScoreDocs == null) { + return Collections.singletonList(key); + } else { + List keys = new ArrayList<>(suggestScoreDocs.size() + 1); + keys.add(key); + for (TopSuggestDocs.SuggestScoreDoc scoreDoc : suggestScoreDocs) { + keys.add(scoreDoc.key); + } + return keys; + } + } + + public List getContexts() { + if (suggestScoreDocs == null) { + if (context != null) { + return Collections.singletonList(context); + } else { + return Collections.emptyList(); + } + } else { + List contexts = new ArrayList<>(suggestScoreDocs.size() + 1); + contexts.add(context); + for (TopSuggestDocs.SuggestScoreDoc scoreDoc : suggestScoreDocs) { + contexts.add(scoreDoc.context); + } + return contexts; + } + } + } + + private final static class SuggestDocPriorityQueue extends PriorityQueue { + + public SuggestDocPriorityQueue(int maxSize) { + super(maxSize); + } + + @Override + protected boolean lessThan(SuggestDoc a, SuggestDoc b) { + if (a.score == b.score) { + int cmp = Lookup.CHARSEQUENCE_COMPARATOR.compare(a.key, b.key); + if (cmp == 0) { + // prefer smaller doc id, in case of a tie + return a.doc > b.doc; + } else { + return cmp > 0; + } + } + return a.score < b.score; + } + + public SuggestDoc[] getResults() { + int size = size(); + SuggestDoc[] res = new SuggestDoc[size]; + for (int i = size - 1; i >= 0; i--) { + res[i] = pop(); + } + return res; + } + } + + private final int num; + private final SuggestDocPriorityQueue pq; + private final Map scoreDocMap; + + public TopDocumentsCollector(int num) { + super(1); // TODO hack, we don't use the underlying pq, so we allocate a size of 1 + this.num = num; + this.scoreDocMap = new LinkedHashMap<>(num); + this.pq = new SuggestDocPriorityQueue(num); + } + @Override - public int compare(Option o1, Option o2) { - return Float.compare(o2.getScore(), o1.getScore()); + public int getCountToCollect() { + // This is only needed because we initialize + // the base class with 1 instead of the actual num + return num; + } + + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + updateResults(); + } + + private void updateResults() { + for (SuggestDoc suggestDoc : scoreDocMap.values()) { + if (pq.insertWithOverflow(suggestDoc) == suggestDoc) { + break; + } + } + scoreDocMap.clear(); + } + + @Override + public void collect(int docID, CharSequence key, CharSequence context, float score) throws IOException { + if (scoreDocMap.containsKey(docID)) { + SuggestDoc suggestDoc = scoreDocMap.get(docID); + suggestDoc.add(key, context, score); + } else if (scoreDocMap.size() <= num) { + scoreDocMap.put(docID, new SuggestDoc(docBase + docID, key, context, score)); + } else { + throw new CollectionTerminatedException(); + } + } + + @Override + public TopSuggestDocs get() throws IOException { + updateResults(); // to empty the last set of collected suggest docs + TopSuggestDocs.SuggestScoreDoc[] suggestScoreDocs = pq.getResults(); + if (suggestScoreDocs.length > 0) { + return new TopSuggestDocs(suggestScoreDocs.length, suggestScoreDocs, suggestScoreDocs[0].score); + } else { + return TopSuggestDocs.EMPTY; + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java index 83515ff74f3..66c21c58162 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java @@ -18,23 +18,37 @@ */ package org.elasticsearch.search.suggest.completion; -import org.elasticsearch.common.bytes.BytesReference; +import org.apache.lucene.search.suggest.Lookup; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.search.suggest.Suggest; import java.io.IOException; -import java.util.Map; +import java.util.*; /** + * Suggestion response for {@link CompletionSuggester} results + * + * Response format for each entry: + * { + * "text" : STRING + * "score" : FLOAT + * "contexts" : CONTEXTS + * } + * + * CONTEXTS : { + * "CONTEXT_NAME" : ARRAY, + * .. + * } * */ -public class CompletionSuggestion extends Suggest.Suggestion { +public final class CompletionSuggestion extends Suggest.Suggestion { - public static final int TYPE = 2; + public static final int TYPE = 4; public CompletionSuggestion() { } @@ -43,6 +57,62 @@ public class CompletionSuggestion extends Suggest.Suggestion { + + private final Comparator comparator; + + OptionPriorityQueue(int maxSize, Comparator comparator) { + super(maxSize); + this.comparator = comparator; + } + + @Override + protected boolean lessThan(Entry.Option a, Entry.Option b) { + int cmp = comparator.compare(a, b); + if (cmp != 0) { + return cmp > 0; + } + return Lookup.CHARSEQUENCE_COMPARATOR.compare(a.getText().string(), b.getText().string()) > 0; + } + + Entry.Option[] get() { + int size = size(); + Entry.Option[] results = new Entry.Option[size]; + for (int i = size - 1; i >= 0; i--) { + results[i] = pop(); + } + return results; + } + } + + @Override + public Suggest.Suggestion reduce(List> toReduce) { + if (toReduce.size() == 1) { + return toReduce.get(0); + } else { + // combine suggestion entries from participating shards on the coordinating node + // the global top size entries are collected from the shard results + // using a priority queue + Comparator optionComparator = sortComparator(); + OptionPriorityQueue priorityQueue = new OptionPriorityQueue(size, sortComparator()); + for (Suggest.Suggestion entries : toReduce) { + assert entries.getEntries().size() == 1 : "CompletionSuggestion must have only one entry"; + for (Entry.Option option : entries.getEntries().get(0)) { + if (option == priorityQueue.insertWithOverflow(option)) { + // if the current option has overflown from pq, + // we can assume all of the successive options + // from this shard result will be overflown as well + break; + } + } + } + Entry options = this.entries.get(0); + options.getOptions().clear(); + Collections.addAll(options.getOptions(), priorityQueue.get()); + return this; + } + } + @Override public int getType() { return TYPE; @@ -53,7 +123,7 @@ public class CompletionSuggestion extends Suggest.Suggestion { + public final static class Entry extends Suggest.Suggestion.Entry { public Entry(Text text, int offset, int length) { super(text, offset, length); @@ -68,41 +138,33 @@ public class CompletionSuggestion extends Suggest.Suggestion> contexts; + private Map> payload; - public Option(Text text, float score, BytesReference payload) { + public Option(Text text, float score, Map> contexts, Map> payload) { super(text, score); this.payload = payload; + this.contexts = contexts; } - protected Option() { super(); } - public void setPayload(BytesReference payload) { - this.payload = payload; + @Override + protected void mergeInto(Suggest.Suggestion.Entry.Option otherOption) { + // Completion suggestions are reduced by + // org.elasticsearch.search.suggest.completion.CompletionSuggestion.reduce() + throw new UnsupportedOperationException(); } - public BytesReference getPayload() { + public Map> getPayload() { return payload; } - public String getPayloadAsString() { - return payload.toUtf8(); - } - - public long getPayloadAsLong() { - return Long.parseLong(payload.toUtf8()); - } - - public double getPayloadAsDouble() { - return Double.parseDouble(payload.toUtf8()); - } - - public Map getPayloadAsMap() { - return XContentHelper.convertToMap(payload, false).v2(); + public Map> getContexts() { + return contexts; } @Override @@ -113,8 +175,27 @@ public class CompletionSuggestion extends Suggest.Suggestion 0) { - builder.rawField("payload", payload); + if (payload.size() > 0) { + builder.startObject("payload"); + for (Map.Entry> entry : payload.entrySet()) { + builder.startArray(entry.getKey()); + for (Object payload : entry.getValue()) { + builder.value(payload); + } + builder.endArray(); + } + builder.endObject(); + } + if (contexts.size() > 0) { + builder.startObject("contexts"); + for (Map.Entry> entry : contexts.entrySet()) { + builder.startArray(entry.getKey()); + for (CharSequence context : entry.getValue()) { + builder.value(context.toString()); + } + builder.endArray(); + } + builder.endObject(); } return builder; } @@ -122,14 +203,78 @@ public class CompletionSuggestion extends Suggest.Suggestion(payloadSize); + for (int i = 0; i < payloadSize; i++) { + String payloadName = in.readString(); + int nValues = in.readVInt(); + List values = new ArrayList<>(nValues); + for (int j = 0; j < nValues; j++) { + values.add(in.readGenericValue()); + } + this.payload.put(payloadName, values); + } + int contextSize = in.readInt(); + this.contexts = new LinkedHashMap<>(contextSize); + for (int i = 0; i < contextSize; i++) { + String contextName = in.readString(); + int nContexts = in.readVInt(); + Set contexts = new HashSet<>(nContexts); + for (int j = 0; j < nContexts; j++) { + contexts.add(in.readString()); + } + this.contexts.put(contextName, contexts); + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeBytesReference(payload); + out.writeInt(payload.size()); + for (Map.Entry> entry : payload.entrySet()) { + out.writeString(entry.getKey()); + List values = entry.getValue(); + out.writeVInt(values.size()); + for (Object value : values) { + out.writeGenericValue(value); + } + } + out.writeInt(contexts.size()); + for (Map.Entry> entry : contexts.entrySet()) { + out.writeString(entry.getKey()); + out.writeVInt(entry.getValue().size()); + for (CharSequence ctx : entry.getValue()) { + out.writeString(ctx.toString()); + } + } } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("text:"); + stringBuilder.append(getText()); + stringBuilder.append(" score:"); + stringBuilder.append(getScore()); + stringBuilder.append(" payload:["); + for (Map.Entry> entry : payload.entrySet()) { + stringBuilder.append(" "); + stringBuilder.append(entry.getKey()); + stringBuilder.append(":"); + stringBuilder.append(entry.getValue()); + } + stringBuilder.append("]"); + stringBuilder.append(" context:["); + for (Map.Entry> entry: contexts.entrySet()) { + stringBuilder.append(" "); + stringBuilder.append(entry.getKey()); + stringBuilder.append(":"); + stringBuilder.append(entry.getValue()); + } + stringBuilder.append("]"); + return stringBuilder.toString(); + } + } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java index 15d04e845e9..fe80f70e260 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java @@ -18,10 +18,21 @@ */ package org.elasticsearch.search.suggest.completion; +import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext; +import org.elasticsearch.search.suggest.completion.context.GeoQueryContext; import java.io.IOException; +import java.util.*; + +import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.*; /** * Defines a suggest command based on a prefix, typically to provide "auto-complete" functionality @@ -30,13 +41,306 @@ import java.io.IOException; * indexing. */ public class CompletionSuggestionBuilder extends SuggestBuilder.SuggestionBuilder { + private FuzzyOptionsBuilder fuzzyOptionsBuilder; + private RegexOptionsBuilder regexOptionsBuilder; + private final Map> queryContexts = new HashMap<>(); + private final Set payloadFields = new HashSet<>(); public CompletionSuggestionBuilder(String name) { super(name, "completion"); } + /** + * Options for fuzzy queries + */ + public static class FuzzyOptionsBuilder implements ToXContent { + private int editDistance = FuzzyCompletionQuery.DEFAULT_MAX_EDITS; + private boolean transpositions = FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS; + private int fuzzyMinLength = FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH; + private int fuzzyPrefixLength = FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX; + private boolean unicodeAware = FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE; + private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES; + + public FuzzyOptionsBuilder() { + } + + /** + * Sets the level of fuzziness used to create suggestions using a {@link Fuzziness} instance. + * The default value is {@link Fuzziness#ONE} which allows for an "edit distance" of one. + */ + public FuzzyOptionsBuilder setFuzziness(int editDistance) { + this.editDistance = editDistance; + return this; + } + + /** + * Sets the level of fuzziness used to create suggestions using a {@link Fuzziness} instance. + * The default value is {@link Fuzziness#ONE} which allows for an "edit distance" of one. + */ + public FuzzyOptionsBuilder setFuzziness(Fuzziness fuzziness) { + this.editDistance = fuzziness.asDistance(); + return this; + } + + /** + * Sets if transpositions (swapping one character for another) counts as one character + * change or two. + * Defaults to true, meaning it uses the fuzzier option of counting transpositions as + * a single change. + */ + public FuzzyOptionsBuilder setTranspositions(boolean transpositions) { + this.transpositions = transpositions; + return this; + } + + /** + * Sets the minimum length of input string before fuzzy suggestions are returned, defaulting + * to 3. + */ + public FuzzyOptionsBuilder setFuzzyMinLength(int fuzzyMinLength) { + this.fuzzyMinLength = fuzzyMinLength; + return this; + } + + /** + * Sets the minimum length of the input, which is not checked for fuzzy alternatives, defaults to 1 + */ + public FuzzyOptionsBuilder setFuzzyPrefixLength(int fuzzyPrefixLength) { + this.fuzzyPrefixLength = fuzzyPrefixLength; + return this; + } + + /** + * Sets the maximum automaton states allowed for the fuzzy expansion + */ + public FuzzyOptionsBuilder setMaxDeterminizedStates(int maxDeterminizedStates) { + this.maxDeterminizedStates = maxDeterminizedStates; + return this; + } + + /** + * Set to true if all measurements (like edit distance, transpositions and lengths) are in unicode + * code points (actual letters) instead of bytes. Default is false. + */ + public FuzzyOptionsBuilder setUnicodeAware(boolean unicodeAware) { + this.unicodeAware = unicodeAware; + return this; + } + + /** + * Returns the maximum number of edits + */ + int getEditDistance() { + return editDistance; + } + + /** + * Returns if transpositions option is set + * + * if transpositions is set, then swapping one character for another counts as one edit instead of two. + */ + boolean isTranspositions() { + return transpositions; + } + + + /** + * Returns the length of input prefix after which edits are applied + */ + int getFuzzyMinLength() { + return fuzzyMinLength; + } + + /** + * Returns the minimum length of the input prefix required to apply any edits + */ + int getFuzzyPrefixLength() { + return fuzzyPrefixLength; + } + + /** + * Returns if all measurements (like edit distance, transpositions and lengths) are in unicode code + * points (actual letters) instead of bytes. + */ + boolean isUnicodeAware() { + return unicodeAware; + } + + /** + * Returns the maximum automaton states allowed for fuzzy expansion + */ + int getMaxDeterminizedStates() { + return maxDeterminizedStates; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("fuzzy"); + builder.field(Fuzziness.FIELD.getPreferredName(), editDistance); + builder.field("transpositions", transpositions); + builder.field("min_length", fuzzyMinLength); + builder.field("prefix_length", fuzzyPrefixLength); + builder.field("unicode_aware", unicodeAware); + builder.field("max_determinized_states", maxDeterminizedStates); + builder.endObject(); + return builder; + } + } + + /** + * Options for regular expression queries + */ + public static class RegexOptionsBuilder implements ToXContent { + private int flagsValue = RegExp.ALL; + private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES; + + public RegexOptionsBuilder() { + } + + /** + * Sets the regular expression syntax flags + * see {@link RegexpFlag} + */ + public RegexOptionsBuilder setFlags(String flags) { + this.flagsValue = RegexpFlag.resolveValue(flags); + return this; + } + + /** + * Sets the maximum automaton states allowed for the regular expression expansion + */ + public RegexOptionsBuilder setMaxDeterminizedStates(int maxDeterminizedStates) { + this.maxDeterminizedStates = maxDeterminizedStates; + return this; + } + + int getFlagsValue() { + return flagsValue; + } + + int getMaxDeterminizedStates() { + return maxDeterminizedStates; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("regex"); + builder.field("flags_value", flagsValue); + builder.field("max_determinized_states", maxDeterminizedStates); + builder.endObject(); + return builder; + } + } + + /** + * Sets the prefix to provide completions for. + * The prefix gets analyzed by the suggest analyzer. + */ + public CompletionSuggestionBuilder prefix(String prefix) { + super.setPrefix(prefix); + return this; + } + + /** + * Same as {@link #prefix(String)} with fuzziness of fuzziness + */ + public CompletionSuggestionBuilder prefix(String prefix, Fuzziness fuzziness) { + super.setPrefix(prefix); + this.fuzzyOptionsBuilder = new FuzzyOptionsBuilder().setFuzziness(fuzziness); + return this; + } + + /** + * Same as {@link #prefix(String)} with full fuzzy options + * see {@link FuzzyOptionsBuilder} + */ + public CompletionSuggestionBuilder prefix(String prefix, FuzzyOptionsBuilder fuzzyOptionsBuilder) { + super.setPrefix(prefix); + this.fuzzyOptionsBuilder = fuzzyOptionsBuilder; + return this; + } + + /** + * Sets a regular expression pattern for prefixes to provide completions for. + */ + public CompletionSuggestionBuilder regex(String regex) { + super.setRegex(regex); + return this; + } + + /** + * Same as {@link #regex(String)} with full regular expression options + * see {@link RegexOptionsBuilder} + */ + public CompletionSuggestionBuilder regex(String regex, RegexOptionsBuilder regexOptionsBuilder) { + this.regex(regex); + this.regexOptionsBuilder = regexOptionsBuilder; + return this; + } + + /** + * Sets the fields to be returned as suggestion payload. + * Note: Only doc values enabled fields are supported + */ + public CompletionSuggestionBuilder payload(String... fields) { + Collections.addAll(this.payloadFields, fields); + return this; + } + + /** + * Sets query contexts for a category context + * @param name of the category context to execute on + * @param queryContexts a list of {@link CategoryQueryContext} + */ + public CompletionSuggestionBuilder categoryContexts(String name, CategoryQueryContext... queryContexts) { + return contexts(name, queryContexts); + } + + /** + * Sets query contexts for a geo context + * @param name of the geo context to execute on + * @param queryContexts a list of {@link GeoQueryContext} + */ + public CompletionSuggestionBuilder geoContexts(String name, GeoQueryContext... queryContexts) { + return contexts(name, queryContexts); + } + + private CompletionSuggestionBuilder contexts(String name, ToXContent... queryContexts) { + List contexts = this.queryContexts.get(name); + if (contexts == null) { + contexts = new ArrayList<>(2); + this.queryContexts.put(name, contexts); + } + Collections.addAll(contexts, queryContexts); + return this; + } + @Override protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + if (payloadFields != null) { + builder.startArray("payload"); + for (String field : payloadFields) { + builder.value(field); + } + builder.endArray(); + } + if (fuzzyOptionsBuilder != null) { + fuzzyOptionsBuilder.toXContent(builder, params); + } + if (regexOptionsBuilder != null) { + regexOptionsBuilder.toXContent(builder, params); + } + if (queryContexts.isEmpty() == false) { + builder.startObject("contexts"); + for (Map.Entry> entry : this.queryContexts.entrySet()) { + builder.startArray(entry.getKey()); + for (ToXContent queryContext : entry.getValue()) { + queryContext.toXContent(builder, params); + } + builder.endArray(); + } + builder.endObject(); + } return builder; } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java index 6c7d3c0ef40..8ffd497eb3a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java @@ -18,14 +18,17 @@ */ package org.elasticsearch.search.suggest.completion; -import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester; +import org.apache.lucene.search.suggest.document.CompletionQuery; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionSearchContext; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; +import org.elasticsearch.search.suggest.completion.context.ContextMapping; +import org.elasticsearch.search.suggest.completion.context.ContextMappings; -import java.util.Collections; -import java.util.List; +import java.util.*; /** * @@ -33,79 +36,89 @@ import java.util.List; public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext { private CompletionFieldMapper.CompletionFieldType fieldType; - private int fuzzyEditDistance = XFuzzySuggester.DEFAULT_MAX_EDITS; - private boolean fuzzyTranspositions = XFuzzySuggester.DEFAULT_TRANSPOSITIONS; - private int fuzzyMinLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH; - private int fuzzyPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX; - private boolean fuzzy = false; - private boolean fuzzyUnicodeAware = XFuzzySuggester.DEFAULT_UNICODE_AWARE; - private List contextQueries = Collections.emptyList(); - - public CompletionSuggestionContext(Suggester suggester) { + private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder; + private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder; + private Map> queryContexts = Collections.emptyMap(); + private final MapperService mapperService; + private final IndexFieldDataService indexFieldDataService; + private Set payloadFields = Collections.emptySet(); + + CompletionSuggestionContext(Suggester suggester, MapperService mapperService, IndexFieldDataService indexFieldDataService) { super(suggester); + this.indexFieldDataService = indexFieldDataService; + this.mapperService = mapperService; } - public CompletionFieldMapper.CompletionFieldType fieldType() { + CompletionFieldMapper.CompletionFieldType getFieldType() { return this.fieldType; } - public void fieldType(CompletionFieldMapper.CompletionFieldType fieldType) { + void setFieldType(CompletionFieldMapper.CompletionFieldType fieldType) { this.fieldType = fieldType; } - public void setFuzzyEditDistance(int fuzzyEditDistance) { - this.fuzzyEditDistance = fuzzyEditDistance; + void setRegexOptionsBuilder(CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder) { + this.regexOptionsBuilder = regexOptionsBuilder; } - public int getFuzzyEditDistance() { - return fuzzyEditDistance; + void setFuzzyOptionsBuilder(CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder) { + this.fuzzyOptionsBuilder = fuzzyOptionsBuilder; } - public void setFuzzyTranspositions(boolean fuzzyTranspositions) { - this.fuzzyTranspositions = fuzzyTranspositions; + void setQueryContexts(Map> queryContexts) { + this.queryContexts = queryContexts; } - public boolean isFuzzyTranspositions() { - return fuzzyTranspositions; + + MapperService getMapperService() { + return mapperService; } - public void setFuzzyMinLength(int fuzzyMinPrefixLength) { - this.fuzzyMinLength = fuzzyMinPrefixLength; + IndexFieldDataService getIndexFieldDataService() { + return indexFieldDataService; } - public int getFuzzyMinLength() { - return fuzzyMinLength; + void setPayloadFields(Set fields) { + this.payloadFields = fields; } - public void setFuzzyPrefixLength(int fuzzyNonPrefixLength) { - this.fuzzyPrefixLength = fuzzyNonPrefixLength; + void setPayloadFields(List fields) { + setPayloadFields(new HashSet(fields)); } - public int getFuzzyPrefixLength() { - return fuzzyPrefixLength; + Set getPayloadFields() { + return payloadFields; } - public void setFuzzy(boolean fuzzy) { - this.fuzzy = fuzzy; - } - - public boolean isFuzzy() { - return fuzzy; - } - - public void setFuzzyUnicodeAware(boolean fuzzyUnicodeAware) { - this.fuzzyUnicodeAware = fuzzyUnicodeAware; - } - - public boolean isFuzzyUnicodeAware() { - return fuzzyUnicodeAware; - } - - public void setContextQuery(List queries) { - this.contextQueries = queries; - } - - public List getContextQueries() { - return this.contextQueries; + CompletionQuery toQuery() { + CompletionFieldMapper.CompletionFieldType fieldType = getFieldType(); + final CompletionQuery query; + if (getPrefix() != null) { + if (fuzzyOptionsBuilder != null) { + query = fieldType.fuzzyQuery(getPrefix().utf8ToString(), + Fuzziness.fromEdits(fuzzyOptionsBuilder.getEditDistance()), + fuzzyOptionsBuilder.getFuzzyPrefixLength(), fuzzyOptionsBuilder.getFuzzyMinLength(), + fuzzyOptionsBuilder.getMaxDeterminizedStates(), fuzzyOptionsBuilder.isTranspositions(), + fuzzyOptionsBuilder.isUnicodeAware()); + } else { + query = fieldType.prefixQuery(getPrefix()); + } + } else if (getRegex() != null) { + if (fuzzyOptionsBuilder != null) { + throw new IllegalArgumentException("can not use 'fuzzy' options with 'regex"); + } + if (regexOptionsBuilder == null) { + regexOptionsBuilder = new CompletionSuggestionBuilder.RegexOptionsBuilder(); + } + query = fieldType.regexpQuery(getRegex(), regexOptionsBuilder.getFlagsValue(), + regexOptionsBuilder.getMaxDeterminizedStates()); + } else { + throw new IllegalArgumentException("'prefix' or 'regex' must be defined"); + } + if (fieldType.hasContextMappings()) { + ContextMappings contextMappings = fieldType.getContextMappings(); + return contextMappings.toContextQuery(query, queryContexts); + } + return query; } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java deleted file mode 100644 index de6bf1365d9..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.suggest.completion; - -import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester; -import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.search.suggest.SuggestBuilder; - -import java.io.IOException; - -/** - * A form of {@link CompletionSuggestionBuilder} that supports fuzzy queries allowing - * matches on typos. - * Various settings control when and how fuzziness is counted. - */ -public class CompletionSuggestionFuzzyBuilder extends SuggestBuilder.SuggestionBuilder { - - public CompletionSuggestionFuzzyBuilder(String name) { - super(name, "completion"); - } - - private Fuzziness fuzziness = Fuzziness.ONE; - private boolean fuzzyTranspositions = XFuzzySuggester.DEFAULT_TRANSPOSITIONS; - private int fuzzyMinLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH; - private int fuzzyPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX; - private boolean unicodeAware = XFuzzySuggester.DEFAULT_UNICODE_AWARE; - - public Fuzziness getFuzziness() { - return fuzziness; - } - - /** - * Sets the level of fuzziness used to create suggestions using a {@link Fuzziness} instance. - * The default value is {@link Fuzziness#ONE} which allows for an "edit distance" of one. - */ - public CompletionSuggestionFuzzyBuilder setFuzziness(Fuzziness fuzziness) { - this.fuzziness = fuzziness; - return this; - } - - public boolean isFuzzyTranspositions() { - return fuzzyTranspositions; - } - - /** - * Sets if transpositions (swapping one character for another) counts as one character - * change or two. - * Defaults to true, meaning it uses the fuzzier option of counting transpositions as - * a single change. - */ - public CompletionSuggestionFuzzyBuilder setFuzzyTranspositions(boolean fuzzyTranspositions) { - this.fuzzyTranspositions = fuzzyTranspositions; - return this; - } - - public int getFuzzyMinLength() { - return fuzzyMinLength; - } - - /** - * Sets the minimum length of input string before fuzzy suggestions are returned, defaulting - * to 3. - */ - public CompletionSuggestionFuzzyBuilder setFuzzyMinLength(int fuzzyMinLength) { - this.fuzzyMinLength = fuzzyMinLength; - return this; - } - - public int getFuzzyPrefixLength() { - return fuzzyPrefixLength; - } - - /** - * Sets the minimum length of the input, which is not checked for fuzzy alternatives, defaults to 1 - */ - public CompletionSuggestionFuzzyBuilder setFuzzyPrefixLength(int fuzzyPrefixLength) { - this.fuzzyPrefixLength = fuzzyPrefixLength; - return this; - } - - public boolean isUnicodeAware() { - return unicodeAware; - } - - /** - * Set to true if all measurements (like edit distance, transpositions and lengths) are in unicode - * code points (actual letters) instead of bytes. Default is false. - */ - public CompletionSuggestionFuzzyBuilder setUnicodeAware(boolean unicodeAware) { - this.unicodeAware = unicodeAware; - return this; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject("fuzzy"); - - if (fuzziness != Fuzziness.ONE) { - fuzziness.toXContent(builder, params); - } - if (fuzzyTranspositions != XFuzzySuggester.DEFAULT_TRANSPOSITIONS) { - builder.field("transpositions", fuzzyTranspositions); - } - if (fuzzyMinLength != XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH) { - builder.field("min_length", fuzzyMinLength); - } - if (fuzzyPrefixLength != XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX) { - builder.field("prefix_length", fuzzyPrefixLength); - } - if (unicodeAware != XFuzzySuggester.DEFAULT_UNICODE_AWARE) { - builder.field("unicode_aware", unicodeAware); - } - - builder.endObject(); - return builder; - } -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java deleted file mode 100644 index 5edf848dda3..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.suggest.completion; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; -import org.apache.lucene.util.*; -import org.apache.lucene.util.fst.Util; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Set; - -/** - * - */ -public final class CompletionTokenStream extends TokenStream { - - private final PayloadAttribute payloadAttr = addAttribute(PayloadAttribute.class); - private final PositionIncrementAttribute posAttr = addAttribute(PositionIncrementAttribute.class); - private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);; - - - private final TokenStream input; - private BytesRef payload; - private Iterator finiteStrings; - private ToFiniteStrings toFiniteStrings; - private int posInc = -1; - private static final int MAX_PATHS = 256; - private CharTermAttribute charTermAttribute; - - public CompletionTokenStream(TokenStream input, BytesRef payload, ToFiniteStrings toFiniteStrings) throws IOException { - // Don't call the super(input) ctor - this is a true delegate and has a new attribute source since we consume - // the input stream entirely in toFiniteStrings(input) - this.input = input; - this.payload = payload; - this.toFiniteStrings = toFiniteStrings; - } - - @Override - public boolean incrementToken() throws IOException { - clearAttributes(); - if (finiteStrings == null) { - Set strings = toFiniteStrings.toFiniteStrings(input); - - if (strings.size() > MAX_PATHS) { - throw new IllegalArgumentException("TokenStream expanded to " + strings.size() + " finite strings. Only <= " + MAX_PATHS - + " finite strings are supported"); - } - posInc = strings.size(); - finiteStrings = strings.iterator(); - } - if (finiteStrings.hasNext()) { - posAttr.setPositionIncrement(posInc); - /* - * this posInc encodes the number of paths that this surface form - * produced. Multi Fields have the same surface form and therefore sum up - */ - posInc = 0; - Util.toBytesRef(finiteStrings.next(), bytesAtt.builder()); // now we have UTF-8 - if (charTermAttribute != null) { - charTermAttribute.setLength(0); - charTermAttribute.append(bytesAtt.toUTF16()); - } - if (payload != null) { - payloadAttr.setPayload(this.payload); - } - return true; - } - - return false; - } - - @Override - public void end() throws IOException { - super.end(); - if (posInc == -1) { - input.end(); - } - } - - @Override - public void close() throws IOException { - input.close(); - } - - public static interface ToFiniteStrings { - public Set toFiniteStrings(TokenStream stream) throws IOException; - } - - @Override - public void reset() throws IOException { - super.reset(); - if (hasAttribute(CharTermAttribute.class)) { - // we only create this if we really need it to safe the UTF-8 to UTF-16 conversion - charTermAttribute = getAttribute(CharTermAttribute.class); - } - finiteStrings = null; - posInc = -1; - } - - public interface ByteTermAttribute extends TermToBytesRefAttribute { - // marker interface - - /** - * Return the builder from which the term is derived. - */ - public BytesRefBuilder builder(); - - public CharSequence toUTF16(); - } - - public static final class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute, TermToBytesRefAttribute { - private final BytesRefBuilder bytes = new BytesRefBuilder(); - private CharsRefBuilder charsRef; - - @Override - public BytesRefBuilder builder() { - return bytes; - } - - @Override - public BytesRef getBytesRef() { - return bytes.get(); - } - - @Override - public void clear() { - bytes.clear(); - } - - @Override - public void copyTo(AttributeImpl target) { - ByteTermAttributeImpl other = (ByteTermAttributeImpl) target; - other.bytes.copyBytes(bytes); - } - - @Override - public CharSequence toUTF16() { - if (charsRef == null) { - charsRef = new CharsRefBuilder(); - } - charsRef.copyUTF8Bytes(getBytesRef()); - return charsRef.get(); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java new file mode 100644 index 00000000000..23c9ca730b9 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java @@ -0,0 +1,209 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ParseContext.Document; + +import java.io.IOException; +import java.util.*; + +/** + * A {@link ContextMapping} that uses a simple string as a criteria + * The suggestions are boosted and/or filtered by their associated + * category (string) value. + * {@link CategoryQueryContext} defines options for constructing + * a unit of query context for this context type + */ +public class CategoryContextMapping extends ContextMapping { + + private static final String FIELD_FIELDNAME = "path"; + + static final String CONTEXT_VALUE = "context"; + static final String CONTEXT_BOOST = "boost"; + static final String CONTEXT_PREFIX = "prefix"; + + private final String fieldName; + + /** + * Create a new {@link CategoryContextMapping} with field + * fieldName + */ + private CategoryContextMapping(String name, String fieldName) { + super(Type.CATEGORY, name); + this.fieldName = fieldName; + } + + /** + * Name of the field to get contexts from at index-time + */ + public String getFieldName() { + return fieldName; + } + + /** + * Loads a named {@link CategoryContextMapping} instance + * from a map. + * see {@link ContextMappings#load(Object, Version)} + * + * Acceptable map param: path + */ + protected static CategoryContextMapping load(String name, Map config) throws ElasticsearchParseException { + CategoryContextMapping.Builder mapping = new CategoryContextMapping.Builder(name); + Object fieldName = config.get(FIELD_FIELDNAME); + if (fieldName != null) { + mapping.field(fieldName.toString()); + config.remove(FIELD_FIELDNAME); + } + return mapping.build(); + } + + @Override + protected XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException { + if (fieldName != null) { + builder.field(FIELD_FIELDNAME, fieldName); + } + return builder; + } + + /** + * Parse a set of {@link CharSequence} contexts at index-time. + * Acceptable formats: + * + *
      + *
    • Array: 
      [<string>, ..]
      • + *
    • String: 
      "string"
      • + *
    + */ + @Override + public Set parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException { + final Set contexts = new HashSet<>(); + Token token = parser.currentToken(); + if (token == Token.VALUE_STRING) { + contexts.add(parser.text()); + } else if (token == Token.START_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { + if (token == Token.VALUE_STRING) { + contexts.add(parser.text()); + } else { + throw new ElasticsearchParseException("context array must have string values"); + } + } + } else { + throw new ElasticsearchParseException("contexts must be a string or a list of strings"); + } + return contexts; + } + + @Override + public Set parseContext(Document document) { + Set values = null; + if (fieldName != null) { + IndexableField[] fields = document.getFields(fieldName); + values = new HashSet<>(fields.length); + for (IndexableField field : fields) { + values.add(field.stringValue()); + } + } + return (values == null) ? Collections.emptySet() : values; + } + + /** + * Parse a list of {@link CategoryQueryContext} + * using parser. A QueryContexts accepts one of the following forms: + * + *
      + *
    • Object: CategoryQueryContext
      • + *
    • String: CategoryQueryContext value with prefix=false and boost=1
      • + *
    • Array: 
      [CategoryQueryContext, ..]
      • + *
    + * + * A CategoryQueryContext has one of the following forms: + *
      + *
    • Object: 
      {"context": <string>, "boost": <int>, "prefix": <boolean>}
      • + *
    • String: 
      "string"
      • + *
    + */ + @Override + public List parseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException { + List queryContexts = new ArrayList<>(); + Token token = parser.nextToken(); + if (token == Token.START_OBJECT || token == Token.VALUE_STRING) { + CategoryQueryContext parse = CategoryQueryContext.parse(parser); + queryContexts.add(new QueryContext(parse.getCategory().toString(), parse.getBoost(), parse.isPrefix())); + } else if (token == Token.START_ARRAY) { + while (parser.nextToken() != Token.END_ARRAY) { + CategoryQueryContext parse = CategoryQueryContext.parse(parser); + queryContexts.add(new QueryContext(parse.getCategory().toString(), parse.getBoost(), parse.isPrefix())); + } + } + return queryContexts; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + CategoryContextMapping mapping = (CategoryContextMapping) o; + return !(fieldName != null ? !fieldName.equals(mapping.fieldName) : mapping.fieldName != null); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), fieldName); + } + + /** + * Builder for {@link CategoryContextMapping} + */ + public static class Builder extends ContextBuilder { + + private String fieldName; + + /** + * Create a builder for + * a named {@link CategoryContextMapping} + * @param name name of the mapping + */ + public Builder(String name) { + super(name); + } + + /** + * Set the name of the field to use + */ + public Builder field(String fieldName) { + this.fieldName = fieldName; + return this; + } + + @Override + public CategoryContextMapping build() { + return new CategoryContextMapping(name, fieldName); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java new file mode 100644 index 00000000000..ee2655ebdda --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java @@ -0,0 +1,140 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.*; + +import java.io.IOException; + +import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_BOOST; +import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_PREFIX; +import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_VALUE; + +/** + * Defines the query context for {@link CategoryContextMapping} + */ +public final class CategoryQueryContext implements ToXContent { + private final CharSequence category; + private final boolean isPrefix; + private final int boost; + + private CategoryQueryContext(CharSequence category, int boost, boolean isPrefix) { + this.category = category; + this.boost = boost; + this.isPrefix = isPrefix; + } + + /** + * Returns the category of the context + */ + public CharSequence getCategory() { + return category; + } + + /** + * Returns if the context should be treated as a prefix + */ + public boolean isPrefix() { + return isPrefix; + } + + /** + * Returns the query-time boost of the context + */ + public int getBoost() { + return boost; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private CharSequence category; + private boolean isPrefix = false; + private int boost = 1; + + public Builder() { + } + + /** + * Sets the category of the context. + * This is a required field + */ + public Builder setCategory(CharSequence context) { + this.category = context; + return this; + } + + /** + * Sets if the context should be treated as a prefix or not. + * Defaults to false + */ + public Builder setPrefix(boolean prefix) { + this.isPrefix = prefix; + return this; + } + + /** + * Sets the query-time boost of the context. + * Defaults to 1. + */ + public Builder setBoost(int boost) { + this.boost = boost; + return this; + } + + public CategoryQueryContext build() { + return new CategoryQueryContext(category, boost, isPrefix); + } + } + + private static ObjectParser CATEGORY_PARSER = new ObjectParser<>("category", null); + static { + CATEGORY_PARSER.declareString(Builder::setCategory, new ParseField("context")); + CATEGORY_PARSER.declareInt(Builder::setBoost, new ParseField("boost")); + CATEGORY_PARSER.declareBoolean(Builder::setPrefix, new ParseField("prefix")); + } + + public static CategoryQueryContext parse(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + Builder builder = builder(); + if (token == XContentParser.Token.START_OBJECT) { + CATEGORY_PARSER.parse(parser, builder); + } else if (token == XContentParser.Token.VALUE_STRING) { + builder.setCategory(parser.text()); + } else { + throw new ElasticsearchParseException("category context must be an object or string"); + } + return builder.build(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONTEXT_VALUE, category); + builder.field(CONTEXT_BOOST, boost); + builder.field(CONTEXT_PREFIX, isPrefix); + builder.endObject(); + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextBuilder.java new file mode 100644 index 00000000000..9e31d8370cb --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextBuilder.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +/** + * Builder for {@link ContextMapping} + */ +public abstract class ContextBuilder { + + protected String name; + + /** + * @param name of the context mapper to build + */ + protected ContextBuilder(String name) { + this.name = name; + } + + public abstract E build(); + + /** + * Create a new {@link GeoContextMapping} + */ + public static GeoContextMapping.Builder geo(String name) { + return new GeoContextMapping.Builder(name); + } + + /** + * Create a new {@link CategoryContextMapping} + */ + public static CategoryContextMapping.Builder category(String name) { + return new CategoryContextMapping.Builder(name); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java new file mode 100644 index 00000000000..b15577d6fb2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java @@ -0,0 +1,157 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.CompletionFieldMapper; + +import java.io.IOException; +import java.util.*; + +/** + * A {@link ContextMapping} defines criteria that can be used to + * filter and/or boost suggestions at query time for {@link CompletionFieldMapper}. + * + * Implementations have to define how contexts are parsed at query/index time + */ +public abstract class ContextMapping implements ToXContent { + + public static final String FIELD_TYPE = "type"; + public static final String FIELD_NAME = "name"; + protected final Type type; + protected final String name; + + public enum Type { + CATEGORY, GEO; + + public static Type fromString(String type) { + if (type.equalsIgnoreCase("category")) { + return CATEGORY; + } else if (type.equalsIgnoreCase("geo")) { + return GEO; + } else { + throw new IllegalArgumentException("No context type for [" + type + "]"); + } + } + } + + /** + * Define a new context mapping of a specific type + * + * @param type type of context mapping, either {@link Type#CATEGORY} or {@link Type#GEO} + * @param name name of context mapping + */ + protected ContextMapping(Type type, String name) { + this.type = type; + this.name = name; + } + + /** + * @return the type name of the context + */ + public Type type() { + return type; + } + + /** + * @return the name/id of the context + */ + public String name() { + return name; + } + + /** + * Parses a set of index-time contexts. + */ + public abstract Set parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException; + + /** + * Retrieves a set of context from a document at index-time. + */ + protected abstract Set parseContext(ParseContext.Document document); + + /** + * Parses query contexts for this mapper + */ + public abstract List parseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException; + + /** + * Implementations should add specific configurations + * that need to be persisted + */ + protected abstract XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException; + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(FIELD_NAME, name); + builder.field(FIELD_TYPE, type.name()); + toInnerXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContextMapping that = (ContextMapping) o; + if (type != that.type) return false; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(type, name); + } + + @Override + public String toString() { + try { + return toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string(); + } catch (IOException e) { + return super.toString(); + } + } + + public static class QueryContext { + public final String context; + public final int boost; + public final boolean isPrefix; + + public QueryContext(String context, int boost, boolean isPrefix) { + this.context = context; + this.boost = boost; + this.isPrefix = isPrefix; + } + + @Override + public String toString() { + return "QueryContext{" + + "context='" + context + '\'' + + ", boost=" + boost + + ", isPrefix=" + isPrefix + + '}'; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java new file mode 100644 index 00000000000..87b702c2ffb --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java @@ -0,0 +1,273 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.apache.lucene.search.suggest.document.CompletionQuery; +import org.apache.lucene.search.suggest.document.ContextQuery; +import org.apache.lucene.search.suggest.document.ContextSuggestField; +import org.apache.lucene.util.CharsRefBuilder; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.CompletionFieldMapper; + +import java.io.IOException; +import java.util.*; + +import static org.elasticsearch.search.suggest.completion.context.ContextMapping.*; + +/** + * ContextMappings indexes context-enabled suggestion fields + * and creates context queries for defined {@link ContextMapping}s + * for a {@link CompletionFieldMapper} + */ +public class ContextMappings implements ToXContent { + private final List contextMappings; + private final Map contextNameMap; + + public ContextMappings(List contextMappings) { + if (contextMappings.size() > 255) { + // we can support more, but max of 255 (1 byte) unique context types per suggest field + // seems reasonable? + throw new UnsupportedOperationException("Maximum of 10 context types are supported was: " + contextMappings.size()); + } + this.contextMappings = contextMappings; + contextNameMap = new HashMap<>(contextMappings.size()); + for (ContextMapping mapping : contextMappings) { + contextNameMap.put(mapping.name(), mapping); + } + } + + /** + * @return number of context mappings + * held by this instance + */ + public int size() { + return contextMappings.size(); + } + + /** + * Returns a context mapping by its name + */ + public ContextMapping get(String name) { + ContextMapping contextMapping = contextNameMap.get(name); + if (contextMapping == null) { + throw new IllegalArgumentException("Unknown context name[" + name + "], must be one of " + contextNameMap.size()); + } + return contextMapping; + } + + /** + * Adds a context-enabled field for all the defined mappings to document + * see {@link org.elasticsearch.search.suggest.completion.context.ContextMappings.TypedContextField} + */ + public void addField(ParseContext.Document document, String name, String input, int weight, Map> contexts) { + document.add(new TypedContextField(name, input, weight, contexts, document)); + } + + /** + * Field prepends context values with a suggestion + * Context values are associated with a type, denoted by + * a type id, which is prepended to the context value. + * + * Every defined context mapping yields a unique type id (index of the + * corresponding context mapping in the context mappings list) + * for all its context values + * + * The type, context and suggestion values are encoded as follows: + *

    + * TYPE_ID | CONTEXT_VALUE | CONTEXT_SEP | SUGGESTION_VALUE + *

    + * + * Field can also use values of other indexed fields as contexts + * at index time + */ + private class TypedContextField extends ContextSuggestField { + private final Map> contexts; + private final ParseContext.Document document; + + public TypedContextField(String name, String value, int weight, Map> contexts, + ParseContext.Document document) { + super(name, value, weight); + this.contexts = contexts; + this.document = document; + } + + @Override + protected Iterable contexts() { + Set typedContexts = new HashSet<>(); + final CharsRefBuilder scratch = new CharsRefBuilder(); + scratch.grow(1); + for (int typeId = 0; typeId < contextMappings.size(); typeId++) { + scratch.setCharAt(0, (char) typeId); + scratch.setLength(1); + ContextMapping mapping = contextMappings.get(typeId); + Set contexts = new HashSet<>(mapping.parseContext(document)); + if (this.contexts.get(mapping.name()) != null) { + contexts.addAll(this.contexts.get(mapping.name())); + } + for (CharSequence context : contexts) { + scratch.append(context); + typedContexts.add(scratch.toCharsRef()); + scratch.setLength(1); + } + } + return typedContexts; + } + } + + /** + * Wraps a {@link CompletionQuery} with context queries + * + * @param query base completion query to wrap + * @param queryContexts a map of context mapping name and collected query contexts + * @return a context-enabled query + */ + public ContextQuery toContextQuery(CompletionQuery query, Map> queryContexts) { + ContextQuery typedContextQuery = new ContextQuery(query); + if (queryContexts.isEmpty() == false) { + CharsRefBuilder scratch = new CharsRefBuilder(); + scratch.grow(1); + for (int typeId = 0; typeId < contextMappings.size(); typeId++) { + scratch.setCharAt(0, (char) typeId); + scratch.setLength(1); + ContextMapping mapping = contextMappings.get(typeId); + List queryContext = queryContexts.get(mapping.name()); + if (queryContext != null) { + for (QueryContext context : queryContext) { + scratch.append(context.context); + typedContextQuery.addContext(scratch.toCharsRef(), context.boost, !context.isPrefix); + scratch.setLength(1); + } + } + } + } + return typedContextQuery; + } + + /** + * Maps an output context list to a map of context mapping names and their values + * + * see {@link org.elasticsearch.search.suggest.completion.context.ContextMappings.TypedContextField} + * @return a map of context names and their values + * + */ + public Map> getNamedContexts(List contexts) { + Map> contextMap = new HashMap<>(contexts.size()); + for (CharSequence typedContext : contexts) { + int typeId = typedContext.charAt(0); + assert typeId < contextMappings.size() : "Returned context has invalid type"; + ContextMapping mapping = contextMappings.get(typeId); + Set contextEntries = contextMap.get(mapping.name()); + if (contextEntries == null) { + contextEntries = new HashSet<>(); + contextMap.put(mapping.name(), contextEntries); + } + contextEntries.add(typedContext.subSequence(1, typedContext.length())); + } + return contextMap; + } + + /** + * Loads {@link ContextMappings} from configuration + * + * Expected configuration: + * List of maps representing {@link ContextMapping} + * [{"name": .., "type": .., ..}, {..}] + * + */ + public static ContextMappings load(Object configuration, Version indexVersionCreated) throws ElasticsearchParseException { + final List contextMappings; + if (configuration instanceof List) { + contextMappings = new ArrayList<>(); + List configurations = (List)configuration; + for (Object contextConfig : configurations) { + contextMappings.add(load((Map) contextConfig, indexVersionCreated)); + } + if (contextMappings.size() == 0) { + throw new ElasticsearchParseException("expected at least one context mapping"); + } + } else if (configuration instanceof Map) { + contextMappings = Collections.singletonList(load(((Map) configuration), indexVersionCreated)); + } else { + throw new ElasticsearchParseException("expected a list or an entry of context mapping"); + } + return new ContextMappings(contextMappings); + } + + private static ContextMapping load(Map contextConfig, Version indexVersionCreated) { + String name = extractRequiredValue(contextConfig, FIELD_NAME); + String type = extractRequiredValue(contextConfig, FIELD_TYPE); + final ContextMapping contextMapping; + switch (Type.fromString(type)) { + case CATEGORY: + contextMapping = CategoryContextMapping.load(name, contextConfig); + break; + case GEO: + contextMapping = GeoContextMapping.load(name, contextConfig); + break; + default: + throw new ElasticsearchParseException("unknown context type[" + type + "]"); + } + DocumentMapperParser.checkNoRemainingFields(name, contextConfig, indexVersionCreated); + return contextMapping; + } + + private static String extractRequiredValue(Map contextConfig, String paramName) { + final Object paramValue = contextConfig.get(paramName); + if (paramValue == null) { + throw new ElasticsearchParseException("missing [" + paramName + "] in context mapping"); + } + contextConfig.remove(paramName); + return paramValue.toString(); + } + + /** + * Writes a list of objects specified by the defined {@link ContextMapping}s + * + * see {@link ContextMapping#toXContent(XContentBuilder, Params)} + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (ContextMapping contextMapping : contextMappings) { + builder.startObject(); + contextMapping.toXContent(builder, params); + builder.endObject(); + } + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(contextMappings); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || (obj instanceof ContextMappings) == false) { + return false; + } + ContextMappings other = ((ContextMappings) obj); + return contextMappings.equals(other.contextMappings); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java new file mode 100644 index 00000000000..57283c1bd05 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java @@ -0,0 +1,371 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; + +import java.io.IOException; +import java.util.*; + +/** + * A {@link ContextMapping} that uses a geo location/area as a + * criteria. + * The suggestions can be boosted and/or filtered depending on + * whether it falls within an area, represented by a query geo hash + * with a specified precision + * + * {@link GeoQueryContext} defines the options for constructing + * a unit of query context for this context type + */ +public class GeoContextMapping extends ContextMapping { + + public static final String FIELD_PRECISION = "precision"; + public static final String FIELD_FIELDNAME = "path"; + + public static final int DEFAULT_PRECISION = 6; + + static final String CONTEXT_VALUE = "context"; + static final String CONTEXT_BOOST = "boost"; + static final String CONTEXT_PRECISION = "precision"; + static final String CONTEXT_NEIGHBOURS = "neighbours"; + + private final int precision; + private final String fieldName; + + private GeoContextMapping(String name, String fieldName, int precision) { + super(Type.GEO, name); + this.precision = precision; + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + public int getPrecision() { + return precision; + } + + protected static GeoContextMapping load(String name, Map config) { + final GeoContextMapping.Builder builder = new GeoContextMapping.Builder(name); + + if (config != null) { + final Object configPrecision = config.get(FIELD_PRECISION); + if (configPrecision != null) { + if (configPrecision instanceof Integer) { + builder.precision((Integer) configPrecision); + } else if (configPrecision instanceof Long) { + builder.precision((Long) configPrecision); + } else if (configPrecision instanceof Double) { + builder.precision((Double) configPrecision); + } else if (configPrecision instanceof Float) { + builder.precision((Float) configPrecision); + } else { + builder.precision(configPrecision.toString()); + } + config.remove(FIELD_PRECISION); + } + + final Object fieldName = config.get(FIELD_FIELDNAME); + if (fieldName != null) { + builder.field(fieldName.toString()); + config.remove(FIELD_FIELDNAME); + } + } + return builder.build(); + } + + @Override + protected XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(FIELD_PRECISION, precision); + if (fieldName != null) { + builder.field(FIELD_FIELDNAME, fieldName); + } + return builder; + } + + /** + * Parse a set of {@link CharSequence} contexts at index-time. + * Acceptable formats: + * + *
      + *
    • Array: 
      [<GEO POINT>, ..]
      • + *
    • String/Object/Array: 
      "GEO POINT"
      • + *
    + * + * see {@link GeoUtils#parseGeoPoint(String, GeoPoint)} for GEO POINT + */ + @Override + public Set parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException { + if (fieldName != null) { + FieldMapper mapper = parseContext.docMapper().mappers().getMapper(fieldName); + if (!(mapper instanceof GeoPointFieldMapper)) { + throw new ElasticsearchParseException("referenced field must be mapped to geo_point"); + } + } + final Set contexts = new HashSet<>(); + Token token = parser.currentToken(); + if (token == Token.START_ARRAY) { + token = parser.nextToken(); + // Test if value is a single point in [lon, lat] format + if (token == Token.VALUE_NUMBER) { + double lon = parser.doubleValue(); + if (parser.nextToken() == Token.VALUE_NUMBER) { + double lat = parser.doubleValue(); + if (parser.nextToken() == Token.END_ARRAY) { + contexts.add(GeoHashUtils.stringEncode(lon, lat, precision)); + } else { + throw new ElasticsearchParseException("only two values [lon, lat] expected"); + } + } else { + throw new ElasticsearchParseException("latitude must be a numeric value"); + } + } else { + while (token != Token.END_ARRAY) { + GeoPoint point = GeoUtils.parseGeoPoint(parser); + contexts.add(GeoHashUtils.stringEncode(point.getLon(), point.getLat(), precision)); + token = parser.nextToken(); + } + } + } else if (token == Token.VALUE_STRING) { + final String geoHash = parser.text(); + final CharSequence truncatedGeoHash = geoHash.subSequence(0, Math.min(geoHash.length(), precision)); + contexts.add(truncatedGeoHash); + } else { + // or a single location + GeoPoint point = GeoUtils.parseGeoPoint(parser); + contexts.add(GeoHashUtils.stringEncode(point.getLon(), point.getLat(), precision)); + } + return contexts; + } + + @Override + public Set parseContext(Document document) { + final Set geohashes = new HashSet<>(); + + if (fieldName != null) { + IndexableField[] fields = document.getFields(fieldName); + GeoPoint spare = new GeoPoint(); + if (fields.length == 0) { + IndexableField[] lonFields = document.getFields(fieldName + ".lon"); + IndexableField[] latFields = document.getFields(fieldName + ".lat"); + if (lonFields.length > 0 && latFields.length > 0) { + for (int i = 0; i < lonFields.length; i++) { + IndexableField lonField = lonFields[i]; + IndexableField latField = latFields[i]; + assert lonField.fieldType().docValuesType() == latField.fieldType().docValuesType(); + // we write doc values fields differently: one field for all values, so we need to only care about indexed fields + if (lonField.fieldType().docValuesType() == DocValuesType.NONE) { + spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue()); + geohashes.add(GeoHashUtils.stringEncode(spare.getLon(), spare.getLat(), precision)); + } + } + } + } else { + for (IndexableField field : fields) { + if (field instanceof StringField) { + spare.resetFromString(field.stringValue()); + } else { + spare.resetFromIndexHash(Long.parseLong(field.stringValue())); + } + geohashes.add(spare.geohash()); + } + } + } + + Set locations = new HashSet<>(); + for (CharSequence geohash : geohashes) { + int precision = Math.min(this.precision, geohash.length()); + CharSequence truncatedGeohash = geohash.subSequence(0, precision); + locations.add(truncatedGeohash); + } + return locations; + } + + /** + * Parse a list of {@link GeoQueryContext} + * using parser. A QueryContexts accepts one of the following forms: + * + *
      + *
    • Object: GeoQueryContext
      • + *
    • String: GeoQueryContext value with boost=1 precision=PRECISION neighbours=[PRECISION]
      • + *
    • Array: 
      [GeoQueryContext, ..]
      • + *
    + * + * A GeoQueryContext has one of the following forms: + *
      + *
    • Object: + *
        + * 
      • GEO POINT
        • + * 
      • {"lat": <double>, "lon": <double>, "precision": <int>, "neighbours": <[int, ..]>}
        • + * 
      • {"context": <string>, "boost": <int>, "precision": <int>, "neighbours": <[int, ..]>}
        • + * 
      • {"context": <GEO POINT>, "boost": <int>, "precision": <int>, "neighbours": <[int, ..]>}
        • + *
      + *
    • String: 
      GEO POINT
      • + *
    + * see {@link GeoUtils#parseGeoPoint(String, GeoPoint)} for GEO POINT + */ + @Override + public List parseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException { + List queryContexts = new ArrayList<>(); + Token token = parser.nextToken(); + if (token == Token.START_OBJECT || token == Token.VALUE_STRING) { + queryContexts.add(GeoQueryContext.parse(parser)); + } else if (token == Token.START_ARRAY) { + while (parser.nextToken() != Token.END_ARRAY) { + queryContexts.add(GeoQueryContext.parse(parser)); + } + } + List queryContextList = new ArrayList<>(); + for (GeoQueryContext queryContext : queryContexts) { + int minPrecision = this.precision; + if (queryContext.getPrecision() != -1) { + minPrecision = Math.min(minPrecision, queryContext.getPrecision()); + } + GeoPoint point = queryContext.getGeoPoint(); + final Collection locations = new HashSet<>(); + String geoHash = GeoHashUtils.stringEncode(point.getLon(), point.getLat(), minPrecision); + locations.add(geoHash); + if (queryContext.getNeighbours().isEmpty() && geoHash.length() == this.precision) { + GeoHashUtils.addNeighbors(geoHash, locations); + } else if (queryContext.getNeighbours().isEmpty() == false) { + for (Integer neighbourPrecision : queryContext.getNeighbours()) { + if (neighbourPrecision < geoHash.length()) { + String truncatedGeoHash = geoHash.substring(0, neighbourPrecision); + locations.add(truncatedGeoHash); + GeoHashUtils.addNeighbors(truncatedGeoHash, locations); + } + } + } + for (String location : locations) { + queryContextList.add(new QueryContext(location, queryContext.getBoost(), location.length() < this.precision)); + } + } + return queryContextList; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + GeoContextMapping that = (GeoContextMapping) o; + if (precision != that.precision) return false; + return !(fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null); + + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), precision, fieldName); + } + + public static class Builder extends ContextBuilder { + + private int precision = DEFAULT_PRECISION; + private String fieldName = null; + + protected Builder(String name) { + super(name); + } + + /** + * Set the precision use o make suggestions + * + * @param precision + * precision as distance with {@link DistanceUnit}. Default: + * meters + * @return this + */ + public Builder precision(String precision) { + return precision(DistanceUnit.parse(precision, DistanceUnit.METERS, DistanceUnit.METERS)); + } + + /** + * Set the precision use o make suggestions + * + * @param precision + * precision value + * @param unit + * {@link DistanceUnit} to use + * @return this + */ + public Builder precision(double precision, DistanceUnit unit) { + return precision(unit.toMeters(precision)); + } + + /** + * Set the precision use o make suggestions + * + * @param meters + * precision as distance in meters + * @return this + */ + public Builder precision(double meters) { + int level = GeoUtils.geoHashLevelsForPrecision(meters); + // Ceiling precision: we might return more results + if (GeoUtils.geoHashCellSize(level) < meters) { + level = Math.max(1, level - 1); + } + return precision(level); + } + + /** + * Set the precision use o make suggestions + * + * @param level + * maximum length of geohashes + * @return this + */ + public Builder precision(int level) { + this.precision = level; + return this; + } + + /** + * Set the name of the field containing a geolocation to use + * @param fieldName name of the field + * @return this + */ + public Builder field(String fieldName) { + this.fieldName = fieldName; + return this; + } + + @Override + public GeoContextMapping build() { + return new GeoContextMapping(name, fieldName, precision); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java new file mode 100644 index 00000000000..75cab1e8e89 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java @@ -0,0 +1,190 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.search.suggest.completion.context.GeoContextMapping.*; + +/** + * Defines the query context for {@link GeoContextMapping} + */ +public final class GeoQueryContext implements ToXContent { + private final GeoPoint geoPoint; + private final int boost; + private final int precision; + private final List neighbours; + + private GeoQueryContext(GeoPoint geoPoint, int boost, int precision, List neighbours) { + this.geoPoint = geoPoint; + this.boost = boost; + this.precision = precision; + this.neighbours = neighbours; + } + + /** + * Returns the geo point of the context + */ + public GeoPoint getGeoPoint() { + return geoPoint; + } + + /** + * Returns the query-time boost of the context + */ + public int getBoost() { + return boost; + } + + /** + * Returns the precision (length) for the geohash + */ + public int getPrecision() { + return precision; + } + + /** + * Returns the precision levels at which geohash cells neighbours are considered + */ + public List getNeighbours() { + return neighbours; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private GeoPoint geoPoint; + private int boost = 1; + private int precision = -1; + private List neighbours = Collections.emptyList(); + + public Builder() { + } + + /** + * Sets the query-time boost for the context + * Defaults to 1 + */ + public Builder setBoost(int boost) { + this.boost = boost; + return this; + } + + /** + * Sets the precision level for computing the geohash from the context geo point. + * Defaults to using index-time precision level + */ + public Builder setPrecision(int precision) { + this.precision = precision; + return this; + } + + /** + * Sets the precision levels at which geohash cells neighbours are considered. + * Defaults to only considering neighbours at the index-time precision level + */ + public Builder setNeighbours(List neighbours) { + this.neighbours = neighbours; + return this; + } + + /** + * Sets the geo point of the context. + * This is a required field + */ + public Builder setGeoPoint(GeoPoint geoPoint) { + this.geoPoint = geoPoint; + return this; + } + + private double lat = Double.NaN; + void setLat(double lat) { + this.lat = lat; + } + + private double lon = Double.NaN; + void setLon(double lon) { + this.lon = lon; + } + + public GeoQueryContext build() { + if (geoPoint == null) { + if (Double.isNaN(lat) == false && Double.isNaN(lon) == false) { + geoPoint = new GeoPoint(lat, lon); + } else { + throw new IllegalArgumentException("no geohash or geo point provided"); + } + } + return new GeoQueryContext(geoPoint, boost, precision, neighbours); + } + } + + private static ObjectParser GEO_CONTEXT_PARSER = new ObjectParser<>("geo", null); + static { + GEO_CONTEXT_PARSER.declareField((parser, geoQueryContext, geoContextMapping) -> geoQueryContext.setGeoPoint(GeoUtils.parseGeoPoint(parser)), new ParseField("context"), ObjectParser.ValueType.OBJECT); + GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setBoost, new ParseField("boost")); + // TODO : add string support for precision for GeoUtils.geoHashLevelsForPrecision() + GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setPrecision, new ParseField("precision")); + // TODO : add string array support for precision for GeoUtils.geoHashLevelsForPrecision() + GEO_CONTEXT_PARSER.declareIntArray(GeoQueryContext.Builder::setNeighbours, new ParseField("neighbours")); + GEO_CONTEXT_PARSER.declareDouble(GeoQueryContext.Builder::setLat, new ParseField("lat")); + GEO_CONTEXT_PARSER.declareDouble(GeoQueryContext.Builder::setLon, new ParseField("lon")); + } + + public static GeoQueryContext parse(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + GeoQueryContext.Builder builder = new Builder(); + if (token == XContentParser.Token.START_OBJECT) { + GEO_CONTEXT_PARSER.parse(parser, builder); + } else if (token == XContentParser.Token.VALUE_STRING) { + builder.setGeoPoint(GeoPoint.fromGeohash(parser.text())); + } else { + throw new ElasticsearchParseException("geo context must be an object or string"); + } + return builder.build(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject(CONTEXT_VALUE); + builder.field("lat", geoPoint.getLat()); + builder.field("lon", geoPoint.getLon()); + builder.endObject(); + builder.field(CONTEXT_BOOST, boost); + builder.field(CONTEXT_NEIGHBOURS, neighbours); + builder.field(CONTEXT_PRECISION, precision); + builder.endObject(); + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java deleted file mode 100644 index 118d95e22d9..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.suggest.context; - -import org.apache.lucene.analysis.PrefixAnalyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.util.automaton.Automata; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.Operations; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.util.iterable.Iterables; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.ParseContext.Document; - -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -/** - * The {@link CategoryContextMapping} is used to define a {@link ContextMapping} that - * references a field within a document. The value of the field in turn will be - * used to setup the suggestions made by the completion suggester. - */ -public class CategoryContextMapping extends ContextMapping { - - protected static final String TYPE = "category"; - - private static final String FIELD_FIELDNAME = "path"; - private static final String DEFAULT_FIELDNAME = "_type"; - - private static final Iterable EMPTY_VALUES = Collections.emptyList(); - - private final String fieldName; - private final Iterable defaultValues; - private final FieldConfig defaultConfig; - - /** - * Create a new {@link CategoryContextMapping} with the default field - * [_type] - */ - public CategoryContextMapping(String name) { - this(name, DEFAULT_FIELDNAME, EMPTY_VALUES); - } - - /** - * Create a new {@link CategoryContextMapping} with the default field - * [_type] - */ - public CategoryContextMapping(String name, String fieldName) { - this(name, fieldName, EMPTY_VALUES); - } - - /** - * Create a new {@link CategoryContextMapping} with the default field - * [_type] - */ - public CategoryContextMapping(String name, Iterable defaultValues) { - this(name, DEFAULT_FIELDNAME, defaultValues); - } - - /** - * Create a new {@link CategoryContextMapping} with the default field - * [_type] - */ - public CategoryContextMapping(String name, String fieldName, Iterable defaultValues) { - super(TYPE, name); - this.fieldName = fieldName; - this.defaultValues = defaultValues; - this.defaultConfig = new FieldConfig(fieldName, defaultValues, null); - } - - /** - * Name of the field used by this {@link CategoryContextMapping} - */ - public String getFieldName() { - return fieldName; - } - - public Iterable getDefaultValues() { - return defaultValues; - } - - @Override - public FieldConfig defaultConfig() { - return defaultConfig; - } - - /** - * Load the specification of a {@link CategoryContextMapping} - * - * @param name - * name of the field to use. If null default field - * will be used - * @return new {@link CategoryContextMapping} - */ - protected static CategoryContextMapping load(String name, Map config) throws ElasticsearchParseException { - CategoryContextMapping.Builder mapping = new CategoryContextMapping.Builder(name); - - Object fieldName = config.get(FIELD_FIELDNAME); - Object defaultValues = config.get(FIELD_MISSING); - - if (fieldName != null) { - mapping.fieldName(fieldName.toString()); - config.remove(FIELD_FIELDNAME); - } - - if (defaultValues != null) { - if (defaultValues instanceof Iterable) { - for (Object value : (Iterable) defaultValues) { - mapping.addDefaultValue(value.toString()); - } - } else { - mapping.addDefaultValue(defaultValues.toString()); - } - config.remove(FIELD_MISSING); - } - - return mapping.build(); - } - - @Override - protected XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException { - if (fieldName != null) { - builder.field(FIELD_FIELDNAME, fieldName); - } - builder.startArray(FIELD_MISSING); - for (CharSequence value : defaultValues) { - builder.value(value); - } - builder.endArray(); - return builder; - } - - @Override - public ContextConfig parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException { - Token token = parser.currentToken(); - if (token == Token.VALUE_NULL) { - return new FieldConfig(fieldName, defaultValues, null); - } else if (token == Token.VALUE_STRING) { - return new FieldConfig(fieldName, null, Collections.singleton(parser.text())); - } else if (token == Token.VALUE_NUMBER) { - return new FieldConfig(fieldName, null, Collections.singleton(parser.text())); - } else if (token == Token.VALUE_BOOLEAN) { - return new FieldConfig(fieldName, null, Collections.singleton(parser.text())); - } else if (token == Token.START_ARRAY) { - ArrayList values = new ArrayList<>(); - while((token = parser.nextToken()) != Token.END_ARRAY) { - values.add(parser.text()); - } - if(values.isEmpty()) { - throw new ElasticsearchParseException("FieldConfig must contain a least one category"); - } - return new FieldConfig(fieldName, null, values); - } else { - throw new ElasticsearchParseException("FieldConfig must be either [null], a string or a list of strings"); - } - } - - @Override - public FieldQuery parseQuery(String name, XContentParser parser) throws IOException, ElasticsearchParseException { - Iterable values; - Token token = parser.currentToken(); - if (token == Token.START_ARRAY) { - ArrayList list = new ArrayList<>(); - while ((token = parser.nextToken()) != Token.END_ARRAY) { - list.add(parser.text()); - } - values = list; - } else if (token == Token.VALUE_NULL) { - values = defaultValues; - } else { - values = Collections.singleton(parser.text()); - } - - return new FieldQuery(name, values); - } - - public static FieldQuery query(String name, CharSequence... fieldvalues) { - return query(name, Arrays.asList(fieldvalues)); - } - - public static FieldQuery query(String name, Iterable fieldvalues) { - return new FieldQuery(name, fieldvalues); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof CategoryContextMapping) { - CategoryContextMapping other = (CategoryContextMapping) obj; - if (this.fieldName.equals(other.fieldName)) { - return Iterables.allElementsAreEqual(this.defaultValues, other.defaultValues); - } - } - return false; - } - - @Override - public int hashCode() { - int hashCode = fieldName.hashCode(); - for (CharSequence seq : defaultValues) { - hashCode = 31 * hashCode + seq.hashCode(); - } - return hashCode; - } - - private static class FieldConfig extends ContextConfig { - - private final String fieldname; - private final Iterable defaultValues; - private final Iterable values; - - public FieldConfig(String fieldname, Iterable defaultValues, Iterable values) { - this.fieldname = fieldname; - this.defaultValues = defaultValues; - this.values = values; - } - - @Override - protected TokenStream wrapTokenStream(Document doc, TokenStream stream) { - if (values != null) { - return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, values); - // if fieldname is default, BUT our default values are set, we take that one - } else if ((doc.getFields(fieldname).length == 0 || fieldname.equals(DEFAULT_FIELDNAME)) && defaultValues.iterator().hasNext()) { - return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, defaultValues); - } else { - IndexableField[] fields = doc.getFields(fieldname); - ArrayList values = new ArrayList<>(fields.length); - for (int i = 0; i < fields.length; i++) { - values.add(fields[i].stringValue()); - } - - return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, values); - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("FieldConfig(" + fieldname + " = ["); - if (this.values != null && this.values.iterator().hasNext()) { - sb.append(delimitValues(this.values)); - } - if (this.defaultValues != null && this.defaultValues.iterator().hasNext()) { - sb.append(" default").append(delimitValues(this.defaultValues)); - } - return sb.append("])").toString(); - } - - private String delimitValues(Iterable values) { - return StreamSupport.stream(values.spliterator(), false).collect(Collectors.joining(", ", "(", ")")); - } - - } - - private static class FieldQuery extends ContextQuery { - - private final Iterable values; - - public FieldQuery(String name, Iterable values) { - super(name); - this.values = values; - } - - @Override - public Automaton toAutomaton() { - List automatons = new ArrayList<>(); - for (CharSequence value : values) { - automatons.add(Automata.makeString(value.toString())); - } - return Operations.union(automatons); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(name); - for (CharSequence value : values) { - builder.value(value); - } - builder.endArray(); - return builder; - } - } - - public static class Builder extends ContextBuilder { - - private String fieldname; - private List defaultValues = new ArrayList<>(); - - public Builder(String name) { - this(name, DEFAULT_FIELDNAME); - } - - public Builder(String name, String fieldname) { - super(name); - this.fieldname = fieldname; - } - - /** - * Set the name of the field to use - */ - public Builder fieldName(String fieldname) { - this.fieldname = fieldname; - return this; - } - - /** - * Add value to the default values of the mapping - */ - public Builder addDefaultValue(CharSequence defaultValue) { - this.defaultValues.add(defaultValue); - return this; - } - - /** - * Add set of default values to the mapping - */ - public Builder addDefaultValues(CharSequence... defaultValues) { - for (CharSequence defaultValue : defaultValues) { - this.defaultValues.add(defaultValue); - } - return this; - } - - /** - * Add set of default values to the mapping - */ - public Builder addDefaultValues(Iterable defaultValues) { - for (CharSequence defaultValue : defaultValues) { - this.defaultValues.add(defaultValue); - } - return this; - } - - @Override - public CategoryContextMapping build() { - return new CategoryContextMapping(name, fieldname, defaultValues); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/context/ContextBuilder.java deleted file mode 100644 index 8b554d957d4..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextBuilder.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.suggest.context; - -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.Version; -import org.elasticsearch.index.mapper.DocumentMapperParser; - -import java.util.Map; -import java.util.Map.Entry; -import java.util.SortedMap; -import java.util.TreeMap; - -public abstract class ContextBuilder { - - protected String name; - - public ContextBuilder(String name) { - this.name = name; - } - - public abstract E build(); - - /** - * Create a new {@link GeolocationContextMapping} - */ - public static GeolocationContextMapping.Builder location(String name) { - return new GeolocationContextMapping.Builder(name); - } - - /** - * Create a new {@link GeolocationContextMapping} with given precision and - * neighborhood usage - * - * @param precision geohash length - * @param neighbors use neighbor cells - */ - public static GeolocationContextMapping.Builder location(String name, int precision, boolean neighbors) { - return new GeolocationContextMapping.Builder(name, neighbors, precision); - } - - /** - * Create a new {@link CategoryContextMapping.Builder} - */ - public static CategoryContextMapping.Builder category(String name) { - return new CategoryContextMapping.Builder(name, null); - } - - /** - * Create a new {@link CategoryContextMapping.Builder} with default category - * - * @param defaultCategory category to use, if it is not provided - */ - public static CategoryContextMapping.Builder category(String name, String defaultCategory) { - return new CategoryContextMapping.Builder(name, null).addDefaultValue(defaultCategory); - } - - /** - * Create a new {@link CategoryContextMapping} - * - * @param fieldname - * name of the field to use - */ - public static CategoryContextMapping.Builder reference(String name, String fieldname) { - return new CategoryContextMapping.Builder(name, fieldname); - } - - /** - * Create a new {@link CategoryContextMapping} - * - * @param fieldname name of the field to use - * @param defaultValues values to use, if the document not provides - * a field with the given name - */ - public static CategoryContextMapping.Builder reference(String name, String fieldname, Iterable defaultValues) { - return new CategoryContextMapping.Builder(name, fieldname).addDefaultValues(defaultValues); - } - - public static SortedMap loadMappings(Object configuration, Version indexVersionCreated) - throws ElasticsearchParseException { - if (configuration instanceof Map) { - Map configurations = (Map)configuration; - SortedMap mappings = new TreeMap<>(); - for (Entry config : configurations.entrySet()) { - String name = config.getKey(); - mappings.put(name, loadMapping(name, (Map) config.getValue(), indexVersionCreated)); - } - return mappings; - } else if (configuration == null) { - return ContextMapping.EMPTY_MAPPING; - } else { - throw new ElasticsearchParseException("no valid context configuration"); - } - } - - protected static ContextMapping loadMapping(String name, Map config, Version indexVersionCreated) - throws ElasticsearchParseException { - final Object argType = config.get(ContextMapping.FIELD_TYPE); - - if (argType == null) { - throw new ElasticsearchParseException("missing [{}] in context mapping", ContextMapping.FIELD_TYPE); - } - - final String type = argType.toString(); - ContextMapping contextMapping; - if (GeolocationContextMapping.TYPE.equals(type)) { - contextMapping = GeolocationContextMapping.load(name, config); - } else if (CategoryContextMapping.TYPE.equals(type)) { - contextMapping = CategoryContextMapping.load(name, config); - } else { - throw new ElasticsearchParseException("unknown context type [{}]", type); - } - config.remove(ContextMapping.FIELD_TYPE); - DocumentMapperParser.checkNoRemainingFields(name, config, indexVersionCreated); - - return contextMapping; - } -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java deleted file mode 100644 index bbdb614c943..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.suggest.context; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; -import org.apache.lucene.util.automaton.Automata; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.Operations; -import org.apache.lucene.util.fst.FST; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.ParseContext.Document; - -import java.io.IOException; -import java.util.*; - -/** - * A {@link ContextMapping} is used t define a context that may used - * in conjunction with a suggester. To define a suggester that depends on a - * specific context derived class of {@link ContextMapping} will be - * used to specify the kind of additional information required in order to make - * suggestions. - */ -public abstract class ContextMapping implements ToXContent { - - /** Character used to separate several contexts */ - public static final char SEPARATOR = '\u001D'; - - /** Dummy Context Mapping that should be used if no context is used*/ - public static final SortedMap EMPTY_MAPPING = new TreeMap<>(); - - /** Dummy Context Config matching the Dummy Mapping by providing an empty context*/ - public static final SortedMap EMPTY_CONFIG = new TreeMap<>(); - - /** Dummy Context matching the Dummy Mapping by not wrapping a {@link TokenStream} */ - public static final Context EMPTY_CONTEXT = new Context(EMPTY_CONFIG, null); - - public static final String FIELD_VALUE = "value"; - public static final String FIELD_MISSING = "default"; - public static final String FIELD_TYPE = "type"; - - protected final String type; // Type of the Contextmapping - protected final String name; - - /** - * Define a new context mapping of a specific type - * - * @param type - * name of the new context mapping - */ - protected ContextMapping(String type, String name) { - super(); - this.type = type; - this.name = name; - } - - /** - * @return the type name of the context - */ - protected String type() { - return type; - } - - /** - * @return the name/id of the context - */ - public String name() { - return name; - } - - @Override - public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(name); - builder.field(FIELD_TYPE, type); - toInnerXContent(builder, params); - builder.endObject(); - return builder; - } - - /** - * A {@link ContextMapping} combined with the information provided by a document - * form a {@link ContextConfig} which is used to build the underlying FST. - * - * @param parseContext context of parsing phase - * @param parser {@link XContentParser} used to read and setup the configuration - * @return A {@link ContextConfig} related to this mapping - */ - public abstract ContextConfig parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException; - - public abstract ContextConfig defaultConfig(); - - /** - * Parse a query according to the context. Parsing starts at parsers current position - * - * @param name name of the context - * @param parser {@link XContentParser} providing the data of the query - * - * @return {@link ContextQuery} according to this mapping - */ - public abstract ContextQuery parseQuery(String name, XContentParser parser) throws IOException, ElasticsearchParseException; - - /** - * Since every context mapping is assumed to have a name given by the field name of an context object, this - * method is used to build the value used to serialize the mapping - * - * @param builder builder to append the mapping to - * @param params parameters passed to the builder - * - * @return the builder used - */ - protected abstract XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException; - - /** - * Test equality of two mapping - * - * @param thisMappings first mapping - * @param otherMappings second mapping - * - * @return true if both arguments are equal - */ - public static boolean mappingsAreEqual(SortedMap thisMappings, SortedMap otherMappings) { - return thisMappings.entrySet().equals(otherMappings.entrySet()); - } - - @Override - public String toString() { - try { - return toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string(); - } catch (IOException e) { - return super.toString(); - } - } - - /** - * A collection of {@link ContextMapping}s, their {@link ContextConfig}uration and a - * Document form a complete {@link Context}. Since this Object provides all information used - * to setup a suggestion, it can be used to wrap the entire {@link TokenStream} used to build a - * path within the {@link FST}. - */ - public static class Context { - - final SortedMap contexts; - final Document doc; - - public Context(SortedMap contexts, Document doc) { - super(); - this.contexts = contexts; - this.doc = doc; - } - - /** - * Wrap the {@link TokenStream} according to the provided informations of {@link ContextConfig} - * and a related {@link Document}. - * - * @param tokenStream {@link TokenStream} to wrap - * - * @return wrapped token stream - */ - public TokenStream wrapTokenStream(TokenStream tokenStream) { - for (ContextConfig context : contexts.values()) { - tokenStream = context.wrapTokenStream(doc, tokenStream); - } - return tokenStream; - } - } - - /** - * A {@link ContextMapping} combined with the information provided by a document - * form a {@link ContextConfig} which is used to build the underlying {@link FST}. This class hold - * a simple method wrapping a {@link TokenStream} by provided document informations. - */ - public static abstract class ContextConfig { - - /** - * Wrap a {@link TokenStream} for building suggestions to use context informations - * provided by a document or a {@link ContextMapping} - * - * @param doc document related to the stream - * @param stream original stream used to build the underlying {@link FST} - * - * @return A new {@link TokenStream} providing additional context information - */ - protected abstract TokenStream wrapTokenStream(Document doc, TokenStream stream); - - } - - /** - * A {@link ContextQuery} defines the context information for a specific {@link ContextMapping} - * defined within a suggestion request. According to the parameters set in the request and the - * {@link ContextMapping} such a query is used to wrap the {@link TokenStream} of the actual - * suggestion request into a {@link TokenStream} with the context settings - */ - public static abstract class ContextQuery implements ToXContent { - - protected final String name; - - protected ContextQuery(String name) { - this.name = name; - } - - public String name() { - return name; - } - - /** - * Create a automaton for a given context query this automaton will be used - * to find the matching paths with the fst - * - * @param preserveSep set an additional char (XAnalyzingSuggester.SEP_LABEL) between each context query - * @param queries list of {@link ContextQuery} defining the lookup context - * - * @return Automaton matching the given Query - */ - public static Automaton toAutomaton(boolean preserveSep, Iterable queries) { - Automaton a = Automata.makeEmptyString(); - - Automaton gap = Automata.makeChar(ContextMapping.SEPARATOR); - if (preserveSep) { - // if separators are preserved the fst contains a SEP_LABEL - // behind each gap. To have a matching automaton, we need to - // include the SEP_LABEL in the query as well - gap = Operations.concatenate(gap, Automata.makeChar(XAnalyzingSuggester.SEP_LABEL)); - } - - for (ContextQuery query : queries) { - a = Operations.concatenate(Arrays.asList(query.toAutomaton(), gap, a)); - } - - // TODO: should we limit this? Do any of our ContextQuery impls really create exponential regexps? GeoQuery looks safe (union - // of strings). - return Operations.determinize(a, Integer.MAX_VALUE); - } - - /** - * Build a LookUp Automaton for this context. - * @return LookUp Automaton - */ - protected abstract Automaton toAutomaton(); - - /** - * Parse a set of {@link ContextQuery} according to a given mapping - * @param mappings List of mapping defined y the suggest field - * @param parser parser holding the settings of the queries. The parsers - * current token is assumed hold an array. The number of elements - * in this array must match the number of elements in the mappings. - * @return List of context queries - * - * @throws IOException if something unexpected happened on the underlying stream - * @throws ElasticsearchParseException if the list of queries could not be parsed - */ - public static List parseQueries(Map mappings, XContentParser parser) - throws IOException, ElasticsearchParseException { - - Map querySet = new HashMap<>(); - Token token = parser.currentToken(); - if(token == Token.START_OBJECT) { - while ((token = parser.nextToken()) != Token.END_OBJECT) { - String name = parser.text(); - ContextMapping mapping = mappings.get(name); - if (mapping == null) { - throw new ElasticsearchParseException("no mapping defined for [{}]", name); - } - parser.nextToken(); - querySet.put(name, mapping.parseQuery(name, parser)); - } - } - - List queries = new ArrayList<>(mappings.size()); - for (ContextMapping mapping : mappings.values()) { - queries.add(querySet.get(mapping.name)); - } - return queries; - } - - @Override - public String toString() { - try { - return toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string(); - } catch (IOException e) { - return super.toString(); - } - } - } -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java deleted file mode 100644 index f2d168f5532..00000000000 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java +++ /dev/null @@ -1,734 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.suggest.context; - -import com.carrotsearch.hppc.IntHashSet; -import org.apache.lucene.analysis.PrefixAnalyzer.PrefixTokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.util.XGeoHashUtils; -import org.apache.lucene.util.automaton.Automata; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.Operations; -import org.apache.lucene.util.fst.FST; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; - -/** - * The {@link GeolocationContextMapping} allows to take GeoInfomation into account - * during building suggestions. The mapping itself works with geohashes - * explicitly and is configured by three parameters: - *
      - *
    • precision: length of the geohash indexed as prefix of the - * completion field
      • - *
    • neighbors: Should the neighbor cells of the deepest geohash - * level also be indexed as alternatives to the actual geohash
      • - *
    • location: (optional) location assumed if it is not provided
      • - *
    - * Internally this mapping wraps the suggestions into a form - * [geohash][suggestion]. If the neighbor option is set the cells - * next to the cell on the deepest geohash level ( precision) will - * be indexed as well. The {@link TokenStream} used to build the {@link FST} for - * suggestion will be wrapped into a {@link PrefixTokenFilter} managing these - * geohases as prefixes. - */ -public class GeolocationContextMapping extends ContextMapping { - - public static final String TYPE = "geo"; - - public static final String FIELD_PRECISION = "precision"; - public static final String FIELD_NEIGHBORS = "neighbors"; - public static final String FIELD_FIELDNAME = "path"; - - private final Collection defaultLocations; - private final int[] precision; - private final boolean neighbors; - private final String fieldName; - private final GeoConfig defaultConfig; - - /** - * Create a new {@link GeolocationContextMapping} with a given precision - * - * @param precision - * length of the geohashes - * @param neighbors - * should neighbors be indexed - * @param defaultLocations - * location to use, if it is not provided by the document - */ - protected GeolocationContextMapping(String name, int[] precision, boolean neighbors, Collection defaultLocations, String fieldName) { - super(TYPE, name); - this.precision = precision; - this.neighbors = neighbors; - this.defaultLocations = defaultLocations; - this.fieldName = fieldName; - this.defaultConfig = new GeoConfig(this, defaultLocations); - } - - /** - * load a {@link GeolocationContextMapping} by configuration. Such a configuration - * can set the parameters - *
      - *
    • precision [String, Double, - * Float or Integer] defines the length of the - * underlying geohash
      • - *
    • defaultLocation [String] defines the location to use if - * it is not provided by the document
      • - *
    • neighbors [Boolean] defines if the last level of the - * geohash should be extended by neighbor cells
      • - *
    - * - * @param config - * Configuration for {@link GeolocationContextMapping} - * @return new {@link GeolocationContextMapping} configured by the parameters of - * config - */ - protected static GeolocationContextMapping load(String name, Map config) { - if (!config.containsKey(FIELD_PRECISION)) { - throw new ElasticsearchParseException("field [precision] is missing"); - } - - final GeolocationContextMapping.Builder builder = new GeolocationContextMapping.Builder(name); - - if (config != null) { - final Object configPrecision = config.get(FIELD_PRECISION); - if (configPrecision == null) { - // ignore precision - } else if (configPrecision instanceof Integer) { - builder.precision((Integer) configPrecision); - config.remove(FIELD_PRECISION); - } else if (configPrecision instanceof Long) { - builder.precision((Long) configPrecision); - config.remove(FIELD_PRECISION); - } else if (configPrecision instanceof Double) { - builder.precision((Double) configPrecision); - config.remove(FIELD_PRECISION); - } else if (configPrecision instanceof Float) { - builder.precision((Float) configPrecision); - config.remove(FIELD_PRECISION); - } else if (configPrecision instanceof Iterable) { - for (Object precision : (Iterable)configPrecision) { - if (precision instanceof Integer) { - builder.precision((Integer) precision); - } else if (precision instanceof Long) { - builder.precision((Long) precision); - } else if (precision instanceof Double) { - builder.precision((Double) precision); - } else if (precision instanceof Float) { - builder.precision((Float) precision); - } else { - builder.precision(precision.toString()); - } - } - config.remove(FIELD_PRECISION); - } else { - builder.precision(configPrecision.toString()); - config.remove(FIELD_PRECISION); - } - - final Object configNeighbors = config.get(FIELD_NEIGHBORS); - if (configNeighbors != null) { - builder.neighbors((Boolean) configNeighbors); - config.remove(FIELD_NEIGHBORS); - } - - final Object def = config.get(FIELD_MISSING); - if (def != null) { - if (def instanceof Iterable) { - for (Object location : (Iterable)def) { - builder.addDefaultLocation(location.toString()); - } - } else if (def instanceof String) { - builder.addDefaultLocation(def.toString()); - } else if (def instanceof Map) { - Map latlonMap = (Map) def; - if (!latlonMap.containsKey("lat") || !(latlonMap.get("lat") instanceof Double)) { - throw new ElasticsearchParseException("field [{}] map must have field lat and a valid latitude", FIELD_MISSING); - } - if (!latlonMap.containsKey("lon") || !(latlonMap.get("lon") instanceof Double)) { - throw new ElasticsearchParseException("field [{}] map must have field lon and a valid longitude", FIELD_MISSING); - } - builder.addDefaultLocation(Double.valueOf(latlonMap.get("lat").toString()), Double.valueOf(latlonMap.get("lon").toString())); - } else { - throw new ElasticsearchParseException("field [{}] must be of type string or list", FIELD_MISSING); - } - config.remove(FIELD_MISSING); - } - - final Object fieldName = config.get(FIELD_FIELDNAME); - if (fieldName != null) { - builder.field(fieldName.toString()); - config.remove(FIELD_FIELDNAME); - } - } - return builder.build(); - } - - @Override - protected XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(FIELD_PRECISION, precision); - builder.field(FIELD_NEIGHBORS, neighbors); - if (defaultLocations != null) { - builder.startArray(FIELD_MISSING); - for (String defaultLocation : defaultLocations) { - builder.value(defaultLocation); - } - builder.endArray(); - } - if (fieldName != null) { - builder.field(FIELD_FIELDNAME, fieldName); - } - return builder; - } - - protected static Collection parseSinglePointOrList(XContentParser parser) throws IOException { - Token token = parser.currentToken(); - if(token == Token.START_ARRAY) { - token = parser.nextToken(); - // Test if value is a single point in [lon, lat] format - if(token == Token.VALUE_NUMBER) { - double lon = parser.doubleValue(); - if(parser.nextToken() == Token.VALUE_NUMBER) { - double lat = parser.doubleValue(); - if(parser.nextToken() == Token.END_ARRAY) { - return Collections.singleton(XGeoHashUtils.stringEncode(lon, lat)); - } else { - throw new ElasticsearchParseException("only two values expected"); - } - } else { - throw new ElasticsearchParseException("latitue must be a numeric value"); - } - } else { - // otherwise it's a list of locations - ArrayList result = new ArrayList<>(); - while (token != Token.END_ARRAY) { - result.add(GeoUtils.parseGeoPoint(parser).geohash()); - token = parser.nextToken(); //infinite loop without this line - } - return result; - } - } else { - // or a single location - return Collections.singleton(GeoUtils.parseGeoPoint(parser).geohash()); - } - } - - @Override - public ContextConfig defaultConfig() { - return defaultConfig; - } - - @Override - public ContextConfig parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException { - - if(fieldName != null) { - FieldMapper mapper = parseContext.docMapper().mappers().getMapper(fieldName); - if(!(mapper instanceof GeoPointFieldMapper)) { - throw new ElasticsearchParseException("referenced field must be mapped to geo_point"); - } - } - - Collection locations; - if(parser.currentToken() == Token.VALUE_NULL) { - locations = null; - } else { - locations = parseSinglePointOrList(parser); - } - return new GeoConfig(this, locations); - } - - /** - * Create a new geolocation query from a given GeoPoint - * - * @param point - * query location - * @return new geolocation query - */ - public static GeoQuery query(String name, GeoPoint point) { - return query(name, point.getGeohash()); - } - - /** - * Create a new geolocation query from a given geocoordinate - * - * @param lat - * latitude of the location - * @param lon - * longitude of the location - * @return new geolocation query - */ - public static GeoQuery query(String name, double lat, double lon, int ... precisions) { - return query(name, XGeoHashUtils.stringEncode(lon, lat), precisions); - } - - public static GeoQuery query(String name, double lat, double lon, String ... precisions) { - int precisionInts[] = new int[precisions.length]; - for (int i = 0 ; i < precisions.length; i++) { - precisionInts[i] = GeoUtils.geoHashLevelsForPrecision(precisions[i]); - } - return query(name, XGeoHashUtils.stringEncode(lon, lat), precisionInts); - } - - /** - * Create a new geolocation query from a given geohash - * - * @param geohash - * geohash of the location - * @return new geolocation query - */ - public static GeoQuery query(String name, String geohash, int ... precisions) { - return new GeoQuery(name, geohash, precisions); - } - - private static final int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException { - switch (parser.currentToken()) { - case VALUE_STRING: - return GeoUtils.geoHashLevelsForPrecision(parser.text()); - case VALUE_NUMBER: - switch (parser.numberType()) { - case INT: - case LONG: - return parser.intValue(); - default: - return GeoUtils.geoHashLevelsForPrecision(parser.doubleValue()); - } - default: - throw new ElasticsearchParseException("invalid precision value"); - } - } - - @Override - public GeoQuery parseQuery(String name, XContentParser parser) throws IOException, ElasticsearchParseException { - if (parser.currentToken() == Token.START_OBJECT) { - double lat = Double.NaN; - double lon = Double.NaN; - GeoPoint point = null; - int[] precision = null; - - while (parser.nextToken() != Token.END_OBJECT) { - final String fieldName = parser.text(); - if("lat".equals(fieldName)) { - if(point == null) { - parser.nextToken(); - switch (parser.currentToken()) { - case VALUE_NUMBER: - case VALUE_STRING: - lat = parser.doubleValue(true); - break; - default: - throw new ElasticsearchParseException("latitude must be a number"); - } - } else { - throw new ElasticsearchParseException("only lat/lon or [{}] is allowed", FIELD_VALUE); - } - } else if ("lon".equals(fieldName)) { - if(point == null) { - parser.nextToken(); - switch (parser.currentToken()) { - case VALUE_NUMBER: - case VALUE_STRING: - lon = parser.doubleValue(true); - break; - default: - throw new ElasticsearchParseException("longitude must be a number"); - } - } else { - throw new ElasticsearchParseException("only lat/lon or [{}] is allowed", FIELD_VALUE); - } - } else if (FIELD_PRECISION.equals(fieldName)) { - if(parser.nextToken() == Token.START_ARRAY) { - IntHashSet precisions = new IntHashSet(); - while(parser.nextToken() != Token.END_ARRAY) { - precisions.add(parsePrecision(parser)); - } - precision = precisions.toArray(); - } else { - precision = new int[] { parsePrecision(parser) }; - } - } else if (FIELD_VALUE.equals(fieldName)) { - if(Double.isNaN(lon) && Double.isNaN(lat)) { - parser.nextToken(); - point = GeoUtils.parseGeoPoint(parser); - } else { - throw new ElasticsearchParseException("only lat/lon or [{}] is allowed", FIELD_VALUE); - } - } else { - throw new ElasticsearchParseException("unexpected fieldname [{}]", fieldName); - } - } - - if (point == null) { - if (Double.isNaN(lat) || Double.isNaN(lon)) { - throw new ElasticsearchParseException("location is missing"); - } else { - point = new GeoPoint(lat, lon); - } - } - - if (precision == null || precision.length == 0) { - precision = this.precision; - } - - return new GeoQuery(name, point.geohash(), precision); - } else { - return new GeoQuery(name, GeoUtils.parseGeoPoint(parser).getGeohash(), precision); - } - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((defaultLocations == null) ? 0 : defaultLocations.hashCode()); - result = prime * result + ((fieldName == null) ? 0 : fieldName.hashCode()); - result = prime * result + (neighbors ? 1231 : 1237); - result = prime * result + Arrays.hashCode(precision); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - GeolocationContextMapping other = (GeolocationContextMapping) obj; - if (defaultLocations == null) { - if (other.defaultLocations != null) - return false; - } else if (!defaultLocations.equals(other.defaultLocations)) - return false; - if (fieldName == null) { - if (other.fieldName != null) - return false; - } else if (!fieldName.equals(other.fieldName)) - return false; - if (neighbors != other.neighbors) - return false; - if (!Arrays.equals(precision, other.precision)) - return false; - return true; - } - - - - - public static class Builder extends ContextBuilder { - - private IntHashSet precisions = new IntHashSet(); - private boolean neighbors; // take neighbor cell on the lowest level into account - private HashSet defaultLocations = new HashSet<>(); - private String fieldName = null; - - protected Builder(String name) { - this(name, true, null); - } - - protected Builder(String name, boolean neighbors, int...levels) { - super(name); - neighbors(neighbors); - if (levels != null) { - for (int level : levels) { - precision(level); - } - } - } - - /** - * Set the precision use o make suggestions - * - * @param precision - * precision as distance with {@link DistanceUnit}. Default: - * meters - * @return this - */ - public Builder precision(String precision) { - return precision(DistanceUnit.parse(precision, DistanceUnit.METERS, DistanceUnit.METERS)); - } - - /** - * Set the precision use o make suggestions - * - * @param precision - * precision value - * @param unit - * {@link DistanceUnit} to use - * @return this - */ - public Builder precision(double precision, DistanceUnit unit) { - return precision(unit.toMeters(precision)); - } - - /** - * Set the precision use o make suggestions - * - * @param meters - * precision as distance in meters - * @return this - */ - public Builder precision(double meters) { - int level = GeoUtils.geoHashLevelsForPrecision(meters); - // Ceiling precision: we might return more results - if (GeoUtils.geoHashCellSize(level) < meters) { - level = Math.max(1, level - 1); - } - return precision(level); - } - - /** - * Set the precision use o make suggestions - * - * @param level - * maximum length of geohashes - * @return this - */ - public Builder precision(int level) { - this.precisions.add(level); - return this; - } - - /** - * Set neighborhood usage - * - * @param neighbors - * should neighbor cells also be valid - * @return this - */ - public Builder neighbors(boolean neighbors) { - this.neighbors = neighbors; - return this; - } - - /** - * Set a default location that should be used, if no location is - * provided by the query - * - * @param geohash - * geohash of the default location - * @return this - */ - public Builder addDefaultLocation(String geohash) { - this.defaultLocations.add(geohash); - return this; - } - - /** - * Set a default location that should be used, if no location is - * provided by the query - * - * @param geohashes - * geohash of the default location - * @return this - */ - public Builder addDefaultLocations(Collection geohashes) { - this.defaultLocations.addAll(geohashes); - return this; - } - - /** - * Set a default location that should be used, if no location is - * provided by the query - * - * @param lat - * latitude of the default location - * @param lon - * longitude of the default location - * @return this - */ - public Builder addDefaultLocation(double lat, double lon) { - this.defaultLocations.add(XGeoHashUtils.stringEncode(lon, lat)); - return this; - } - - /** - * Set a default location that should be used, if no location is - * provided by the query - * - * @param point - * location - * @return this - */ - public Builder defaultLocation(GeoPoint point) { - this.defaultLocations.add(point.geohash()); - return this; - } - - /** - * Set the name of the field containing a geolocation to use - * @param fieldName name of the field - * @return this - */ - public Builder field(String fieldName) { - this.fieldName = fieldName; - return this; - } - - @Override - public GeolocationContextMapping build() { - if(precisions.isEmpty()) { - precisions.add(XGeoHashUtils.PRECISION); - } - int[] precisionArray = precisions.toArray(); - Arrays.sort(precisionArray); - return new GeolocationContextMapping(name, precisionArray, neighbors, defaultLocations, fieldName); - } - - } - - private static class GeoConfig extends ContextConfig { - - private final GeolocationContextMapping mapping; - private final Collection locations; - - public GeoConfig(GeolocationContextMapping mapping, Collection locations) { - this.locations = locations; - this.mapping = mapping; - } - - @Override - protected TokenStream wrapTokenStream(Document doc, TokenStream stream) { - Collection geohashes; - - if (locations == null || locations.size() == 0) { - if(mapping.fieldName != null) { - IndexableField[] fields = doc.getFields(mapping.fieldName); - if(fields.length == 0) { - IndexableField[] lonFields = doc.getFields(mapping.fieldName + ".lon"); - IndexableField[] latFields = doc.getFields(mapping.fieldName + ".lat"); - if (lonFields.length > 0 && latFields.length > 0) { - geohashes = new ArrayList<>(fields.length); - GeoPoint spare = new GeoPoint(); - for (int i = 0 ; i < lonFields.length ; i++) { - IndexableField lonField = lonFields[i]; - IndexableField latField = latFields[i]; - assert lonField.fieldType().docValuesType() == latField.fieldType().docValuesType(); - // we write doc values fields differently: one field for all values, so we need to only care about indexed fields - if (lonField.fieldType().docValuesType() == DocValuesType.NONE) { - spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue()); - geohashes.add(spare.geohash()); - } - } - } else { - geohashes = mapping.defaultLocations; - } - } else { - geohashes = new ArrayList<>(fields.length); - GeoPoint spare = new GeoPoint(); - for (IndexableField field : fields) { - spare.resetFromString(field.stringValue()); - geohashes.add(spare.geohash()); - } - } - } else { - geohashes = mapping.defaultLocations; - } - } else { - geohashes = locations; - } - - Collection locations = new HashSet<>(); - for (String geohash : geohashes) { - for (int p : mapping.precision) { - int precision = Math.min(p, geohash.length()); - String truncatedGeohash = geohash.substring(0, precision); - if(mapping.neighbors) { - XGeoHashUtils.addNeighbors(truncatedGeohash, precision, locations); - } - locations.add(truncatedGeohash); - } - } - - return new PrefixTokenFilter(stream, ContextMapping.SEPARATOR, locations); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("GeoConfig(location = ["); - Iterator location = this.locations.iterator(); - if (location.hasNext()) { - sb.append(location.next()); - while (location.hasNext()) { - sb.append(", ").append(location.next()); - } - } - return sb.append("])").toString(); - } - } - - private static class GeoQuery extends ContextQuery { - private final String location; - private final int[] precisions; - - public GeoQuery(String name, String location, int...precisions) { - super(name); - this.location = location; - this.precisions = precisions; - } - - @Override - public Automaton toAutomaton() { - Automaton automaton; - if(precisions == null || precisions.length == 0) { - automaton = Automata.makeString(location); - } else { - automaton = Automata.makeString(location.substring(0, Math.max(1, Math.min(location.length(), precisions[0])))); - for (int i = 1; i < precisions.length; i++) { - final String cell = location.substring(0, Math.max(1, Math.min(location.length(), precisions[i]))); - automaton = Operations.union(automaton, Automata.makeString(cell)); - } - } - return automaton; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if(precisions == null || precisions.length == 0) { - builder.field(name, location); - } else { - builder.startObject(name); - builder.field(FIELD_VALUE, location); - builder.field(FIELD_PRECISION, precisions); - builder.endObject(); - } - return builder; - } - } -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java index 13149e20e4d..4bbdaf9c49e 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java @@ -27,9 +27,9 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.Template; @@ -49,10 +49,10 @@ public final class PhraseSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, - IndexQueryParserService queryParserService, HasContextAndHeaders headersContext) throws IOException { + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService, + HasContextAndHeaders headersContext) throws IOException { PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester); - suggestion.setQueryParserService(queryParserService); + ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher(); XContentParser.Token token; String fieldName = null; boolean gramSizeSet = false; @@ -60,7 +60,7 @@ public final class PhraseSuggestParser implements SuggestContextParser { if (token == XContentParser.Token.FIELD_NAME) { fieldName = parser.currentName(); } else if (token.isValue()) { - if (!SuggestUtils.parseSuggestContext(parser, mapperService, fieldName, suggestion, queryParserService.parseFieldMatcher())) { + if (!SuggestUtils.parseSuggestContext(parser, mapperService, fieldName, suggestion, parseFieldMatcher)) { if ("real_word_error_likelihood".equals(fieldName) || "realWorldErrorLikelihood".equals(fieldName)) { suggestion.setRealWordErrorLikelihood(parser.floatValue()); if (suggestion.realworldErrorLikelyhood() <= 0.0) { @@ -106,7 +106,7 @@ public final class PhraseSuggestParser implements SuggestContextParser { fieldName = parser.currentName(); } if (token.isValue()) { - parseCandidateGenerator(parser, mapperService, fieldName, generator, queryParserService.parseFieldMatcher()); + parseCandidateGenerator(parser, mapperService, fieldName, generator, parseFieldMatcher); } } verifyGenerator(generator); @@ -141,7 +141,7 @@ public final class PhraseSuggestParser implements SuggestContextParser { if (suggestion.getCollateQueryScript() != null) { throw new IllegalArgumentException("suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]"); } - Template template = Template.parse(parser, queryParserService.parseFieldMatcher()); + Template template = Template.parse(parser, parseFieldMatcher); CompiledScript compiledScript = suggester.scriptService().compile(template, ScriptContext.Standard.SEARCH, headersContext); suggestion.setCollateQueryScript(compiledScript); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index 6e7a91dffc3..fccf9ebc30e 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -29,12 +29,13 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.Lucene.EarlyTerminatingCollector; import org.elasticsearch.common.text.StringText; import org.elasticsearch.common.text.Text; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptService; @@ -55,10 +56,11 @@ public final class PhraseSuggester extends Suggester { private final BytesRef SEPARATOR = new BytesRef(" "); private static final String SUGGESTION_TEMPLATE_VAR_NAME = "suggestion"; private final ScriptService scriptService; + private final IndicesService indicesService; - @Inject - public PhraseSuggester(ScriptService scriptService) { + public PhraseSuggester(ScriptService scriptService, IndicesService indicesService) { this.scriptService = scriptService; + this.indicesService = indicesService; } /* @@ -117,7 +119,9 @@ public final class PhraseSuggester extends Suggester { vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString()); final ExecutableScript executable = scriptService.executable(collateScript, vars); final BytesReference querySource = (BytesReference) executable.run(); - final ParsedQuery parsedQuery = suggestion.getQueryParserService().parse(querySource); + IndexService indexService = indicesService.indexService(suggestion.getIndex()); + IndexShard shard = indexService.getShard(suggestion.getShard()); + final ParsedQuery parsedQuery = shard.getQueryShardContext().parse(querySource); collateMatch = Lucene.exists(searcher, parsedQuery.query()); } if (!collateMatch && !collatePrune) { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java index 14ecb887582..8d2a6fdd123 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java @@ -25,7 +25,6 @@ import java.util.Map; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; import org.elasticsearch.search.suggest.Suggester; @@ -33,7 +32,6 @@ import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContex class PhraseSuggestionContext extends SuggestionContext { private final BytesRef SEPARATOR = new BytesRef(" "); - private IndexQueryParserService queryParserService; private float maxErrors = 0.5f; private BytesRef separator = SEPARATOR; private float realworldErrorLikelihood = 0.95f; @@ -112,14 +110,6 @@ class PhraseSuggestionContext extends SuggestionContext { return scorer; } - public void setQueryParserService(IndexQueryParserService queryParserService) { - this.queryParserService = queryParserService; - } - - public IndexQueryParserService getQueryParserService() { - return queryParserService; - } - static class DirectCandidateGenerator extends DirectSpellcheckerSettings { private Analyzer preFilter; private Analyzer postFilter; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java index 86f50927e73..a0e0e289540 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java @@ -21,8 +21,8 @@ package org.elasticsearch.search.suggest.term; import org.elasticsearch.common.HasContextAndHeaders; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestUtils; @@ -39,8 +39,8 @@ public final class TermSuggestParser implements SuggestContextParser { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, - IndexQueryParserService queryParserService, HasContextAndHeaders headersContext) throws IOException { + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService, + HasContextAndHeaders headersContext) throws IOException { XContentParser.Token token; String fieldName = null; TermSuggestionContext suggestion = new TermSuggestionContext(suggester); @@ -49,7 +49,7 @@ public final class TermSuggestParser implements SuggestContextParser { if (token == XContentParser.Token.FIELD_NAME) { fieldName = parser.currentName(); } else if (token.isValue()) { - parseTokenValue(parser, mapperService, fieldName, suggestion, settings, queryParserService.parseFieldMatcher()); + parseTokenValue(parser, mapperService, fieldName, suggestion, settings, mapperService.getIndexSettings().getParseFieldMatcher()); } else { throw new IllegalArgumentException("suggester[term] doesn't support field [" + fieldName + "]"); } diff --git a/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index 6e881cb1473..1ce27f97cff 100644 --- a/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -19,6 +19,8 @@ package org.elasticsearch.search.warmer; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; @@ -27,12 +29,17 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentGenerator; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -67,10 +74,10 @@ public class IndexWarmersMetaData extends AbstractDiffable public static class Entry { private final String name; private final String[] types; - private final BytesReference source; + private final SearchSource source; private final Boolean requestCache; - public Entry(String name, String[] types, Boolean requestCache, BytesReference source) { + public Entry(String name, String[] types, Boolean requestCache, SearchSource source) { this.name = name; this.types = types == null ? Strings.EMPTY_ARRAY : types; this.source = source; @@ -86,7 +93,7 @@ public class IndexWarmersMetaData extends AbstractDiffable } @Nullable - public BytesReference source() { + public SearchSource source() { return this.source; } @@ -141,9 +148,9 @@ public class IndexWarmersMetaData extends AbstractDiffable for (int i = 0; i < entries.length; i++) { String name = in.readString(); String[] types = in.readStringArray(); - BytesReference source = null; + SearchSource source = null; if (in.readBoolean()) { - source = in.readBytesReference(); + source = new SearchSource(in); } Boolean queryCache; queryCache = in.readOptionalBoolean(); @@ -162,7 +169,7 @@ public class IndexWarmersMetaData extends AbstractDiffable out.writeBoolean(false); } else { out.writeBoolean(true); - out.writeBytesReference(entry.source()); + entry.source.writeTo(out); } out.writeOptionalBoolean(entry.requestCache()); } @@ -194,7 +201,7 @@ public class IndexWarmersMetaData extends AbstractDiffable } else if (token == XContentParser.Token.START_OBJECT) { String name = currentFieldName; List types = new ArrayList<>(2); - BytesReference source = null; + SearchSource source = null; Boolean queryCache = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -207,12 +214,15 @@ public class IndexWarmersMetaData extends AbstractDiffable } } else if (token == XContentParser.Token.START_OBJECT) { if ("source".equals(currentFieldName)) { - XContentBuilder builder = XContentFactory.jsonBuilder().map(parser.mapOrdered()); - source = builder.bytes(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (XContentGenerator generator = XContentType.JSON.xContent().createGenerator(out)) { + generator.copyCurrentStructure(parser); + } + source = new SearchSource(new BytesArray(out.toByteArray())); } } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { if ("source".equals(currentFieldName)) { - source = new BytesArray(parser.binaryValue()); + source = new SearchSource(new BytesArray(parser.binaryValue())); } } else if (token.isValue()) { if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) { @@ -239,22 +249,12 @@ public class IndexWarmersMetaData extends AbstractDiffable } public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { - boolean binary = params.paramAsBoolean("binary", false); builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE); builder.field("types", entry.types()); if (entry.requestCache() != null) { builder.field("requestCache", entry.requestCache()); } - builder.field("source"); - if (binary) { - builder.value(entry.source()); - } else { - Map mapping; - try (XContentParser parser = XContentFactory.xContent(entry.source()).createParser(entry.source())) { - mapping = parser.mapOrdered(); - } - builder.map(mapping); - } + builder.field("source", entry.source()); builder.endObject(); } @@ -277,4 +277,78 @@ public class IndexWarmersMetaData extends AbstractDiffable } return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()])); } + + public static class SearchSource extends ToXContentToBytes implements Writeable { + private final BytesReference binary; + private SearchSourceBuilder cached; + + public SearchSource(BytesReference bytesArray) { + if (bytesArray == null) { + throw new IllegalArgumentException("bytesArray must not be null"); + } + this.binary = bytesArray; + } + + public SearchSource(StreamInput input) throws IOException { + this(input.readBytesReference()); + } + + public SearchSource(SearchSourceBuilder source) { + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + source.toXContent(builder, ToXContent.EMPTY_PARAMS); + binary = builder.bytes(); + } catch (IOException ex) { + throw new ElasticsearchException("failed to generate XContent", ex); + } + } + + public SearchSourceBuilder build(QueryParseContext ctx) throws IOException { + if (cached == null) { + try (XContentParser parser = XContentFactory.xContent(binary).createParser(binary)) { + ctx.reset(parser); + cached = SearchSourceBuilder.parseSearchSource(parser, ctx); + } + } + return cached; + } + + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (binary == null) { + cached.toXContent(builder, params); + } else { + try (XContentParser parser = XContentFactory.xContent(binary).createParser(binary)) { + builder.copyCurrentStructure(parser); + } + } + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBytesReference(binary); + } + + @Override + public SearchSource readFrom(StreamInput in) throws IOException { + return new SearchSource(in); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + SearchSource that = (SearchSource) o; + + return binary.equals(that.binary); + + } + + @Override + public int hashCode() { + return binary.hashCode(); + } + } } diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java index 866cd38d9d5..59d55dfaf4c 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java +++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java @@ -190,7 +190,7 @@ public class RestoreInfo implements ToXContent, Streamable { * @return restore info */ public static RestoreInfo readOptionalRestoreInfo(StreamInput in) throws IOException { - return in.readOptionalStreamable(new RestoreInfo()); + return in.readOptionalStreamable(RestoreInfo::new); } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 2964a8d496b..cd710d52cdc 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -22,32 +22,15 @@ import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.hppc.IntSet; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.RestoreInProgress.ShardRestoreStatus; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; -import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; -import org.elasticsearch.cluster.metadata.RepositoriesMetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.cluster.metadata.*; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RestoreSource; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.settings.ClusterDynamicSettings; @@ -70,35 +53,16 @@ import org.elasticsearch.index.snapshots.IndexShardRepository; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.EmptyTransportResponseHandler; -import org.elasticsearch.transport.TransportChannel; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestHandler; -import org.elasticsearch.transport.TransportResponse; -import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.*; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CopyOnWriteArrayList; import static java.util.Collections.unmodifiableSet; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_INDEX_UUID; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_MINIMUM_COMPATIBLE; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_UPGRADED; +import static org.elasticsearch.cluster.metadata.IndexMetaData.*; import static org.elasticsearch.common.util.set.Sets.newHashSet; /** @@ -113,7 +77,7 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet; * method. *

    * Individual shards are getting restored as part of normal recovery process in - * {@link IndexShard#restoreFromRepository(ShardRouting, IndexShardRepository, DiscoveryNode)} )} + * {@link IndexShard#restoreFromRepository(IndexShardRepository, DiscoveryNode)} )} * method, which detects that shard should be restored from snapshot rather than recovered from gateway by looking * at the {@link org.elasticsearch.cluster.routing.ShardRouting#restoreSource()} property. *

    @@ -251,14 +215,14 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis // Index doesn't exist - create it and start recovery // Make sure that the index we are about to create has a validate name createIndexService.validateIndexName(renamedIndex, currentState); - createIndexService.validateIndexSettings(renamedIndex, snapshotIndexMetaData.settings()); + createIndexService.validateIndexSettings(renamedIndex, snapshotIndexMetaData.getSettings()); IndexMetaData.Builder indexMdBuilder = IndexMetaData.builder(snapshotIndexMetaData).state(IndexMetaData.State.OPEN).index(renamedIndex); - indexMdBuilder.settings(Settings.settingsBuilder().put(snapshotIndexMetaData.settings()).put(IndexMetaData.SETTING_INDEX_UUID, Strings.randomBase64UUID())); - if (!request.includeAliases() && !snapshotIndexMetaData.aliases().isEmpty()) { + indexMdBuilder.settings(Settings.settingsBuilder().put(snapshotIndexMetaData.getSettings()).put(IndexMetaData.SETTING_INDEX_UUID, Strings.randomBase64UUID())); + if (!request.includeAliases() && !snapshotIndexMetaData.getAliases().isEmpty()) { // Remove all aliases - they shouldn't be restored indexMdBuilder.removeAllAliases(); } else { - for (ObjectCursor alias : snapshotIndexMetaData.aliases().keys()) { + for (ObjectCursor alias : snapshotIndexMetaData.getAliases().keys()) { aliases.add(alias.value); } } @@ -273,22 +237,22 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis validateExistingIndex(currentIndexMetaData, snapshotIndexMetaData, renamedIndex, partial); // Index exists and it's closed - open it in metadata and start recovery IndexMetaData.Builder indexMdBuilder = IndexMetaData.builder(snapshotIndexMetaData).state(IndexMetaData.State.OPEN); - indexMdBuilder.version(Math.max(snapshotIndexMetaData.version(), currentIndexMetaData.version() + 1)); + indexMdBuilder.version(Math.max(snapshotIndexMetaData.getVersion(), currentIndexMetaData.getVersion() + 1)); if (!request.includeAliases()) { // Remove all snapshot aliases - if (!snapshotIndexMetaData.aliases().isEmpty()) { + if (!snapshotIndexMetaData.getAliases().isEmpty()) { indexMdBuilder.removeAllAliases(); } /// Add existing aliases - for (ObjectCursor alias : currentIndexMetaData.aliases().values()) { + for (ObjectCursor alias : currentIndexMetaData.getAliases().values()) { indexMdBuilder.putAlias(alias.value); } } else { - for (ObjectCursor alias : snapshotIndexMetaData.aliases().keys()) { + for (ObjectCursor alias : snapshotIndexMetaData.getAliases().keys()) { aliases.add(alias.value); } } - indexMdBuilder.settings(Settings.settingsBuilder().put(snapshotIndexMetaData.settings()).put(IndexMetaData.SETTING_INDEX_UUID, currentIndexMetaData.indexUUID())); + indexMdBuilder.settings(Settings.settingsBuilder().put(snapshotIndexMetaData.getSettings()).put(IndexMetaData.SETTING_INDEX_UUID, currentIndexMetaData.getIndexUUID())); IndexMetaData updatedIndexMetaData = indexMdBuilder.index(renamedIndex).build(); rtBuilder.addAsRestore(updatedIndexMetaData, restoreSource); blocks.updateBlocks(updatedIndexMetaData); @@ -324,7 +288,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis RoutingTable rt = rtBuilder.build(); ClusterState updatedState = builder.metaData(mdBuilder).blocks(blocks).routingTable(rt).build(); - RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(rt).build()); + RoutingAllocation.Result routingResult = allocationService.reroute( + ClusterState.builder(updatedState).routingTable(rt).build(), + "restored snapshot [" + snapshotId + "]"); return ClusterState.builder(updatedState).routingResult(routingResult).build(); } @@ -359,7 +325,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis private void validateExistingIndex(IndexMetaData currentIndexMetaData, IndexMetaData snapshotIndexMetaData, String renamedIndex, boolean partial) { // Index exist - checking that it's closed - if (currentIndexMetaData.state() != IndexMetaData.State.CLOSE) { + if (currentIndexMetaData.getState() != IndexMetaData.State.CLOSE) { // TODO: Enable restore for open indices throw new SnapshotRestoreException(snapshotId, "cannot restore index [" + renamedIndex + "] because it's open"); } @@ -384,7 +350,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } Settings normalizedChangeSettings = Settings.settingsBuilder().put(changeSettings).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData); - Map settingsMap = new HashMap<>(indexMetaData.settings().getAsMap()); + Map settingsMap = new HashMap<>(indexMetaData.getSettings().getAsMap()); List simpleMatchPatterns = new ArrayList<>(); for (String ignoredSetting : ignoreSettings) { if (!Regex.isSimpleMatchPattern(ignoredSetting)) { diff --git a/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java b/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java index d7e0a064048..1206ef53501 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java +++ b/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java @@ -29,8 +29,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static java.util.Collections.*; - /** * Represent information about snapshot */ @@ -93,7 +91,7 @@ public class Snapshot implements Comparable, ToXContent, FromXContentB * Special constructor for the prototype object */ private Snapshot() { - this("", (List) EMPTY_LIST, 0); + this("", Collections.emptyList(), 0); } private static SnapshotState snapshotState(String reason, List shardFailures) { @@ -227,7 +225,7 @@ public class Snapshot implements Comparable, ToXContent, FromXContentB @Override public int hashCode() { int result = name.hashCode(); - result = 31 * result + (int) (startTime ^ (startTime >>> 32)); + result = 31 * result + Long.hashCode(startTime); return result; } diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 57b974ce7ae..3033a0ff801 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -324,7 +324,7 @@ public class SnapshotInfo implements ToXContent, Streamable { * @return deserialized snapshot info or null */ public static SnapshotInfo readOptionalSnapshotInfo(StreamInput in) throws IOException { - return in.readOptionalStreamable(new SnapshotInfo()); + return in.readOptionalStreamable(SnapshotInfo::new); } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 0b4d0419ce1..bf3af7394dd 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -141,7 +141,7 @@ public class SnapshotsService extends AbstractLifecycleComponent snapshots(String repositoryName) { + public List snapshots(String repositoryName, boolean ignoreUnavailable) { Set snapshotSet = new HashSet<>(); List entries = currentSnapshots(repositoryName, null); for (SnapshotsInProgress.Entry entry : entries) { @@ -150,8 +150,17 @@ public class SnapshotsService extends AbstractLifecycleComponent snapshotIds = repository.snapshots(); for (SnapshotId snapshotId : snapshotIds) { - snapshotSet.add(repository.readSnapshot(snapshotId)); + try { + snapshotSet.add(repository.readSnapshot(snapshotId)); + } catch (Exception ex) { + if (ignoreUnavailable) { + logger.warn("failed to get snapshot [{}]", ex, snapshotId); + } else { + throw new SnapshotException(snapshotId, "Snapshot could not be read", ex); + } + } } + ArrayList snapshotList = new ArrayList<>(snapshotSet); CollectionUtil.timSort(snapshotList); return Collections.unmodifiableList(snapshotList); @@ -1023,13 +1032,13 @@ public class SnapshotsService extends AbstractLifecycleComponent TYPE_MAP; + + static { + Map typeMap = new HashMap<>(); + for (ThreadPoolType threadPoolType : ThreadPoolType.values()) { + typeMap.put(threadPoolType.getType(), threadPoolType); + } + TYPE_MAP = Collections.unmodifiableMap(typeMap); + } + + public static ThreadPoolType fromType(String type) { + ThreadPoolType threadPoolType = TYPE_MAP.get(type); + if (threadPoolType == null) { + throw new IllegalArgumentException("no ThreadPoolType for " + type); + } + return threadPoolType; + } + } + + public static Map THREAD_POOL_TYPES; + + static { + HashMap map = new HashMap<>(); + map.put(Names.SAME, ThreadPoolType.DIRECT); + map.put(Names.GENERIC, ThreadPoolType.CACHED); + map.put(Names.LISTENER, ThreadPoolType.FIXED); + map.put(Names.GET, ThreadPoolType.FIXED); + map.put(Names.INDEX, ThreadPoolType.FIXED); + map.put(Names.BULK, ThreadPoolType.FIXED); + map.put(Names.SEARCH, ThreadPoolType.FIXED); + map.put(Names.SUGGEST, ThreadPoolType.FIXED); + map.put(Names.PERCOLATE, ThreadPoolType.FIXED); + map.put(Names.MANAGEMENT, ThreadPoolType.SCALING); + map.put(Names.FLUSH, ThreadPoolType.SCALING); + map.put(Names.REFRESH, ThreadPoolType.SCALING); + map.put(Names.WARMER, ThreadPoolType.SCALING); + map.put(Names.SNAPSHOT, ThreadPoolType.SCALING); + map.put(Names.FORCE_MERGE, ThreadPoolType.FIXED); + map.put(Names.FETCH_SHARD_STARTED, ThreadPoolType.SCALING); + map.put(Names.FETCH_SHARD_STORE, ThreadPoolType.SCALING); + THREAD_POOL_TYPES = Collections.unmodifiableMap(map); + } + + private static void add(Map executorSettings, ExecutorSettingsBuilder builder) { + Settings settings = builder.build(); + String name = settings.get("name"); + executorSettings.put(name, settings); + } + + private static class ExecutorSettingsBuilder { + Map settings = new HashMap<>(); + + public ExecutorSettingsBuilder(String name) { + settings.put("name", name); + settings.put("type", THREAD_POOL_TYPES.get(name).getType()); + } + + public ExecutorSettingsBuilder size(int availableProcessors) { + return add("size", Integer.toString(availableProcessors)); + } + + public ExecutorSettingsBuilder queueSize(int queueSize) { + return add("queue_size", Integer.toString(queueSize)); + } + + public ExecutorSettingsBuilder keepAlive(String keepAlive) { + return add("keep_alive", keepAlive); + } + + private ExecutorSettingsBuilder add(String key, String value) { + settings.put(key, value); + return this; + } + + public Settings build() { + return settingsBuilder().put(settings).build(); + } + } + public static final String THREADPOOL_GROUP = "threadpool."; private volatile Map executors; @@ -102,7 +188,6 @@ public class ThreadPool extends AbstractComponent { static final Executor DIRECT_EXECUTOR = command -> command.run(); - public ThreadPool(String name) { this(Settings.builder().put("name", name).build()); } @@ -112,42 +197,31 @@ public class ThreadPool extends AbstractComponent { assert settings.get("name") != null : "ThreadPool's settings should contain a name"; - Map groupSettings = settings.getGroups(THREADPOOL_GROUP); + Map groupSettings = getThreadPoolSettingsGroup(settings); int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings); int halfProcMaxAt5 = Math.min(((availableProcessors + 1) / 2), 5); int halfProcMaxAt10 = Math.min(((availableProcessors + 1) / 2), 10); Map defaultExecutorTypeSettings = new HashMap<>(); - defaultExecutorTypeSettings.put(Names.GENERIC, settingsBuilder().put("type", "cached").put("keep_alive", "30s").build()); - defaultExecutorTypeSettings.put(Names.INDEX, - settingsBuilder().put("type", "fixed").put("size", availableProcessors).put("queue_size", 200).build()); - defaultExecutorTypeSettings.put(Names.BULK, - settingsBuilder().put("type", "fixed").put("size", availableProcessors).put("queue_size", 50).build()); - defaultExecutorTypeSettings.put(Names.GET, - settingsBuilder().put("type", "fixed").put("size", availableProcessors).put("queue_size", 1000).build()); - defaultExecutorTypeSettings.put(Names.SEARCH, - settingsBuilder().put("type", "fixed").put("size", ((availableProcessors * 3) / 2) + 1).put("queue_size", 1000).build()); - defaultExecutorTypeSettings.put(Names.SUGGEST, - settingsBuilder().put("type", "fixed").put("size", availableProcessors).put("queue_size", 1000).build()); - defaultExecutorTypeSettings.put(Names.PERCOLATE, - settingsBuilder().put("type", "fixed").put("size", availableProcessors).put("queue_size", 1000).build()); - defaultExecutorTypeSettings .put(Names.MANAGEMENT, settingsBuilder().put("type", "scaling").put("keep_alive", "5m").put("size", 5).build()); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GENERIC).keepAlive("30s")); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.INDEX).size(availableProcessors).queueSize(200)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.BULK).size(availableProcessors).queueSize(50)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GET).size(availableProcessors).queueSize(1000)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SEARCH).size(((availableProcessors * 3) / 2) + 1).queueSize(1000)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SUGGEST).size(availableProcessors).queueSize(1000)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.PERCOLATE).size(availableProcessors).queueSize(1000)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.MANAGEMENT).size(5).keepAlive("5m")); // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded // the assumption here is that the listeners should be very lightweight on the listeners side - defaultExecutorTypeSettings.put(Names.LISTENER, settingsBuilder().put("type", "fixed").put("size", halfProcMaxAt10).build()); - defaultExecutorTypeSettings.put(Names.FLUSH, - settingsBuilder().put("type", "scaling").put("keep_alive", "5m").put("size", halfProcMaxAt5).build()); - defaultExecutorTypeSettings.put(Names.REFRESH, - settingsBuilder().put("type", "scaling").put("keep_alive", "5m").put("size", halfProcMaxAt10).build()); - defaultExecutorTypeSettings.put(Names.WARMER, - settingsBuilder().put("type", "scaling").put("keep_alive", "5m").put("size", halfProcMaxAt5).build()); - defaultExecutorTypeSettings.put(Names.SNAPSHOT, - settingsBuilder().put("type", "scaling").put("keep_alive", "5m").put("size", halfProcMaxAt5).build()); - defaultExecutorTypeSettings.put(Names.OPTIMIZE, settingsBuilder().put("type", "fixed").put("size", 1).build()); - defaultExecutorTypeSettings.put(Names.FETCH_SHARD_STARTED, - settingsBuilder().put("type", "scaling").put("keep_alive", "5m").put("size", availableProcessors * 2).build()); - defaultExecutorTypeSettings.put(Names.FETCH_SHARD_STORE, - settingsBuilder().put("type", "scaling").put("keep_alive", "5m").put("size", availableProcessors * 2).build()); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.LISTENER).size(halfProcMaxAt10)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.FLUSH).size(halfProcMaxAt5).keepAlive("5m")); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.REFRESH).size(halfProcMaxAt10).keepAlive("5m")); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.WARMER).size(halfProcMaxAt5).keepAlive("5m")); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SNAPSHOT).size(halfProcMaxAt5).keepAlive("5m")); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.FORCE_MERGE).size(1)); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.FETCH_SHARD_STARTED).size(availableProcessors * 2).keepAlive("5m")); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.FETCH_SHARD_STORE).size(availableProcessors * 2).keepAlive("5m")); + this.defaultExecutorTypeSettings = unmodifiableMap(defaultExecutorTypeSettings); Map executors = new HashMap<>(); @@ -163,8 +237,8 @@ public class ThreadPool extends AbstractComponent { executors.put(entry.getKey(), build(entry.getKey(), entry.getValue(), Settings.EMPTY)); } - executors.put(Names.SAME, new ExecutorHolder(DIRECT_EXECUTOR, new Info(Names.SAME, "same"))); - if (!executors.get(Names.GENERIC).info.getType().equals("cached")) { + executors.put(Names.SAME, new ExecutorHolder(DIRECT_EXECUTOR, new Info(Names.SAME, ThreadPoolType.DIRECT))); + if (!executors.get(Names.GENERIC).info.getThreadPoolType().equals(ThreadPoolType.CACHED)) { throw new IllegalArgumentException("generic thread pool must be of type cached"); } this.executors = unmodifiableMap(executors); @@ -178,6 +252,12 @@ public class ThreadPool extends AbstractComponent { this.estimatedTimeThread.start(); } + private Map getThreadPoolSettingsGroup(Settings settings) { + Map groupSettings = settings.getGroups(THREADPOOL_GROUP); + validate(groupSettings); + return groupSettings; + } + public void setNodeSettingsService(NodeSettingsService nodeSettingsService) { if(settingsListenerIsSet) { throw new IllegalStateException("the node settings listener was set more then once"); @@ -326,24 +406,28 @@ public class ThreadPool extends AbstractComponent { settings = Settings.Builder.EMPTY_SETTINGS; } Info previousInfo = previousExecutorHolder != null ? previousExecutorHolder.info : null; - String type = settings.get("type", previousInfo != null ? previousInfo.getType() : defaultSettings.get("type")); + String type = settings.get("type", previousInfo != null ? previousInfo.getThreadPoolType().getType() : defaultSettings.get("type")); + ThreadPoolType threadPoolType = ThreadPoolType.fromType(type); ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(this.settings, name); - if ("same".equals(type)) { + if (ThreadPoolType.DIRECT == threadPoolType) { if (previousExecutorHolder != null) { logger.debug("updating thread_pool [{}], type [{}]", name, type); } else { logger.debug("creating thread_pool [{}], type [{}]", name, type); } - return new ExecutorHolder(DIRECT_EXECUTOR, new Info(name, type)); - } else if ("cached".equals(type)) { + return new ExecutorHolder(DIRECT_EXECUTOR, new Info(name, threadPoolType)); + } else if (ThreadPoolType.CACHED == threadPoolType) { + if (!Names.GENERIC.equals(name)) { + throw new IllegalArgumentException("thread pool type cached is reserved only for the generic thread pool and can not be applied to [" + name + "]"); + } TimeValue defaultKeepAlive = defaultSettings.getAsTime("keep_alive", timeValueMinutes(5)); if (previousExecutorHolder != null) { - if ("cached".equals(previousInfo.getType())) { + if (ThreadPoolType.CACHED == previousInfo.getThreadPoolType()) { TimeValue updatedKeepAlive = settings.getAsTime("keep_alive", previousInfo.getKeepAlive()); if (!previousInfo.getKeepAlive().equals(updatedKeepAlive)) { logger.debug("updating thread_pool [{}], type [{}], keep_alive [{}]", name, type, updatedKeepAlive); ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setKeepAliveTime(updatedKeepAlive.millis(), TimeUnit.MILLISECONDS); - return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, type, -1, -1, updatedKeepAlive, null)); + return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, threadPoolType, -1, -1, updatedKeepAlive, null)); } return previousExecutorHolder; } @@ -358,13 +442,13 @@ public class ThreadPool extends AbstractComponent { logger.debug("creating thread_pool [{}], type [{}], keep_alive [{}]", name, type, keepAlive); } Executor executor = EsExecutors.newCached(name, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory); - return new ExecutorHolder(executor, new Info(name, type, -1, -1, keepAlive, null)); - } else if ("fixed".equals(type)) { + return new ExecutorHolder(executor, new Info(name, threadPoolType, -1, -1, keepAlive, null)); + } else if (ThreadPoolType.FIXED == threadPoolType) { int defaultSize = defaultSettings.getAsInt("size", EsExecutors.boundedNumberOfProcessors(settings)); SizeValue defaultQueueSize = getAsSizeOrUnbounded(defaultSettings, "queue", getAsSizeOrUnbounded(defaultSettings, "queue_size", null)); if (previousExecutorHolder != null) { - if ("fixed".equals(previousInfo.getType())) { + if (ThreadPoolType.FIXED == previousInfo.getThreadPoolType()) { SizeValue updatedQueueSize = getAsSizeOrUnbounded(settings, "capacity", getAsSizeOrUnbounded(settings, "queue", getAsSizeOrUnbounded(settings, "queue_size", previousInfo.getQueueSize()))); if (Objects.equals(previousInfo.getQueueSize(), updatedQueueSize)) { int updatedSize = settings.getAsInt("size", previousInfo.getMax()); @@ -378,7 +462,7 @@ public class ThreadPool extends AbstractComponent { ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setCorePoolSize(updatedSize); ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setMaximumPoolSize(updatedSize); } - return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, type, updatedSize, updatedSize, null, updatedQueueSize)); + return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, threadPoolType, updatedSize, updatedSize, null, updatedQueueSize)); } return previousExecutorHolder; } @@ -393,13 +477,13 @@ public class ThreadPool extends AbstractComponent { SizeValue queueSize = getAsSizeOrUnbounded(settings, "capacity", getAsSizeOrUnbounded(settings, "queue", getAsSizeOrUnbounded(settings, "queue_size", defaultQueueSize))); logger.debug("creating thread_pool [{}], type [{}], size [{}], queue_size [{}]", name, type, size, queueSize); Executor executor = EsExecutors.newFixed(name, size, queueSize == null ? -1 : (int) queueSize.singles(), threadFactory); - return new ExecutorHolder(executor, new Info(name, type, size, size, null, queueSize)); - } else if ("scaling".equals(type)) { + return new ExecutorHolder(executor, new Info(name, threadPoolType, size, size, null, queueSize)); + } else if (ThreadPoolType.SCALING == threadPoolType) { TimeValue defaultKeepAlive = defaultSettings.getAsTime("keep_alive", timeValueMinutes(5)); int defaultMin = defaultSettings.getAsInt("min", 1); int defaultSize = defaultSettings.getAsInt("size", EsExecutors.boundedNumberOfProcessors(settings)); if (previousExecutorHolder != null) { - if ("scaling".equals(previousInfo.getType())) { + if (ThreadPoolType.SCALING == previousInfo.getThreadPoolType()) { TimeValue updatedKeepAlive = settings.getAsTime("keep_alive", previousInfo.getKeepAlive()); int updatedMin = settings.getAsInt("min", previousInfo.getMin()); int updatedSize = settings.getAsInt("max", settings.getAsInt("size", previousInfo.getMax())); @@ -414,7 +498,7 @@ public class ThreadPool extends AbstractComponent { if (previousInfo.getMax() != updatedSize) { ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setMaximumPoolSize(updatedSize); } - return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, type, updatedMin, updatedSize, updatedKeepAlive, null)); + return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, threadPoolType, updatedMin, updatedSize, updatedKeepAlive, null)); } return previousExecutorHolder; } @@ -437,13 +521,13 @@ public class ThreadPool extends AbstractComponent { logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], keep_alive [{}]", name, type, min, size, keepAlive); } Executor executor = EsExecutors.newScaling(name, min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory); - return new ExecutorHolder(executor, new Info(name, type, min, size, keepAlive, null)); + return new ExecutorHolder(executor, new Info(name, threadPoolType, min, size, keepAlive, null)); } throw new IllegalArgumentException("No type found [" + type + "], for [" + name + "]"); } public void updateSettings(Settings settings) { - Map groupSettings = settings.getGroups("threadpool"); + Map groupSettings = getThreadPoolSettingsGroup(settings); if (groupSettings.isEmpty()) { return; } @@ -490,6 +574,20 @@ public class ThreadPool extends AbstractComponent { } } + private void validate(Map groupSettings) { + for (String key : groupSettings.keySet()) { + if (!THREAD_POOL_TYPES.containsKey(key)) { + continue; + } + String type = groupSettings.get(key).get("type"); + ThreadPoolType correctThreadPoolType = THREAD_POOL_TYPES.get(key); + // TODO: the type equality check can be removed after #3760/#6732 are addressed + if (type != null && !correctThreadPoolType.getType().equals(type)) { + throw new IllegalArgumentException("setting " + THREADPOOL_GROUP + key + ".type to " + type + " is not permitted; must be " + correctThreadPoolType.getType()); + } + } + } + /** * A thread pool size can also be unbounded and is represented by -1, which is not supported by SizeValue (which only supports positive numbers) */ @@ -643,7 +741,7 @@ public class ThreadPool extends AbstractComponent { public static class Info implements Streamable, ToXContent { private String name; - private String type; + private ThreadPoolType type; private int min; private int max; private TimeValue keepAlive; @@ -653,15 +751,15 @@ public class ThreadPool extends AbstractComponent { } - public Info(String name, String type) { + public Info(String name, ThreadPoolType type) { this(name, type, -1); } - public Info(String name, String type, int size) { + public Info(String name, ThreadPoolType type, int size) { this(name, type, size, size, null, null); } - public Info(String name, String type, int min, int max, @Nullable TimeValue keepAlive, @Nullable SizeValue queueSize) { + public Info(String name, ThreadPoolType type, int min, int max, @Nullable TimeValue keepAlive, @Nullable SizeValue queueSize) { this.name = name; this.type = type; this.min = min; @@ -674,7 +772,7 @@ public class ThreadPool extends AbstractComponent { return this.name; } - public String getType() { + public ThreadPoolType getThreadPoolType() { return this.type; } @@ -699,7 +797,7 @@ public class ThreadPool extends AbstractComponent { @Override public void readFrom(StreamInput in) throws IOException { name = in.readString(); - type = in.readString(); + type = ThreadPoolType.fromType(in.readString()); min = in.readInt(); max = in.readInt(); if (in.readBoolean()) { @@ -716,7 +814,7 @@ public class ThreadPool extends AbstractComponent { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); - out.writeString(type); + out.writeString(type.getType()); out.writeInt(min); out.writeInt(max); if (keepAlive == null) { @@ -739,7 +837,7 @@ public class ThreadPool extends AbstractComponent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(name, XContentBuilder.FieldCaseConversion.NONE); - builder.field(Fields.TYPE, type); + builder.field(Fields.TYPE, type.getType()); if (min != -1) { builder.field(Fields.MIN, min); } @@ -814,4 +912,37 @@ public class ThreadPool extends AbstractComponent { return false; } + public static ThreadPoolTypeSettingsValidator THREAD_POOL_TYPE_SETTINGS_VALIDATOR = new ThreadPoolTypeSettingsValidator(); + private static class ThreadPoolTypeSettingsValidator implements Validator { + @Override + public String validate(String setting, String value, ClusterState clusterState) { + // TODO: the type equality validation can be removed after #3760/#6732 are addressed + Matcher matcher = Pattern.compile("threadpool\\.(.*)\\.type").matcher(setting); + if (!matcher.matches()) { + return null; + } else { + String threadPool = matcher.group(1); + ThreadPool.ThreadPoolType defaultThreadPoolType = ThreadPool.THREAD_POOL_TYPES.get(threadPool); + ThreadPool.ThreadPoolType threadPoolType; + try { + threadPoolType = ThreadPool.ThreadPoolType.fromType(value); + } catch (IllegalArgumentException e) { + return e.getMessage(); + } + if (defaultThreadPoolType.equals(threadPoolType)) { + return null; + } else { + return String.format( + Locale.ROOT, + "thread pool type for [%s] can only be updated to [%s] but was [%s]", + threadPool, + defaultThreadPoolType.getType(), + threadPoolType.getType() + ); + } + } + + } + } + } diff --git a/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java b/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java index 8cbe6b4c960..c0577e48623 100644 --- a/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java +++ b/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java @@ -59,7 +59,7 @@ public class PlainTransportFuture extends BaseFutur try { return get(timeout, unit); } catch (TimeoutException e) { - throw new ElasticsearchTimeoutException(e.getMessage()); + throw new ElasticsearchTimeoutException(e); } catch (InterruptedException e) { throw new IllegalStateException("Future got interrupted", e); } catch (ExecutionException e) { diff --git a/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java b/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java index a79e57441fe..375fbb8bfcb 100644 --- a/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java +++ b/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java @@ -31,7 +31,7 @@ import java.io.IOException; public class ResponseHandlerFailureTransportException extends TransportException { public ResponseHandlerFailureTransportException(Throwable cause) { - super(cause.getMessage(), cause); + super(cause); } public ResponseHandlerFailureTransportException(StreamInput in) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/transport/TransportException.java b/core/src/main/java/org/elasticsearch/transport/TransportException.java index a672fb939e5..7cc1c54c236 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportException.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportException.java @@ -28,6 +28,10 @@ import java.io.IOException; * */ public class TransportException extends ElasticsearchException { + public TransportException(Throwable cause) { + super(cause); + } + public TransportException(StreamInput in) throws IOException { super(in); } diff --git a/core/src/main/java/org/elasticsearch/transport/TransportRequestOptions.java b/core/src/main/java/org/elasticsearch/transport/TransportRequestOptions.java index 0d92d00f144..879d6aec661 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportRequestOptions.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportRequestOptions.java @@ -21,64 +21,16 @@ package org.elasticsearch.transport; import org.elasticsearch.common.unit.TimeValue; -/** - * - */ public class TransportRequestOptions { - public static final TransportRequestOptions EMPTY = options(); + private final TimeValue timeout; + private final boolean compress; + private final Type type; - public static TransportRequestOptions options() { - return new TransportRequestOptions(); - } - - public static enum Type { - RECOVERY, - BULK, - REG, - STATE, - PING; - - public static Type fromString(String type) { - if ("bulk".equalsIgnoreCase(type)) { - return BULK; - } else if ("reg".equalsIgnoreCase(type)) { - return REG; - } else if ("state".equalsIgnoreCase(type)) { - return STATE; - } else if ("recovery".equalsIgnoreCase(type)) { - return RECOVERY; - } else if ("ping".equalsIgnoreCase(type)) { - return PING; - } else { - throw new IllegalArgumentException("failed to match transport type for [" + type + "]"); - } - } - } - - private TimeValue timeout; - - private boolean compress; - - private Type type = Type.REG; - - public TransportRequestOptions withTimeout(long timeout) { - return withTimeout(TimeValue.timeValueMillis(timeout)); - } - - public TransportRequestOptions withTimeout(TimeValue timeout) { + private TransportRequestOptions(TimeValue timeout, boolean compress, Type type) { this.timeout = timeout; - return this; - } - - public TransportRequestOptions withCompress(boolean compress) { this.compress = compress; - return this; - } - - public TransportRequestOptions withType(Type type) { this.type = type; - return this; } public TimeValue timeout() { @@ -92,4 +44,57 @@ public class TransportRequestOptions { public Type type() { return this.type; } + + public static final TransportRequestOptions EMPTY = new TransportRequestOptions.Builder().build(); + + public enum Type { + RECOVERY, + BULK, + REG, + STATE, + PING + } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(TransportRequestOptions options) { + return new Builder() + .withTimeout(options.timeout) + .withCompress(options.compress) + .withType(options.type()); + } + + public static class Builder { + private TimeValue timeout; + private boolean compress; + private Type type = Type.REG; + + private Builder() { + } + + public Builder withTimeout(long timeout) { + return withTimeout(TimeValue.timeValueMillis(timeout)); + } + + public Builder withTimeout(TimeValue timeout) { + this.timeout = timeout; + return this; + } + + public Builder withCompress(boolean compress) { + this.compress = compress; + return this; + } + + public Builder withType(Type type) { + this.type = type; + return this; + } + + public TransportRequestOptions build() { + return new TransportRequestOptions(timeout, compress, type); + } + } } diff --git a/core/src/main/java/org/elasticsearch/transport/TransportResponseOptions.java b/core/src/main/java/org/elasticsearch/transport/TransportResponseOptions.java index 32dbf528b74..eb163641749 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportResponseOptions.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportResponseOptions.java @@ -24,20 +24,37 @@ package org.elasticsearch.transport; */ public class TransportResponseOptions { - public static final TransportResponseOptions EMPTY = options(); + private final boolean compress; - public static TransportResponseOptions options() { - return new TransportResponseOptions(); - } - - private boolean compress; - - public TransportResponseOptions withCompress(boolean compress) { + private TransportResponseOptions(boolean compress) { this.compress = compress; - return this; } public boolean compress() { return this.compress; } + + public static final TransportResponseOptions EMPTY = TransportResponseOptions.builder().build(); + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(TransportResponseOptions options) { + return new Builder() + .withCompress(options.compress); + } + + public static class Builder { + private boolean compress; + + public Builder withCompress(boolean compress) { + this.compress = compress; + return this; + } + + public TransportResponseOptions build() { + return new TransportResponseOptions(compress); + } + } } diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java index 964cfacc8c1..14fc9029b00 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java @@ -484,7 +484,7 @@ public class TransportService extends AbstractLifecycleComponent implem private final static ConcurrentMap transports = newConcurrentMap(); private static final AtomicLong transportAddressIdGenerator = new AtomicLong(); private final ConcurrentMap connectedNodes = newConcurrentMap(); - private final NamedWriteableRegistry namedWriteableRegistry; + protected final NamedWriteableRegistry namedWriteableRegistry; public static final String TRANSPORT_LOCAL_ADDRESS = "transport.local.address"; public static final String TRANSPORT_LOCAL_WORKERS = "transport.local.workers"; @@ -138,7 +154,7 @@ public class LocalTransport extends AbstractLifecycleComponent implem @Override public Map profileBoundAddresses() { - return Collections.EMPTY_MAP; + return Collections.emptyMap(); } @Override diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java index 853497d59ae..2f1c52a0ac2 100644 --- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java +++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java @@ -24,6 +24,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.bytes.ReleasablePagedBytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.compress.CompressorFactory; @@ -86,6 +87,7 @@ import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory; import org.jboss.netty.util.HashedWheelTimer; import java.io.IOException; +import java.net.BindException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -205,6 +207,8 @@ public class NettyTransport extends AbstractLifecycleComponent implem final ScheduledPing scheduledPing; @Inject + @SuppressForbidden(reason = "sets org.jboss.netty.epollBugWorkaround based on netty.epollBugWorkaround") + // TODO: why be confusing like this? just let the user do it with the netty parameter instead! public NettyTransport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays, Version version, NamedWriteableRegistry namedWriteableRegistry) { super(settings); this.threadPool = threadPool; @@ -343,14 +347,6 @@ public class NettyTransport extends AbstractLifecycleComponent implem return unmodifiableMap(new HashMap<>(profileBoundAddresses)); } - private InetSocketAddress createPublishAddress(String publishHost, int publishPort) { - try { - return new InetSocketAddress(networkService.resolvePublishHostAddress(publishHost), publishPort); - } catch (Exception e) { - throw new BindTransportException("Failed to resolve publish address", e); - } - } - private ClientBootstrap createClientBootstrap() { if (blockingClient) { @@ -436,11 +432,11 @@ public class NettyTransport extends AbstractLifecycleComponent implem private void bindServerBootstrap(final String name, final Settings settings) { // Bind and start to accept incoming connections. InetAddress hostAddresses[]; - String bindHost = settings.get("bind_host"); + String bindHosts[] = settings.getAsArray("bind_host", null); try { - hostAddresses = networkService.resolveBindHostAddress(bindHost); + hostAddresses = networkService.resolveBindHostAddresses(bindHosts); } catch (IOException e) { - throw new BindTransportException("Failed to resolve host [" + bindHost + "]", e); + throw new BindTransportException("Failed to resolve host " + Arrays.toString(bindHosts) + "", e); } if (logger.isDebugEnabled()) { String[] addresses = new String[hostAddresses.length]; @@ -449,14 +445,24 @@ public class NettyTransport extends AbstractLifecycleComponent implem } logger.debug("binding server bootstrap to: {}", (Object)addresses); } + + assert hostAddresses.length > 0; + + List boundAddresses = new ArrayList<>(); for (InetAddress hostAddress : hostAddresses) { - bindServerBootstrap(name, hostAddress, settings); + boundAddresses.add(bindToPort(name, hostAddress, settings.get("port"))); + } + + final BoundTransportAddress boundTransportAddress = createBoundTransportAddress(name, settings, boundAddresses); + + if (DEFAULT_PROFILE.equals(name)) { + this.boundAddress = boundTransportAddress; + } else { + profileBoundAddresses.put(name, boundTransportAddress); } } - private void bindServerBootstrap(final String name, final InetAddress hostAddress, Settings profileSettings) { - - String port = profileSettings.get("port"); + private InetSocketAddress bindToPort(final String name, final InetAddress hostAddress, String port) { PortsRange portsRange = new PortsRange(port); final AtomicReference lastException = new AtomicReference<>(); final AtomicReference boundSocket = new AtomicReference<>(); @@ -485,48 +491,64 @@ public class NettyTransport extends AbstractLifecycleComponent implem throw new BindTransportException("Failed to bind to [" + port + "]", lastException.get()); } - InetSocketAddress boundAddress = boundSocket.get(); - // TODO: We can remove the special casing for the default profile and store it in the profile map to reduce the complexity here - if (!DEFAULT_PROFILE.equals(name)) { - // check to see if an address is already bound for this profile - BoundTransportAddress boundTransportAddress = profileBoundAddresses().get(name); - if (boundTransportAddress == null) { - // no address is bound, so lets create one with the publish address information from the settings or the bound address as a fallback - int publishPort = profileSettings.getAsInt("publish_port", boundAddress.getPort()); - String publishHost = profileSettings.get("publish_host", boundAddress.getHostString()); - InetSocketAddress publishAddress = createPublishAddress(publishHost, publishPort); - profileBoundAddresses.put(name, new BoundTransportAddress(new TransportAddress[]{new InetSocketTransportAddress(boundAddress)}, new InetSocketTransportAddress(publishAddress))); - } else { - // TODO: support real multihoming with publishing. Today we update the bound addresses so only the prioritized address is published - // an address already exists. add the new bound address to the end of a new array and create a new BoundTransportAddress with the array and existing publish address - // the new bound address is appended in order to preserve the ordering/priority of bound addresses - TransportAddress[] existingBoundAddress = boundTransportAddress.boundAddresses(); - TransportAddress[] updatedBoundAddresses = Arrays.copyOf(existingBoundAddress, existingBoundAddress.length + 1); - updatedBoundAddresses[updatedBoundAddresses.length - 1] = new InetSocketTransportAddress(boundAddress); - profileBoundAddresses.put(name, new BoundTransportAddress(updatedBoundAddresses, boundTransportAddress.publishAddress())); - } - } else { - if (this.boundAddress == null) { - // this is the first address that has been bound for the default profile so we get the publish address information and create a new BoundTransportAddress - // these calls are different from the profile ones due to the way the settings for a profile are created. If we want to merge the code for the default profile and - // other profiles together, we need to change how the profileSettings are built for the default profile... - int publishPort = settings.getAsInt("transport.netty.publish_port", settings.getAsInt("transport.publish_port", boundAddress.getPort())); - String publishHost = settings.get("transport.netty.publish_host", settings.get("transport.publish_host", settings.get("transport.host"))); - InetSocketAddress publishAddress = createPublishAddress(publishHost, publishPort); - this.boundAddress = new BoundTransportAddress(new TransportAddress[]{new InetSocketTransportAddress(boundAddress)}, new InetSocketTransportAddress(publishAddress)); - } else { - // the default profile is already bound to one address and has the publish address, copy the existing bound addresses as is and append the new address. - // the new bound address is appended in order to preserve the ordering/priority of bound addresses - TransportAddress[] existingBoundAddress = this.boundAddress.boundAddresses(); - TransportAddress[] updatedBoundAddresses = Arrays.copyOf(existingBoundAddress, existingBoundAddress.length + 1); - updatedBoundAddresses[updatedBoundAddresses.length - 1] = new InetSocketTransportAddress(boundAddress); - this.boundAddress = new BoundTransportAddress(updatedBoundAddresses, this.boundAddress.publishAddress()); - } - } - if (logger.isDebugEnabled()) { logger.debug("Bound profile [{}] to address {{}}", name, NetworkAddress.format(boundSocket.get())); } + + return boundSocket.get(); + } + + private BoundTransportAddress createBoundTransportAddress(String name, Settings profileSettings, List boundAddresses) { + String[] boundAddressesHostStrings = new String[boundAddresses.size()]; + TransportAddress[] transportBoundAddresses = new TransportAddress[boundAddresses.size()]; + for (int i = 0; i < boundAddresses.size(); i++) { + InetSocketAddress boundAddress = boundAddresses.get(i); + boundAddressesHostStrings[i] = boundAddress.getHostString(); + transportBoundAddresses[i] = new InetSocketTransportAddress(boundAddress); + } + + final String[] publishHosts; + if (DEFAULT_PROFILE.equals(name)) { + publishHosts = settings.getAsArray("transport.netty.publish_host", settings.getAsArray("transport.publish_host", settings.getAsArray("transport.host", null))); + } else { + publishHosts = profileSettings.getAsArray("publish_host", boundAddressesHostStrings); + } + + final InetAddress publishInetAddress; + try { + publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts); + } catch (Exception e) { + throw new BindTransportException("Failed to resolve publish address", e); + } + + Integer publishPort; + if (DEFAULT_PROFILE.equals(name)) { + publishPort = settings.getAsInt("transport.netty.publish_port", settings.getAsInt("transport.publish_port", null)); + } else { + publishPort = profileSettings.getAsInt("publish_port", null); + } + + // if port not explicitly provided, search for port of address in boundAddresses that matches publishInetAddress + if (publishPort == null) { + for (InetSocketAddress boundAddress : boundAddresses) { + InetAddress boundInetAddress = boundAddress.getAddress(); + if (boundInetAddress.isAnyLocalAddress() || boundInetAddress.equals(publishInetAddress)) { + publishPort = boundAddress.getPort(); + break; + } + } + } + + // if port still not matches, just take port of first bound address + if (publishPort == null) { + // TODO: In case of DEFAULT_PROFILE we should probably fail here, as publish address does not match any bound address + // In case of a custom profile, we might use the publish address of the default profile + publishPort = boundAddresses.get(0).getPort(); + logger.warn("Publish port not found by matching publish address [{}] to bound addresses [{}], falling back to port [{}] of first bound address", publishInetAddress, boundAddresses, publishPort); + } + + final TransportAddress publishAddress = new InetSocketTransportAddress(new InetSocketAddress(publishInetAddress, publishPort)); + return new BoundTransportAddress(transportBoundAddresses, publishAddress); } private void createServerBootstrap(String name, Settings settings) { @@ -742,6 +764,11 @@ public class NettyTransport extends AbstractLifecycleComponent implem // close the channel as safe measure, which will cause a node to be disconnected if relevant ctx.getChannel().close(); disconnectFromNodeChannel(ctx.getChannel(), e.getCause()); + } else if (e.getCause() instanceof BindException) { + logger.trace("bind exception caught on transport layer [{}]", e.getCause(), ctx.getChannel()); + // close the channel as safe measure, which will cause a node to be disconnected if relevant + ctx.getChannel().close(); + disconnectFromNodeChannel(ctx.getChannel(), e.getCause()); } else if (e.getCause() instanceof CancelledKeyException) { logger.trace("cancelled key exception caught on transport layer [{}], disconnecting from relevant node", e.getCause(), ctx.getChannel()); // close the channel as safe measure, which will cause a node to be disconnected if relevant @@ -794,7 +821,7 @@ public class NettyTransport extends AbstractLifecycleComponent implem Channel targetChannel = nodeChannel(node, options); if (compress) { - options.withCompress(true); + options = TransportRequestOptions.builder(options).withCompress(true).build(); } byte status = 0; diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java index e601d8016d2..fe3a941f665 100644 --- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java +++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java @@ -78,7 +78,7 @@ public class NettyTransportChannel implements TransportChannel { @Override public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException { if (transport.compress) { - options.withCompress(true); + options = TransportResponseOptions.builder(options).withCompress(transport.compress).build(); } byte status = 0; diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java b/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java new file mode 100644 index 00000000000..688dfe5a92d --- /dev/null +++ b/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tribe; + +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collections; + +/** + * An internal node that connects to a remove cluster, as part of a tribe node. + */ +class TribeClientNode extends Node { + TribeClientNode(Settings settings) { + super(new Environment(settings), Version.CURRENT, Collections.>emptyList()); + } +} diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java index 75b81762dd7..f577415ee6b 100644 --- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java +++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java @@ -20,7 +20,6 @@ package org.elasticsearch.tribe; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -46,8 +45,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.discovery.DiscoveryService; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.rest.RestStatus; import java.util.EnumSet; @@ -132,14 +129,14 @@ public class TribeService extends AbstractLifecycleComponent { nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client for (Map.Entry entry : nodesSettings.entrySet()) { Settings.Builder sb = Settings.builder().put(entry.getValue()); - sb.put("node.name", settings.get("name") + "/" + entry.getKey()); + sb.put("name", settings.get("name") + "/" + entry.getKey()); sb.put("path.home", settings.get("path.home")); // pass through ES home dir sb.put(TRIBE_NAME, entry.getKey()); - sb.put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true); if (sb.get("http.enabled") == null) { sb.put("http.enabled", false); } - nodes.add(NodeBuilder.nodeBuilder().settings(sb).client(true).build()); + sb.put("node.client", true); + nodes.add(new TribeClientNode(sb.build())); } String[] blockIndicesWrite = Strings.EMPTY_ARRAY; @@ -187,7 +184,7 @@ public class TribeService extends AbstractLifecycleComponent { if (e instanceof RuntimeException) { throw (RuntimeException) e; } - throw new ElasticsearchException(e.getMessage(), e); + throw new ElasticsearchException(e); } } } @@ -261,17 +258,17 @@ public class TribeService extends AbstractLifecycleComponent { RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable()); // go over existing indices, and see if they need to be removed for (IndexMetaData index : currentState.metaData()) { - String markedTribeName = index.settings().get(TRIBE_NAME); + String markedTribeName = index.getSettings().get(TRIBE_NAME); if (markedTribeName != null && markedTribeName.equals(tribeName)) { - IndexMetaData tribeIndex = tribeState.metaData().index(index.index()); - if (tribeIndex == null || tribeIndex.state() == IndexMetaData.State.CLOSE) { - logger.info("[{}] removing index [{}]", tribeName, index.index()); + IndexMetaData tribeIndex = tribeState.metaData().index(index.getIndex()); + if (tribeIndex == null || tribeIndex.getState() == IndexMetaData.State.CLOSE) { + logger.info("[{}] removing index [{}]", tribeName, index.getIndex()); removeIndex(blocks, metaData, routingTable, index); } else { // always make sure to update the metadata and routing table, in case // there are changes in them (new mapping, shards moving from initializing to started) - routingTable.add(tribeState.routingTable().index(index.index())); - Settings tribeSettings = Settings.builder().put(tribeIndex.settings()).put(TRIBE_NAME, tribeName).build(); + routingTable.add(tribeState.routingTable().index(index.getIndex())); + Settings tribeSettings = Settings.builder().put(tribeIndex.getSettings()).put(TRIBE_NAME, tribeName).build(); metaData.put(IndexMetaData.builder(tribeIndex).settings(tribeSettings)); } } @@ -279,15 +276,15 @@ public class TribeService extends AbstractLifecycleComponent { // go over tribe one, and see if they need to be added for (IndexMetaData tribeIndex : tribeState.metaData()) { // if there is no routing table yet, do nothing with it... - IndexRoutingTable table = tribeState.routingTable().index(tribeIndex.index()); + IndexRoutingTable table = tribeState.routingTable().index(tribeIndex.getIndex()); if (table == null) { continue; } - final IndexMetaData indexMetaData = currentState.metaData().index(tribeIndex.index()); + final IndexMetaData indexMetaData = currentState.metaData().index(tribeIndex.getIndex()); if (indexMetaData == null) { - if (!droppedIndices.contains(tribeIndex.index())) { + if (!droppedIndices.contains(tribeIndex.getIndex())) { // a new index, add it, and add the tribe name as a setting - logger.info("[{}] adding index [{}]", tribeName, tribeIndex.index()); + logger.info("[{}] adding index [{}]", tribeName, tribeIndex.getIndex()); addNewIndex(tribeState, blocks, metaData, routingTable, tribeIndex); } } else { @@ -298,15 +295,15 @@ public class TribeService extends AbstractLifecycleComponent { // we chose any tribe, carry on } else if (ON_CONFLICT_DROP.equals(onConflict)) { // drop the indices, there is a conflict - logger.info("[{}] dropping index [{}] due to conflict with [{}]", tribeName, tribeIndex.index(), existingFromTribe); + logger.info("[{}] dropping index [{}] due to conflict with [{}]", tribeName, tribeIndex.getIndex(), existingFromTribe); removeIndex(blocks, metaData, routingTable, tribeIndex); - droppedIndices.add(tribeIndex.index()); + droppedIndices.add(tribeIndex.getIndex()); } else if (onConflict.startsWith(ON_CONFLICT_PREFER)) { // on conflict, prefer a tribe... String preferredTribeName = onConflict.substring(ON_CONFLICT_PREFER.length()); if (tribeName.equals(preferredTribeName)) { // the new one is hte preferred one, replace... - logger.info("[{}] adding index [{}], preferred over [{}]", tribeName, tribeIndex.index(), existingFromTribe); + logger.info("[{}] adding index [{}], preferred over [{}]", tribeName, tribeIndex.getIndex(), existingFromTribe); removeIndex(blocks, metaData, routingTable, tribeIndex); addNewIndex(tribeState, blocks, metaData, routingTable, tribeIndex); } // else: either the existing one is the preferred one, or we haven't seen one, carry on @@ -319,23 +316,23 @@ public class TribeService extends AbstractLifecycleComponent { } private void removeIndex(ClusterBlocks.Builder blocks, MetaData.Builder metaData, RoutingTable.Builder routingTable, IndexMetaData index) { - metaData.remove(index.index()); - routingTable.remove(index.index()); - blocks.removeIndexBlocks(index.index()); + metaData.remove(index.getIndex()); + routingTable.remove(index.getIndex()); + blocks.removeIndexBlocks(index.getIndex()); } private void addNewIndex(ClusterState tribeState, ClusterBlocks.Builder blocks, MetaData.Builder metaData, RoutingTable.Builder routingTable, IndexMetaData tribeIndex) { - Settings tribeSettings = Settings.builder().put(tribeIndex.settings()).put(TRIBE_NAME, tribeName).build(); + Settings tribeSettings = Settings.builder().put(tribeIndex.getSettings()).put(TRIBE_NAME, tribeName).build(); metaData.put(IndexMetaData.builder(tribeIndex).settings(tribeSettings)); - routingTable.add(tribeState.routingTable().index(tribeIndex.index())); - if (Regex.simpleMatch(blockIndicesMetadata, tribeIndex.index())) { - blocks.addIndexBlock(tribeIndex.index(), IndexMetaData.INDEX_METADATA_BLOCK); + routingTable.add(tribeState.routingTable().index(tribeIndex.getIndex())); + if (Regex.simpleMatch(blockIndicesMetadata, tribeIndex.getIndex())) { + blocks.addIndexBlock(tribeIndex.getIndex(), IndexMetaData.INDEX_METADATA_BLOCK); } - if (Regex.simpleMatch(blockIndicesRead, tribeIndex.index())) { - blocks.addIndexBlock(tribeIndex.index(), IndexMetaData.INDEX_READ_BLOCK); + if (Regex.simpleMatch(blockIndicesRead, tribeIndex.getIndex())) { + blocks.addIndexBlock(tribeIndex.getIndex(), IndexMetaData.INDEX_READ_BLOCK); } - if (Regex.simpleMatch(blockIndicesWrite, tribeIndex.index())) { - blocks.addIndexBlock(tribeIndex.index(), IndexMetaData.INDEX_WRITE_BLOCK); + if (Regex.simpleMatch(blockIndicesWrite, tribeIndex.getIndex())) { + blocks.addIndexBlock(tribeIndex.getIndex(), IndexMetaData.INDEX_WRITE_BLOCK); } } diff --git a/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat index 06b50d314be..2c92f0ecd3f 100644 --- a/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat +++ b/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat @@ -1 +1 @@ -org.elasticsearch.search.suggest.completion.Completion090PostingsFormat \ No newline at end of file +org.apache.lucene.search.suggest.document.Completion50PostingsFormat diff --git a/core/src/main/resources/es-build.properties b/core/src/main/resources/es-build.properties deleted file mode 100644 index 563ecddcaf6..00000000000 --- a/core/src/main/resources/es-build.properties +++ /dev/null @@ -1,3 +0,0 @@ -version=${project.version} -hash=${buildNumber} -timestamp=${timestamp} diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 7e7f347ce1b..26785010110 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -19,56 +19,24 @@ // Default security policy file. // On startup, BootStrap reads environment and adds additional permissions -// for configured paths to these. +// for configured paths and network binding to these. -//// System code permissions: -//// These permissions apply to the JDK itself: +//// SecurityManager impl: +//// Must have all permissions to properly perform access checks -grant codeBase "file:${{java.ext.dirs}}/*" { +grant codeBase "${codebase.securesm-1.0.jar}" { permission java.security.AllPermission; }; //// Very special jar permissions: //// These are dangerous permissions that we don't want to grant to everything. -grant codeBase "${es.security.jar.lucene.core}" { +grant codeBase "${codebase.lucene-core-5.5.0-snapshot-1719088.jar}" { // needed to allow MMapDirectory's "unmap hack" permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; -}; - -//// test framework permissions. -//// These are mock objects and test management that we allow test framework libs -//// to provide on our behalf. But tests themselves cannot do this stuff! - -grant codeBase "${es.security.jar.elasticsearch.securemock}" { - // needed to access ReflectionFactory (see below) - permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect"; - // needed to support creation of mocks - permission java.lang.RuntimePermission "reflectionFactoryAccess"; - // needed for spy interception, etc - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; -}; - -grant codeBase "${es.security.jar.lucene.testframework}" { - // needed by RamUsageTester - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; -}; - -grant codeBase "${es.security.jar.randomizedtesting.runner}" { - // optionally needed for access to private test methods (e.g. beforeClass) - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; - - // needed for top threads handling - permission java.lang.RuntimePermission "modifyThreadGroup"; -}; - -grant codeBase "${es.security.jar.randomizedtesting.junit4}" { - // needed for gson serialization - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; - - // needed for stream redirection - permission java.lang.RuntimePermission "setIO"; + // NOTE: also needed for RAMUsageEstimator size calculations + permission java.lang.RuntimePermission "accessDeclaredMembers"; }; //// Everything else: @@ -80,37 +48,61 @@ grant { permission org.elasticsearch.SpecialPermission; // Allow connecting to the internet anywhere - permission java.net.SocketPermission "*", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "*", "accept,connect,resolve"; - // Allow read/write to all system properties - permission java.util.PropertyPermission "*", "read,write"; + // Allow read access to all system properties + permission java.util.PropertyPermission "*", "read"; - // needed by lucene SPI currently - permission java.lang.RuntimePermission "getClassLoader"; + // TODO: clean all these property writes up, and don't allow any more in. these are all bogus! + + // LuceneTestCase randomization (locale/timezone/cpus/ssd) + // TODO: put these in doPrivileged and move these to test-framework.policy + permission java.util.PropertyPermission "user.language", "write"; + permission java.util.PropertyPermission "user.timezone", "write"; + permission java.util.PropertyPermission "lucene.cms.override_core_count", "write"; + permission java.util.PropertyPermission "lucene.cms.override_spins", "write"; + // messiness in LuceneTestCase: do the above, or clean this up, or simply allow to fail if its denied + permission java.util.PropertyPermission "solr.solr.home", "write"; + permission java.util.PropertyPermission "solr.data.dir", "write"; + permission java.util.PropertyPermission "solr.directoryFactory", "write"; + + // set by ESTestCase to improve test reproducibility + // TODO: set this with gradle or some other way that repros with seed? + permission java.util.PropertyPermission "es.processors.override", "write"; + // set by CLIToolTestCase + // TODO: do this differently? or test commandline tools differently? + permission java.util.PropertyPermission "es.default.path.home", "write"; + + // TODO: these simply trigger a noisy warning if its unable to clear the properties + // fix that in randomizedtesting + permission java.util.PropertyPermission "junit4.childvm.count", "write"; + permission java.util.PropertyPermission "junit4.childvm.id", "write"; + + // set by NettyTransport/NettyHttpServerTransport based on another parameter + // TODO: look into this and decide if users should simply set the actual sysprop?! + permission java.util.PropertyPermission "org.jboss.netty.epollBugWorkaround", "write"; + + // Netty SelectorUtil wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 + // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! + permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; // needed by Settings permission java.lang.RuntimePermission "getenv.*"; - // needed by ES threadpool termination... clean this up - // otherwise can be provided only to test libraries + // thread permission for the same thread group and ancestor groups + // (this logic is more strict than the JDK, see SecureSM) permission java.lang.RuntimePermission "modifyThread"; + permission java.lang.RuntimePermission "modifyThreadGroup"; // needed by ExceptionSerializationTests and RestTestCase for // some hackish things they do. otherwise only needed by groovy // (TODO: clean this up?) permission java.lang.RuntimePermission "getProtectionDomain"; - // likely not low hanging fruit... - permission java.lang.RuntimePermission "accessDeclaredMembers"; - // needed by HotThreads and potentially more // otherwise can be provided only to test libraries permission java.lang.RuntimePermission "getStackTrace"; - // needed by ESTestCase for leniency of thread exceptions (?!) - // otherwise can be provided only to test libraries - permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler"; - // needed by JMX instead of getFileSystemAttributes, seems like a bug... permission java.lang.RuntimePermission "getFileStoreAttributes"; @@ -118,18 +110,9 @@ grant { // otherwise can be provided only to test libraries permission java.lang.RuntimePermission "fileSystemProvider"; - // needed by plugin manager to set unix permissions - permission java.lang.RuntimePermission "accessUserInformation"; - // needed by jvminfo for monitoring the jvm permission java.lang.management.ManagementPermission "monitor"; // needed by JDKESLoggerTests permission java.util.logging.LoggingPermission "control"; - - // needed to install SSLFactories, advanced SSL configuration, etc. - permission java.lang.RuntimePermission "setFactory"; - - // needed to allow installation of bouncycastle crypto provider - permission java.security.SecurityPermission "putProviderProperty.BC"; }; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy new file mode 100644 index 00000000000..b5f9c24d04f --- /dev/null +++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//// additional test framework permissions. +//// These are mock objects and test management that we allow test framework libs +//// to provide on our behalf. But tests themselves cannot do this stuff! + +grant codeBase "${codebase.securemock-1.2.jar}" { + // needed to access ReflectionFactory (see below) + permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect"; + // needed to support creation of mocks + permission java.lang.RuntimePermission "reflectionFactoryAccess"; + // needed for spy interception, etc + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +grant codeBase "${codebase.lucene-test-framework-5.5.0-snapshot-1719088.jar}" { + // needed by RamUsageTester + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +grant codeBase "${codebase.randomizedtesting-runner-2.3.2.jar}" { + // optionally needed for access to private test methods (e.g. beforeClass) + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + // needed to fail tests on uncaught exceptions from other threads + permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler"; + // needed for top threads handling + permission org.elasticsearch.ThreadPermission "modifyArbitraryThreadGroup"; + // needed for TestClass creation + permission java.lang.RuntimePermission "accessDeclaredMembers"; +}; + +grant codeBase "${codebase.junit4-ant-2.3.2.jar}" { + // needed for stream redirection + permission java.lang.RuntimePermission "setIO"; +}; + +grant codeBase "${codebase.junit-4.11.jar}" { + // needed for TestClass creation + permission java.lang.RuntimePermission "accessDeclaredMembers"; +}; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy b/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy index 2475c56e814..8e7ca8d8b6e 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy @@ -19,13 +19,21 @@ /* * Limited security policy for scripts. - * This is what is needed for invokeDynamic functionality to work. + * This is what is needed for basic functionality to work. */ grant { // groovy IndyInterface bootstrap requires this property for indy logging permission java.util.PropertyPermission "groovy.indy.logging", "read"; + // groovy JsonOutput, just allow it to read these props so it works (unsafe is not allowed) + permission java.util.PropertyPermission "groovy.json.faststringutils.disable", "read"; + permission java.util.PropertyPermission "groovy.json.faststringutils.write.to.final.fields", "read"; + + // needed by Rhino engine exception handling + permission java.util.PropertyPermission "rhino.stack.style", "read"; + // needed IndyInterface selectMethod (setCallSiteTarget) + // TODO: clean this up / only give it to engines that really must have it permission java.lang.RuntimePermission "getClassLoader"; }; diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help index 7486d98bcb6..8c73e3837a4 100644 --- a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help +++ b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help @@ -43,13 +43,14 @@ OFFICIAL PLUGINS - discovery-ec2 - discovery-gce - discovery-multicast - - lang-expression - - lang-groovy - lang-javascript + - lang-plan-a - lang-python + - mapper-attachments - mapper-murmur3 - mapper-size - repository-azure + - repository-hdfs - repository-s3 - store-smb diff --git a/core/src/test/eclipse-build.gradle b/core/src/test/eclipse-build.gradle new file mode 100644 index 00000000000..f180aec4287 --- /dev/null +++ b/core/src/test/eclipse-build.gradle @@ -0,0 +1,7 @@ + +// this is just shell gradle file for eclipse to have separate projects for core src and tests +apply from: '../../build.gradle' + +dependencies { + testCompile project(':core') +} diff --git a/core/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java b/core/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java index 6a563afc002..3c77142221d 100644 --- a/core/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java +++ b/core/src/test/java/org/apache/lucene/analysis/miscellaneous/TruncateTokenFilterTests.java @@ -25,7 +25,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -34,9 +33,7 @@ import static org.hamcrest.Matchers.equalTo; */ public class TruncateTokenFilterTests extends ESTestCase { - - @Test - public void simpleTest() throws IOException { + public void testSimple() throws IOException { Analyzer analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { diff --git a/core/src/test/java/org/apache/lucene/analysis/miscellaneous/UniqueTokenFilterTests.java b/core/src/test/java/org/apache/lucene/analysis/miscellaneous/UniqueTokenFilterTests.java index 25c4a688fe0..7756933a781 100644 --- a/core/src/test/java/org/apache/lucene/analysis/miscellaneous/UniqueTokenFilterTests.java +++ b/core/src/test/java/org/apache/lucene/analysis/miscellaneous/UniqueTokenFilterTests.java @@ -25,7 +25,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -34,9 +33,7 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class UniqueTokenFilterTests extends ESTestCase { - - @Test - public void simpleTest() throws IOException { + public void testSimple() throws IOException { Analyzer analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { diff --git a/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java b/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java index 91eaeb2607b..a287ec119e7 100644 --- a/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java +++ b/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java @@ -31,7 +31,6 @@ import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.TestUtil; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.*; @@ -42,8 +41,6 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class BlendedTermQueryTests extends ESTestCase { - - @Test public void testBooleanQuery() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); @@ -97,7 +94,6 @@ public class BlendedTermQueryTests extends ESTestCase { } - @Test public void testDismaxQuery() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); @@ -171,7 +167,6 @@ public class BlendedTermQueryTests extends ESTestCase { dir.close(); } - @Test public void testBasics() { final int iters = scaledRandomIntBetween(5, 25); for (int j = 0; j < iters; j++) { @@ -209,7 +204,6 @@ public class BlendedTermQueryTests extends ESTestCase { return searcher; } - @Test public void testExtractTerms() throws IOException { Set terms = new HashSet<>(); int num = scaledRandomIntBetween(1, 10); diff --git a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java index dc176ae5620..fcddc58f77a 100644 --- a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java +++ b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java @@ -23,16 +23,12 @@ import org.apache.lucene.search.highlight.DefaultEncoder; import org.apache.lucene.search.highlight.SimpleHTMLEncoder; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.MatcherAssert.assertThat; public class CustomPassageFormatterTests extends ESTestCase { - - @Test public void testSimpleFormat() { String content = "This is a really cool highlighter. Postings highlighter gives nice snippets back. No matches here."; @@ -74,7 +70,6 @@ public class CustomPassageFormatterTests extends ESTestCase { assertThat(fragments[2].isHighlighted(), equalTo(false)); } - @Test public void testHtmlEncodeFormat() { String content = "This is a really cool highlighter. Postings highlighter gives nice snippets back."; diff --git a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java index 58728d8e258..737b3df41ac 100644 --- a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java +++ b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java @@ -24,21 +24,25 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.DefaultEncoder; import org.apache.lucene.store.Directory; import org.elasticsearch.search.highlight.HighlightUtils; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.CoreMatchers.equalTo; public class CustomPostingsHighlighterTests extends ESTestCase { - - @Test public void testCustomPostingsHighlighter() throws Exception { - Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setMergePolicy(newLogMergePolicy()); @@ -106,7 +110,6 @@ public class CustomPostingsHighlighterTests extends ESTestCase { dir.close(); } - @Test public void testNoMatchSize() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); diff --git a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIteratorTests.java b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIteratorTests.java index 1be578f1003..ac3a24346ac 100644 --- a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIteratorTests.java +++ b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIteratorTests.java @@ -21,7 +21,6 @@ package org.apache.lucene.search.postingshighlight; import org.elasticsearch.search.highlight.HighlightUtils; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.text.BreakIterator; import java.text.CharacterIterator; @@ -31,8 +30,6 @@ import java.util.Locale; import static org.hamcrest.CoreMatchers.equalTo; public class CustomSeparatorBreakIteratorTests extends ESTestCase { - - @Test public void testBreakOnCustomSeparator() throws Exception { Character separator = randomSeparator(); BreakIterator bi = new CustomSeparatorBreakIterator(separator); @@ -69,7 +66,6 @@ public class CustomSeparatorBreakIteratorTests extends ESTestCase { assertThat(source.substring(0, bi.next(3)), equalTo("this" + separator + "is" + separator + "the" + separator)); } - @Test public void testSingleSentences() throws Exception { BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT); BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator()); @@ -79,7 +75,6 @@ public class CustomSeparatorBreakIteratorTests extends ESTestCase { assertSameBreaks("", expected, actual); } - @Test public void testSliceEnd() throws Exception { BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT); BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator()); @@ -89,7 +84,6 @@ public class CustomSeparatorBreakIteratorTests extends ESTestCase { assertSameBreaks("000", 0, 0, expected, actual); } - @Test public void testSliceStart() throws Exception { BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT); BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator()); @@ -99,7 +93,6 @@ public class CustomSeparatorBreakIteratorTests extends ESTestCase { assertSameBreaks("000", 3, 0, expected, actual); } - @Test public void testSliceMiddle() throws Exception { BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT); BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator()); @@ -110,7 +103,6 @@ public class CustomSeparatorBreakIteratorTests extends ESTestCase { } /** the current position must be ignored, initial position is always first() */ - @Test public void testFirstPosition() throws Exception { BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT); BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator()); diff --git a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java b/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java index f7d43fd1305..abfc7c005e3 100644 --- a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java +++ b/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java @@ -22,13 +22,10 @@ package org.apache.lucene.util; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.number.IsCloseTo.closeTo; public class SloppyMathTests extends ESTestCase { - - @Test public void testAccuracy() { for (double lat1 = -89; lat1 <= 89; lat1+=1) { final double lon1 = randomLongitude(); @@ -42,7 +39,6 @@ public class SloppyMathTests extends ESTestCase { } } - @Test public void testSloppyMath() { testSloppyMath(DistanceUnit.METERS, 0.01, 5, 45, 90); testSloppyMath(DistanceUnit.KILOMETERS, 0.01, 5, 45, 90); @@ -53,7 +49,7 @@ public class SloppyMathTests extends ESTestCase { private static double maxError(double distance) { return distance / 1000.0; } - + private void testSloppyMath(DistanceUnit unit, double...deltaDeg) { final double lat1 = randomLatitude(); final double lon1 = randomLongitude(); @@ -68,12 +64,12 @@ public class SloppyMathTests extends ESTestCase { final double accurate = GeoDistance.ARC.calculate(lat1, lon1, lat2, lon2, unit); final double dist = GeoDistance.SLOPPY_ARC.calculate(lat1, lon1, lat2, lon2, unit); - + assertThat("distance between("+lat1+", "+lon1+") and ("+lat2+", "+lon2+"))", dist, closeTo(accurate, maxError(accurate))); } } } - + private static void assertAccurate(double lat1, double lon1, double lat2, double lon2) { double accurate = GeoDistance.ARC.calculate(lat1, lon1, lat2, lon2, DistanceUnit.METERS); double sloppy = GeoDistance.SLOPPY_ARC.calculate(lat1, lon1, lat2, lon2, DistanceUnit.METERS); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/PayloadProcessor.java b/core/src/test/java/org/elasticsearch/BuildTests.java similarity index 55% rename from core/src/main/java/org/elasticsearch/search/suggest/completion/PayloadProcessor.java rename to core/src/test/java/org/elasticsearch/BuildTests.java index 544d9052a0e..d55f5bb9760 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/PayloadProcessor.java +++ b/core/src/test/java/org/elasticsearch/BuildTests.java @@ -17,22 +17,23 @@ * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.nio.file.AccessMode; +import java.nio.file.Path; -interface PayloadProcessor { +public class BuildTests extends ESTestCase { - BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException; - - void parsePayload(BytesRef payload, SuggestPayload ref) throws IOException; - - static class SuggestPayload { - final BytesRefBuilder payload = new BytesRefBuilder(); - long weight = 0; - final BytesRefBuilder surfaceForm = new BytesRefBuilder(); + /** Asking for the jar metadata should not throw exception in tests, no matter how configured */ + public void testJarMetadata() throws IOException { + Path path = Build.getElasticsearchCodebase(); + // throws exception if does not exist, or we cannot access it + path.getFileSystem().provider().checkAccess(path, AccessMode.READ); + // these should never be null + assertNotNull(Build.CURRENT.date()); + assertNotNull(Build.CURRENT.shortHash()); } } diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java index 5bb3bf4c13a..91be1f339dd 100644 --- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java +++ b/core/src/test/java/org/elasticsearch/ESExceptionTests.java @@ -37,7 +37,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.SearchShardTarget; @@ -47,7 +47,6 @@ import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.transport.RemoteTransportException; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.EOFException; import java.io.FileNotFoundException; @@ -59,7 +58,6 @@ import static org.hamcrest.Matchers.equalTo; public class ESExceptionTests extends ESTestCase { private static final ToXContent.Params PARAMS = ToXContent.EMPTY_PARAMS; - @Test public void testStatus() { ElasticsearchException exception = new ElasticsearchException("test"); assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); @@ -142,7 +140,7 @@ public class ESExceptionTests extends ESTestCase { new SearchShardTarget("node_1", "foo", 1)); ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), new SearchShardTarget("node_1", "foo", 2)); - SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1}); + SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", randomBoolean() ? failure1.getCause() : failure.getCause(), new ShardSearchFailure[]{failure, failure1}); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); ex.toXContent(builder, PARAMS); @@ -165,6 +163,21 @@ public class ESExceptionTests extends ESTestCase { String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}},{\"shard\":1,\"index\":\"foo1\",\"node\":\"node_1\",\"reason\":{\"type\":\"query_shard_exception\",\"reason\":\"foobar\",\"index\":\"foo1\"}}]}"; assertEquals(expected, builder.string()); } + { + ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), + new SearchShardTarget("node_1", "foo", 1)); + ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), + new SearchShardTarget("node_1", "foo", 2)); + NullPointerException nullPointerException = new NullPointerException(); + SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", nullPointerException, new ShardSearchFailure[]{failure, failure1}); + assertEquals(nullPointerException, ex.getCause()); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + ex.toXContent(builder, PARAMS); + builder.endObject(); + String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}],\"caused_by\":{\"type\":\"null_pointer_exception\",\"reason\":null}}"; + assertEquals(expected, builder.string()); + } } public void testGetRootCause() { @@ -323,16 +336,11 @@ public class ESExceptionTests extends ESTestCase { } else { assertEquals(e.getCause().getClass(), NotSerializableExceptionWrapper.class); } - // TODO: fix this test - // on java 9, expected: - // but was: - if (!Constants.JRE_IS_MINIMUM_JAVA9) { - assertArrayEquals(e.getStackTrace(), ex.getStackTrace()); - } + assertArrayEquals(e.getStackTrace(), ex.getStackTrace()); assertTrue(e.getStackTrace().length > 1); ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(getRandom()), t); ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(getRandom()), ex); ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(getRandom()), e); } } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 55dc2e42113..46cdea3dadf 100644 --- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.AlreadyExpiredException; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.IndexFailedEngineException; import org.elasticsearch.index.engine.RecoveryEngineException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShardState; @@ -137,7 +136,7 @@ public class ExceptionSerializationTests extends ESTestCase { } else if (ElasticsearchException.isRegistered((Class) clazz)) { registered.add(clazz); try { - if (clazz.getDeclaredMethod("writeTo", StreamOutput.class) != null) { + if (clazz.getMethod("writeTo", StreamOutput.class) != null) { hasDedicatedWrite.add(clazz); } } catch (Exception e) { @@ -275,11 +274,6 @@ public class ExceptionSerializationTests extends ESTestCase { assertEquals(-3, alreadyExpiredException.now()); } - public void testMergeMappingException() throws IOException { - MergeMappingException ex = serialize(new MergeMappingException(new String[]{"one", "two"})); - assertArrayEquals(ex.failures(), new String[]{"one", "two"}); - } - public void testActionNotFoundTransportException() throws IOException { ActionNotFoundTransportException ex = serialize(new ActionNotFoundTransportException("AACCCTION")); assertEquals("AACCCTION", ex.action()); @@ -561,15 +555,12 @@ public class ExceptionSerializationTests extends ESTestCase { } Throwable deserialized = serialize(t); assertTrue(deserialized instanceof NotSerializableExceptionWrapper); - // TODO: fix this test for more java 9 differences - if (!Constants.JRE_IS_MINIMUM_JAVA9) { - assertArrayEquals(t.getStackTrace(), deserialized.getStackTrace()); - assertEquals(t.getSuppressed().length, deserialized.getSuppressed().length); - if (t.getSuppressed().length > 0) { - assertTrue(deserialized.getSuppressed()[0] instanceof NotSerializableExceptionWrapper); - assertArrayEquals(t.getSuppressed()[0].getStackTrace(), deserialized.getSuppressed()[0].getStackTrace()); - assertTrue(deserialized.getSuppressed()[1] instanceof NullPointerException); - } + assertArrayEquals(t.getStackTrace(), deserialized.getStackTrace()); + assertEquals(t.getSuppressed().length, deserialized.getSuppressed().length); + if (t.getSuppressed().length > 0) { + assertTrue(deserialized.getSuppressed()[0] instanceof NotSerializableExceptionWrapper); + assertArrayEquals(t.getSuppressed()[0].getStackTrace(), deserialized.getSuppressed()[0].getStackTrace()); + assertTrue(deserialized.getSuppressed()[1] instanceof NullPointerException); } } } @@ -728,7 +719,6 @@ public class ExceptionSerializationTests extends ESTestCase { ids.put(84, org.elasticsearch.transport.NodeDisconnectedException.class); ids.put(85, org.elasticsearch.index.AlreadyExpiredException.class); ids.put(86, org.elasticsearch.search.aggregations.AggregationExecutionException.class); - ids.put(87, org.elasticsearch.index.mapper.MergeMappingException.class); ids.put(88, org.elasticsearch.indices.InvalidIndexTemplateException.class); ids.put(89, org.elasticsearch.percolator.PercolateException.class); ids.put(90, org.elasticsearch.index.engine.RefreshFailedEngineException.class); diff --git a/core/src/test/java/org/elasticsearch/NamingConventionTests.java b/core/src/test/java/org/elasticsearch/NamingConventionTests.java index 40868fc281c..912f8922b07 100644 --- a/core/src/test/java/org/elasticsearch/NamingConventionTests.java +++ b/core/src/test/java/org/elasticsearch/NamingConventionTests.java @@ -19,15 +19,14 @@ package org.elasticsearch; import junit.framework.TestCase; + import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; -import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.net.URISyntaxException; import java.nio.file.FileVisitResult; @@ -91,7 +90,7 @@ public class NamingConventionTests extends ESTestCase { } else if (Modifier.isAbstract(clazz.getModifiers()) == false && Modifier.isInterface(clazz.getModifiers()) == false) { if (isTestCase(clazz)) { missingSuffix.add(clazz); - } else if (junit.framework.Test.class.isAssignableFrom(clazz) || hasTestAnnotation(clazz)) { + } else if (junit.framework.Test.class.isAssignableFrom(clazz)) { pureUnitTest.add(clazz); } } @@ -102,16 +101,6 @@ public class NamingConventionTests extends ESTestCase { return FileVisitResult.CONTINUE; } - private boolean hasTestAnnotation(Class clazz) { - for (Method method : clazz.getDeclaredMethods()) { - if (method.getAnnotation(Test.class) != null) { - return true; - } - } - return false; - - } - private boolean isTestCase(Class clazz) { return LuceneTestCase.class.isAssignableFrom(clazz); } @@ -145,7 +134,6 @@ public class NamingConventionTests extends ESTestCase { assertTrue(innerClasses.remove(InnerTests.class)); assertTrue(notImplementing.remove(NotImplementingTests.class)); assertTrue(pureUnitTest.remove(PlainUnit.class)); - assertTrue(pureUnitTest.remove(PlainUnitTheSecond.class)); String classesToSubclass = String.join( ",", @@ -187,11 +175,4 @@ public class NamingConventionTests extends ESTestCase { public static final class WrongNameTheSecond extends ESTestCase {} public static final class PlainUnit extends TestCase {} - - public static final class PlainUnitTheSecond { - @Test - public void foo() { - } - } - } diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java index 3adb6d98c0c..52508f8dc83 100644 --- a/core/src/test/java/org/elasticsearch/VersionTests.java +++ b/core/src/test/java/org/elasticsearch/VersionTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; -import org.junit.Test; import java.lang.reflect.Modifier; import java.util.HashMap; @@ -36,6 +35,7 @@ import static org.elasticsearch.Version.V_0_20_0; import static org.elasticsearch.Version.V_0_90_0; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; @@ -102,24 +102,41 @@ public class VersionTests extends ESTestCase { } } - @Test(expected = IllegalArgumentException.class) public void testTooLongVersionFromString() { - Version.fromString("1.0.0.1.3"); + try { + Version.fromString("1.0.0.1.3"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision")); + } } - @Test(expected = IllegalArgumentException.class) public void testTooShortVersionFromString() { - Version.fromString("1.0"); + try { + Version.fromString("1.0"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision")); + } + } - @Test(expected = IllegalArgumentException.class) public void testWrongVersionFromString() { - Version.fromString("WRONG.VERSION"); + try { + Version.fromString("WRONG.VERSION"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("needs to contain major, minor, and revision")); + } } - @Test(expected = IllegalStateException.class) public void testVersionNoPresentInSettings() { - Version.indexCreated(Settings.builder().build()); + try { + Version.indexCreated(Settings.builder().build()); + fail("Expected IllegalArgumentException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), containsString("[index.version.created] is not present")); + } } public void testIndexCreatedVersion() { @@ -174,7 +191,7 @@ public class VersionTests extends ESTestCase { public void testAllVersionsMatchId() throws Exception { Map maxBranchVersions = new HashMap<>(); - for (java.lang.reflect.Field field : Version.class.getDeclaredFields()) { + for (java.lang.reflect.Field field : Version.class.getFields()) { if (field.getName().endsWith("_ID")) { assertTrue(field.getName() + " should be static", Modifier.isStatic(field.getModifiers())); assertTrue(field.getName() + " should be final", Modifier.isFinal(field.getModifiers())); diff --git a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java index 7d52c6c77ca..f68cb76c955 100644 --- a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java +++ b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -33,10 +32,7 @@ import java.util.concurrent.atomic.AtomicReference; /** */ public class ListenerActionIT extends ESIntegTestCase { - - @Test - public void verifyThreadedListeners() throws Throwable { - + public void testThreadedListeners() throws Throwable { final CountDownLatch latch = new CountDownLatch(1); final AtomicReference failure = new AtomicReference<>(); final AtomicReference threadName = new AtomicReference<>(); diff --git a/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java b/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java index becdb933867..a19905c99d8 100644 --- a/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java +++ b/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -36,7 +35,6 @@ public class OriginalIndicesTests extends ESTestCase { IndicesOptions.lenientExpandOpen() , IndicesOptions.strictExpand(), IndicesOptions.strictExpandOpen(), IndicesOptions.strictExpandOpenAndForbidClosed(), IndicesOptions.strictSingleIndexNoExpandForbidClosed()}; - @Test public void testOriginalIndicesSerialization() throws IOException { int iterations = iterations(10, 30); for (int i = 0; i < iterations; i++) { diff --git a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java index 3c59f677f55..fb0283db48f 100644 --- a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java +++ b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Locale; import java.util.concurrent.CopyOnWriteArrayList; @@ -56,8 +55,7 @@ public class RejectionActionIT extends ESIntegTestCase { } - @Test - public void simulateSearchRejectionLoad() throws Throwable { + public void testSimulatedSearchRejectionLoad() throws Throwable { for (int i = 0; i < 10; i++) { client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "1").get(); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java b/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java index 47fcdff2774..a6217d7ea64 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsReq import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -41,8 +40,6 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.lessThan; public class HotThreadsIT extends ESIntegTestCase { - - @Test public void testHotThreadsDontFail() throws ExecutionException, InterruptedException { /** * This test just checks if nothing crashes or gets stuck etc. diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java index 86ead20d414..a4d089c0f82 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java @@ -19,179 +19,28 @@ package org.elasticsearch.action.admin.cluster.health; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.*; public class ClusterHealthResponsesTests extends ESTestCase { - private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); - - private void assertIndexHealth(ClusterIndexHealth indexHealth, ShardCounter counter, IndexMetaData indexMetaData) { - assertThat(indexHealth.getStatus(), equalTo(counter.status())); - assertThat(indexHealth.getNumberOfShards(), equalTo(indexMetaData.getNumberOfShards())); - assertThat(indexHealth.getNumberOfReplicas(), equalTo(indexMetaData.getNumberOfReplicas())); - assertThat(indexHealth.getActiveShards(), equalTo(counter.active)); - assertThat(indexHealth.getRelocatingShards(), equalTo(counter.relocating)); - assertThat(indexHealth.getInitializingShards(), equalTo(counter.initializing)); - assertThat(indexHealth.getUnassignedShards(), equalTo(counter.unassigned)); - assertThat(indexHealth.getShards().size(), equalTo(indexMetaData.getNumberOfShards())); - assertThat(indexHealth.getValidationFailures(), empty()); - int totalShards = 0; - for (ClusterShardHealth shardHealth : indexHealth.getShards().values()) { - totalShards += shardHealth.getActiveShards() + shardHealth.getInitializingShards() + shardHealth.getUnassignedShards(); - } - - assertThat(totalShards, equalTo(indexMetaData.getNumberOfShards() * (1 + indexMetaData.getNumberOfReplicas()))); - } - - protected class ShardCounter { - public int active; - public int relocating; - public int initializing; - public int unassigned; - public int primaryActive; - public int primaryInactive; - - public ClusterHealthStatus status() { - if (primaryInactive > 0) { - return ClusterHealthStatus.RED; - } - if (unassigned > 0 || initializing > 0) { - return ClusterHealthStatus.YELLOW; - } - return ClusterHealthStatus.GREEN; - } - - public void update(ShardRouting shardRouting) { - if (shardRouting.active()) { - active++; - if (shardRouting.primary()) { - primaryActive++; - } - if (shardRouting.relocating()) { - relocating++; - } - return; - } - - if (shardRouting.primary()) { - primaryInactive++; - } - if (shardRouting.initializing()) { - initializing++; - } else { - unassigned++; - } - } - } - - static int node_id = 1; - - private ShardRouting genShardRouting(String index, int shardId, boolean primary) { - - ShardRoutingState state; - - int i = randomInt(40); - if (i > 5) { - state = ShardRoutingState.STARTED; - } else if (i > 3) { - state = ShardRoutingState.RELOCATING; - } else { - state = ShardRoutingState.INITIALIZING; - } - - switch (state) { - case STARTED: - return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, primary, ShardRoutingState.STARTED, 1); - case INITIALIZING: - return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, primary, ShardRoutingState.INITIALIZING, 1); - case RELOCATING: - return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), "node_" + Integer.toString(node_id++), null, primary, ShardRoutingState.RELOCATING, 1); - default: - throw new ElasticsearchException("Unknown state: " + state.name()); - } - - } - - private IndexShardRoutingTable genShardRoutingTable(String index, int shardId, int replicas, ShardCounter counter) { - IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(new ShardId(index, shardId)); - ShardRouting shardRouting = genShardRouting(index, shardId, true); - counter.update(shardRouting); - builder.addShard(shardRouting); - for (; replicas > 0; replicas--) { - shardRouting = genShardRouting(index, shardId, false); - counter.update(shardRouting); - builder.addShard(shardRouting); - } - - return builder.build(); - } - - IndexRoutingTable genIndexRoutingTable(IndexMetaData indexMetaData, ShardCounter counter) { - IndexRoutingTable.Builder builder = IndexRoutingTable.builder(indexMetaData.index()); - for (int shard = 0; shard < indexMetaData.numberOfShards(); shard++) { - builder.addIndexShard(genShardRoutingTable(indexMetaData.index(), shard, indexMetaData.getNumberOfReplicas(), counter)); - } - return builder.build(); - } - - @Test - public void testClusterIndexHealth() { - int numberOfShards = randomInt(3) + 1; - int numberOfReplicas = randomInt(4); - IndexMetaData indexMetaData = IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas(numberOfReplicas).build(); - ShardCounter counter = new ShardCounter(); - IndexRoutingTable indexRoutingTable = genIndexRoutingTable(indexMetaData, counter); - - ClusterIndexHealth indexHealth = new ClusterIndexHealth(indexMetaData, indexRoutingTable); - logger.info("index status: {}, expected {}", indexHealth.getStatus(), counter.status()); - assertIndexHealth(indexHealth, counter, indexMetaData); - } - - private void assertClusterHealth(ClusterHealthResponse clusterHealth, ShardCounter counter) { - assertThat(clusterHealth.getStatus(), equalTo(counter.status())); - assertThat(clusterHealth.getActiveShards(), equalTo(counter.active)); - assertThat(clusterHealth.getActivePrimaryShards(), equalTo(counter.primaryActive)); - assertThat(clusterHealth.getInitializingShards(), equalTo(counter.initializing)); - assertThat(clusterHealth.getRelocatingShards(), equalTo(counter.relocating)); - assertThat(clusterHealth.getUnassignedShards(), equalTo(counter.unassigned)); - assertThat(clusterHealth.getValidationFailures(), empty()); - } - public void testIsTimeout() throws IOException { ClusterHealthResponse res = new ClusterHealthResponse(); for (int i = 0; i < 5; i++) { - res.timedOut = randomBoolean(); + res.setTimedOut(randomBoolean()); if (res.isTimedOut()) { assertEquals(RestStatus.REQUEST_TIMEOUT, res.status()); } else { @@ -200,28 +49,15 @@ public class ClusterHealthResponsesTests extends ESTestCase { } } - @Test public void testClusterHealth() throws IOException { - ShardCounter counter = new ShardCounter(); - RoutingTable.Builder routingTable = RoutingTable.builder(); - MetaData.Builder metaData = MetaData.builder(); - for (int i = randomInt(4); i >= 0; i--) { - int numberOfShards = randomInt(3) + 1; - int numberOfReplicas = randomInt(4); - IndexMetaData indexMetaData = IndexMetaData.builder("test_" + Integer.toString(i)).settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas(numberOfReplicas).build(); - IndexRoutingTable indexRoutingTable = genIndexRoutingTable(indexMetaData, counter); - metaData.put(indexMetaData, true); - routingTable.add(indexRoutingTable); - } - ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable.build()).build(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).build(); int pendingTasks = randomIntBetween(0, 200); int inFlight = randomIntBetween(0, 200); int delayedUnassigned = randomIntBetween(0, 200); TimeValue pendingTaskInQueueTime = TimeValue.timeValueMillis(randomIntBetween(1000, 100000)); - ClusterHealthResponse clusterHealth = new ClusterHealthResponse("bla", indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), (String[]) null), clusterState, pendingTasks, inFlight, delayedUnassigned, pendingTaskInQueueTime); - logger.info("cluster status: {}, expected {}", clusterHealth.getStatus(), counter.status()); + ClusterHealthResponse clusterHealth = new ClusterHealthResponse("bla", new String[] {MetaData.ALL}, clusterState, pendingTasks, inFlight, delayedUnassigned, pendingTaskInQueueTime); clusterHealth = maybeSerialize(clusterHealth); - assertClusterHealth(clusterHealth, counter); + assertClusterHealth(clusterHealth); assertThat(clusterHealth.getNumberOfPendingTasks(), Matchers.equalTo(pendingTasks)); assertThat(clusterHealth.getNumberOfInFlightFetch(), Matchers.equalTo(inFlight)); assertThat(clusterHealth.getDelayedUnassignedShards(), Matchers.equalTo(delayedUnassigned)); @@ -229,6 +65,19 @@ public class ClusterHealthResponsesTests extends ESTestCase { assertThat(clusterHealth.getActiveShardsPercent(), is(allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0)))); } + private void assertClusterHealth(ClusterHealthResponse clusterHealth) { + ClusterStateHealth clusterStateHealth = clusterHealth.getClusterStateHealth(); + + assertThat(clusterHealth.getValidationFailures(), Matchers.equalTo(clusterStateHealth.getValidationFailures())); + assertThat(clusterHealth.getActiveShards(), Matchers.equalTo(clusterStateHealth.getActiveShards())); + assertThat(clusterHealth.getRelocatingShards(), Matchers.equalTo(clusterStateHealth.getRelocatingShards())); + assertThat(clusterHealth.getActivePrimaryShards(), Matchers.equalTo(clusterStateHealth.getActivePrimaryShards())); + assertThat(clusterHealth.getInitializingShards(), Matchers.equalTo(clusterStateHealth.getInitializingShards())); + assertThat(clusterHealth.getUnassignedShards(), Matchers.equalTo(clusterStateHealth.getUnassignedShards())); + assertThat(clusterHealth.getNumberOfNodes(), Matchers.equalTo(clusterStateHealth.getNumberOfNodes())); + assertThat(clusterHealth.getNumberOfDataNodes(), Matchers.equalTo(clusterStateHealth.getNumberOfDataNodes())); + } + ClusterHealthResponse maybeSerialize(ClusterHealthResponse clusterHealth) throws IOException { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); @@ -238,25 +87,4 @@ public class ClusterHealthResponsesTests extends ESTestCase { } return clusterHealth; } - - @Test - public void testValidations() throws IOException { - IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(2).build(); - ShardCounter counter = new ShardCounter(); - IndexRoutingTable indexRoutingTable = genIndexRoutingTable(indexMetaData, counter); - indexMetaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(3).build(); - - ClusterIndexHealth indexHealth = new ClusterIndexHealth(indexMetaData, indexRoutingTable); - assertThat(indexHealth.getValidationFailures(), Matchers.hasSize(2)); - - RoutingTable.Builder routingTable = RoutingTable.builder(); - MetaData.Builder metaData = MetaData.builder(); - metaData.put(indexMetaData, true); - routingTable.add(indexRoutingTable); - ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable.build()).build(); - ClusterHealthResponse clusterHealth = new ClusterHealthResponse("bla", indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), (String[]) null), clusterState, 0, 0, 0, TimeValue.timeValueMillis(0)); - clusterHealth = maybeSerialize(clusterHealth); - // currently we have no cluster level validation failures as index validation issues are reported per index. - assertThat(clusterHealth.getValidationFailures(), Matchers.hasSize(0)); - } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java index caa88ddfb78..9c554da781a 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; @@ -39,8 +38,6 @@ import static org.hamcrest.Matchers.hasSize; */ @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class RepositoryBlocksIT extends ESIntegTestCase { - - @Test public void testPutRepositoryWithBlocks() { logger.info("--> registering a repository is blocked when the cluster is read only"); try { @@ -60,7 +57,6 @@ public class RepositoryBlocksIT extends ESIntegTestCase { .setSettings(Settings.settingsBuilder().put("location", randomRepoPath()))); } - @Test public void testVerifyRepositoryWithBlocks() { assertAcked(client().admin().cluster().preparePutRepository("test-repo-blocks") .setType("fs") @@ -77,7 +73,6 @@ public class RepositoryBlocksIT extends ESIntegTestCase { } } - @Test public void testDeleteRepositoryWithBlocks() { assertAcked(client().admin().cluster().preparePutRepository("test-repo-blocks") .setType("fs") @@ -96,7 +91,6 @@ public class RepositoryBlocksIT extends ESIntegTestCase { assertAcked(client().admin().cluster().prepareDeleteRepository("test-repo-blocks")); } - @Test public void testGetRepositoryWithBlocks() { assertAcked(client().admin().cluster().preparePutRepository("test-repo-blocks") .setType("fs") diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index 2516310c7a3..f3a23be919d 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -30,10 +30,9 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.junit.Before; -import org.junit.Test; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; @@ -85,7 +84,6 @@ public class SnapshotBlocksIT extends ESIntegTestCase { ensureSearchable(); } - @Test public void testCreateSnapshotWithBlocks() { logger.info("--> creating a snapshot is allowed when the cluster is read only"); try { @@ -102,7 +100,6 @@ public class SnapshotBlocksIT extends ESIntegTestCase { assertThat(response.status(), equalTo(RestStatus.OK)); } - @Test public void testCreateSnapshotWithIndexBlocks() { logger.info("--> creating a snapshot is not blocked when an index is read only"); try { @@ -123,7 +120,6 @@ public class SnapshotBlocksIT extends ESIntegTestCase { } } - @Test public void testDeleteSnapshotWithBlocks() { logger.info("--> deleting a snapshot is allowed when the cluster is read only"); try { @@ -134,7 +130,6 @@ public class SnapshotBlocksIT extends ESIntegTestCase { } } - @Test public void testRestoreSnapshotWithBlocks() { assertAcked(client().admin().indices().prepareDelete(INDEX_NAME, OTHER_INDEX_NAME)); assertFalse(client().admin().indices().prepareExists(INDEX_NAME, OTHER_INDEX_NAME).get().isExists()); @@ -156,7 +151,6 @@ public class SnapshotBlocksIT extends ESIntegTestCase { assertTrue(client().admin().indices().prepareExists(OTHER_INDEX_NAME).get().isExists()); } - @Test public void testGetSnapshotWithBlocks() { // This test checks that the Get Snapshot operation is never blocked, even if the cluster is read only. try { @@ -169,7 +163,6 @@ public class SnapshotBlocksIT extends ESIntegTestCase { } } - @Test public void testSnapshotStatusWithBlocks() { // This test checks that the Snapshot Status operation is never blocked, even if the cluster is read only. try { diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java index 5c0627555ad..a2d838bc3fd 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java @@ -21,11 +21,10 @@ package org.elasticsearch.action.admin.cluster.state; import org.elasticsearch.Version; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; -import org.junit.Test; import static org.hamcrest.CoreMatchers.equalTo; @@ -33,8 +32,6 @@ import static org.hamcrest.CoreMatchers.equalTo; * Unit tests for the {@link ClusterStateRequest}. */ public class ClusterStateRequestTests extends ESTestCase { - - @Test public void testSerialization() throws Exception { int iterations = randomIntBetween(5, 20); for (int i = 0; i < iterations; i++) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index 2be808ebef8..55b0ba86aca 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -21,21 +21,22 @@ package org.elasticsearch.action.admin.cluster.stats; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.store.Store; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0) @@ -55,7 +56,6 @@ public class ClusterStatsIT extends ESIntegTestCase { assertThat(actionGet.isTimedOut(), is(false)); } - @Test public void testNodeCounts() { ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); assertCounts(response.getNodesStats().getCounts(), 1, 0, 0, 1, 0); @@ -84,12 +84,10 @@ public class ClusterStatsIT extends ESIntegTestCase { assertThat(stats.getReplication(), Matchers.equalTo(replicationFactor)); } - @Test public void testIndicesShardStats() { ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); - prepareCreate("test1").setSettings("number_of_shards", 2, "number_of_replicas", 1).get(); ensureYellow(); response = client().admin().cluster().prepareClusterStats().get(); @@ -129,7 +127,6 @@ public class ClusterStatsIT extends ESIntegTestCase { } - @Test public void testValuesSmokeScreen() throws IOException { internalCluster().ensureAtMostNumDataNodes(5); internalCluster().ensureAtLeastNumDataNodes(1); @@ -161,4 +158,31 @@ public class ClusterStatsIT extends ESIntegTestCase { assertThat(msg, response.nodesStats.getProcess().getMaxOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L)); } + + public void testAllocatedProcessors() throws Exception { + // stop all other nodes + internalCluster().ensureAtMostNumDataNodes(0); + + // start one node with 7 processors. + internalCluster().startNodesAsync(Settings.builder().put(EsExecutors.PROCESSORS, 7).build()).get(); + waitForNodes(1); + + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + assertThat(response.getNodesStats().getOs().getAllocatedProcessors(), equalTo(7)); + } + + public void testClusterStatusWhenStateNotRecovered() throws Exception { + // stop all other nodes + internalCluster().ensureAtMostNumDataNodes(0); + + internalCluster().startNode(Settings.builder().put("gateway.recover_after_nodes", 2).build()); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED)); + + internalCluster().ensureAtLeastNumDataNodes(3); + // wait for the cluster status to settle + ensureGreen(); + response = client().admin().cluster().prepareClusterStats().get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java index a02860e8a8f..95fa5b2600f 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java @@ -21,16 +21,16 @@ package org.elasticsearch.action.admin.cluster.tasks; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class PendingTasksBlocksIT extends ESIntegTestCase { - - @Test public void testPendingTasksWithBlocks() { createIndex("test"); ensureGreen("test"); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java new file mode 100644 index 00000000000..cb0e0fa0f78 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -0,0 +1,242 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action.admin.indices; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.internal.AllFieldMapper; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +public class TransportAnalyzeActionTests extends ESTestCase { + + private AnalysisService analysisService; + private AnalysisRegistry registry; + private Environment environment; + + @Override + public void setUp() throws Exception { + super.setUp(); + Settings settings = Settings.builder().put("path.home", createTempDir().toString()).build(); + + Settings indexSettings = settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") + .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter").build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), indexSettings); + environment = new Environment(settings); + registry = new AnalysisRegistry(null, environment); + analysisService = registry.build(idxSettings); + } + + public void testNoAnalysisService() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the quick brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.tokenFilters("lowercase", "word_delimiter"); + request.text("the qu1ck brown fox"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? analysisService : null, registry, environment); + tokens = analyze.getTokens(); + assertEquals(6, tokens.size()); + assertEquals("qu", tokens.get(1).getTerm()); + assertEquals("1", tokens.get(2).getTerm()); + assertEquals("ck", tokens.get(3).getTerm()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.charFilters("html_strip"); + request.tokenFilters("lowercase", "word_delimiter"); + request.text("

    the qu1ck brown fox

    "); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? analysisService : null, registry, environment); + tokens = analyze.getTokens(); + assertEquals(6, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("qu", tokens.get(1).getTerm()); + assertEquals("1", tokens.get(2).getTerm()); + assertEquals("ck", tokens.get(3).getTerm()); + assertEquals("brown", tokens.get(4).getTerm()); + assertEquals("fox", tokens.get(5).getTerm()); + } + + public void testFillsAttributes() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the 1 brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals(0, tokens.get(0).getStartOffset()); + assertEquals(3, tokens.get(0).getEndOffset()); + assertEquals(0, tokens.get(0).getPosition()); + assertEquals("", tokens.get(0).getType()); + + assertEquals("1", tokens.get(1).getTerm()); + assertEquals(4, tokens.get(1).getStartOffset()); + assertEquals(5, tokens.get(1).getEndOffset()); + assertEquals(1, tokens.get(1).getPosition()); + assertEquals("", tokens.get(1).getType()); + + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals(6, tokens.get(2).getStartOffset()); + assertEquals(11, tokens.get(2).getEndOffset()); + assertEquals(2, tokens.get(2).getPosition()); + assertEquals("", tokens.get(2).getType()); + + assertEquals("fox", tokens.get(3).getTerm()); + assertEquals(12, tokens.get(3).getStartOffset()); + assertEquals(15, tokens.get(3).getEndOffset()); + assertEquals(3, tokens.get(3).getPosition()); + assertEquals("", tokens.get(3).getType()); + } + + public void testWithAnalysisService() throws IOException { + + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the quick brown fox"); + request.analyzer("custom_analyzer"); + request.text("the qu1ck brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + + request.analyzer("whitespace"); + request.text("the qu1ck brown fox-dog"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment); + tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + + request.analyzer("custom_analyzer"); + request.text("the qu1ck brown fox-dog"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment); + tokens = analyze.getTokens(); + assertEquals(5, tokens.size()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.tokenFilters("lowercase", "wordDelimiter"); + request.text("the qu1ck brown fox-dog"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, analysisService, registry, environment); + tokens = analyze.getTokens(); + assertEquals(5, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("qu1ck", tokens.get(1).getTerm()); + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals("fox", tokens.get(3).getTerm()); + assertEquals("dog", tokens.get(4).getTerm()); + } + + public void testGetIndexAnalyserWithoutAnalysisService() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("custom_analyzer"); + request.text("the qu1ck brown fox-dog"); + try { + TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + fail("no analysis service provided"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "failed to find global analyzer [custom_analyzer]"); + } + } + + public void testUnknown() throws IOException { + boolean notGlobal = randomBoolean(); + try { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("foobar"); + request.text("the qu1ck brown fox"); + TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment); + fail("no such analyzer"); + } catch (IllegalArgumentException e) { + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find analyzer [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global analyzer [foobar]"); + } + } + try { + AnalyzeRequest request = new AnalyzeRequest(); + request.tokenizer("foobar"); + request.text("the qu1ck brown fox"); + TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment); + fail("no such analyzer"); + } catch (IllegalArgumentException e) { + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global tokenizer under [foobar]"); + } + } + + try { + AnalyzeRequest request = new AnalyzeRequest(); + request.tokenizer("whitespace"); + request.tokenFilters("foobar"); + request.text("the qu1ck brown fox"); + TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment); + fail("no such analyzer"); + } catch (IllegalArgumentException e) { + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find token filter under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global token filter under [foobar]"); + } + } + + try { + AnalyzeRequest request = new AnalyzeRequest(); + request.tokenizer("whitespace"); + request.tokenFilters("lowercase"); + request.charFilters("foobar"); + request.text("the qu1ck brown fox"); + TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, notGlobal ? analysisService : null, registry, environment); + fail("no such analyzer"); + } catch (IllegalArgumentException e) { + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find char filter under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global char filter under [foobar]"); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java index deb57cfa241..dbc7e5cddc3 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java @@ -21,19 +21,19 @@ package org.elasticsearch.action.admin.indices.cache.clear; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class ClearIndicesCacheBlocksIT extends ESIntegTestCase { - - @Test public void testClearIndicesCacheWithBlocks() { createIndex("test"); ensureGreen("test"); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 1c810d7eee7..bb154218215 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -19,30 +19,40 @@ package org.elasticsearch.action.admin.indices.create; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.util.HashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.core.IsNull.notNullValue; @ClusterScope(scope = Scope.TEST) public class CreateIndexIT extends ESIntegTestCase { - - @Test - public void testCreationDate_Given() { + public void testCreationDateGiven() { prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 4l)).get(); ClusterStateResponse response = client().admin().cluster().prepareState().get(); ClusterState state = response.getState(); @@ -54,11 +64,10 @@ public class CreateIndexIT extends ESIntegTestCase { assertThat(indices.size(), equalTo(1)); IndexMetaData index = indices.get("test"); assertThat(index, notNullValue()); - assertThat(index.creationDate(), equalTo(4l)); + assertThat(index.getCreationDate(), equalTo(4l)); } - @Test - public void testCreationDate_Generated() { + public void testCreationDateGenerated() { long timeBeforeRequest = System.currentTimeMillis(); prepareCreate("test").get(); long timeAfterRequest = System.currentTimeMillis(); @@ -72,10 +81,9 @@ public class CreateIndexIT extends ESIntegTestCase { assertThat(indices.size(), equalTo(1)); IndexMetaData index = indices.get("test"); assertThat(index, notNullValue()); - assertThat(index.creationDate(), allOf(lessThanOrEqualTo(timeAfterRequest), greaterThanOrEqualTo(timeBeforeRequest))); + assertThat(index.getCreationDate(), allOf(lessThanOrEqualTo(timeAfterRequest), greaterThanOrEqualTo(timeBeforeRequest))); } - @Test public void testDoubleAddMapping() throws Exception { try { prepareCreate("test") @@ -103,12 +111,11 @@ public class CreateIndexIT extends ESIntegTestCase { } } - @Test public void testInvalidShardCountSettings() throws Exception { try { prepareCreate("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(-10, 0)) - .build()) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(-10, 0)) + .build()) .get(); fail("should have thrown an exception about the primary shard count"); } catch (IllegalArgumentException e) { @@ -118,8 +125,8 @@ public class CreateIndexIT extends ESIntegTestCase { try { prepareCreate("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(-10, -1)) - .build()) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(-10, -1)) + .build()) .get(); fail("should have thrown an exception about the replica shard count"); } catch (IllegalArgumentException e) { @@ -129,9 +136,9 @@ public class CreateIndexIT extends ESIntegTestCase { try { prepareCreate("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(-10, 0)) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(-10, -1)) - .build()) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(-10, 0)) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(-10, -1)) + .build()) .get(); fail("should have thrown an exception about the shard count"); } catch (IllegalArgumentException e) { @@ -142,7 +149,6 @@ public class CreateIndexIT extends ESIntegTestCase { } } - @Test public void testCreateIndexWithBlocks() { try { setClusterReadOnly(true); @@ -152,14 +158,12 @@ public class CreateIndexIT extends ESIntegTestCase { } } - @Test public void testCreateIndexWithMetadataBlocks() { assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_BLOCKS_METADATA, true))); assertBlocked(client().admin().indices().prepareGetSettings("test"), IndexMetaData.INDEX_METADATA_BLOCK); disableIndexBlock("test", IndexMetaData.SETTING_BLOCKS_METADATA); } - @Test public void testInvalidShardCountSettingsWithoutPrefix() throws Exception { try { prepareCreate("test").setSettings(Settings.builder() @@ -196,4 +200,89 @@ public class CreateIndexIT extends ESIntegTestCase { } } + public void testCreateAndDeleteIndexConcurrently() throws InterruptedException { + createIndex("test"); + final AtomicInteger indexVersion = new AtomicInteger(0); + final Object indexVersionLock = new Object(); + final CountDownLatch latch = new CountDownLatch(1); + int numDocs = randomIntBetween(1, 10); + for (int i = 0; i < numDocs; i++) { + client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get(); + } + synchronized (indexVersionLock) { // not necessarily needed here but for completeness we lock here too + indexVersion.incrementAndGet(); + } + client().admin().indices().prepareDelete("test").execute(new ActionListener() { // this happens async!!! + @Override + public void onResponse(DeleteIndexResponse deleteIndexResponse) { + Thread thread = new Thread() { + @Override + public void run() { + try { + client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get(); // recreate that index + synchronized (indexVersionLock) { + // we sync here since we have to ensure that all indexing operations below for a given ID are done before we increment the + // index version otherwise a doc that is in-flight could make it into an index that it was supposed to be deleted for and our assertion fail... + indexVersion.incrementAndGet(); + } + assertAcked(client().admin().indices().prepareDelete("test").get()); // from here on all docs with index_version == 0|1 must be gone!!!! only 2 are ok; + } finally { + latch.countDown(); + } + } + }; + thread.start(); + } + + @Override + public void onFailure(Throwable e) { + throw new RuntimeException(e); + } + } + ); + numDocs = randomIntBetween(100, 200); + for (int i = 0; i < numDocs; i++) { + try { + synchronized (indexVersionLock) { + client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get(); + } + } catch (IndexNotFoundException inf) { + // fine + } + } + latch.await(); + refresh(); + + // we only really assert that we never reuse segments of old indices or anything like this here and that nothing fails with crazy exceptions + SearchResponse expected = client().prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(new RangeQueryBuilder("index_version").from(indexVersion.get(), true)).get(); + SearchResponse all = client().prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()).get(); + assertEquals(expected + " vs. " + all, expected.getHits().getTotalHits(), all.getHits().getTotalHits()); + logger.info("total: {}", expected.getHits().getTotalHits()); + } + + /** + * Asserts that the root cause of mapping conflicts is readable. + */ + public void testMappingConflictRootCause() throws Exception { + CreateIndexRequestBuilder b = prepareCreate("test"); + b.addMapping("type1", jsonBuilder().startObject().startObject("properties") + .startObject("text") + .field("type", "string") + .field("analyzer", "standard") + .field("search_analyzer", "whitespace") + .endObject().endObject().endObject()); + b.addMapping("type2", jsonBuilder().humanReadable(true).startObject().startObject("properties") + .startObject("text") + .field("type", "string") + .endObject().endObject().endObject()); + try { + b.get(); + } catch (MapperParsingException e) { + StringBuilder messages = new StringBuilder(); + for (Exception rootCause: e.guessRootCauses()) { + messages.append(rootCause.getMessage()); + } + assertThat(messages.toString(), containsString("mapper [text] is used by multiple types")); + } + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java index 98569b7db8f..97375061de5 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.rest.NoOpClient; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -56,7 +55,6 @@ public class CreateIndexRequestBuilderTests extends ESTestCase { /** * test setting the source with available setters */ - @Test public void testSetSource() throws IOException { CreateIndexRequestBuilder builder = new CreateIndexRequestBuilder(this.testClient, CreateIndexAction.INSTANCE); builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}"); @@ -82,7 +80,6 @@ public class CreateIndexRequestBuilderTests extends ESTestCase { /** * test setting the settings with available setters */ - @Test public void testSetSettings() throws IOException { CreateIndexRequestBuilder builder = new CreateIndexRequestBuilder(this.testClient, CreateIndexAction.INSTANCE); builder.setSettings(KEY, VALUE); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java index 85e0072bfe3..a83c209a3c2 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java @@ -21,14 +21,11 @@ package org.elasticsearch.action.admin.indices.delete; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class DeleteIndexBlocksIT extends ESIntegTestCase { - - @Test public void testDeleteIndexWithBlocks() { createIndex("test"); ensureGreen("test"); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java index 803262f8292..7a55b22b600 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java @@ -21,19 +21,19 @@ package org.elasticsearch.action.admin.indices.flush; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class FlushBlocksIT extends ESIntegTestCase { - - @Test public void testFlushWithBlocks() { createIndex("test"); ensureGreen("test"); @@ -49,7 +49,7 @@ public class FlushBlocksIT extends ESIntegTestCase { for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE)) { try { enableIndexBlock("test", blockSetting); - FlushResponse response = client().admin().indices().prepareFlush("test").execute().actionGet(); + FlushResponse response = client().admin().indices().prepareFlush("test").setWaitIfOngoing(true).execute().actionGet(); assertNoFailures(response); assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards)); } finally { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/optimize/OptimizeBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java similarity index 74% rename from core/src/test/java/org/elasticsearch/action/admin/indices/optimize/OptimizeBlocksIT.java rename to core/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java index 6b6e663b293..e1f498b09bb 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/optimize/OptimizeBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java @@ -17,24 +17,25 @@ * under the License. */ -package org.elasticsearch.action.admin.indices.optimize; +package org.elasticsearch.action.admin.indices.forcemerge; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) -public class OptimizeBlocksIT extends ESIntegTestCase { +public class ForceMergeBlocksIT extends ESIntegTestCase { - @Test - public void testOptimizeWithBlocks() { + public void testForceMergeWithBlocks() { createIndex("test"); ensureGreen("test"); @@ -49,7 +50,7 @@ public class OptimizeBlocksIT extends ESIntegTestCase { for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE)) { try { enableIndexBlock("test", blockSetting); - OptimizeResponse response = client().admin().indices().prepareOptimize("test").execute().actionGet(); + ForceMergeResponse response = client().admin().indices().prepareForceMerge("test").execute().actionGet(); assertNoFailures(response); assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards)); } finally { @@ -61,22 +62,22 @@ public class OptimizeBlocksIT extends ESIntegTestCase { for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) { try { enableIndexBlock("test", blockSetting); - assertBlocked(client().admin().indices().prepareOptimize("test")); + assertBlocked(client().admin().indices().prepareForceMerge("test")); } finally { disableIndexBlock("test", blockSetting); } } - // Optimizing all indices is blocked when the cluster is read-only + // Merging all indices is blocked when the cluster is read-only try { - OptimizeResponse response = client().admin().indices().prepareOptimize().execute().actionGet(); + ForceMergeResponse response = client().admin().indices().prepareForceMerge().execute().actionGet(); assertNoFailures(response); assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards)); setClusterReadOnly(true); - assertBlocked(client().admin().indices().prepareOptimize()); + assertBlocked(client().admin().indices().prepareForceMerge()); } finally { setClusterReadOnly(false); } } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java index 9484c5e07f1..e878a3df45c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java @@ -28,22 +28,25 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.search.warmer.IndexWarmersMetaData.Entry; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_METADATA_BLOCK; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.SuiteScopeTestCase public class GetIndexIT extends ESIntegTestCase { - - private static final String[] allFeatures = { "_alias", "_aliases", "_mapping", "_mappings", "_settings", "_warmer", "_warmers" }; - @Override protected void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate("idx").addAlias(new Alias("alias_idx")).addMapping("type1", "{\"type1\":{}}") @@ -54,7 +57,6 @@ public class GetIndexIT extends ESIntegTestCase { ensureSearchable("idx", "empty_idx"); } - @Test public void testSimple() { GetIndexResponse response = client().admin().indices().prepareGetIndex().addIndices("idx").get(); String[] indices = response.indices(); @@ -67,12 +69,15 @@ public class GetIndexIT extends ESIntegTestCase { assertWarmers(response, "idx"); } - @Test(expected=IndexNotFoundException.class) public void testSimpleUnknownIndex() { - client().admin().indices().prepareGetIndex().addIndices("missing_idx").get(); + try { + client().admin().indices().prepareGetIndex().addIndices("missing_idx").get(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test public void testEmpty() { GetIndexResponse response = client().admin().indices().prepareGetIndex().addIndices("empty_idx").get(); String[] indices = response.indices(); @@ -85,7 +90,6 @@ public class GetIndexIT extends ESIntegTestCase { assertEmptyWarmers(response); } - @Test public void testSimpleMapping() { GetIndexResponse response = runWithRandomFeatureMethod(client().admin().indices().prepareGetIndex().addIndices("idx"), Feature.MAPPINGS); @@ -99,7 +103,6 @@ public class GetIndexIT extends ESIntegTestCase { assertEmptyWarmers(response); } - @Test public void testSimpleAlias() { GetIndexResponse response = runWithRandomFeatureMethod(client().admin().indices().prepareGetIndex().addIndices("idx"), Feature.ALIASES); @@ -113,7 +116,6 @@ public class GetIndexIT extends ESIntegTestCase { assertEmptyWarmers(response); } - @Test public void testSimpleSettings() { GetIndexResponse response = runWithRandomFeatureMethod(client().admin().indices().prepareGetIndex().addIndices("idx"), Feature.SETTINGS); @@ -127,7 +129,6 @@ public class GetIndexIT extends ESIntegTestCase { assertEmptyWarmers(response); } - @Test public void testSimpleWarmer() { GetIndexResponse response = runWithRandomFeatureMethod(client().admin().indices().prepareGetIndex().addIndices("idx"), Feature.WARMERS); @@ -141,7 +142,6 @@ public class GetIndexIT extends ESIntegTestCase { assertEmptySettings(response); } - @Test public void testSimpleMixedFeatures() { int numFeatures = randomIntBetween(1, Feature.values().length); List features = new ArrayList(numFeatures); @@ -176,7 +176,6 @@ public class GetIndexIT extends ESIntegTestCase { } } - @Test public void testEmptyMixedFeatures() { int numFeatures = randomIntBetween(1, Feature.values().length); List features = new ArrayList(numFeatures); @@ -203,7 +202,6 @@ public class GetIndexIT extends ESIntegTestCase { assertEmptyWarmers(response); } - @Test public void testGetIndexWithBlocks() { for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) { try { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java index 5815ce8c266..33e03010d59 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java @@ -22,19 +22,19 @@ package org.elasticsearch.action.admin.indices.refresh; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class RefreshBlocksIT extends ESIntegTestCase { - - @Test public void testRefreshWithBlocks() { createIndex("test"); ensureGreen("test"); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java index fcb7a509d5c..035c760d84b 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java @@ -21,18 +21,18 @@ package org.elasticsearch.action.admin.indices.segments; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndicesSegmentsBlocksIT extends ESIntegTestCase { - - @Test public void testIndicesSegmentsWithBlocks() { createIndex("test-blocks"); ensureGreen("test-blocks"); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java index 9db47f6ab37..0b7d9017811 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java @@ -22,19 +22,22 @@ package org.elasticsearch.action.admin.indices.segments; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.Segment; +import org.elasticsearch.index.shard.MergePolicyConfig; +import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; -import org.junit.Test; import java.util.List; +import static org.hamcrest.Matchers.is; + public class IndicesSegmentsRequestTests extends ESSingleNodeTestCase { - + @Before public void setupIndex() { Settings settings = Settings.builder() // don't allow any merges so that the num docs is the expected segments - .put("index.merge.policy.segments_per_tier", 1000000f) + .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) .build(); createIndex("test", settings); @@ -43,28 +46,32 @@ public class IndicesSegmentsRequestTests extends ESSingleNodeTestCase { String id = Integer.toString(j); client().prepareIndex("test", "type1", id).setSource("text", "sometext").get(); } - client().admin().indices().prepareFlush("test").get(); + client().admin().indices().prepareFlush("test").setWaitIfOngoing(true).get(); } public void testBasic() { IndicesSegmentResponse rsp = client().admin().indices().prepareSegments("test").get(); List segments = rsp.getIndices().get("test").iterator().next().getShards()[0].getSegments(); - assertNull(segments.get(0).ramTree); + assertNull(segments.get(0).toString(), segments.get(0).ramTree); } - + public void testVerbose() { IndicesSegmentResponse rsp = client().admin().indices().prepareSegments("test").setVerbose(true).get(); List segments = rsp.getIndices().get("test").iterator().next().getShards()[0].getSegments(); - assertNotNull(segments.get(0).ramTree); + assertNotNull(segments.get(0).toString(), segments.get(0).ramTree); } /** * with the default IndicesOptions inherited from BroadcastOperationRequest this will raise an exception */ - @Test(expected=org.elasticsearch.indices.IndexClosedException.class) public void testRequestOnClosedIndex() { client().admin().indices().prepareClose("test").get(); - client().admin().indices().prepareSegments("test").get(); + try { + client().admin().indices().prepareSegments("test").get(); + fail("Expected IndexClosedException"); + } catch (IndexClosedException e) { + assertThat(e.getMessage(), is("closed")); + } } /** diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index de9eadaf057..ffb9e630b70 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -20,14 +20,16 @@ package org.elasticsearch.action.admin.indices.shards; import com.carrotsearch.hppc.cursors.IntObjectCursor; - import com.carrotsearch.hppc.cursors.ObjectCursor; + import org.apache.lucene.index.CorruptIndexException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -36,28 +38,32 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.test.store.MockFSDirectoryService; -import org.junit.Test; +import org.elasticsearch.test.store.MockFSIndexStore; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.function.Predicate; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndicesShardStoreRequestIT extends ESIntegTestCase { - - @Test public void testEmpty() { ensureGreen(); IndicesShardStoresResponse rsp = client().admin().indices().prepareShardStores().get(); assertThat(rsp.getStoreStatuses().size(), equalTo(0)); } - @Test @TestLogging("action.admin.indices.shards:TRACE,cluster.service:TRACE") public void testBasic() throws Exception { String index = "test"; @@ -108,7 +114,6 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase { enableAllocation(index); } - @Test public void testIndices() throws Exception { String index1 = "test1"; String index2 = "test2"; @@ -137,14 +142,13 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase { assertThat(shardStatuses.get(index1).size(), equalTo(2)); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12416") public void testCorruptedShards() throws Exception { String index = "test"; internalCluster().ensureAtLeastNumDataNodes(2); assertAcked(prepareCreate(index).setSettings(Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "5") - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) )); indexRandomData(index); ensureGreen(index); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java index 777555f5b73..cf197a27faf 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.xcontent.*; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.NodeDisconnectedException; -import org.junit.Test; import java.io.IOException; import java.util.*; @@ -37,8 +36,6 @@ import java.util.*; import static org.hamcrest.Matchers.equalTo; public class IndicesShardStoreResponseTests extends ESTestCase { - - @Test public void testBasicSerialization() throws Exception { ImmutableOpenMap.Builder>> indexStoreStatuses = ImmutableOpenMap.builder(); List failures = new ArrayList<>(); @@ -104,7 +101,6 @@ public class IndicesShardStoreResponseTests extends ESTestCase { } } - @Test public void testStoreStatusOrdering() throws Exception { DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, Version.CURRENT); List orderedStoreStatuses = new ArrayList<>(); @@ -115,7 +111,7 @@ public class IndicesShardStoreResponseTests extends ESTestCase { orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 3, IndicesShardStoresResponse.StoreStatus.Allocation.REPLICA, new IOException("corrupted"))); List storeStatuses = new ArrayList<>(orderedStoreStatuses); - Collections.shuffle(storeStatuses); + Collections.shuffle(storeStatuses, random()); CollectionUtil.timSort(storeStatuses); assertThat(storeStatuses, equalTo(orderedStoreStatuses)); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java index 125c4e46021..25fdb7a84db 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java @@ -23,16 +23,15 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndicesStatsBlocksIT extends ESIntegTestCase { - - @Test public void testIndicesStatsWithBlocks() { createIndex("ro"); ensureGreen("ro"); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java index 86a9bbc1f3f..c642bdb1e79 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java @@ -20,6 +20,8 @@ package org.elasticsearch.action.admin.indices.template.put; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.cluster.metadata.AliasValidator; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService; @@ -27,19 +29,14 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutReques import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; public class MetaDataIndexTemplateServiceTests extends ESTestCase { - @Test public void testIndexTemplateInvalidNumberOfShards() { PutRequest request = new PutRequest("test", "test_shards"); request.template("test_shards*"); @@ -54,7 +51,6 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase { assertThat(throwables.get(0).getMessage(), containsString("index must have 1 or more primary shards")); } - @Test public void testIndexTemplateValidationAccumulatesValidationErrors() { PutRequest request = new PutRequest("test", "putTemplate shards"); request.template("_test_shards*"); @@ -71,21 +67,29 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase { assertThat(throwables.get(0).getMessage(), containsString("index must have 1 or more primary shards")); } + public void testIndexTemplateWithAliasNameEqualToTemplatePattern() { + PutRequest request = new PutRequest("api", "foobar_template"); + request.template("foobar"); + request.aliases(Collections.singleton(new Alias("foobar"))); + + List errors = putTemplate(request); + assertThat(errors.size(), equalTo(1)); + assertThat(errors.get(0), instanceOf(IllegalArgumentException.class)); + assertThat(errors.get(0).getMessage(), equalTo("Alias [foobar] cannot be the same as the template pattern [foobar]")); + } + private static List putTemplate(PutRequest request) { MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService( Settings.EMPTY, null, null, null, - null, - null, Version.CURRENT, null, new HashSet<>(), null, - null - ); - MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, null); + null); + MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY)); final List throwables = new ArrayList<>(); service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/warmer/put/PutWarmerRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/warmer/put/PutWarmerRequestTests.java index 1a17a4c78ac..f20564e1712 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/warmer/put/PutWarmerRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/warmer/put/PutWarmerRequestTests.java @@ -20,14 +20,12 @@ package org.elasticsearch.action.admin.indices.warmer.put; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.hasSize; public class PutWarmerRequestTests extends ESTestCase { - - @Test // issue 4196 + // issue 4196 public void testThatValidationWithoutSpecifyingSearchRequestFails() { PutWarmerRequest putWarmerRequest = new PutWarmerRequest("foo"); ActionRequestValidationException validationException = putWarmerRequest.validate(); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 7d946ed3787..4300a629fbd 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -20,17 +20,14 @@ package org.elasticsearch.action.bulk; -import java.nio.charset.StandardCharsets; - import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; + +import java.nio.charset.StandardCharsets; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; public class BulkIntegrationIT extends ESIntegTestCase { - - @Test public void testBulkIndexCreatesMapping() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/bulk-log.json"); BulkRequestBuilder bulkBuilder = client().prepareBulk(); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index b26a10f100e..35fb73b7bfa 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -23,12 +23,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class BulkProcessorClusterSettingsIT extends ESIntegTestCase { - - @Test public void testBulkProcessorAutoCreateRestrictions() throws Exception { // See issue #8125 Settings settings = Settings.settingsBuilder().put("action.auto_create_index", false).build(); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java index dd39f85338b..237f3a2e821 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.HashSet; @@ -45,13 +44,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class BulkProcessorIT extends ESIntegTestCase { - - @Test public void testThatBulkProcessorCountIsCorrect() throws InterruptedException { - final CountDownLatch latch = new CountDownLatch(1); BulkProcessorTestListener listener = new BulkProcessorTestListener(latch); @@ -74,7 +76,6 @@ public class BulkProcessorIT extends ESIntegTestCase { } } - @Test public void testBulkProcessorFlush() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); BulkProcessorTestListener listener = new BulkProcessorTestListener(latch); @@ -101,11 +102,10 @@ public class BulkProcessorIT extends ESIntegTestCase { } } - @Test public void testBulkProcessorConcurrentRequests() throws Exception { int bulkActions = randomIntBetween(10, 100); int numDocs = randomIntBetween(bulkActions, bulkActions + 100); - int concurrentRequests = randomIntBetween(0, 10); + int concurrentRequests = randomIntBetween(0, 7); int expectedBulkActions = numDocs / bulkActions; @@ -141,7 +141,7 @@ public class BulkProcessorIT extends ESIntegTestCase { Set ids = new HashSet<>(); for (BulkItemResponse bulkItemResponse : listener.bulkItems) { - assertThat(bulkItemResponse.isFailed(), equalTo(false)); + assertThat(bulkItemResponse.getFailureMessage(), bulkItemResponse.isFailed(), equalTo(false)); assertThat(bulkItemResponse.getIndex(), equalTo("test")); assertThat(bulkItemResponse.getType(), equalTo("test")); //with concurrent requests > 1 we can't rely on the order of the bulk requests @@ -153,7 +153,6 @@ public class BulkProcessorIT extends ESIntegTestCase { assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs); } - @Test //https://github.com/elasticsearch/elasticsearch/issues/5038 public void testBulkProcessorConcurrentRequestsNoNodeAvailableException() throws Exception { //we create a transport client with no nodes to make sure it throws NoNodeAvailableException @@ -196,7 +195,6 @@ public class BulkProcessorIT extends ESIntegTestCase { transportClient.close(); } - @Test public void testBulkProcessorWaitOnClose() throws Exception { BulkProcessorTestListener listener = new BulkProcessorTestListener(); @@ -205,7 +203,7 @@ public class BulkProcessorIT extends ESIntegTestCase { //let's make sure that the bulk action limit trips, one single execution will index all the documents .setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs) .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(randomIntBetween(1, 10), - (ByteSizeUnit)RandomPicks.randomFrom(getRandom(), ByteSizeUnit.values()))) + RandomPicks.randomFrom(getRandom(), ByteSizeUnit.values()))) .build(); MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs); @@ -227,7 +225,6 @@ public class BulkProcessorIT extends ESIntegTestCase { assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs); } - @Test public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception { createIndex("test-ro"); assertAcked(client().admin().indices().prepareUpdateSettings("test-ro") diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index 207ebdb9f6c..78f96bab7b2 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -19,10 +19,9 @@ package org.elasticsearch.action.bulk; -import java.nio.charset.StandardCharsets; - import org.apache.lucene.util.Constants; import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; @@ -31,20 +30,16 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.script.Script; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.*; public class BulkRequestTests extends ESTestCase { - - @Test public void testSimpleBulk1() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json"); // translate Windows line endings (\r\n) to standard ones (\n) @@ -59,7 +54,6 @@ public class BulkRequestTests extends ESTestCase { assertThat(((IndexRequest) bulkRequest.requests().get(2)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }").toBytes())); } - @Test public void testSimpleBulk2() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk2.json"); BulkRequest bulkRequest = new BulkRequest(); @@ -67,7 +61,6 @@ public class BulkRequestTests extends ESTestCase { assertThat(bulkRequest.numberOfActions(), equalTo(3)); } - @Test public void testSimpleBulk3() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk3.json"); BulkRequest bulkRequest = new BulkRequest(); @@ -75,7 +68,6 @@ public class BulkRequestTests extends ESTestCase { assertThat(bulkRequest.numberOfActions(), equalTo(3)); } - @Test public void testSimpleBulk4() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk4.json"); BulkRequest bulkRequest = new BulkRequest(); @@ -98,7 +90,6 @@ public class BulkRequestTests extends ESTestCase { assertThat(((UpdateRequest) bulkRequest.requests().get(1)).upsertRequest().source().toUtf8(), equalTo("{\"counter\":1}")); } - @Test public void testBulkAllowExplicitIndex() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json"); try { @@ -112,7 +103,6 @@ public class BulkRequestTests extends ESTestCase { new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false); } - @Test public void testBulkAddIterable() { BulkRequest bulkRequest = Requests.bulkRequest(); List requests = new ArrayList<>(); @@ -126,7 +116,6 @@ public class BulkRequestTests extends ESTestCase { assertThat(bulkRequest.requests().get(2), instanceOf(DeleteRequest.class)); } - @Test public void testSimpleBulk6() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk6.json"); BulkRequest bulkRequest = new BulkRequest(); @@ -139,7 +128,6 @@ public class BulkRequestTests extends ESTestCase { } } - @Test public void testSimpleBulk7() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk7.json"); BulkRequest bulkRequest = new BulkRequest(); @@ -152,7 +140,6 @@ public class BulkRequestTests extends ESTestCase { } } - @Test public void testSimpleBulk8() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk8.json"); BulkRequest bulkRequest = new BulkRequest(); @@ -165,7 +152,6 @@ public class BulkRequestTests extends ESTestCase { } } - @Test public void testSimpleBulk9() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk9.json"); BulkRequest bulkRequest = new BulkRequest(); @@ -178,11 +164,45 @@ public class BulkRequestTests extends ESTestCase { } } - @Test public void testSimpleBulk10() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk10.json"); BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); assertThat(bulkRequest.numberOfActions(), equalTo(9)); } + + // issue 7361 + public void testBulkRequestWithRefresh() throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + // We force here a "id is missing" validation error + bulkRequest.add(new DeleteRequest("index", "type", null).refresh(true)); + // We force here a "type is missing" validation error + bulkRequest.add(new DeleteRequest("index", null, "id")); + bulkRequest.add(new DeleteRequest("index", "type", "id").refresh(true)); + bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").refresh(true)); + bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").refresh(true)); + ActionRequestValidationException validate = bulkRequest.validate(); + assertThat(validate, notNullValue()); + assertThat(validate.validationErrors(), not(empty())); + assertThat(validate.validationErrors(), contains( + "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", + "id is missing", + "type is missing", + "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", + "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", + "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.")); + } + + // issue 15120 + public void testBulkNoSource() throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new UpdateRequest("index", "type", "id")); + bulkRequest.add(new IndexRequest("index", "type", "id")); + ActionRequestValidationException validate = bulkRequest.validate(); + assertThat(validate, notNullValue()); + assertThat(validate.validationErrors(), not(empty())); + assertThat(validate.validationErrors(), contains( + "script or doc is missing", + "source is missing")); + } } diff --git a/core/src/test/java/org/elasticsearch/action/count/CountRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/count/CountRequestBuilderTests.java deleted file mode 100644 index 5d77247d38a..00000000000 --- a/core/src/test/java/org/elasticsearch/action/count/CountRequestBuilderTests.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.count; - -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.test.ESTestCase; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; - -public class CountRequestBuilderTests extends ESTestCase { - - private static Client client; - - @BeforeClass - public static void initClient() { - //this client will not be hit by any request, but it needs to be a non null proper client - //that is why we create it but we don't add any transport address to it - Settings settings = Settings.builder() - .put("path.home", createTempDir().toString()) - .build(); - client = TransportClient.builder().settings(settings).build(); - } - - @AfterClass - public static void closeClient() { - client.close(); - client = null; - } - - @Test - public void testEmptySourceToString() { - CountRequestBuilder countRequestBuilder = client.prepareCount(); - assertThat(countRequestBuilder.toString(), equalTo(new QuerySourceBuilder().toString())); - } - - @Test - public void testQueryBuilderQueryToString() { - CountRequestBuilder countRequestBuilder = client.prepareCount(); - countRequestBuilder.setQuery(QueryBuilders.matchAllQuery()); - assertThat(countRequestBuilder.toString(), equalTo(new QuerySourceBuilder().setQuery(QueryBuilders.matchAllQuery()).toString())); - } - - @Test - public void testStringQueryToString() { - CountRequestBuilder countRequestBuilder = client.prepareCount(); - String query = "{ \"match_all\" : {} }"; - countRequestBuilder.setQuery(new BytesArray(query)); - assertThat(countRequestBuilder.toString(), containsString("\"query\":{ \"match_all\" : {} }")); - } - - @Test - public void testXContentBuilderQueryToString() throws IOException { - CountRequestBuilder countRequestBuilder = client.prepareCount(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - xContentBuilder.startObject(); - xContentBuilder.startObject("match_all"); - xContentBuilder.endObject(); - xContentBuilder.endObject(); - countRequestBuilder.setQuery(xContentBuilder); - assertThat(countRequestBuilder.toString(), equalTo(new QuerySourceBuilder().setQuery(xContentBuilder.bytes()).toString())); - } - - @Test - public void testStringSourceToString() { - CountRequestBuilder countRequestBuilder = client.prepareCount(); - String query = "{ \"query\": { \"match_all\" : {} } }"; - countRequestBuilder.setSource(new BytesArray(query)); - assertThat(countRequestBuilder.toString(), equalTo("{ \"query\": { \"match_all\" : {} } }")); - } - - @Test - public void testXContentBuilderSourceToString() throws IOException { - CountRequestBuilder countRequestBuilder = client.prepareCount(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - xContentBuilder.startObject(); - xContentBuilder.startObject("match_all"); - xContentBuilder.endObject(); - xContentBuilder.endObject(); - countRequestBuilder.setSource(xContentBuilder.bytes()); - assertThat(countRequestBuilder.toString(), equalTo(XContentHelper.convertToJson(xContentBuilder.bytes(), false, true))); - } - - @Test - public void testThatToStringDoesntWipeSource() { - String source = "{\n" + - " \"query\" : {\n" + - " \"match\" : {\n" + - " \"field\" : {\n" + - " \"query\" : \"value\"" + - " }\n" + - " }\n" + - " }\n" + - " }"; - CountRequestBuilder countRequestBuilder = client.prepareCount().setSource(new BytesArray(source)); - String preToString = countRequestBuilder.request().source().toUtf8(); - assertThat(countRequestBuilder.toString(), equalTo(source)); - String postToString = countRequestBuilder.request().source().toUtf8(); - assertThat(preToString, equalTo(postToString)); - } -} diff --git a/core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java b/core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java deleted file mode 100644 index 407cfbaec91..00000000000 --- a/core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.count; - -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.test.ESTestCase; -import org.junit.Test; - -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; - -public class CountRequestTests extends ESTestCase { - - @Test - public void testToSearchRequest() { - CountRequest countRequest; - if (randomBoolean()) { - countRequest = new CountRequest(randomStringArray()); - } else { - countRequest = new CountRequest(); - } - if (randomBoolean()) { - countRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); - } - if (randomBoolean()) { - countRequest.types(randomStringArray()); - } - if (randomBoolean()) { - countRequest.routing(randomStringArray()); - } - if (randomBoolean()) { - countRequest.preference(randomAsciiOfLengthBetween(1, 10)); - } - if (randomBoolean()) { - countRequest.source(new QuerySourceBuilder().setQuery(QueryBuilders.termQuery("field", "value"))); - } - if (randomBoolean()) { - countRequest.minScore(randomFloat()); - } - if (randomBoolean()) { - countRequest.terminateAfter(randomIntBetween(1, 1000)); - } - - SearchRequest searchRequest = countRequest.toSearchRequest(); - assertThat(searchRequest.indices(), equalTo(countRequest.indices())); - assertThat(searchRequest.indicesOptions(), equalTo(countRequest.indicesOptions())); - assertThat(searchRequest.types(), equalTo(countRequest.types())); - assertThat(searchRequest.routing(), equalTo(countRequest.routing())); - assertThat(searchRequest.preference(), equalTo(countRequest.preference())); - - if (countRequest.source() == null) { - assertThat(searchRequest.source(), nullValue()); - } else { - Map sourceMap = XContentHelper.convertToMap(searchRequest.source(), false).v2(); - assertThat(sourceMap.size(), equalTo(1)); - assertThat(sourceMap.get("query"), notNullValue()); - } - - Map extraSourceMap = XContentHelper.convertToMap(searchRequest.extraSource(), false).v2(); - int count = 1; - assertThat((Integer)extraSourceMap.get("size"), equalTo(0)); - if (countRequest.minScore() == CountRequest.DEFAULT_MIN_SCORE) { - assertThat(extraSourceMap.get("min_score"), nullValue()); - } else { - assertThat(((Number)extraSourceMap.get("min_score")).floatValue(), equalTo(countRequest.minScore())); - count++; - } - if (countRequest.terminateAfter() == SearchContext.DEFAULT_TERMINATE_AFTER) { - assertThat(extraSourceMap.get("terminate_after"), nullValue()); - } else { - assertThat((Integer)extraSourceMap.get("terminate_after"), equalTo(countRequest.terminateAfter())); - count++; - } - assertThat(extraSourceMap.size(), equalTo(count)); - } - - private static String[] randomStringArray() { - int count = randomIntBetween(1, 5); - String[] indices = new String[count]; - for (int i = 0; i < count; i++) { - indices[i] = randomAsciiOfLengthBetween(1, 10); - } - return indices; - } -} diff --git a/core/src/test/java/org/elasticsearch/action/count/CountResponseTests.java b/core/src/test/java/org/elasticsearch/action/count/CountResponseTests.java deleted file mode 100644 index 11fd191c780..00000000000 --- a/core/src/test/java/org/elasticsearch/action/count/CountResponseTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.count; - -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.search.internal.InternalSearchHits; -import org.elasticsearch.search.internal.InternalSearchResponse; -import org.elasticsearch.test.ESTestCase; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; - -public class CountResponseTests extends ESTestCase { - - @Test - public void testFromSearchResponse() { - InternalSearchResponse internalSearchResponse = new InternalSearchResponse(new InternalSearchHits(null, randomLong(), randomFloat()), null, null, randomBoolean(), randomBoolean()); - ShardSearchFailure[] shardSearchFailures = new ShardSearchFailure[randomIntBetween(0, 5)]; - for (int i = 0; i < shardSearchFailures.length; i++) { - shardSearchFailures[i] = new ShardSearchFailure(new IllegalArgumentException()); - } - SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, randomIntBetween(0, 100), randomIntBetween(0, 100), randomIntBetween(0, 100), shardSearchFailures); - - CountResponse countResponse = new CountResponse(searchResponse); - assertThat(countResponse.getTotalShards(), equalTo(searchResponse.getTotalShards())); - assertThat(countResponse.getSuccessfulShards(), equalTo(searchResponse.getSuccessfulShards())); - assertThat(countResponse.getFailedShards(), equalTo(searchResponse.getFailedShards())); - assertThat(countResponse.getShardFailures(), equalTo((ShardOperationFailedException[])searchResponse.getShardFailures())); - assertThat(countResponse.getCount(), equalTo(searchResponse.getHits().totalHits())); - assertThat(countResponse.terminatedEarly(), equalTo(searchResponse.isTerminatedEarly())); - } -} diff --git a/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java index e33fba69b8b..937cfb7b948 100644 --- a/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java @@ -42,7 +42,7 @@ public class FieldStatsRequestTests extends ESTestCase { assertThat(request.getFields()[3], equalTo("field4")); assertThat(request.getFields()[4], equalTo("field5")); - assertThat(request.getIndexConstraints().length, equalTo(6)); + assertThat(request.getIndexConstraints().length, equalTo(8)); assertThat(request.getIndexConstraints()[0].getField(), equalTo("field2")); assertThat(request.getIndexConstraints()[0].getValue(), equalTo("9")); assertThat(request.getIndexConstraints()[0].getProperty(), equalTo(MAX)); @@ -67,6 +67,16 @@ public class FieldStatsRequestTests extends ESTestCase { assertThat(request.getIndexConstraints()[5].getValue(), equalTo("9")); assertThat(request.getIndexConstraints()[5].getProperty(), equalTo(MAX)); assertThat(request.getIndexConstraints()[5].getComparison(), equalTo(LT)); + assertThat(request.getIndexConstraints()[6].getField(), equalTo("field1")); + assertThat(request.getIndexConstraints()[6].getValue(), equalTo("2014-01-01")); + assertThat(request.getIndexConstraints()[6].getProperty(), equalTo(MIN)); + assertThat(request.getIndexConstraints()[6].getComparison(), equalTo(GTE)); + assertThat(request.getIndexConstraints()[6].getOptionalFormat(), equalTo("date_optional_time")); + assertThat(request.getIndexConstraints()[7].getField(), equalTo("field1")); + assertThat(request.getIndexConstraints()[7].getValue(), equalTo("2015-01-01")); + assertThat(request.getIndexConstraints()[7].getProperty(), equalTo(MAX)); + assertThat(request.getIndexConstraints()[7].getComparison(), equalTo(LT)); + assertThat(request.getIndexConstraints()[7].getOptionalFormat(), equalTo("date_optional_time")); } } diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java index c48f65be4b8..451ade62584 100644 --- a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.index.VersionType; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -32,8 +31,6 @@ import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; public class MultiGetShardRequestTests extends ESTestCase { - - @Test public void testSerialization() throws IOException { MultiGetRequest multiGetRequest = new MultiGetRequest(); if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java index f9dc86b59e9..badb79e21b7 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.rest.NoOpClient; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.ByteArrayOutputStream; import java.util.HashMap; @@ -55,7 +54,6 @@ public class IndexRequestBuilderTests extends ESTestCase { /** * test setting the source for the request with different available setters */ - @Test public void testSetSource() throws Exception { IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(this.testClient, IndexAction.INSTANCE); Map source = new HashMap<>(); diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index 7c08a0db359..1d3a9e18757 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -18,9 +18,10 @@ */ package org.elasticsearch.action.index; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.VersionType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.HashSet; @@ -31,8 +32,6 @@ import static org.hamcrest.Matchers.*; /** */ public class IndexRequestTests extends ESTestCase { - - @Test public void testIndexRequestOpTypeFromString() throws Exception { String create = "create"; String index = "index"; @@ -45,10 +44,13 @@ public class IndexRequestTests extends ESTestCase { assertThat(IndexRequest.OpType.fromString(indexUpper), equalTo(IndexRequest.OpType.INDEX)); } - @Test(expected = IllegalArgumentException.class) public void testReadBogusString() { - String foobar = "foobar"; - IndexRequest.OpType.fromString(foobar); + try { + IndexRequest.OpType.fromString("foobar"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("opType [foobar] not allowed")); + } } public void testCreateOperationRejectsVersions() { @@ -64,4 +66,43 @@ public class IndexRequestTests extends ESTestCase { request.version(randomIntBetween(0, Integer.MAX_VALUE)); assertThat(request.validate().validationErrors(), not(empty())); } + + public void testSetTTLAsTimeValue() { + IndexRequest indexRequest = new IndexRequest(); + TimeValue ttl = TimeValue.parseTimeValue(randomTimeValue(), null, "ttl"); + indexRequest.ttl(ttl); + assertThat(indexRequest.ttl(), equalTo(ttl)); + } + + public void testSetTTLAsString() { + IndexRequest indexRequest = new IndexRequest(); + String ttlAsString = randomTimeValue(); + TimeValue ttl = TimeValue.parseTimeValue(ttlAsString, null, "ttl"); + indexRequest.ttl(ttlAsString); + assertThat(indexRequest.ttl(), equalTo(ttl)); + } + + public void testSetTTLAsLong() { + IndexRequest indexRequest = new IndexRequest(); + String ttlAsString = randomTimeValue(); + TimeValue ttl = TimeValue.parseTimeValue(ttlAsString, null, "ttl"); + indexRequest.ttl(ttl.millis()); + assertThat(indexRequest.ttl(), equalTo(ttl)); + } + + public void testValidateTTL() { + IndexRequest indexRequest = new IndexRequest("index", "type"); + if (randomBoolean()) { + indexRequest.ttl(randomIntBetween(Integer.MIN_VALUE, -1)); + } else { + if (randomBoolean()) { + indexRequest.ttl(new TimeValue(randomIntBetween(Integer.MIN_VALUE, -1))); + } else { + indexRequest.ttl(randomIntBetween(Integer.MIN_VALUE, -1) + "ms"); + } + } + ActionRequestValidationException validate = indexRequest.validate(); + assertThat(validate, notNullValue()); + assertThat(validate.getMessage(), containsString("ttl must not be negative")); + } } diff --git a/core/src/test/java/org/elasticsearch/action/indexedscripts/get/GetIndexedScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/indexedscripts/get/GetIndexedScriptRequestTests.java index ea1d3ba8d92..af09af9a642 100644 --- a/core/src/test/java/org/elasticsearch/action/indexedscripts/get/GetIndexedScriptRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/indexedscripts/get/GetIndexedScriptRequestTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.index.VersionType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -31,8 +30,6 @@ import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; public class GetIndexedScriptRequestTests extends ESTestCase { - - @Test public void testGetIndexedScriptRequestSerialization() throws IOException { GetIndexedScriptRequest request = new GetIndexedScriptRequest("lang", "id"); if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/action/percolate/MultiPercolatorRequestTests.java b/core/src/test/java/org/elasticsearch/action/percolate/MultiPercolatorRequestTests.java index 48c75d8267b..16251463d57 100644 --- a/core/src/test/java/org/elasticsearch/action/percolate/MultiPercolatorRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/percolate/MultiPercolatorRequestTests.java @@ -20,20 +20,19 @@ package org.elasticsearch.action.percolate; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.StreamsUtils; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** */ public class MultiPercolatorRequestTests extends ESTestCase { - - @Test public void testParseBulkRequests() throws Exception { byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/percolate/mpercolate1.json"); MultiPercolateRequest request = new MultiPercolateRequest().add(data, 0, data.length); @@ -150,8 +149,7 @@ public class MultiPercolatorRequestTests extends ESTestCase { assertThat(sourceMap.get("doc"), nullValue()); } - @Test - public void testParseBulkRequests_defaults() throws Exception { + public void testParseBulkRequestsDefaults() throws Exception { byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/percolate/mpercolate2.json"); MultiPercolateRequest request = new MultiPercolateRequest(); request.indices("my-index1").documentType("my-type1").indicesOptions(IndicesOptions.lenientExpandOpen()); diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index 5fd9baea068..ee0ceef1721 100644 --- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -20,24 +20,32 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.test.StreamsUtils; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.MatchAllQueryParser; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.rest.action.search.RestMultiSearchAction; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.StreamsUtils; import java.io.IOException; +import java.util.Collections; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public class MultiSearchRequestTests extends ESTestCase { - - @Test - public void simpleAdd() throws Exception { + public void testSimpleAdd() throws Exception { + IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, Collections.singleton(new MatchAllQueryParser()), new NamedWriteableRegistry()); byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch1.json"); - MultiSearchRequest request = new MultiSearchRequest().add(data, 0, data.length, false, null, null, null); + MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null, + null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY); assertThat(request.requests().size(), equalTo(8)); assertThat(request.requests().get(0).indices()[0], equalTo("test")); assertThat(request.requests().get(0).indicesOptions(), equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed()))); @@ -60,10 +68,11 @@ public class MultiSearchRequestTests extends ESTestCase { assertThat(request.requests().get(7).types().length, equalTo(0)); } - @Test - public void simpleAdd2() throws Exception { + public void testSimpleAdd2() throws Exception { + IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, Collections.singleton(new MatchAllQueryParser()), new NamedWriteableRegistry()); byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch2.json"); - MultiSearchRequest request = new MultiSearchRequest().add(data, 0, data.length, false, null, null, null); + MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null, + null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY); assertThat(request.requests().size(), equalTo(5)); assertThat(request.requests().get(0).indices()[0], equalTo("test")); assertThat(request.requests().get(0).types().length, equalTo(0)); @@ -78,10 +87,11 @@ public class MultiSearchRequestTests extends ESTestCase { assertThat(request.requests().get(4).types().length, equalTo(0)); } - @Test - public void simpleAdd3() throws Exception { + public void testSimpleAdd3() throws Exception { + IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, Collections.singleton(new MatchAllQueryParser()), new NamedWriteableRegistry()); byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch3.json"); - MultiSearchRequest request = new MultiSearchRequest().add(data, 0, data.length, false, null, null, null); + MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null, + null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY); assertThat(request.requests().size(), equalTo(4)); assertThat(request.requests().get(0).indices()[0], equalTo("test0")); assertThat(request.requests().get(0).indices()[1], equalTo("test1")); @@ -97,10 +107,11 @@ public class MultiSearchRequestTests extends ESTestCase { assertThat(request.requests().get(3).searchType(), equalTo(SearchType.DFS_QUERY_THEN_FETCH)); } - @Test - public void simpleAdd4() throws Exception { + public void testSimpleAdd4() throws Exception { + IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, Collections.singleton(new MatchAllQueryParser()), new NamedWriteableRegistry()); byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch4.json"); - MultiSearchRequest request = new MultiSearchRequest().add(data, 0, data.length, false, null, null, null); + MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null, + null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY); assertThat(request.requests().size(), equalTo(3)); assertThat(request.requests().get(0).indices()[0], equalTo("test0")); assertThat(request.requests().get(0).indices()[1], equalTo("test1")); @@ -118,10 +129,11 @@ public class MultiSearchRequestTests extends ESTestCase { assertThat(request.requests().get(2).routing(), equalTo("123")); } - @Test - public void simpleAdd5() throws Exception { + public void testSimpleAdd5() throws Exception { + IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, Collections.singleton(new MatchAllQueryParser()), new NamedWriteableRegistry()); byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch5.json"); - MultiSearchRequest request = new MultiSearchRequest().add(data, 0, data.length, true, null, null, null); + MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), true, null, null, + null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY); assertThat(request.requests().size(), equalTo(3)); assertThat(request.requests().get(0).indices()[0], equalTo("test0")); assertThat(request.requests().get(0).indices()[1], equalTo("test1")); @@ -137,6 +149,18 @@ public class MultiSearchRequestTests extends ESTestCase { assertThat(request.requests().get(2).types()[0], equalTo("type2")); assertThat(request.requests().get(2).types()[1], equalTo("type1")); assertThat(request.requests().get(2).routing(), equalTo("123")); + assertNotNull(request.requests().get(0).template()); + assertNotNull(request.requests().get(1).template()); + assertNotNull(request.requests().get(2).template()); + assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(0).template().getType()); + assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(1).template().getType()); + assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(2).template().getType()); + assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(0).template().getScript()); + assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(1).template().getScript()); + assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(2).template().getScript()); + assertEquals(1, request.requests().get(0).template().getParams().size()); + assertEquals(1, request.requests().get(1).template().getParams().size()); + assertEquals(1, request.requests().get(2).template().getParams().size()); } public void testResponseErrorToXContent() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java index 1a0579445f8..fc6453318cf 100644 --- a/core/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java @@ -21,26 +21,16 @@ package org.elasticsearch.action.search; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Test; -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; public class SearchRequestBuilderTests extends ESTestCase { - private static Client client; @BeforeClass @@ -59,76 +49,28 @@ public class SearchRequestBuilderTests extends ESTestCase { client = null; } - @Test public void testEmptySourceToString() { SearchRequestBuilder searchRequestBuilder = client.prepareSearch(); assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder().toString())); } - @Test public void testQueryBuilderQueryToString() { SearchRequestBuilder searchRequestBuilder = client.prepareSearch(); searchRequestBuilder.setQuery(QueryBuilders.matchAllQuery()); assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString())); } - @Test - public void testXContentBuilderQueryToString() throws IOException { + public void testSearchSourceBuilderToString() { SearchRequestBuilder searchRequestBuilder = client.prepareSearch(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - xContentBuilder.startObject(); - xContentBuilder.startObject("match_all"); - xContentBuilder.endObject(); - xContentBuilder.endObject(); - searchRequestBuilder.setQuery(xContentBuilder); - assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder().query(xContentBuilder).toString())); + searchRequestBuilder.setSource(new SearchSourceBuilder().query(QueryBuilders.termQuery("field", "value"))); + assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder().query(QueryBuilders.termQuery("field", "value")).toString())); } - @Test - public void testStringQueryToString() { - SearchRequestBuilder searchRequestBuilder = client.prepareSearch(); - String query = "{ \"match_all\" : {} }"; - searchRequestBuilder.setQuery(query); - assertThat(searchRequestBuilder.toString(), containsString("\"query\":{ \"match_all\" : {} }")); - } - - @Test - public void testStringSourceToString() { - SearchRequestBuilder searchRequestBuilder = client.prepareSearch(); - String source = "{ \"query\" : { \"match_all\" : {} } }"; - searchRequestBuilder.setSource(new BytesArray(source)); - assertThat(searchRequestBuilder.toString(), equalTo(source)); - } - - @Test - public void testXContentBuilderSourceToString() throws IOException { - SearchRequestBuilder searchRequestBuilder = client.prepareSearch(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - xContentBuilder.startObject(); - xContentBuilder.startObject("query"); - xContentBuilder.startObject("match_all"); - xContentBuilder.endObject(); - xContentBuilder.endObject(); - xContentBuilder.endObject(); - searchRequestBuilder.setSource(xContentBuilder.bytes()); - assertThat(searchRequestBuilder.toString(), equalTo(XContentHelper.convertToJson(xContentBuilder.bytes(), false, true))); - } - - @Test public void testThatToStringDoesntWipeRequestSource() { - String source = "{\n" + - " \"query\" : {\n" + - " \"match\" : {\n" + - " \"field\" : {\n" + - " \"query\" : \"value\"" + - " }\n" + - " }\n" + - " }\n" + - " }"; - SearchRequestBuilder searchRequestBuilder = client.prepareSearch().setSource(new BytesArray(source)); - String preToString = searchRequestBuilder.request().source().toUtf8(); - assertThat(searchRequestBuilder.toString(), equalTo(source)); - String postToString = searchRequestBuilder.request().source().toUtf8(); + SearchRequestBuilder searchRequestBuilder = client.prepareSearch().setSource(new SearchSourceBuilder().query(QueryBuilders.termQuery("field", "value"))); + String preToString = searchRequestBuilder.request().toString(); + assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder().query(QueryBuilders.termQuery("field", "value")).toString())); + String postToString = searchRequestBuilder.request().toString(); assertThat(preToString, equalTo(postToString)); } } diff --git a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java index c681a6107c3..d5ed5302b97 100644 --- a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java @@ -23,14 +23,11 @@ import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; public class IndicesOptionsTests extends ESTestCase { - - @Test public void testSerialization() throws Exception { int iterations = randomIntBetween(5, 20); for (int i = 0; i < iterations; i++) { @@ -55,7 +52,6 @@ public class IndicesOptionsTests extends ESTestCase { } } - @Test public void testFromOptions() { int iterations = randomIntBetween(5, 20); for (int i = 0; i < iterations; i++) { diff --git a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java index 148fc70d122..f21013b7fbe 100644 --- a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import org.junit.Test; import java.util.ArrayList; import java.util.Collections; @@ -41,7 +40,9 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.notNullValue; public class TransportActionFilterChainTests extends ESTestCase { @@ -52,9 +53,7 @@ public class TransportActionFilterChainTests extends ESTestCase { counter = new AtomicInteger(); } - @Test public void testActionFiltersRequest() throws ExecutionException, InterruptedException { - int numFilters = randomInt(10); Set orders = new HashSet<>(numFilters); while (orders.size() < numFilters) { @@ -134,9 +133,7 @@ public class TransportActionFilterChainTests extends ESTestCase { } } - @Test public void testActionFiltersResponse() throws ExecutionException, InterruptedException { - int numFilters = randomInt(10); Set orders = new HashSet<>(numFilters); while (orders.size() < numFilters) { @@ -216,9 +213,7 @@ public class TransportActionFilterChainTests extends ESTestCase { } } - @Test public void testTooManyContinueProcessingRequest() throws ExecutionException, InterruptedException { - final int additionalContinueCount = randomInt(10); RequestTestFilter testFilter = new RequestTestFilter(randomInt(), new RequestCallback() { @@ -274,9 +269,7 @@ public class TransportActionFilterChainTests extends ESTestCase { } } - @Test public void testTooManyContinueProcessingResponse() throws ExecutionException, InterruptedException { - final int additionalContinueCount = randomInt(10); ResponseTestFilter testFilter = new ResponseTestFilter(randomInt(), new ResponseCallback() { diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index 18a86b1dea0..6f5be649451 100644 --- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.broadcast.BroadcastRequest; @@ -85,7 +84,6 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { private TestClusterService clusterService; private CapturingTransport transport; - private TransportService transportService; private TestTransportBroadcastByNodeAction action; @@ -184,13 +182,13 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { super.setUp(); transport = new CapturingTransport(); clusterService = new TestClusterService(THREAD_POOL); - transportService = new TransportService(transport, THREAD_POOL); + final TransportService transportService = new TransportService(transport, THREAD_POOL); transportService.start(); setClusterState(clusterService, TEST_INDEX); action = new TestTransportBroadcastByNodeAction( Settings.EMPTY, transportService, - new ActionFilters(new HashSet()), + new ActionFilters(new HashSet<>()), new MyResolver(), Request::new, ThreadPool.Names.SAME @@ -240,13 +238,13 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { PlainActionFuture listener = new PlainActionFuture<>(); ClusterBlocks.Builder block = ClusterBlocks.builder() - .addGlobalBlock(new ClusterBlock(1, "", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); + .addGlobalBlock(new ClusterBlock(1, "test-block", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block)); try { action.new AsyncAction(request, listener).start(); fail("expected ClusterBlockException"); } catch (ClusterBlockException expected) { - + assertEquals("blocked by: [SERVICE_UNAVAILABLE/1/test-block];", expected.getMessage()); } } @@ -261,7 +259,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { action.new AsyncAction(request, listener).start(); fail("expected ClusterBlockException"); } catch (ClusterBlockException expected) { - + assertEquals("blocked by: [SERVICE_UNAVAILABLE/1/test-block];", expected.getMessage()); } } @@ -289,6 +287,44 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { } } + // simulate the master being removed from the cluster but before a new master is elected + // as such, the shards assigned to the master will still show up in the cluster state as assigned to a node but + // that node will not be in the local cluster state on any node that has detected the master as failing + // in this case, such a shard should be treated as unassigned + public void testRequestsAreNotSentToFailedMaster() { + Request request = new Request(new String[]{TEST_INDEX}); + PlainActionFuture listener = new PlainActionFuture<>(); + + DiscoveryNode masterNode = clusterService.state().nodes().masterNode(); + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(clusterService.state().getNodes()); + builder.remove(masterNode.id()); + + clusterService.setState(ClusterState.builder(clusterService.state()).nodes(builder)); + + action.new AsyncAction(request, listener).start(); + + Map> capturedRequests = transport.capturedRequestsByTargetNode(); + + // the master should not be in the list of nodes that requests were sent to + ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[]{TEST_INDEX}); + Set set = new HashSet<>(); + for (ShardRouting shard : shardIt.asUnordered()) { + if (!shard.currentNodeId().equals(masterNode.id())) { + set.add(shard.currentNodeId()); + } + } + + // check a request was sent to the right number of nodes + assertEquals(set.size(), capturedRequests.size()); + + // check requests were sent to the right nodes + assertEquals(set, capturedRequests.keySet()); + for (Map.Entry> entry : capturedRequests.entrySet()) { + // check one request was sent to each non-master node + assertEquals(1, entry.getValue().size()); + } + } + public void testOperationExecution() throws Exception { ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[]{TEST_INDEX}); Set shards = new HashSet<>(); @@ -340,6 +376,18 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { Request request = new Request(new String[]{TEST_INDEX}); PlainActionFuture listener = new PlainActionFuture<>(); + // simulate removing the master + final boolean simulateFailedMasterNode = rarely(); + DiscoveryNode failedMasterNode = null; + if (simulateFailedMasterNode) { + failedMasterNode = clusterService.state().nodes().masterNode(); + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(clusterService.state().getNodes()); + builder.remove(failedMasterNode.id()); + builder.masterNodeId(null); + + clusterService.setState(ClusterState.builder(clusterService.state()).nodes(builder)); + } + action.new AsyncAction(request, listener).start(); Map> capturedRequests = transport.capturedRequestsByTargetNode(); transport.clear(); @@ -348,7 +396,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { Map> map = new HashMap<>(); for (ShardRouting shard : shardIt.asUnordered()) { if (!map.containsKey(shard.currentNodeId())) { - map.put(shard.currentNodeId(), new ArrayList()); + map.put(shard.currentNodeId(), new ArrayList<>()); } map.get(shard.currentNodeId()).add(shard); } @@ -382,6 +430,9 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { transport.handleResponse(requestId, nodeResponse); } } + if (simulateFailedMasterNode) { + totalShards += map.get(failedMasterNode.id()).size(); + } Response response = listener.get(); assertEquals("total shards", totalShards, response.getTotalShards()); diff --git a/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java b/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java new file mode 100644 index 00000000000..825e3e40894 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java @@ -0,0 +1,115 @@ +package org.elasticsearch.action.support.master; + +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.DiscoverySettings; +import org.elasticsearch.discovery.zen.elect.ElectMasterService; +import org.elasticsearch.discovery.zen.fd.FaultDetection; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.disruption.NetworkDisconnectPartition; +import org.elasticsearch.test.disruption.NetworkPartition; +import org.elasticsearch.test.transport.MockTransportService; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; + +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +@ESIntegTestCase.SuppressLocalMode +public class IndexingMasterFailoverIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + final HashSet> classes = new HashSet<>(super.nodePlugins()); + classes.add(MockTransportService.TestPlugin.class); + return classes; + } + + /** + * Indexing operations which entail mapping changes require a blocking request to the master node to update the mapping. + * If the master node is being disrupted or if it cannot commit cluster state changes, it needs to retry within timeout limits. + * This retry logic is implemented in TransportMasterNodeAction and tested by the following master failover scenario. + */ + public void testMasterFailoverDuringIndexingWithMappingChanges() throws Throwable { + logger.info("--> start 4 nodes, 3 master, 1 data"); + + final Settings sharedSettings = Settings.builder() + .put(FaultDetection.SETTING_PING_TIMEOUT, "1s") // for hitting simulated network failures quickly + .put(FaultDetection.SETTING_PING_RETRIES, "1") // for hitting simulated network failures quickly + .put("discovery.zen.join_timeout", "10s") // still long to induce failures but to long so test won't time out + .put(DiscoverySettings.PUBLISH_TIMEOUT, "1s") // <-- for hitting simulated network failures quickly + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, 2) + .build(); + + internalCluster().startMasterOnlyNodesAsync(3, sharedSettings).get(); + + String dataNode = internalCluster().startDataOnlyNode(sharedSettings); + + logger.info("--> wait for all nodes to join the cluster"); + ensureStableCluster(4); + + // We index data with mapping changes into cluster and have master failover at same time + client().admin().indices().prepareCreate("myindex") + .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) + .get(); + ensureGreen("myindex"); + + final CyclicBarrier barrier = new CyclicBarrier(2); + + Thread indexingThread = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + } catch (InterruptedException e) { + logger.warn("Barrier interrupted", e); + return; + } catch (BrokenBarrierException e) { + logger.warn("Broken barrier", e); + return; + } + for (int i = 0; i < 10; i++) { + // index data with mapping changes + IndexResponse response = client(dataNode).prepareIndex("myindex", "mytype").setSource("field_" + i, "val").get(); + assertThat(response.isCreated(), equalTo(true)); + } + } + }); + indexingThread.setName("indexingThread"); + indexingThread.start(); + + barrier.await(); + + // interrupt communication between master and other nodes in cluster + String master = internalCluster().getMasterName(); + Set otherNodes = new HashSet<>(Arrays.asList(internalCluster().getNodeNames())); + otherNodes.remove(master); + + NetworkPartition partition = new NetworkDisconnectPartition(Collections.singleton(master), otherNodes, random()); + internalCluster().setDisruptionScheme(partition); + + logger.info("--> disrupting network"); + partition.startDisrupting(); + + logger.info("--> waiting for new master to be elected"); + ensureStableCluster(3, dataNode); + + partition.stopDisrupting(); + logger.info("--> waiting to heal"); + ensureStableCluster(4); + + indexingThread.join(); + + ensureGreen("myindex"); + refresh(); + assertThat(client().prepareSearch("myindex").get().getHits().getTotalHits(), equalTo(10L)); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index d3dd75c8def..b66196ae7d5 100644 --- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -38,6 +39,8 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; @@ -49,7 +52,6 @@ import org.elasticsearch.transport.TransportService; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import java.util.HashSet; import java.util.Set; @@ -146,7 +148,6 @@ public class TransportMasterNodeActionTests extends ESTestCase { } } - @Test public void testLocalOperationWithoutBlocks() throws ExecutionException, InterruptedException { final boolean masterOperationFailure = randomBoolean(); @@ -182,7 +183,6 @@ public class TransportMasterNodeActionTests extends ESTestCase { } } - @Test public void testLocalOperationWithBlocks() throws ExecutionException, InterruptedException { final boolean retryableBlock = randomBoolean(); final boolean unblockBeforeTimeout = randomBoolean(); @@ -214,10 +214,19 @@ public class TransportMasterNodeActionTests extends ESTestCase { } assertTrue(listener.isDone()); - assertListenerThrows("ClusterBlockException should be thrown", listener, ClusterBlockException.class); + if (retryableBlock) { + try { + listener.get(); + fail("Expected exception but returned proper result"); + } catch (ExecutionException ex) { + assertThat(ex.getCause(), instanceOf(MasterNotDiscoveredException.class)); + assertThat(ex.getCause().getCause(), instanceOf(ClusterBlockException.class)); + } + } else { + assertListenerThrows("ClusterBlockException should be thrown", listener, ClusterBlockException.class); + } } - @Test public void testForceLocalOperation() throws ExecutionException, InterruptedException { Request request = new Request(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -235,7 +244,6 @@ public class TransportMasterNodeActionTests extends ESTestCase { listener.get(); } - @Test public void testMasterNotAvailable() throws ExecutionException, InterruptedException { Request request = new Request().masterNodeTimeout(TimeValue.timeValueSeconds(0)); clusterService.setState(ClusterStateCreationUtils.state(localNode, null, allNodes)); @@ -245,7 +253,6 @@ public class TransportMasterNodeActionTests extends ESTestCase { assertListenerThrows("MasterNotDiscoveredException should be thrown", listener, MasterNotDiscoveredException.class); } - @Test public void testMasterBecomesAvailable() throws ExecutionException, InterruptedException { Request request = new Request(); clusterService.setState(ClusterStateCreationUtils.state(localNode, null, allNodes)); @@ -257,23 +264,13 @@ public class TransportMasterNodeActionTests extends ESTestCase { listener.get(); } - @Test public void testDelegateToMaster() throws ExecutionException, InterruptedException { Request request = new Request(); clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); PlainActionFuture listener = new PlainActionFuture<>(); - final AtomicBoolean delegationToMaster = new AtomicBoolean(); + new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener); - new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { - @Override - protected void processBeforeDelegationToMaster(Request request, ClusterState state) { - logger.debug("Delegation to master called"); - delegationToMaster.set(true); - } - }.execute(request, listener); - - assertTrue("processBeforeDelegationToMaster not called", delegationToMaster.get()); assertThat(transport.capturedRequests().length, equalTo(1)); CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; assertTrue(capturedRequest.node.isMasterNode()); @@ -286,24 +283,14 @@ public class TransportMasterNodeActionTests extends ESTestCase { assertThat(listener.get(), equalTo(response)); } - @Test public void testDelegateToFailingMaster() throws ExecutionException, InterruptedException { boolean failsWithConnectTransportException = randomBoolean(); Request request = new Request().masterNodeTimeout(TimeValue.timeValueSeconds(failsWithConnectTransportException ? 60 : 0)); clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); PlainActionFuture listener = new PlainActionFuture<>(); - final AtomicBoolean delegationToMaster = new AtomicBoolean(); + new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener); - new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { - @Override - protected void processBeforeDelegationToMaster(Request request, ClusterState state) { - logger.debug("Delegation to master called"); - delegationToMaster.set(true); - } - }.execute(request, listener); - - assertTrue("processBeforeDelegationToMaster not called", delegationToMaster.get()); assertThat(transport.capturedRequests().length, equalTo(1)); CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; assertTrue(capturedRequest.node.isMasterNode()); @@ -328,4 +315,35 @@ public class TransportMasterNodeActionTests extends ESTestCase { } } } + + public void testMasterFailoverAfterStepDown() throws ExecutionException, InterruptedException { + Request request = new Request().masterNodeTimeout(TimeValue.timeValueHours(1)); + PlainActionFuture listener = new PlainActionFuture<>(); + + final Response response = new Response(); + + clusterService.setState(ClusterStateCreationUtils.state(localNode, localNode, allNodes)); + + new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + // The other node has become master, simulate failures of this node while publishing cluster state through ZenDiscovery + TransportMasterNodeActionTests.this.clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); + Throwable failure = randomBoolean() + ? new Discovery.FailedToCommitClusterStateException("Fake error") + : new NotMasterException("Fake error"); + listener.onFailure(failure); + } + }.execute(request, listener); + + assertThat(transport.capturedRequests().length, equalTo(1)); + CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; + assertTrue(capturedRequest.node.isMasterNode()); + assertThat(capturedRequest.request, equalTo(request)); + assertThat(capturedRequest.action, equalTo("testAction")); + + transport.handleResponse(capturedRequest.requestId, response); + assertTrue(listener.isDone()); + assertThat(listener.get(), equalTo(response)); + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java index 2fe04bb9238..4d17155f611 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.action.support.replication; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.indices.flush.FlushRequest; @@ -48,7 +48,6 @@ import org.elasticsearch.transport.local.LocalTransport; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import java.io.IOException; import java.util.Date; @@ -59,8 +58,12 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state; +import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithAssignedPrimariesAndOneReplica; +import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithNoShard; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class BroadcastReplicationTests extends ESTestCase { @@ -92,14 +95,13 @@ public class BroadcastReplicationTests extends ESTestCase { threadPool = null; } - @Test public void testNotStartedPrimary() throws InterruptedException, ExecutionException, IOException { final String index = "test"; clusterService.setState(state(index, randomBoolean(), randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED, ShardRoutingState.UNASSIGNED)); logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); Future response = (broadcastReplicationAction.execute(new BroadcastRequest().indices(index))); - for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { + for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { if (randomBoolean()) { shardRequests.v2().onFailure(new NoShardAvailableActionException(shardRequests.v1())); } else { @@ -112,23 +114,21 @@ public class BroadcastReplicationTests extends ESTestCase { assertBroadcastResponse(2, 0, 0, response.get(), null); } - @Test public void testStartedPrimary() throws InterruptedException, ExecutionException, IOException { final String index = "test"; clusterService.setState(state(index, randomBoolean(), ShardRoutingState.STARTED)); logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); Future response = (broadcastReplicationAction.execute(new BroadcastRequest().indices(index))); - for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { - ActionWriteResponse actionWriteResponse = new ActionWriteResponse(); - actionWriteResponse.setShardInfo(new ActionWriteResponse.ShardInfo(1, 1, new ActionWriteResponse.ShardInfo.Failure[0])); - shardRequests.v2().onResponse(actionWriteResponse); + for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { + ReplicationResponse replicationResponse = new ReplicationResponse(); + replicationResponse.setShardInfo(new ReplicationResponse.ShardInfo(1, 1, new ReplicationResponse.ShardInfo.Failure[0])); + shardRequests.v2().onResponse(replicationResponse); } logger.info("total shards: {}, ", response.get().getTotalShards()); assertBroadcastResponse(1, 1, 0, response.get(), null); } - @Test public void testResultCombine() throws InterruptedException, ExecutionException, IOException { final String index = "test"; int numShards = randomInt(3); @@ -137,20 +137,20 @@ public class BroadcastReplicationTests extends ESTestCase { Future response = (broadcastReplicationAction.execute(new BroadcastRequest().indices(index))); int succeeded = 0; int failed = 0; - for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { + for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { if (randomBoolean()) { - ActionWriteResponse.ShardInfo.Failure[] failures = new ActionWriteResponse.ShardInfo.Failure[0]; + ReplicationResponse.ShardInfo.Failure[] failures = new ReplicationResponse.ShardInfo.Failure[0]; int shardsSucceeded = randomInt(1) + 1; succeeded += shardsSucceeded; - ActionWriteResponse actionWriteResponse = new ActionWriteResponse(); + ReplicationResponse replicationResponse = new ReplicationResponse(); if (shardsSucceeded == 1 && randomBoolean()) { //sometimes add failure (no failure means shard unavailable) - failures = new ActionWriteResponse.ShardInfo.Failure[1]; - failures[0] = new ActionWriteResponse.ShardInfo.Failure(index, shardRequests.v1().id(), null, new Exception("pretend shard failed"), RestStatus.GATEWAY_TIMEOUT, false); + failures = new ReplicationResponse.ShardInfo.Failure[1]; + failures[0] = new ReplicationResponse.ShardInfo.Failure(index, shardRequests.v1().id(), null, new Exception("pretend shard failed"), RestStatus.GATEWAY_TIMEOUT, false); failed++; } - actionWriteResponse.setShardInfo(new ActionWriteResponse.ShardInfo(2, shardsSucceeded, failures)); - shardRequests.v2().onResponse(actionWriteResponse); + replicationResponse.setShardInfo(new ReplicationResponse.ShardInfo(2, shardsSucceeded, failures)); + shardRequests.v2().onResponse(replicationResponse); } else { // sometimes fail failed += 2; @@ -161,7 +161,6 @@ public class BroadcastReplicationTests extends ESTestCase { assertBroadcastResponse(2 * numShards, succeeded, failed, response.get(), Exception.class); } - @Test public void testNoShards() throws InterruptedException, ExecutionException, IOException { clusterService.setState(stateWithNoShard()); logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); @@ -169,7 +168,6 @@ public class BroadcastReplicationTests extends ESTestCase { assertBroadcastResponse(0, 0, 0, response, null); } - @Test public void testShardsList() throws InterruptedException, ExecutionException { final String index = "test"; final ShardId shardId = new ShardId(index, 0); @@ -181,16 +179,16 @@ public class BroadcastReplicationTests extends ESTestCase { assertThat(shards.get(0), equalTo(shardId)); } - private class TestBroadcastReplicationAction extends TransportBroadcastReplicationAction { - protected final Set>> capturedShardRequests = ConcurrentCollections.newConcurrentSet(); + private class TestBroadcastReplicationAction extends TransportBroadcastReplicationAction { + protected final Set>> capturedShardRequests = ConcurrentCollections.newConcurrentSet(); public TestBroadcastReplicationAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, TransportReplicationAction replicatedBroadcastShardAction) { super("test-broadcast-replication-action", BroadcastRequest::new, settings, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, replicatedBroadcastShardAction); } @Override - protected ActionWriteResponse newShardResponse() { - return new ActionWriteResponse(); + protected ReplicationResponse newShardResponse() { + return new ReplicationResponse(); } @Override @@ -204,7 +202,7 @@ public class BroadcastReplicationTests extends ESTestCase { } @Override - protected void shardExecute(BroadcastRequest request, ShardId shardId, ActionListener shardActionListener) { + protected void shardExecute(BroadcastRequest request, ShardId shardId, ActionListener shardActionListener) { capturedShardRequests.add(new Tuple<>(shardId, shardActionListener)); } } diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 913d52d5b17..406e476b4e0 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -27,12 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.index.shard.ShardId; @@ -40,10 +35,7 @@ import org.elasticsearch.index.shard.ShardId; import java.util.HashSet; import java.util.Set; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; +import static org.elasticsearch.cluster.metadata.IndexMetaData.*; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomIntBetween; diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ShardReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java similarity index 66% rename from core/src/test/java/org/elasticsearch/action/support/replication/ShardReplicationTests.java rename to core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 8d4591730f1..5834b2662ad 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ShardReplicationTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.action.support.replication; import org.apache.lucene.index.CorruptIndexException; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.support.ActionFilter; @@ -28,7 +28,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -46,24 +45,23 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.shard.IndexShardNotStartedException; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardNotFoundException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.cluster.TestClusterService; import org.elasticsearch.test.transport.CapturingTransport; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportChannel; -import org.elasticsearch.transport.TransportResponse; -import org.elasticsearch.transport.TransportResponseOptions; -import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.*; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -80,7 +78,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class ShardReplicationTests extends ESTestCase { +public class TransportReplicationActionTests extends ESTestCase { private static ThreadPool threadPool; @@ -125,7 +123,6 @@ public class ShardReplicationTests extends ESTestCase { } } - @Test public void testBlocks() throws ExecutionException, InterruptedException { Request request = new Request(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -133,22 +130,22 @@ public class ShardReplicationTests extends ESTestCase { ClusterBlocks.Builder block = ClusterBlocks.builder() .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block)); - TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, listener); - assertFalse("primary phase should stop execution", primaryPhase.checkBlocks()); + TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(request, listener); + reroutePhase.run(); assertListenerThrows("primary phase should fail operation", listener, ClusterBlockException.class); block = ClusterBlocks.builder() .addGlobalBlock(new ClusterBlock(1, "retryable", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block)); listener = new PlainActionFuture<>(); - primaryPhase = action.new PrimaryPhase(new Request().timeout("5ms"), listener); - assertFalse("primary phase should stop execution on retryable block", primaryPhase.checkBlocks()); + reroutePhase = action.new ReroutePhase(new Request().timeout("5ms"), listener); + reroutePhase.run(); assertListenerThrows("failed to timeout on retryable block", listener, ClusterBlockException.class); listener = new PlainActionFuture<>(); - primaryPhase = action.new PrimaryPhase(new Request(), listener); - assertFalse("primary phase should stop execution on retryable block", primaryPhase.checkBlocks()); + reroutePhase = action.new ReroutePhase(new Request(), listener); + reroutePhase.run(); assertFalse("primary phase should wait on retryable block", listener.isDone()); block = ClusterBlocks.builder() @@ -162,7 +159,6 @@ public class ShardReplicationTests extends ESTestCase { assertEquals(1, count.get()); } - @Test public void testNotStartedPrimary() throws InterruptedException, ExecutionException { final String index = "test"; final ShardId shardId = new ShardId(index, 0); @@ -174,26 +170,47 @@ public class ShardReplicationTests extends ESTestCase { Request request = new Request(shardId).timeout("1ms"); PlainActionFuture listener = new PlainActionFuture<>(); - TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, listener); - primaryPhase.run(); + TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(request, listener); + reroutePhase.run(); assertListenerThrows("unassigned primary didn't cause a timeout", listener, UnavailableShardsException.class); request = new Request(shardId); listener = new PlainActionFuture<>(); - primaryPhase = action.new PrimaryPhase(request, listener); - primaryPhase.run(); + reroutePhase = action.new ReroutePhase(request, listener); + reroutePhase.run(); assertFalse("unassigned primary didn't cause a retry", listener.isDone()); clusterService.setState(state(index, true, ShardRoutingState.STARTED)); logger.debug("--> primary assigned state:\n{}", clusterService.state().prettyPrint()); - listener.get(); - assertTrue("request wasn't processed on primary, despite of it being assigned", request.processedOnPrimary.get()); + final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id()); + final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId(); + final List capturedRequests = transport.capturedRequestsByTargetNode().get(primaryNodeId); + assertThat(capturedRequests, notNullValue()); + assertThat(capturedRequests.size(), equalTo(1)); + assertThat(capturedRequests.get(0).action, equalTo("testAction[p]")); assertIndexShardCounter(1); } - @Test - public void testRoutingToPrimary() { + public void testUnknownIndexOrShardOnReroute() throws InterruptedException { + final String index = "test"; + // no replicas in oder to skip the replication part + clusterService.setState(state(index, true, + randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED)); + logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); + Request request = new Request(new ShardId("unknown_index", 0)).timeout("1ms"); + PlainActionFuture listener = new PlainActionFuture<>(); + TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(request, listener); + reroutePhase.run(); + assertListenerThrows("must throw index not found exception", listener, IndexNotFoundException.class); + request = new Request(new ShardId(index, 10)).timeout("1ms"); + listener = new PlainActionFuture<>(); + reroutePhase = action.new ReroutePhase(request, listener); + reroutePhase.run(); + assertListenerThrows("must throw shard not found exception", listener, ShardNotFoundException.class); + } + + public void testRoutePhaseExecutesRequest() { final String index = "test"; final ShardId shardId = new ShardId(index, 0); @@ -206,28 +223,128 @@ public class ShardReplicationTests extends ESTestCase { Request request = new Request(shardId); PlainActionFuture listener = new PlainActionFuture<>(); - TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, listener); - assertTrue(primaryPhase.checkBlocks()); - primaryPhase.routeRequestOrPerformLocally(shardRoutingTable.primaryShard(), shardRoutingTable.shardsIt()); - if (primaryNodeId.equals(clusterService.localNode().id())) { - logger.info("--> primary is assigned locally, testing for execution"); - assertTrue("request failed to be processed on a local primary", request.processedOnPrimary.get()); - if (transport.capturedRequests().length > 0) { - assertIndexShardCounter(2); - } else { - assertIndexShardCounter(1); - } + TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(request, listener); + reroutePhase.run(); + assertThat(request.shardId(), equalTo(shardId)); + logger.info("--> primary is assigned to [{}], checking request forwarded", primaryNodeId); + final List capturedRequests = transport.capturedRequestsByTargetNode().get(primaryNodeId); + assertThat(capturedRequests, notNullValue()); + assertThat(capturedRequests.size(), equalTo(1)); + if (clusterService.state().nodes().localNodeId().equals(primaryNodeId)) { + assertThat(capturedRequests.get(0).action, equalTo("testAction[p]")); } else { - logger.info("--> primary is assigned to [{}], checking request forwarded", primaryNodeId); - final List capturedRequests = transport.capturedRequestsByTargetNode().get(primaryNodeId); - assertThat(capturedRequests, notNullValue()); - assertThat(capturedRequests.size(), equalTo(1)); assertThat(capturedRequests.get(0).action, equalTo("testAction")); - assertIndexShardUninitialized(); + } + assertIndexShardUninitialized(); + } + + public void testPrimaryPhaseExecutesRequest() throws InterruptedException, ExecutionException { + final String index = "test"; + final ShardId shardId = new ShardId(index, 0); + clusterService.setState(state(index, true, ShardRoutingState.STARTED, ShardRoutingState.STARTED)); + Request request = new Request(shardId).timeout("1ms"); + PlainActionFuture listener = new PlainActionFuture<>(); + TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, createTransportChannel(listener)); + primaryPhase.run(); + assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true)); + final String replicaNodeId = clusterService.state().getRoutingTable().shardRoutingTable(index, shardId.id()).replicaShards().get(0).currentNodeId(); + final List requests = transport.capturedRequestsByTargetNode().get(replicaNodeId); + assertThat(requests, notNullValue()); + assertThat(requests.size(), equalTo(1)); + assertThat("replica request was not sent", requests.get(0).action, equalTo("testAction[r]")); + } + + public void testAddedReplicaAfterPrimaryOperation() { + final String index = "test"; + final ShardId shardId = new ShardId(index, 0); + // start with no replicas + clusterService.setState(stateWithStartedPrimary(index, true, 0)); + logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); + final ClusterState stateWithAddedReplicas = state(index, true, ShardRoutingState.STARTED, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.STARTED); + + final Action actionWithAddedReplicaAfterPrimaryOp = new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { + @Override + protected Tuple shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Throwable { + final Tuple operationOnPrimary = super.shardOperationOnPrimary(metaData, shardRequest); + // add replicas after primary operation + ((TestClusterService) clusterService).setState(stateWithAddedReplicas); + logger.debug("--> state after primary operation:\n{}", clusterService.state().prettyPrint()); + return operationOnPrimary; + } + }; + + Request request = new Request(shardId); + PlainActionFuture listener = new PlainActionFuture<>(); + TransportReplicationAction.PrimaryPhase primaryPhase = actionWithAddedReplicaAfterPrimaryOp.new PrimaryPhase(request, createTransportChannel(listener)); + primaryPhase.run(); + assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true)); + for (ShardRouting replica : stateWithAddedReplicas.getRoutingTable().shardRoutingTable(index, shardId.id()).replicaShards()) { + List requests = transport.capturedRequestsByTargetNode().get(replica.currentNodeId()); + assertThat(requests, notNullValue()); + assertThat(requests.size(), equalTo(1)); + assertThat("replica request was not sent", requests.get(0).action, equalTo("testAction[r]")); } } - @Test + public void testRelocatingReplicaAfterPrimaryOperation() { + final String index = "test"; + final ShardId shardId = new ShardId(index, 0); + // start with a replica + clusterService.setState(state(index, true, ShardRoutingState.STARTED, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.STARTED)); + logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); + final ClusterState stateWithRelocatingReplica = state(index, true, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING); + + final Action actionWithRelocatingReplicasAfterPrimaryOp = new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { + @Override + protected Tuple shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Throwable { + final Tuple operationOnPrimary = super.shardOperationOnPrimary(metaData, shardRequest); + // set replica to relocating + ((TestClusterService) clusterService).setState(stateWithRelocatingReplica); + logger.debug("--> state after primary operation:\n{}", clusterService.state().prettyPrint()); + return operationOnPrimary; + } + }; + + Request request = new Request(shardId); + PlainActionFuture listener = new PlainActionFuture<>(); + TransportReplicationAction.PrimaryPhase primaryPhase = actionWithRelocatingReplicasAfterPrimaryOp.new PrimaryPhase(request, createTransportChannel(listener)); + primaryPhase.run(); + assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true)); + ShardRouting relocatingReplicaShard = stateWithRelocatingReplica.getRoutingTable().shardRoutingTable(index, shardId.id()).replicaShards().get(0); + for (String node : new String[] {relocatingReplicaShard.currentNodeId(), relocatingReplicaShard.relocatingNodeId()}) { + List requests = transport.capturedRequestsByTargetNode().get(node); + assertThat(requests, notNullValue()); + assertThat(requests.size(), equalTo(1)); + assertThat("replica request was not sent to replica", requests.get(0).action, equalTo("testAction[r]")); + } + } + + public void testIndexDeletedAfterPrimaryOperation() { + final String index = "test"; + final ShardId shardId = new ShardId(index, 0); + clusterService.setState(state(index, true, ShardRoutingState.STARTED, ShardRoutingState.STARTED)); + logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); + final ClusterState stateWithDeletedIndex = state(index + "_new", true, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING); + + final Action actionWithDeletedIndexAfterPrimaryOp = new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) { + @Override + protected Tuple shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Throwable { + final Tuple operationOnPrimary = super.shardOperationOnPrimary(metaData, shardRequest); + // delete index after primary op + ((TestClusterService) clusterService).setState(stateWithDeletedIndex); + logger.debug("--> state after primary operation:\n{}", clusterService.state().prettyPrint()); + return operationOnPrimary; + } + }; + + Request request = new Request(shardId); + PlainActionFuture listener = new PlainActionFuture<>(); + TransportReplicationAction.PrimaryPhase primaryPhase = actionWithDeletedIndexAfterPrimaryOp.new PrimaryPhase(request, createTransportChannel(listener)); + primaryPhase.run(); + assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true)); + assertThat("replication phase should be skipped if index gets deleted after primary operation", transport.capturedRequestsByTargetNode().size(), equalTo(0)); + } + public void testWriteConsistency() throws ExecutionException, InterruptedException { action = new ActionWithConsistency(Settings.EMPTY, "testActionWithConsistency", transportService, clusterService, threadPool); final String index = "test"; @@ -270,10 +387,9 @@ public class ShardReplicationTests extends ESTestCase { final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id()); PlainActionFuture listener = new PlainActionFuture<>(); - - TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, listener); + TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, createTransportChannel(listener)); if (passesWriteConsistency) { - assertThat(primaryPhase.checkWriteConsistency(shardRoutingTable.primaryShard()), nullValue()); + assertThat(primaryPhase.checkWriteConsistency(shardRoutingTable.primaryShard().shardId()), nullValue()); primaryPhase.run(); assertTrue("operations should have been perform, consistency level is met", request.processedOnPrimary.get()); if (assignedReplicas > 0) { @@ -282,20 +398,23 @@ public class ShardReplicationTests extends ESTestCase { assertIndexShardCounter(1); } } else { - assertThat(primaryPhase.checkWriteConsistency(shardRoutingTable.primaryShard()), notNullValue()); + assertThat(primaryPhase.checkWriteConsistency(shardRoutingTable.primaryShard().shardId()), notNullValue()); primaryPhase.run(); assertFalse("operations should not have been perform, consistency level is *NOT* met", request.processedOnPrimary.get()); + assertListenerThrows("should throw exception to trigger retry", listener, UnavailableShardsException.class); assertIndexShardUninitialized(); for (int i = 0; i < replicaStates.length; i++) { replicaStates[i] = ShardRoutingState.STARTED; } clusterService.setState(state(index, true, ShardRoutingState.STARTED, replicaStates)); + listener = new PlainActionFuture<>(); + primaryPhase = action.new PrimaryPhase(request, createTransportChannel(listener)); + primaryPhase.run(); assertTrue("once the consistency level met, operation should continue", request.processedOnPrimary.get()); assertIndexShardCounter(2); } } - @Test public void testReplication() throws ExecutionException, InterruptedException { final String index = "test"; final ShardId shardId = new ShardId(index, 0); @@ -319,14 +438,13 @@ public class ShardReplicationTests extends ESTestCase { runReplicateTest(shardRoutingTable, assignedReplicas, totalShards); } - @Test public void testReplicationWithShadowIndex() throws ExecutionException, InterruptedException { final String index = "test"; final ShardId shardId = new ShardId(index, 0); ClusterState state = stateWithStartedPrimary(index, true, randomInt(5)); MetaData.Builder metaData = MetaData.builder(state.metaData()); - Settings.Builder settings = Settings.builder().put(metaData.get(index).settings()); + Settings.Builder settings = Settings.builder().put(metaData.get(index).getSettings()); settings.put(IndexMetaData.SETTING_SHADOW_REPLICAS, true); metaData.put(IndexMetaData.builder(metaData.get(index)).settings(settings)); clusterService.setState(ClusterState.builder(state).metaData(metaData)); @@ -346,22 +464,19 @@ public class ShardReplicationTests extends ESTestCase { protected void runReplicateTest(IndexShardRoutingTable shardRoutingTable, int assignedReplicas, int totalShards) throws InterruptedException, ExecutionException { - final ShardRouting primaryShard = shardRoutingTable.primaryShard(); final ShardIterator shardIt = shardRoutingTable.shardsIt(); final ShardId shardId = shardIt.shardId(); - final Request request = new Request(); - PlainActionFuture listener = new PlainActionFuture<>(); - + final Request request = new Request(shardId); + final PlainActionFuture listener = new PlainActionFuture<>(); logger.debug("expecting [{}] assigned replicas, [{}] total shards. using state: \n{}", assignedReplicas, totalShards, clusterService.state().prettyPrint()); - final TransportReplicationAction.InternalRequest internalRequest = action.new InternalRequest(request); - internalRequest.concreteIndex(shardId.index().name()); Releasable reference = getOrCreateIndexShardOperationsCounter(); assertIndexShardCounter(2); + // TODO: set a default timeout TransportReplicationAction.ReplicationPhase replicationPhase = - action.new ReplicationPhase(shardIt, request, - new Response(), new ClusterStateObserver(clusterService, logger), - primaryShard, internalRequest, listener, reference); + action.new ReplicationPhase(request, + new Response(), + request.shardId(), createTransportChannel(listener), reference, null); assertThat(replicationPhase.totalShards(), equalTo(totalShards)); assertThat(replicationPhase.pending(), equalTo(assignedReplicas)); @@ -374,11 +489,13 @@ public class ShardReplicationTests extends ESTestCase { } int pending = replicationPhase.pending(); int criticalFailures = 0; // failures that should fail the shard - int successfull = 1; + int successful = 1; + List failures = new ArrayList<>(); for (CapturingTransport.CapturedRequest capturedRequest : capturedRequests) { if (randomBoolean()) { Throwable t; - if (randomBoolean()) { + boolean criticalFailure = randomBoolean(); + if (criticalFailure) { t = new CorruptIndexException("simulated", (String) null); criticalFailures++; } else { @@ -386,23 +503,31 @@ public class ShardReplicationTests extends ESTestCase { } logger.debug("--> simulating failure on {} with [{}]", capturedRequest.node, t.getClass().getSimpleName()); transport.handleResponse(capturedRequest.requestId, t); + if (criticalFailure) { + CapturingTransport.CapturedRequest[] shardFailedRequests = transport.capturedRequests(); + transport.clear(); + assertEquals(1, shardFailedRequests.length); + CapturingTransport.CapturedRequest shardFailedRequest = shardFailedRequests[0]; + failures.add(shardFailedRequest); + transport.handleResponse(shardFailedRequest.requestId, TransportResponse.Empty.INSTANCE); + } } else { - successfull++; + successful++; transport.handleResponse(capturedRequest.requestId, TransportResponse.Empty.INSTANCE); } pending--; assertThat(replicationPhase.pending(), equalTo(pending)); - assertThat(replicationPhase.successful(), equalTo(successfull)); + assertThat(replicationPhase.successful(), equalTo(successful)); } assertThat(listener.isDone(), equalTo(true)); Response response = listener.get(); - final ActionWriteResponse.ShardInfo shardInfo = response.getShardInfo(); + final ReplicationResponse.ShardInfo shardInfo = response.getShardInfo(); assertThat(shardInfo.getFailed(), equalTo(criticalFailures)); assertThat(shardInfo.getFailures(), arrayWithSize(criticalFailures)); - assertThat(shardInfo.getSuccessful(), equalTo(successfull)); + assertThat(shardInfo.getSuccessful(), equalTo(successful)); assertThat(shardInfo.getTotal(), equalTo(totalShards)); - assertThat("failed to see enough shard failures", transport.capturedRequests().length, equalTo(criticalFailures)); + assertThat("failed to see enough shard failures", failures.size(), equalTo(criticalFailures)); for (CapturingTransport.CapturedRequest capturedRequest : transport.capturedRequests()) { assertThat(capturedRequest.action, equalTo(ShardStateAction.SHARD_FAILED_ACTION_NAME)); } @@ -410,7 +535,6 @@ public class ShardReplicationTests extends ESTestCase { assertIndexShardCounter(1); } - @Test public void testCounterOnPrimary() throws InterruptedException, ExecutionException, IOException { final String index = "test"; final ShardId shardId = new ShardId(index, 0); @@ -429,7 +553,7 @@ public class ShardReplicationTests extends ESTestCase { * However, this failure would only become apparent once listener.get is called. Seems a little implicit. * */ action = new ActionWithDelay(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool); - final TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, listener); + final TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, createTransportChannel(listener)); Thread t = new Thread() { @Override public void run() { @@ -451,7 +575,6 @@ public class ShardReplicationTests extends ESTestCase { assertThat(transport.capturedRequests().length, equalTo(0)); } - @Test public void testCounterIncrementedWhileReplicationOngoing() throws InterruptedException, ExecutionException, IOException { final String index = "test"; final ShardId shardId = new ShardId(index, 0); @@ -461,7 +584,7 @@ public class ShardReplicationTests extends ESTestCase { logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); Request request = new Request(shardId).timeout("100ms"); PlainActionFuture listener = new PlainActionFuture<>(); - TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, listener); + TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, createTransportChannel(listener)); primaryPhase.run(); assertIndexShardCounter(2); assertThat(transport.capturedRequests().length, equalTo(1)); @@ -470,16 +593,21 @@ public class ShardReplicationTests extends ESTestCase { assertIndexShardCounter(1); transport.clear(); request = new Request(shardId).timeout("100ms"); - primaryPhase = action.new PrimaryPhase(request, listener); + primaryPhase = action.new PrimaryPhase(request, createTransportChannel(listener)); primaryPhase.run(); assertIndexShardCounter(2); - assertThat(transport.capturedRequests().length, equalTo(1)); + CapturingTransport.CapturedRequest[] replicationRequests = transport.capturedRequests(); + transport.clear(); + assertThat(replicationRequests.length, equalTo(1)); // try with failure response - transport.handleResponse(transport.capturedRequests()[0].requestId, new CorruptIndexException("simulated", (String) null)); + transport.handleResponse(replicationRequests[0].requestId, new CorruptIndexException("simulated", (String) null)); + CapturingTransport.CapturedRequest[] shardFailedRequests = transport.capturedRequests(); + transport.clear(); + assertEquals(1, shardFailedRequests.length); + transport.handleResponse(shardFailedRequests[0].requestId, TransportResponse.Empty.INSTANCE); assertIndexShardCounter(1); } - @Test public void testReplicasCounter() throws Exception { final ShardId shardId = new ShardId("test", 0); clusterService.setState(state(shardId.index().getName(), true, @@ -490,7 +618,7 @@ public class ShardReplicationTests extends ESTestCase { @Override public void run() { try { - replicaOperationTransportHandler.messageReceived(new Request(), createTransportChannel()); + replicaOperationTransportHandler.messageReceived(new Request(), createTransportChannel(new PlainActionFuture<>())); } catch (Exception e) { } } @@ -507,14 +635,13 @@ public class ShardReplicationTests extends ESTestCase { action = new ActionWithExceptions(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool); final Action.ReplicaOperationTransportHandler replicaOperationTransportHandlerForException = action.new ReplicaOperationTransportHandler(); try { - replicaOperationTransportHandlerForException.messageReceived(new Request(shardId), createTransportChannel()); + replicaOperationTransportHandlerForException.messageReceived(new Request(shardId), createTransportChannel(new PlainActionFuture<>())); fail(); } catch (Throwable t2) { } assertIndexShardCounter(1); } - @Test public void testCounterDecrementedIfShardOperationThrowsException() throws InterruptedException, ExecutionException, IOException { action = new ActionWithExceptions(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool); final String index = "test"; @@ -524,7 +651,7 @@ public class ShardReplicationTests extends ESTestCase { logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); Request request = new Request(shardId).timeout("100ms"); PlainActionFuture listener = new PlainActionFuture<>(); - TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, listener); + TransportReplicationAction.PrimaryPhase primaryPhase = action.new PrimaryPhase(request, createTransportChannel(listener)); primaryPhase.run(); // no replica request should have been sent yet assertThat(transport.capturedRequests().length, equalTo(0)); @@ -552,7 +679,6 @@ public class ShardReplicationTests extends ESTestCase { } public static class Request extends ReplicationRequest { - int shardId; public AtomicBoolean processedOnPrimary = new AtomicBoolean(); public AtomicInteger processedOnReplicas = new AtomicInteger(); @@ -561,25 +687,23 @@ public class ShardReplicationTests extends ESTestCase { Request(ShardId shardId) { this(); - this.shardId = shardId.id(); - this.index(shardId.index().name()); + this.shardId = shardId; + this.index = shardId.getIndex(); // keep things simple } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeVInt(shardId); } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - shardId = in.readVInt(); } } - static class Response extends ActionWriteResponse { + static class Response extends ReplicationResponse { } class Action extends TransportReplicationAction { @@ -598,22 +722,17 @@ public class ShardReplicationTests extends ESTestCase { } @Override - protected Tuple shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { - boolean executedBefore = shardRequest.request.processedOnPrimary.getAndSet(true); + protected Tuple shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Throwable { + boolean executedBefore = shardRequest.processedOnPrimary.getAndSet(true); assert executedBefore == false : "request has already been executed on the primary"; - return new Tuple<>(new Response(), shardRequest.request); + return new Tuple<>(new Response(), shardRequest); } @Override - protected void shardOperationOnReplica(ShardId shardId, Request request) { + protected void shardOperationOnReplica(Request request) { request.processedOnReplicas.incrementAndGet(); } - @Override - protected ShardIterator shards(ClusterState clusterState, InternalRequest request) { - return clusterState.getRoutingTable().index(request.concreteIndex()).shard(request.request().shardId).shardsIt(); - } - @Override protected boolean checkWriteConsistency() { return false; @@ -652,8 +771,8 @@ public class ShardReplicationTests extends ESTestCase { } @Override - protected Tuple shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { - return throwException(shardRequest.shardId); + protected Tuple shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Throwable { + return throwException(shardRequest.shardId()); } private Tuple throwException(ShardId shardId) { @@ -674,8 +793,8 @@ public class ShardReplicationTests extends ESTestCase { } @Override - protected void shardOperationOnReplica(ShardId shardId, Request shardRequest) { - throwException(shardRequest.internalShardId); + protected void shardOperationOnReplica(Request shardRequest) { + throwException(shardRequest.shardId()); } } @@ -690,9 +809,9 @@ public class ShardReplicationTests extends ESTestCase { } @Override - protected Tuple shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { + protected Tuple shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Throwable { awaitLatch(); - return new Tuple<>(new Response(), shardRequest.request); + return new Tuple<>(new Response(), shardRequest); } private void awaitLatch() throws InterruptedException { @@ -701,7 +820,7 @@ public class ShardReplicationTests extends ESTestCase { } @Override - protected void shardOperationOnReplica(ShardId shardId, Request shardRequest) { + protected void shardOperationOnReplica(Request shardRequest) { try { awaitLatch(); } catch (InterruptedException e) { @@ -713,7 +832,7 @@ public class ShardReplicationTests extends ESTestCase { /* * Transport channel that is needed for replica operation testing. * */ - public TransportChannel createTransportChannel() { + public TransportChannel createTransportChannel(final PlainActionFuture listener) { return new TransportChannel() { @Override @@ -728,14 +847,17 @@ public class ShardReplicationTests extends ESTestCase { @Override public void sendResponse(TransportResponse response) throws IOException { + listener.onResponse(((Response) response)); } @Override public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException { + listener.onResponse(((Response) response)); } @Override public void sendResponse(Throwable error) throws IOException { + listener.onFailure(error); } }; } diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java index 1d0c317f5ad..0eb7c0757e7 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; @@ -59,7 +58,6 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { .build(); } - @Test public void testSimpleTermVectors() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties") @@ -143,8 +141,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - BytesStream bytesStream = xBuilder.bytesStream(); - String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -200,8 +197,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - BytesStream bytesStream = xBuilder.bytesStream(); - String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -260,8 +256,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS); xBuilder.endObject(); - BytesStream bytesStream = xBuilder.bytesStream(); - String utf8 = bytesStream.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java index 6f046974633..5507686e355 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.termvectors; import com.carrotsearch.hppc.ObjectIntHashMap; + import org.apache.lucene.analysis.payloads.PayloadHelper; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DirectoryReader; @@ -41,7 +42,6 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.FieldMapper; import org.hamcrest.Matcher; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -63,8 +63,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class GetTermVectorsIT extends AbstractTermVectorsTestCase { - - @Test public void testNoSuchDoc() throws Exception { XContentBuilder mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties") @@ -91,7 +89,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testExistingFieldWithNoTermVectorsNoNPE() throws Exception { XContentBuilder mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties") @@ -119,7 +116,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { assertThat(actionGet.getFields().terms("existingfield"), nullValue()); } - @Test public void testExistingFieldButNotInDocNPE() throws Exception { XContentBuilder mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties") @@ -150,7 +146,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { assertThat(actionGet.getFields().terms("existingfield"), nullValue()); } - @Test public void testNotIndexedField() throws Exception { // must be of type string and indexed. assertAcked(prepareCreate("test") @@ -193,7 +188,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testSimpleTermVectors() throws IOException { XContentBuilder mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties") @@ -231,7 +225,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testRandomSingleTermVectors() throws IOException { FieldType ft = new FieldType(); int config = randomInt(6); @@ -392,7 +385,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { return ret; } - @Test public void testDuelESLucene() throws Exception { TestFieldSetting[] testFieldSettings = getFieldSettings(); createIndexBasedOnFieldSettings("test", "alias", testFieldSettings); @@ -419,7 +411,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testRandomPayloadWithDelimitedPayloadTokenFilter() throws IOException { //create the test document int encoding = randomIntBetween(0, 2); @@ -587,7 +578,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } // like testSimpleTermVectors but we create fields with no term vectors - @Test public void testSimpleTermVectorsWithGenerate() throws IOException { String[] fieldNames = new String[10]; for (int i = 0; i < fieldNames.length; i++) { @@ -680,7 +670,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { assertThat(iterator.next(), nullValue()); } - @Test public void testDuelWithAndWithoutTermVectors() throws IOException, ExecutionException, InterruptedException { // setup indices String[] indexNames = new String[] {"with_tv", "without_tv"}; @@ -769,7 +758,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { assertThat(iter1.next(), nullValue()); } - @Test public void testSimpleWildCards() throws IOException { int numFields = 25; @@ -797,7 +785,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { assertThat("All term vectors should have been generated", response.getFields().size(), equalTo(numFields)); } - @Test public void testArtificialVsExisting() throws ExecutionException, InterruptedException, IOException { // setup indices Settings.Builder settings = settingsBuilder() @@ -856,7 +843,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testArtificialNoDoc() throws IOException { // setup indices Settings.Builder settings = settingsBuilder() @@ -885,7 +871,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { checkBrownFoxTermVector(resp.getFields(), "field1", false); } - @Test public void testArtificialNonExistingField() throws Exception { // setup indices Settings.Builder settings = settingsBuilder() @@ -933,7 +918,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testPerFieldAnalyzer() throws IOException { int numFields = 25; @@ -1030,7 +1014,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { return randomBoolean() ? "test" : "alias"; } - @Test public void testDfs() throws ExecutionException, InterruptedException, IOException { logger.info("Setting up the index ..."); Settings.Builder settings = settingsBuilder() @@ -1135,7 +1118,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { return lessThan(value); } - @Test public void testTermVectorsWithVersion() { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1))); @@ -1239,7 +1221,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { assertThat(response.getVersion(), equalTo(2l)); } - @Test public void testFilterLength() throws ExecutionException, InterruptedException, IOException { logger.info("Setting up the index ..."); Settings.Builder settings = settingsBuilder() @@ -1278,7 +1259,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testFilterTermFreq() throws ExecutionException, InterruptedException, IOException { logger.info("Setting up the index ..."); Settings.Builder settings = settingsBuilder() @@ -1319,7 +1299,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { } } - @Test public void testFilterDocFreq() throws ExecutionException, InterruptedException, IOException { logger.info("Setting up the index ..."); Settings.Builder settings = settingsBuilder() diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java index e70937ed571..516eaf371e0 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java @@ -28,16 +28,16 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.VersionConflictEngineException; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class MultiTermVectorsIT extends AbstractTermVectorsTestCase { - - @Test public void testDuelESLucene() throws Exception { AbstractTermVectorsTestCase.TestFieldSetting[] testFieldSettings = getFieldSettings(); createIndexBasedOnFieldSettings("test", "alias", testFieldSettings); @@ -73,7 +73,6 @@ public class MultiTermVectorsIT extends AbstractTermVectorsTestCase { } - @Test public void testMissingIndexThrowsMissingIndex() throws Exception { TermVectorsRequestBuilder requestBuilder = client().prepareTermVectors("testX", "typeX", Integer.toString(1)); MultiTermVectorsRequestBuilder mtvBuilder = client().prepareMultiTermVectors(); @@ -84,7 +83,6 @@ public class MultiTermVectorsIT extends AbstractTermVectorsTestCase { assertThat(response.getResponses()[0].getFailure().getCause().getMessage(), equalTo("no such index")); } - @Test public void testMultiTermVectorsWithVersion() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1))); diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java index 82809d1c5cd..cab27df6936 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java @@ -20,9 +20,17 @@ package org.elasticsearch.action.termvectors; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.*; -import org.apache.lucene.index.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -44,7 +52,6 @@ import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.StreamsUtils; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -57,10 +64,7 @@ import java.util.Set; import static org.hamcrest.Matchers.equalTo; public class TermVectorsUnitTests extends ESTestCase { - - @Test - public void streamResponse() throws Exception { - + public void testStreamResponse() throws Exception { TermVectorsResponse outResponse = new TermVectorsResponse("a", "b", "c"); outResponse.setExists(true); writeStandardTermVector(outResponse); @@ -169,7 +173,6 @@ public class TermVectorsUnitTests extends ESTestCase { assertThat(fields.size(), equalTo(2)); } - @Test public void testRestRequestParsing() throws Exception { BytesReference inputBytes = new BytesArray( " {\"fields\" : [\"a\", \"b\",\"c\"], \"offsets\":false, \"positions\":false, \"payloads\":true}"); @@ -207,7 +210,6 @@ public class TermVectorsUnitTests extends ESTestCase { } - @Test public void testRequestParsingThrowsException() throws Exception { BytesReference inputBytes = new BytesArray( " {\"fields\" : \"a, b,c \", \"offsets\":false, \"positions\":false, \"payloads\":true, \"meaningless_term\":2}"); @@ -223,9 +225,7 @@ public class TermVectorsUnitTests extends ESTestCase { } - @Test - public void streamRequest() throws IOException { - + public void testStreamRequest() throws IOException { for (int i = 0; i < 10; i++) { TermVectorsRequest request = new TermVectorsRequest("index", "type", "id"); request.offsets(random().nextBoolean()); @@ -259,8 +259,7 @@ public class TermVectorsUnitTests extends ESTestCase { } } - - @Test + public void testFieldTypeToTermVectorString() throws Exception { FieldType ft = new FieldType(); ft.setStoreTermVectorOffsets(false); @@ -279,7 +278,6 @@ public class TermVectorsUnitTests extends ESTestCase { assertThat("TypeParsers.parseTermVector should accept string with_positions_payloads but does not.", exceptiontrown, equalTo(false)); } - @Test public void testTermVectorStringGenerationWithoutPositions() throws Exception { FieldType ft = new FieldType(); ft.setStoreTermVectorOffsets(true); @@ -290,14 +288,13 @@ public class TermVectorsUnitTests extends ESTestCase { assertThat(ftOpts, equalTo("with_offsets")); } - @Test public void testMultiParser() throws Exception { byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest1.json"); BytesReference bytes = new BytesArray(data); MultiTermVectorsRequest request = new MultiTermVectorsRequest(); request.add(new TermVectorsRequest(), bytes); checkParsedParameters(request); - + data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest2.json"); bytes = new BytesArray(data); request = new MultiTermVectorsRequest(); @@ -326,7 +323,7 @@ public class TermVectorsUnitTests extends ESTestCase { } } - @Test // issue #12311 + // issue #12311 public void testMultiParserFilter() throws Exception { byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/termvectors/multiRequest3.json"); BytesReference bytes = new BytesArray(data); diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 6be5fe95fd2..6cf7a3384ab 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -23,24 +23,20 @@ import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.*; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; public class UpdateRequestTests extends ESTestCase { - - @Test public void testUpdateRequest() throws Exception { UpdateRequest request = new UpdateRequest("test", "type", "1"); // simple script @@ -126,9 +122,9 @@ public class UpdateRequestTests extends ESTestCase { assertThat(((Map) doc.get("compound")).get("field2").toString(), equalTo("value2")); } - @Test // Related to issue 3256 + // Related to issue 3256 public void testUpdateRequestWithTTL() throws Exception { - long providedTTLValue = randomIntBetween(500, 1000); + TimeValue providedTTLValue = TimeValue.parseTimeValue(randomTimeValue(), null, "ttl"); Settings settings = settings(Version.CURRENT).build(); UpdateHelper updateHelper = new UpdateHelper(settings, null); diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 6bbec12dc12..1cbe05da6a0 100644 --- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -44,7 +44,6 @@ import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.HashSet; @@ -80,8 +79,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class IndexAliasesIT extends ESIntegTestCase { - - @Test public void testAliases() throws Exception { logger.info("--> creating index [test]"); createIndex("test"); @@ -108,7 +105,6 @@ public class IndexAliasesIT extends ESIntegTestCase { assertThat(indexResponse.getIndex(), equalTo("test_x")); } - @Test public void testFailedFilter() throws Exception { logger.info("--> creating index [test]"); createIndex("test"); @@ -134,7 +130,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test public void testFilteringAliases() throws Exception { logger.info("--> creating index [test]"); assertAcked(prepareCreate("test").addMapping("type", "user", "type=string")); @@ -149,11 +144,10 @@ public class IndexAliasesIT extends ESIntegTestCase { logger.info("--> making sure that filter was stored with alias [alias1] and filter [user:kimchy]"); ClusterState clusterState = admin().cluster().prepareState().get().getState(); IndexMetaData indexMd = clusterState.metaData().index("test"); - assertThat(indexMd.aliases().get("alias1").filter().string(), equalTo("{\"term\":{\"user\":{\"value\":\"kimchy\",\"boost\":1.0}}}")); + assertThat(indexMd.getAliases().get("alias1").filter().string(), equalTo("{\"term\":{\"user\":{\"value\":\"kimchy\",\"boost\":1.0}}}")); } - @Test public void testEmptyFilter() throws Exception { logger.info("--> creating index [test]"); createIndex("test"); @@ -163,7 +157,6 @@ public class IndexAliasesIT extends ESIntegTestCase { assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1", "{}")); } - @Test public void testSearchingFilteringAliasesSingleIndex() throws Exception { logger.info("--> creating index [test]"); assertAcked(prepareCreate("test").addMapping("type1", "id", "type=string", "name", "type=string")); @@ -244,7 +237,6 @@ public class IndexAliasesIT extends ESIntegTestCase { assertHits(searchResponse.getHits(), "1", "2", "3", "4"); } - @Test public void testSearchingFilteringAliasesTwoIndices() throws Exception { logger.info("--> creating index [test1]"); assertAcked(prepareCreate("test1").addMapping("type1", "name", "type=string")); @@ -280,35 +272,34 @@ public class IndexAliasesIT extends ESIntegTestCase { logger.info("--> checking filtering alias for two indices"); SearchResponse searchResponse = client().prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "1", "5"); - assertThat(client().prepareCount("foos").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(2L)); + assertThat(client().prepareSearch("foos").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(2L)); logger.info("--> checking filtering alias for one index"); searchResponse = client().prepareSearch("bars").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "2"); - assertThat(client().prepareCount("bars").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(1L)); + assertThat(client().prepareSearch("bars").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(1L)); logger.info("--> checking filtering alias for two indices and one complete index"); searchResponse = client().prepareSearch("foos", "test1").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "1", "2", "3", "4", "5"); - assertThat(client().prepareCount("foos", "test1").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(5L)); + assertThat(client().prepareSearch("foos", "test1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(5L)); logger.info("--> checking filtering alias for two indices and non-filtering alias for one index"); searchResponse = client().prepareSearch("foos", "aliasToTest1").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "1", "2", "3", "4", "5"); - assertThat(client().prepareCount("foos", "aliasToTest1").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(5L)); + assertThat(client().prepareSearch("foos", "aliasToTest1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(5L)); logger.info("--> checking filtering alias for two indices and non-filtering alias for both indices"); searchResponse = client().prepareSearch("foos", "aliasToTests").setQuery(QueryBuilders.matchAllQuery()).get(); assertThat(searchResponse.getHits().totalHits(), equalTo(8L)); - assertThat(client().prepareCount("foos", "aliasToTests").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(8L)); + assertThat(client().prepareSearch("foos", "aliasToTests").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(8L)); logger.info("--> checking filtering alias for two indices and non-filtering alias for both indices"); searchResponse = client().prepareSearch("foos", "aliasToTests").setQuery(QueryBuilders.termQuery("name", "something")).get(); assertHits(searchResponse.getHits(), "4", "8"); - assertThat(client().prepareCount("foos", "aliasToTests").setQuery(QueryBuilders.termQuery("name", "something")).get().getCount(), equalTo(2L)); + assertThat(client().prepareSearch("foos", "aliasToTests").setSize(0).setQuery(QueryBuilders.termQuery("name", "something")).get().getHits().totalHits(), equalTo(2L)); } - @Test public void testSearchingFilteringAliasesMultipleIndices() throws Exception { logger.info("--> creating indices"); createIndex("test1", "test2", "test3"); @@ -350,30 +341,29 @@ public class IndexAliasesIT extends ESIntegTestCase { logger.info("--> checking filtering alias for multiple indices"); SearchResponse searchResponse = client().prepareSearch("filter23", "filter13").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "21", "31", "13", "33"); - assertThat(client().prepareCount("filter23", "filter13").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(4L)); + assertThat(client().prepareSearch("filter23", "filter13").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(4L)); searchResponse = client().prepareSearch("filter23", "filter1").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "21", "31", "11", "12", "13"); - assertThat(client().prepareCount("filter23", "filter1").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(5L)); + assertThat(client().prepareSearch("filter23", "filter1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(5L)); searchResponse = client().prepareSearch("filter13", "filter1").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "11", "12", "13", "33"); - assertThat(client().prepareCount("filter13", "filter1").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(4L)); + assertThat(client().prepareSearch("filter13", "filter1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(4L)); searchResponse = client().prepareSearch("filter13", "filter1", "filter23").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "11", "12", "13", "21", "31", "33"); - assertThat(client().prepareCount("filter13", "filter1", "filter23").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(6L)); + assertThat(client().prepareSearch("filter13", "filter1", "filter23").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(6L)); searchResponse = client().prepareSearch("filter23", "filter13", "test2").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "21", "22", "23", "31", "13", "33"); - assertThat(client().prepareCount("filter23", "filter13", "test2").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(6L)); + assertThat(client().prepareSearch("filter23", "filter13", "test2").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(6L)); searchResponse = client().prepareSearch("filter23", "filter13", "test1", "test2").setQuery(QueryBuilders.matchAllQuery()).get(); assertHits(searchResponse.getHits(), "11", "12", "13", "21", "22", "23", "31", "33"); - assertThat(client().prepareCount("filter23", "filter13", "test1", "test2").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(8L)); + assertThat(client().prepareSearch("filter23", "filter13", "test1", "test2").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(8L)); } - @Test public void testDeletingByQueryFilteringAliases() throws Exception { logger.info("--> creating index [test1] and [test2"); assertAcked(prepareCreate("test1").addMapping("type1", "name", "type=string")); @@ -408,12 +398,9 @@ public class IndexAliasesIT extends ESIntegTestCase { refresh(); logger.info("--> checking counts before delete"); - assertThat(client().prepareCount("bars").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(1L)); + assertThat(client().prepareSearch("bars").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get().getHits().totalHits(), equalTo(1L)); } - - - @Test public void testDeleteAliases() throws Exception { logger.info("--> creating index [test1] and [test2]"); assertAcked(prepareCreate("test1").addMapping("type", "name", "type=string")); @@ -442,8 +429,6 @@ public class IndexAliasesIT extends ESIntegTestCase { assertThat(response.exists(), equalTo(false)); } - - @Test public void testWaitForAliasCreationMultipleShards() throws Exception { logger.info("--> creating index [test]"); createIndex("test"); @@ -456,7 +441,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test public void testWaitForAliasCreationSingleShard() throws Exception { logger.info("--> creating index [test]"); assertAcked(admin().indices().create(createIndexRequest("test").settings(settingsBuilder().put("index.numberOfReplicas", 0).put("index.numberOfShards", 1))).get()); @@ -469,7 +453,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test public void testWaitForAliasSimultaneousUpdate() throws Exception { final int aliasCount = 10; @@ -497,8 +480,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - - @Test public void testSameAlias() throws Exception { logger.info("--> creating index [test]"); assertAcked(prepareCreate("test").addMapping("type", "name", "type=string")); @@ -540,18 +521,20 @@ public class IndexAliasesIT extends ESIntegTestCase { } - @Test(expected = AliasesNotFoundException.class) public void testIndicesRemoveNonExistingAliasResponds404() throws Exception { logger.info("--> creating index [test]"); createIndex("test"); ensureGreen(); logger.info("--> deleting alias1 which does not exist"); - assertAcked((admin().indices().prepareAliases().removeAlias("test", "alias1"))); + try { + admin().indices().prepareAliases().removeAlias("test", "alias1").get(); + fail("Expected AliasesNotFoundException"); + } catch (AliasesNotFoundException e) { + assertThat(e.getMessage(), containsString("[alias1] missing")); + } } - @Test public void testIndicesGetAliases() throws Exception { - logger.info("--> creating indices [foobar, test, test123, foobarbaz, bazbar]"); createIndex("foobar"); createIndex("test"); @@ -736,7 +719,6 @@ public class IndexAliasesIT extends ESIntegTestCase { assertThat(existsResponse.exists(), equalTo(false)); } - @Test public void testAddAliasNullWithoutExistingIndices() { try { assertAcked(admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction(null, "alias1"))); @@ -747,7 +729,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test public void testAddAliasNullWithExistingIndices() throws Exception { logger.info("--> creating index [test]"); createIndex("test"); @@ -764,64 +745,89 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test(expected = ActionRequestValidationException.class) public void testAddAliasEmptyIndex() { - admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("", "alias1")).get(); + try { + admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("", "alias1")).get(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("[index] may not be empty string")); + } } - @Test(expected = ActionRequestValidationException.class) public void testAddAliasNullAlias() { - admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("index1", null)).get(); + try { + admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("index1", null)).get(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("[alias] may not be empty string")); + } } - @Test(expected = ActionRequestValidationException.class) public void testAddAliasEmptyAlias() { - admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("index1", "")).get(); + try { + admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("index1", "")).get(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("[alias] may not be empty string")); + } } - @Test public void testAddAliasNullAliasNullIndex() { try { admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction(null, null)).get(); - assertTrue("Should throw " + ActionRequestValidationException.class.getSimpleName(), false); + fail("Should throw " + ActionRequestValidationException.class.getSimpleName()); } catch (ActionRequestValidationException e) { assertThat(e.validationErrors(), notNullValue()); assertThat(e.validationErrors().size(), equalTo(2)); } } - @Test public void testAddAliasEmptyAliasEmptyIndex() { try { admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("", "")).get(); - assertTrue("Should throw " + ActionRequestValidationException.class.getSimpleName(), false); + fail("Should throw " + ActionRequestValidationException.class.getSimpleName()); } catch (ActionRequestValidationException e) { assertThat(e.validationErrors(), notNullValue()); assertThat(e.validationErrors().size(), equalTo(2)); } } - @Test(expected = ActionRequestValidationException.class) - public void tesRemoveAliasNullIndex() { - admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction(null, "alias1")).get(); + public void testRemoveAliasNullIndex() { + try { + admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction(null, "alias1")).get(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("[index] may not be empty string")); + } } - @Test(expected = ActionRequestValidationException.class) - public void tesRemoveAliasEmptyIndex() { - admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction("", "alias1")).get(); + public void testRemoveAliasEmptyIndex() { + try { + admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction("", "alias1")).get(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("[index] may not be empty string")); + } } - @Test(expected = ActionRequestValidationException.class) - public void tesRemoveAliasNullAlias() { - admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction("index1", null)).get(); + public void testRemoveAliasNullAlias() { + try { + admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction("index1", null)).get(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("[alias] may not be empty string")); + } } - @Test(expected = ActionRequestValidationException.class) - public void tesRemoveAliasEmptyAlias() { - admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction("index1", "")).get(); + public void testRemoveAliasEmptyAlias() { + try { + admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction("index1", "")).get(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("[alias] may not be empty string")); + } } - @Test public void testRemoveAliasNullAliasNullIndex() { try { admin().indices().prepareAliases().addAliasAction(AliasAction.newRemoveAliasAction(null, null)).get(); @@ -832,7 +838,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test public void testRemoveAliasEmptyAliasEmptyIndex() { try { admin().indices().prepareAliases().addAliasAction(AliasAction.newAddAliasAction("", "")).get(); @@ -843,7 +848,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test public void testGetAllAliasesWorks() { createIndex("index1"); createIndex("index2"); @@ -857,18 +861,16 @@ public class IndexAliasesIT extends ESIntegTestCase { assertThat(response.getAliases(), hasKey("index1")); } - @Test public void testCreateIndexWithAliases() throws Exception { assertAcked(prepareCreate("test") .addMapping("type", "field", "type=string") .addAlias(new Alias("alias1")) - .addAlias(new Alias("alias2").filter(QueryBuilders.missingQuery("field"))) + .addAlias(new Alias("alias2").filter(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("field")))) .addAlias(new Alias("alias3").indexRouting("index").searchRouting("search"))); checkAliases(); } - @Test public void testCreateIndexWithAliasesInSource() throws Exception { assertAcked(prepareCreate("test").setSource("{\n" + " \"aliases\" : {\n" + @@ -881,7 +883,6 @@ public class IndexAliasesIT extends ESIntegTestCase { checkAliases(); } - @Test public void testCreateIndexWithAliasesSource() throws Exception { assertAcked(prepareCreate("test") .addMapping("type", "field", "type=string") @@ -894,7 +895,6 @@ public class IndexAliasesIT extends ESIntegTestCase { checkAliases(); } - @Test public void testCreateIndexWithAliasesFilterNotValid() { //non valid filter, invalid json CreateIndexRequestBuilder createIndexRequestBuilder = prepareCreate("test").addAlias(new Alias("alias2").filter("f")); @@ -917,7 +917,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test // Before 2.0 alias filters were parsed at alias creation time, in order // for filters to work correctly ES required that fields mentioned in those // filters exist in the mapping. @@ -936,7 +935,6 @@ public class IndexAliasesIT extends ESIntegTestCase { .get(); } - @Test public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception { assertAcked(prepareCreate("my-index").addMapping("my-type", "_timestamp", "enabled=true")); assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("_timestamp").from("now-1d").to("now"))); @@ -956,7 +954,6 @@ public class IndexAliasesIT extends ESIntegTestCase { } } - @Test public void testAliasesFilterWithHasChildQuery() throws Exception { assertAcked(prepareCreate("my-index") .addMapping("parent") @@ -977,7 +974,6 @@ public class IndexAliasesIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).id(), equalTo("2")); } - @Test public void testAliasesWithBlocks() { createIndex("test"); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/benchmark/aliases/AliasesBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/aliases/AliasesBenchmark.java deleted file mode 100644 index 7b5d489e45f..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/aliases/AliasesBenchmark.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.aliases; - -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.monitor.jvm.JvmStats; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; - -import java.io.IOException; -import java.util.List; - -/** - */ -public class AliasesBenchmark { - - private final static String INDEX_NAME = "my-index"; - - public static void main(String[] args) throws IOException { - int NUM_ADDITIONAL_NODES = 1; - int BASE_ALIAS_COUNT = 100000; - int NUM_ADD_ALIAS_REQUEST = 1000; - - Settings settings = Settings.settingsBuilder() - .put("node.master", false).build(); - Node node1 = NodeBuilder.nodeBuilder().settings( - Settings.settingsBuilder().put(settings).put("node.master", true) - ).node(); - - Node[] otherNodes = new Node[NUM_ADDITIONAL_NODES]; - for (int i = 0; i < otherNodes.length; i++) { - otherNodes[i] = NodeBuilder.nodeBuilder().settings(settings).node(); - } - - Client client = node1.client(); - try { - client.admin().indices().prepareCreate(INDEX_NAME).execute().actionGet(); - } catch (IndexAlreadyExistsException e) {} - client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); - int numberOfAliases = countAliases(client); - System.out.println("Number of aliases: " + numberOfAliases); - - if (numberOfAliases < BASE_ALIAS_COUNT) { - int diff = BASE_ALIAS_COUNT - numberOfAliases; - System.out.println("Adding " + diff + " more aliases to get to the start amount of " + BASE_ALIAS_COUNT + " aliases"); - IndicesAliasesRequestBuilder builder = client.admin().indices().prepareAliases(); - for (int i = 1; i <= diff; i++) { - builder.addAlias(INDEX_NAME, Strings.randomBase64UUID()); - if (i % 1000 == 0) { - builder.execute().actionGet(); - builder = client.admin().indices().prepareAliases(); - } - } - if (!builder.request().getAliasActions().isEmpty()) { - builder.execute().actionGet(); - } - } else if (numberOfAliases > BASE_ALIAS_COUNT) { - IndicesAliasesRequestBuilder builder = client.admin().indices().prepareAliases(); - int diff = numberOfAliases - BASE_ALIAS_COUNT; - System.out.println("Removing " + diff + " aliases to get to the start amount of " + BASE_ALIAS_COUNT + " aliases"); - List aliases= client.admin().indices().prepareGetAliases("*") - .addIndices(INDEX_NAME) - .execute().actionGet().getAliases().get(INDEX_NAME); - for (int i = 0; i <= diff; i++) { - builder.removeAlias(INDEX_NAME, aliases.get(i).alias()); - if (i % 1000 == 0) { - builder.execute().actionGet(); - builder = client.admin().indices().prepareAliases(); - } - } - if (!builder.request().getAliasActions().isEmpty()) { - builder.execute().actionGet(); - } - } - - numberOfAliases = countAliases(client); - System.out.println("Number of aliases: " + numberOfAliases); - - long totalTime = 0; - int max = numberOfAliases + NUM_ADD_ALIAS_REQUEST; - for (int i = numberOfAliases; i <= max; i++) { - if (i != numberOfAliases && i % 100 == 0) { - long avgTime = totalTime / 100; - System.out.println("Added [" + (i - numberOfAliases) + "] aliases. Avg create time: " + avgTime + " ms"); - System.out.println("Heap used [" + JvmStats.jvmStats().getMem().getHeapUsed() + "]"); - totalTime = 0; - } - - long time = System.currentTimeMillis(); -// String filter = termFilter("field" + i, "value" + i).toXContent(XContentFactory.jsonBuilder(), null).string(); - client.admin().indices().prepareAliases().addAlias(INDEX_NAME, Strings.randomBase64UUID()/*, filter*/) - .execute().actionGet(); - totalTime += System.currentTimeMillis() - time; - } - System.gc(); - System.out.println("Final heap used [" + JvmStats.jvmStats().getMem().getHeapUsed() + "]"); - System.out.println("Number of aliases: " + countAliases(client)); - - client.close(); - node1.close(); - for (Node otherNode : otherNodes) { - otherNode.close(); - } - } - - private static int countAliases(Client client) { - GetAliasesResponse response = client.admin().indices().prepareGetAliases("*") - .addIndices(INDEX_NAME) - .execute().actionGet(); - if (response.getAliases().isEmpty()) { - return 0; - } else { - return response.getAliases().get(INDEX_NAME).size(); - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/bloom/BloomBench.java b/core/src/test/java/org/elasticsearch/benchmark/bloom/BloomBench.java deleted file mode 100644 index 15745fc931d..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/bloom/BloomBench.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.bloom; - -import org.apache.lucene.codecs.bloom.FuzzySet; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.util.BloomFilter; - -import java.security.SecureRandom; - -/** - */ -public class BloomBench { - - public static void main(String[] args) throws Exception { - SecureRandom random = new SecureRandom(); - final int ELEMENTS = (int) SizeValue.parseSizeValue("1m").singles(); - final double fpp = 0.01; - BloomFilter gFilter = BloomFilter.create(ELEMENTS, fpp); - System.out.println("G SIZE: " + new ByteSizeValue(gFilter.getSizeInBytes())); - - FuzzySet lFilter = FuzzySet.createSetBasedOnMaxMemory((int) gFilter.getSizeInBytes()); - //FuzzySet lFilter = FuzzySet.createSetBasedOnQuality(ELEMENTS, 0.97f); - - for (int i = 0; i < ELEMENTS; i++) { - BytesRef bytesRef = new BytesRef(Strings.randomBase64UUID(random)); - gFilter.put(bytesRef); - lFilter.addValue(bytesRef); - } - - int lFalse = 0; - int gFalse = 0; - for (int i = 0; i < ELEMENTS; i++) { - BytesRef bytesRef = new BytesRef(Strings.randomBase64UUID(random)); - if (gFilter.mightContain(bytesRef)) { - gFalse++; - } - if (lFilter.contains(bytesRef) == FuzzySet.ContainsResult.MAYBE) { - lFalse++; - } - } - System.out.println("Failed positives, g[" + gFalse + "], l[" + lFalse + "]"); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/breaker/CircuitBreakerBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/breaker/CircuitBreakerBenchmark.java deleted file mode 100644 index f6b0497b090..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/breaker/CircuitBreakerBenchmark.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.breaker; - -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; - -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; - -import static junit.framework.Assert.assertNotNull; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; - -/** - * Benchmarks for different implementations of the circuit breaker - */ -public class CircuitBreakerBenchmark { - - private static final String INDEX = UUID.randomUUID().toString(); - private static final int QUERIES = 100; - private static final int BULK_SIZE = 100; - private static final int NUM_DOCS = 2_000_000; - private static final int AGG_SIZE = 25; - - private static void switchToNoop(Client client) { - Settings settings = settingsBuilder() - .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING, CircuitBreaker.Type.NOOP) - .build(); - client.admin().cluster().prepareUpdateSettings().setTransientSettings(settings).execute().actionGet(); - } - - private static void switchToMemory(Client client) { - Settings settings = settingsBuilder() - .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING, CircuitBreaker.Type.MEMORY) - .build(); - client.admin().cluster().prepareUpdateSettings().setTransientSettings(settings).execute().actionGet(); - } - - private static void runSingleThreadedQueries(Client client) { - long totalTime = 0; - for (int i = 0; i < QUERIES; i++) { - if (i % 10 == 0) { - System.out.println("--> query #" + i); - } - SearchResponse resp = client.prepareSearch(INDEX).setQuery(matchAllQuery()) - .addAggregation( - terms("myterms") - .size(AGG_SIZE) - .field("num") - ).setSize(0).get(); - Terms terms = resp.getAggregations().get("myterms"); - assertNotNull("term aggs were calculated", terms); - totalTime += resp.getTookInMillis(); - } - - System.out.println("--> single threaded average time: " + (totalTime / QUERIES) + "ms"); - } - - private static void runMultiThreadedQueries(final Client client) throws Exception { - final AtomicLong totalThreadedTime = new AtomicLong(0); - int THREADS = 10; - Thread threads[] = new Thread[THREADS]; - for (int i = 0; i < THREADS; i++) { - threads[i] = new Thread(new Runnable() { - @Override - public void run() { - long tid = Thread.currentThread().getId(); - for (int i = 0; i < QUERIES; i++) { - if (i % 30 == 0) { - System.out.println("--> [" + tid + "] query # "+ i); - } - SearchResponse resp = client.prepareSearch(INDEX).setQuery(matchAllQuery()) - .addAggregation( - terms("myterms") - .size(AGG_SIZE) - .field("num") - ).setSize(0).get(); - Terms terms = resp.getAggregations().get("myterms"); - assertNotNull("term aggs were calculated", terms); - totalThreadedTime.addAndGet(resp.getTookInMillis()); - } - } - }); - } - - System.out.println("--> starting " + THREADS + " threads for parallel aggregating"); - for (Thread t : threads) { - t.start(); - } - - for (Thread t : threads) { - t.join(); - } - - System.out.println("--> threaded average time: " + (totalThreadedTime.get() / (THREADS * QUERIES)) + "ms"); - } - - public static void main(String args[]) throws Exception { - Node node = NodeBuilder.nodeBuilder().settings(Settings.settingsBuilder()).node(); - final Client client = node.client(); - try { - try { - client.admin().indices().prepareDelete(INDEX).get(); - } catch (Exception e) { - // Ignore - } - try { - client.admin().indices().prepareCreate(INDEX).setSettings( - settingsBuilder().put("number_of_shards", 2).put("number_of_replicas", 0)).get(); - } catch (IndexAlreadyExistsException e) {} - client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); - - - System.out.println("--> indexing: " + NUM_DOCS + " documents..."); - BulkRequestBuilder bulkBuilder = client.prepareBulk(); - for (int i = 0; i < NUM_DOCS; i++) { - bulkBuilder.add(client.prepareIndex(INDEX, "doc").setSource("num", i)); - if (i % BULK_SIZE == 0) { - // Send off bulk request - bulkBuilder.get(); - // Create a new holder - bulkBuilder = client.prepareBulk(); - } - } - bulkBuilder.get(); - client.admin().indices().prepareRefresh(INDEX).get(); - SearchResponse countResp = client.prepareSearch(INDEX).setQuery(matchAllQuery()).setSize(0).get(); - assert countResp.getHits().getTotalHits() == NUM_DOCS : "all docs should be indexed"; - - final int warmupCount = 100; - for (int i = 0; i < warmupCount; i++) { - if (i % 15 == 0) { - System.out.println("--> warmup #" + i); - } - SearchResponse resp = client.prepareSearch(INDEX).setQuery(matchAllQuery()) - .addAggregation( - terms("myterms") - .size(AGG_SIZE) - .field("num") - ).setSize(0).get(); - Terms terms = resp.getAggregations().get("myterms"); - assertNotNull("term aggs were calculated", terms); - } - - System.out.println("--> running single-threaded tests"); - runSingleThreadedQueries(client); - System.out.println("--> switching to NOOP breaker"); - switchToNoop(client); - runSingleThreadedQueries(client); - switchToMemory(client); - - System.out.println("--> running multi-threaded tests"); - runMultiThreadedQueries(client); - System.out.println("--> switching to NOOP breaker"); - switchToNoop(client); - runMultiThreadedQueries(client); - } finally { - client.close(); - node.close(); - } - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/checksum/ChecksumBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/checksum/ChecksumBenchmark.java deleted file mode 100644 index 660d042e5ef..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/checksum/ChecksumBenchmark.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.checksum; - -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; - -import java.security.MessageDigest; -import java.util.zip.Adler32; -import java.util.zip.CRC32; - -/** - * - */ -public class ChecksumBenchmark { - - public static final int BATCH_SIZE = 16 * 1024; - - public static void main(String[] args) throws Exception { - System.out.println("Warning up"); - long warmSize = ByteSizeValue.parseBytesSizeValue("1g", null).bytes(); - crc(warmSize); - adler(warmSize); - md5(warmSize); - - long dataSize = ByteSizeValue.parseBytesSizeValue("10g", null).bytes(); - System.out.println("Running size: " + dataSize); - crc(dataSize); - adler(dataSize); - md5(dataSize); - } - - private static void crc(long dataSize) { - long start = System.currentTimeMillis(); - CRC32 crc = new CRC32(); - byte[] data = new byte[BATCH_SIZE]; - long iter = dataSize / BATCH_SIZE; - for (long i = 0; i < iter; i++) { - crc.update(data); - } - crc.getValue(); - System.out.println("CRC took " + new TimeValue(System.currentTimeMillis() - start)); - } - - private static void adler(long dataSize) { - long start = System.currentTimeMillis(); - Adler32 crc = new Adler32(); - byte[] data = new byte[BATCH_SIZE]; - long iter = dataSize / BATCH_SIZE; - for (long i = 0; i < iter; i++) { - crc.update(data); - } - crc.getValue(); - System.out.println("Adler took " + new TimeValue(System.currentTimeMillis() - start)); - } - - private static void md5(long dataSize) throws Exception { - long start = System.currentTimeMillis(); - byte[] data = new byte[BATCH_SIZE]; - long iter = dataSize / BATCH_SIZE; - MessageDigest digest = MessageDigest.getInstance("MD5"); - for (long i = 0; i < iter; i++) { - digest.update(data); - } - digest.digest(); - System.out.println("md5 took " + new TimeValue(System.currentTimeMillis() - start)); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/cluster/ClusterAllocationRerouteBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/cluster/ClusterAllocationRerouteBenchmark.java deleted file mode 100644 index 6b2608c06f7..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/cluster/ClusterAllocationRerouteBenchmark.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.cluster; - -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.test.ESAllocationTestCase; - -import java.util.Random; - -import static java.util.Collections.singletonMap; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; - -public class ClusterAllocationRerouteBenchmark { - - private static final ESLogger logger = Loggers.getLogger(ClusterAllocationRerouteBenchmark.class); - - public static void main(String[] args) { - final int numberOfRuns = 1; - final int numIndices = 5 * 365; // five years - final int numShards = 6; - final int numReplicas = 2; - final int numberOfNodes = 30; - final int numberOfTags = 2; - AllocationService strategy = ESAllocationTestCase.createAllocationService(Settings.builder() - .put("cluster.routing.allocation.awareness.attributes", "tag") - .build(), new Random(1)); - - MetaData.Builder mb = MetaData.builder(); - for (int i = 1; i <= numIndices; i++) { - mb.put(IndexMetaData.builder("test_" + i).numberOfShards(numShards).numberOfReplicas(numReplicas)); - } - MetaData metaData = mb.build(); - RoutingTable.Builder rb = RoutingTable.builder(); - for (int i = 1; i <= numIndices; i++) { - rb.addAsNew(metaData.index("test_" + i)); - } - RoutingTable routingTable = rb.build(); - DiscoveryNodes.Builder nb = DiscoveryNodes.builder(); - for (int i = 1; i <= numberOfNodes; i++) { - nb.put(ESAllocationTestCase.newNode("node" + i, singletonMap("tag", "tag_" + (i % numberOfTags)))); - } - ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes(nb).build(); - - long start = System.currentTimeMillis(); - for (int i = 0; i < numberOfRuns; i++) { - logger.info("[{}] starting... ", i); - long runStart = System.currentTimeMillis(); - ClusterState clusterState = initialClusterState; - while (clusterState.getRoutingNodes().hasUnassignedShards()) { - logger.info("[{}] remaining unassigned {}", i, clusterState.getRoutingNodes().unassigned().size()); - RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)); - clusterState = ClusterState.builder(clusterState).routingResult(result).build(); - result = strategy.reroute(clusterState); - clusterState = ClusterState.builder(clusterState).routingResult(result).build(); - } - logger.info("[{}] took {}", i, TimeValue.timeValueMillis(System.currentTimeMillis() - runStart)); - } - long took = System.currentTimeMillis() - start; - logger.info("total took {}, AVG {}", TimeValue.timeValueMillis(took), TimeValue.timeValueMillis(took / numberOfRuns)); - } -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/benchmark/common/lucene/uidscan/LuceneUidScanBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/common/lucene/uidscan/LuceneUidScanBenchmark.java deleted file mode 100644 index fe548b9ee4c..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/common/lucene/uidscan/LuceneUidScanBenchmark.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.common.lucene.uidscan; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.*; -import org.apache.lucene.store.FSDirectory; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.uid.Versions; -import org.elasticsearch.common.unit.SizeValue; - -import java.nio.file.Paths; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ThreadLocalRandom; - -/** - * - */ -public class LuceneUidScanBenchmark { - - public static void main(String[] args) throws Exception { - - FSDirectory dir = FSDirectory.open(PathUtils.get("work/test")); - IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - - final int NUMBER_OF_THREADS = 2; - final long INDEX_COUNT = SizeValue.parseSizeValue("1m").singles(); - final long SCAN_COUNT = SizeValue.parseSizeValue("100k").singles(); - final long startUid = 1000000; - - long LIMIT = startUid + INDEX_COUNT; - StopWatch watch = new StopWatch().start(); - System.out.println("Indexing " + INDEX_COUNT + " docs..."); - for (long i = startUid; i < LIMIT; i++) { - Document doc = new Document(); - doc.add(new StringField("_uid", Long.toString(i), Store.NO)); - doc.add(new NumericDocValuesField("_version", i)); - writer.addDocument(doc); - } - System.out.println("Done indexing, took " + watch.stop().lastTaskTime()); - - final IndexReader reader = DirectoryReader.open(writer, true); - - final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS); - Thread[] threads = new Thread[NUMBER_OF_THREADS]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(new Runnable() { - @Override - public void run() { - try { - for (long i = 0; i < SCAN_COUNT; i++) { - long id = startUid + (Math.abs(ThreadLocalRandom.current().nextInt()) % INDEX_COUNT); - final long version = Versions.loadVersion(reader, new Term("_uid", Long.toString(id))); - if (version != id) { - System.err.println("wrong id..."); - break; - } - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - latch.countDown(); - } - } - }); - } - - watch = new StopWatch().start(); - for (int i = 0; i < threads.length; i++) { - threads[i].start(); - } - latch.await(); - watch.stop(); - System.out.println("Scanned in " + watch.totalTime() + " TP Seconds " + ((SCAN_COUNT * NUMBER_OF_THREADS) / watch.totalTime().secondsFrac())); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/common/recycler/RecyclerBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/common/recycler/RecyclerBenchmark.java deleted file mode 100644 index 9710605aa6d..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/common/recycler/RecyclerBenchmark.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.common.recycler; - -import org.elasticsearch.common.recycler.AbstractRecyclerC; -import org.elasticsearch.common.recycler.Recycler; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import static org.elasticsearch.common.recycler.Recyclers.concurrent; -import static org.elasticsearch.common.recycler.Recyclers.concurrentDeque; -import static org.elasticsearch.common.recycler.Recyclers.deque; -import static org.elasticsearch.common.recycler.Recyclers.dequeFactory; -import static org.elasticsearch.common.recycler.Recyclers.locked; -import static org.elasticsearch.common.recycler.Recyclers.none; - -/** Benchmark that tries to measure the overhead of object recycling depending on concurrent access. */ -public class RecyclerBenchmark { - - private static final long NUM_RECYCLES = 5000000L; - private static final Random RANDOM = new Random(0); - - private static long bench(final Recycler recycler, long numRecycles, int numThreads) throws InterruptedException { - final AtomicLong recycles = new AtomicLong(numRecycles); - final CountDownLatch latch = new CountDownLatch(1); - final Thread[] threads = new Thread[numThreads]; - for (int i = 0; i < numThreads; ++i){ - // Thread ids happen to be generated sequentially, so we also generate random threads so that distribution of IDs - // is not perfect for the concurrent recycler - for (int j = RANDOM.nextInt(5); j >= 0; --j) { - new Thread(); - } - - threads[i] = new Thread() { - @Override - public void run() { - try { - latch.await(); - } catch (InterruptedException e) { - return; - } - while (recycles.getAndDecrement() > 0) { - final Recycler.V v = recycler.obtain(); - v.close(); - } - } - }; - } - for (Thread thread : threads) { - thread.start(); - } - final long start = System.nanoTime(); - latch.countDown(); - for (Thread thread : threads) { - thread.join(); - } - return System.nanoTime() - start; - } - - public static void main(String[] args) throws InterruptedException { - final int limit = 100; - final Recycler.C c = new AbstractRecyclerC() { - - @Override - public Object newInstance(int sizing) { - return new Object(); - } - - @Override - public void recycle(Object value) { - // do nothing - } - }; - - Map> recyclers = new HashMap<>(); - recyclers.put("none", none(c)); - recyclers.put("concurrent-queue", concurrentDeque(c, limit)); - recyclers.put("locked", locked(deque(c, limit))); - recyclers.put("concurrent", concurrent(dequeFactory(c, limit), Runtime.getRuntime().availableProcessors())); - - // warmup - final long start = System.nanoTime(); - while (System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)) { - for (Recycler recycler : recyclers.values()) { - bench(recycler, NUM_RECYCLES, 2); - } - } - - // run - for (int numThreads = 1; numThreads <= 4 * Runtime.getRuntime().availableProcessors(); numThreads *= 2) { - System.out.println("## " + numThreads + " threads\n"); - System.gc(); - Thread.sleep(1000); - for (Recycler recycler : recyclers.values()) { - bench(recycler, NUM_RECYCLES, numThreads); - } - for (int i = 0; i < 5; ++i) { - for (Map.Entry> entry : recyclers.entrySet()) { - System.out.println(entry.getKey() + "\t" + TimeUnit.NANOSECONDS.toMillis(bench(entry.getValue(), NUM_RECYCLES, numThreads))); - } - System.out.println(); - } - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/counter/SimpleCounterBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/counter/SimpleCounterBenchmark.java deleted file mode 100644 index ea1e589f7d5..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/counter/SimpleCounterBenchmark.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.counter; - -import org.elasticsearch.common.StopWatch; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicLong; - -/** - * - */ -public class SimpleCounterBenchmark { - - private static long NUMBER_OF_ITERATIONS = 10000000; - private static int NUMBER_OF_THREADS = 100; - - public static void main(String[] args) throws Exception { - final AtomicLong counter = new AtomicLong(); - StopWatch stopWatch = new StopWatch().start(); - System.out.println("Running " + NUMBER_OF_ITERATIONS); - for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) { - counter.incrementAndGet(); - } - System.out.println("Took " + stopWatch.stop().totalTime() + " TP Millis " + (NUMBER_OF_ITERATIONS / stopWatch.totalTime().millisFrac())); - - System.out.println("Running using " + NUMBER_OF_THREADS + " threads with " + NUMBER_OF_ITERATIONS + " iterations"); - final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS); - Thread[] threads = new Thread[NUMBER_OF_THREADS]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(new Runnable() { - @Override - public void run() { - for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) { - counter.incrementAndGet(); - } - latch.countDown(); - } - }); - } - stopWatch = new StopWatch().start(); - for (Thread thread : threads) { - thread.start(); - } - latch.await(); - stopWatch.stop(); - System.out.println("Took " + stopWatch.totalTime() + " TP Millis " + ((NUMBER_OF_ITERATIONS * NUMBER_OF_THREADS) / stopWatch.totalTime().millisFrac())); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java deleted file mode 100644 index 06fc39deaba..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.fs; - -import org.apache.lucene.util.IOUtils; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.unit.ByteSizeValue; - -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.util.Random; - -/** - * - */ -public class FsAppendBenchmark { - - public static void main(String[] args) throws Exception { - Path path = PathUtils.get("work/test.log"); - IOUtils.deleteFilesIgnoringExceptions(path); - - int CHUNK = (int) ByteSizeValue.parseBytesSizeValue("1k", "CHUNK").bytes(); - long DATA = ByteSizeValue.parseBytesSizeValue("10gb", "DATA").bytes(); - - byte[] data = new byte[CHUNK]; - new Random().nextBytes(data); - - StopWatch watch = new StopWatch().start("write"); - try (FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) { - long position = 0; - while (position < DATA) { - channel.write(ByteBuffer.wrap(data), position); - position += data.length; - } - watch.stop().start("flush"); - channel.force(true); - } - watch.stop(); - System.out.println("Wrote [" + (new ByteSizeValue(DATA)) + "], chunk [" + (new ByteSizeValue(CHUNK)) + "], in " + watch); - } - - private static final ByteBuffer fill = ByteBuffer.allocateDirect(1); - -// public static long padLogFile(long position, long currentSize, long preAllocSize) throws IOException { -// if (position + 4096 >= currentSize) { -// currentSize = currentSize + preAllocSize; -// fill.position(0); -// f.getChannel().write(fill, currentSize - fill.remaining()); -// } -// return currentSize; -// } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/get/SimpleGetActionBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/get/SimpleGetActionBenchmark.java deleted file mode 100644 index d78df7f6aa8..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/get/SimpleGetActionBenchmark.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.get; - -import org.elasticsearch.client.Client; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; - -// simple test for embedded / single remote lookup -public class SimpleGetActionBenchmark { - - public static void main(String[] args) { - long OPERATIONS = SizeValue.parseSizeValue("300k").singles(); - - Node node = NodeBuilder.nodeBuilder().node(); - - Client client; - if (false) { - client = NodeBuilder.nodeBuilder().client(true).node().client(); - } else { - client = node.client(); - } - - client.prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet(); - - StopWatch stopWatch = new StopWatch().start(); - for (long i = 0; i < OPERATIONS; i++) { - client.prepareGet("test", "type1", "1").execute().actionGet(); - } - stopWatch.stop(); - - System.out.println("Ran in " + stopWatch.totalTime() + ", per second: " + (((double) OPERATIONS) / stopWatch.totalTime().secondsFrac())); - - node.close(); - } -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/benchmark/hppc/StringMapAdjustOrPutBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/hppc/StringMapAdjustOrPutBenchmark.java deleted file mode 100644 index e51ba31b6d1..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/hppc/StringMapAdjustOrPutBenchmark.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.hppc; - -import com.carrotsearch.hppc.IntIntHashMap; -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.ObjectIntHashMap; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.unit.SizeValue; - -import java.util.HashMap; -import java.util.IdentityHashMap; -import java.util.concurrent.ThreadLocalRandom; - -// TODO: these benchmarks aren't too good and may be easily skewed by jit doing -// escape analysis/ side-effects/ local -// optimisations. Proper benchmarks with JMH (bulk ops, single-shot mode) -// should be better here. -// https://github.com/carrotsearch/hppc/blob/master/hppc-benchmarks/src/main/java/com/carrotsearch/hppc/benchmarks/B003_HashSet_Contains.java - -public class StringMapAdjustOrPutBenchmark { - - public static void main(String[] args) { - - int NUMBER_OF_KEYS = (int) SizeValue.parseSizeValue("20").singles(); - int STRING_SIZE = 5; - long PUT_OPERATIONS = SizeValue.parseSizeValue("5m").singles(); - long ITERATIONS = 10; - boolean REUSE = true; - - - String[] values = new String[NUMBER_OF_KEYS]; - for (int i = 0; i < values.length; i++) { - values[i] = RandomStrings.randomAsciiOfLength(ThreadLocalRandom.current(), STRING_SIZE); - } - - StopWatch stopWatch; - - stopWatch = new StopWatch().start(); - ObjectIntHashMap map = new ObjectIntHashMap<>(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - map.clear(); - } else { - map = new ObjectIntHashMap<>(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - map.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1); - } - } - map.clear(); - map = null; - - stopWatch.stop(); - System.out.println("TObjectIntHashMap: " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - - stopWatch = new StopWatch().start(); -// TObjectIntCustomHashMap iMap = new TObjectIntCustomHashMap(new StringIdentityHashingStrategy()); - ObjectIntHashMap iMap = new ObjectIntHashMap<>(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - iMap.clear(); - } else { - iMap = new ObjectIntHashMap<>(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - iMap.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1); - } - } - stopWatch.stop(); - System.out.println("TObjectIntCustomHashMap(StringIdentity): " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - iMap.clear(); - iMap = null; - - stopWatch = new StopWatch().start(); - iMap = new ObjectIntHashMap<>(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - iMap.clear(); - } else { - iMap = new ObjectIntHashMap<>(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - iMap.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1); - } - } - stopWatch.stop(); - System.out.println("TObjectIntCustomHashMap(PureIdentity): " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - iMap.clear(); - iMap = null; - - // now test with THashMap - stopWatch = new StopWatch().start(); - ObjectObjectHashMap tMap = new ObjectObjectHashMap<>(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - tMap.clear(); - } else { - tMap = new ObjectObjectHashMap<>(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - String key = values[(int) (i % NUMBER_OF_KEYS)]; - StringEntry stringEntry = tMap.get(key); - if (stringEntry == null) { - stringEntry = new StringEntry(key, 1); - tMap.put(key, stringEntry); - } else { - stringEntry.counter++; - } - } - } - - tMap.clear(); - tMap = null; - - stopWatch.stop(); - System.out.println("THashMap: " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - - stopWatch = new StopWatch().start(); - HashMap hMap = new HashMap<>(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - hMap.clear(); - } else { - hMap = new HashMap<>(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - String key = values[(int) (i % NUMBER_OF_KEYS)]; - StringEntry stringEntry = hMap.get(key); - if (stringEntry == null) { - stringEntry = new StringEntry(key, 1); - hMap.put(key, stringEntry); - } else { - stringEntry.counter++; - } - } - } - - hMap.clear(); - hMap = null; - - stopWatch.stop(); - System.out.println("HashMap: " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - - - stopWatch = new StopWatch().start(); - IdentityHashMap ihMap = new IdentityHashMap<>(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - ihMap.clear(); - } else { - hMap = new HashMap<>(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - String key = values[(int) (i % NUMBER_OF_KEYS)]; - StringEntry stringEntry = ihMap.get(key); - if (stringEntry == null) { - stringEntry = new StringEntry(key, 1); - ihMap.put(key, stringEntry); - } else { - stringEntry.counter++; - } - } - } - stopWatch.stop(); - System.out.println("IdentityHashMap: " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - - ihMap.clear(); - ihMap = null; - - int[] iValues = new int[NUMBER_OF_KEYS]; - for (int i = 0; i < values.length; i++) { - iValues[i] = ThreadLocalRandom.current().nextInt(); - } - - stopWatch = new StopWatch().start(); - IntIntHashMap intMap = new IntIntHashMap(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - intMap.clear(); - } else { - intMap = new IntIntHashMap(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - int key = iValues[(int) (i % NUMBER_OF_KEYS)]; - intMap.addTo(key, 1); - } - } - stopWatch.stop(); - System.out.println("TIntIntHashMap: " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - - intMap.clear(); - intMap = null; - - // now test with THashMap - stopWatch = new StopWatch().start(); - IntObjectHashMap tIntMap = new IntObjectHashMap<>(); - for (long iter = 0; iter < ITERATIONS; iter++) { - if (REUSE) { - tIntMap.clear(); - } else { - tIntMap = new IntObjectHashMap<>(); - } - for (long i = 0; i < PUT_OPERATIONS; i++) { - int key = iValues[(int) (i % NUMBER_OF_KEYS)]; - IntEntry intEntry = tIntMap.get(key); - if (intEntry == null) { - intEntry = new IntEntry(key, 1); - tIntMap.put(key, intEntry); - } else { - intEntry.counter++; - } - } - } - - tIntMap.clear(); - tIntMap = null; - - stopWatch.stop(); - System.out.println("TIntObjectHashMap: " + stopWatch.totalTime() + ", " + stopWatch.totalTime().millisFrac() / ITERATIONS + "ms"); - } - - - static class StringEntry { - String key; - int counter; - - StringEntry(String key, int counter) { - this.key = key; - this.counter = counter; - } - } - - static class IntEntry { - int key; - int counter; - - IntEntry(int key, int counter) { - this.key = key; - this.counter = counter; - } - } -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/benchmark/mapping/ManyMappingsBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/mapping/ManyMappingsBenchmark.java deleted file mode 100644 index 840c3c1dc67..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/mapping/ManyMappingsBenchmark.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.mapping; - -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.bootstrap.BootstrapForTesting; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.node.Node; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - */ -public class ManyMappingsBenchmark { - - private static final String MAPPING = "{\n" + - " \"dynamic_templates\": [\n" + - " {\n" + - " \"t1\": {\n" + - " \"mapping\": {\n" + - " \"store\": false,\n" + - " \"norms\": {\n" + - " \"enabled\": false\n" + - " },\n" + - " \"type\": \"string\"\n" + - " },\n" + - " \"match\": \"*_ss\"\n" + - " }\n" + - " },\n" + - " {\n" + - " \"t2\": {\n" + - " \"mapping\": {\n" + - " \"store\": false,\n" + - " \"type\": \"date\"\n" + - " },\n" + - " \"match\": \"*_dt\"\n" + - " }\n" + - " },\n" + - " {\n" + - " \"t3\": {\n" + - " \"mapping\": {\n" + - " \"store\": false,\n" + - " \"type\": \"integer\"\n" + - " },\n" + - " \"match\": \"*_i\"\n" + - " }\n" + - " }\n" + - " ],\n" + - " \"_source\": {\n" + - " \"enabled\": false\n" + - " },\n" + - " \"properties\": {}\n" + - " }"; - - private static final String INDEX_NAME = "index"; - private static final String TYPE_NAME = "type"; - private static final int FIELD_COUNT = 100000; - private static final int DOC_COUNT = 10000000; - private static final boolean TWO_NODES = true; - - public static void main(String[] args) throws Exception { - System.setProperty("es.logger.prefix", ""); - BootstrapForTesting.ensureInitialized(); - Settings settings = settingsBuilder() - .put("") - .put(SETTING_NUMBER_OF_SHARDS, 5) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = ManyMappingsBenchmark.class.getSimpleName(); - Node node = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings)) - .node(); - if (TWO_NODES) { - Node node2 = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings)) - .node(); - } - - Client client = node.client(); - - client.admin().indices().prepareDelete(INDEX_NAME) - .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .get(); - client.admin().indices().prepareCreate(INDEX_NAME) - .addMapping(TYPE_NAME, MAPPING) - .get(); - - BulkRequestBuilder builder = client.prepareBulk(); - int fieldCount = 0; - long time = System.currentTimeMillis(); - final int PRINT = 1000; - for (int i = 0; i < DOC_COUNT; i++) { - XContentBuilder sourceBuilder = jsonBuilder().startObject(); - sourceBuilder.field(++fieldCount + "_ss", "xyz"); - sourceBuilder.field(++fieldCount + "_dt", System.currentTimeMillis()); - sourceBuilder.field(++fieldCount + "_i", i % 100); - sourceBuilder.endObject(); - - if (fieldCount >= FIELD_COUNT) { - fieldCount = 0; - System.out.println("dynamic fields rolled up"); - } - - builder.add( - client.prepareIndex(INDEX_NAME, TYPE_NAME, String.valueOf(i)) - .setSource(sourceBuilder) - ); - - if (builder.numberOfActions() >= 1000) { - builder.get(); - builder = client.prepareBulk(); - } - - if (i % PRINT == 0) { - long took = System.currentTimeMillis() - time; - time = System.currentTimeMillis(); - System.out.println("Indexed " + i + " docs, in " + TimeValue.timeValueMillis(took)); - } - } - if (builder.numberOfActions() > 0) { - builder.get(); - } - - - - } - -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/benchmark/monitor/os/OsProbeBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/monitor/os/OsProbeBenchmark.java deleted file mode 100644 index c475c10c8f8..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/monitor/os/OsProbeBenchmark.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.monitor.os; - -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.monitor.os.OsProbe; - -public class OsProbeBenchmark { - - private static final int ITERATIONS = 100_000; - - public static void main(String[] args) { - System.setProperty("es.logger.prefix", ""); - final ESLogger logger = ESLoggerFactory.getLogger("benchmark"); - - logger.info("--> loading OS probe"); - OsProbe probe = OsProbe.getInstance(); - - logger.info("--> warming up..."); - for (int i = 0; i < ITERATIONS; i++) { - probe.getTotalPhysicalMemorySize(); - probe.getFreePhysicalMemorySize(); - probe.getTotalSwapSpaceSize(); - probe.getFreeSwapSpaceSize(); - probe.getSystemLoadAverage(); - } - logger.info("--> warmed up"); - - - - - logger.info("--> testing 'getTotalPhysicalMemorySize' method..."); - long start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getTotalPhysicalMemorySize(); - } - long elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getFreePhysicalMemorySize' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getFreePhysicalMemorySize(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getTotalSwapSpaceSize' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getTotalSwapSpaceSize(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getFreeSwapSpaceSize' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getFreeSwapSpaceSize(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getSystemLoadAverage' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getSystemLoadAverage(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/monitor/process/ProcessProbeBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/monitor/process/ProcessProbeBenchmark.java deleted file mode 100644 index b91b516d96e..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/monitor/process/ProcessProbeBenchmark.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.monitor.process; - -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.monitor.process.ProcessProbe; - -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadMXBean; - -@SuppressForbidden(reason = "use of om.sun.management.ThreadMXBean to compare performance") -public class ProcessProbeBenchmark { - - private static final int ITERATIONS = 100_000; - - public static void main(String[] args) { - System.setProperty("es.logger.prefix", ""); - final ESLogger logger = ESLoggerFactory.getLogger("benchmark"); - - logger.info("--> loading process probe"); - ProcessProbe probe = ProcessProbe.getInstance(); - - logger.info("--> warming up..."); - for (int i = 0; i < ITERATIONS; i++) { - probe.getOpenFileDescriptorCount(); - probe.getMaxFileDescriptorCount(); - probe.getTotalVirtualMemorySize(); - probe.getProcessCpuPercent(); - probe.getProcessCpuTotalTime(); - } - logger.info("--> warmed up"); - - - - - logger.info("--> testing 'getOpenFileDescriptorCount' method..."); - long start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getOpenFileDescriptorCount(); - } - long elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getMaxFileDescriptorCount' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getMaxFileDescriptorCount(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getTotalVirtualMemorySize' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getTotalVirtualMemorySize(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getProcessCpuPercent' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getProcessCpuPercent(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - logger.info("--> testing 'getProcessCpuTotalTime' method..."); - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - probe.getProcessCpuTotalTime(); - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS)); - - - - - logger.info("--> calculating process CPU user time with 'getAllThreadIds + getThreadUserTime' methods..."); - final ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); - final long[] threadIds = threadMxBean.getAllThreadIds(); - long sum = 0; - - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - for (long threadId : threadIds) { - sum += threadMxBean.getThreadUserTime(threadId); - } - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> execution time [total: {} ms, avg: {} ms] for {} iterations with average result of {}", - elapsed, (elapsed / (double)ITERATIONS), ITERATIONS, (sum / (double)ITERATIONS)); - - if (threadMxBean instanceof com.sun.management.ThreadMXBean) { - logger.info("--> calculating process CPU user time with 'getAllThreadIds + getThreadUserTime(long[])' methods..."); - final com.sun.management.ThreadMXBean threadMxBean2 = (com.sun.management.ThreadMXBean)threadMxBean; - sum = 0; - - start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - long[] user = threadMxBean2.getThreadUserTime(threadIds); - for (int n = 0 ; n != threadIds.length; ++n) { - sum += user[n]; - } - } - elapsed = System.currentTimeMillis() - start; - logger.info("--> execution time [total: {} ms, avg: {} ms] for {} iterations with average result of {}", - elapsed, (elapsed / (double)ITERATIONS), ITERATIONS, (sum / (double)ITERATIONS)); - - } - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/percolator/PercolatorStressBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/percolator/PercolatorStressBenchmark.java deleted file mode 100644 index 1f9863da7b1..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/percolator/PercolatorStressBenchmark.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.percolator; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.percolate.PercolateResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.node.Node; -import org.elasticsearch.percolator.PercolatorService; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class PercolatorStressBenchmark { - - public static void main(String[] args) throws Exception { - Settings settings = settingsBuilder() - .put(SETTING_NUMBER_OF_SHARDS, 4) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node[] nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node" + i)).node(); - } - - Node clientNode = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - Client client = clientNode.client(); - - client.admin().indices().create(createIndexRequest("test")).actionGet(); - ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth("test") - .setWaitForGreenStatus() - .execute().actionGet(); - if (healthResponse.isTimedOut()) { - System.err.println("Quiting, because cluster health requested timed out..."); - return; - } else if (healthResponse.getStatus() != ClusterHealthStatus.GREEN) { - System.err.println("Quiting, because cluster state isn't green..."); - return; - } - - int COUNT = 200000; - int QUERIES = 100; - int TERM_QUERIES = QUERIES / 2; - int RANGE_QUERIES = QUERIES - TERM_QUERIES; - - client.prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject().field("numeric1", 1).endObject()).execute().actionGet(); - - // register queries - int i = 0; - for (; i < TERM_QUERIES; i++) { - client.prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i)) - .setSource(jsonBuilder().startObject() - .field("query", termQuery("name", "value")) - .endObject()) - .execute().actionGet(); - } - - int[] numbers = new int[RANGE_QUERIES]; - for (; i < QUERIES; i++) { - client.prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i)) - .setSource(jsonBuilder().startObject() - .field("query", rangeQuery("numeric1").from(i).to(i)) - .endObject()) - .execute().actionGet(); - numbers[i - TERM_QUERIES] = i; - } - - StopWatch stopWatch = new StopWatch().start(); - System.out.println("Percolating [" + COUNT + "] ..."); - for (i = 1; i <= COUNT; i++) { - XContentBuilder source; - int expectedMatches; - if (i % 2 == 0) { - source = source(Integer.toString(i), "value"); - expectedMatches = TERM_QUERIES; - } else { - int number = numbers[i % RANGE_QUERIES]; - source = source(Integer.toString(i), number); - expectedMatches = 1; - } - PercolateResponse percolate = client.preparePercolate() - .setIndices("test").setDocumentType("type1") - .setSource(source) - .execute().actionGet(); - if (percolate.getMatches().length != expectedMatches) { - System.err.println("No matching number of queries"); - } - - if ((i % 10000) == 0) { - System.out.println("Percolated " + i + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - System.out.println("Percolation took " + stopWatch.totalTime() + ", TPS " + (((double) COUNT) / stopWatch.totalTime().secondsFrac())); - - clientNode.close(); - for (Node node : nodes) { - node.close(); - } - } - - private static XContentBuilder source(String id, String nameValue) throws IOException { - return jsonBuilder().startObject().startObject("doc") - .field("id", id) - .field("name", nameValue) - .endObject().endObject(); - } - - private static XContentBuilder source(String id, int number) throws IOException { - return jsonBuilder().startObject().startObject("doc") - .field("id", id) - .field("numeric1", number) - .field("numeric2", number) - .field("numeric3", number) - .field("numeric4", number) - .field("numeric5", number) - .field("numeric6", number) - .field("numeric7", number) - .field("numeric8", number) - .field("numeric9", number) - .field("numeric10", number) - .endObject().endObject(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/recovery/ReplicaRecoveryBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/recovery/ReplicaRecoveryBenchmark.java deleted file mode 100644 index 555a3326a04..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/recovery/ReplicaRecoveryBenchmark.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.recovery; - -import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.bootstrap.BootstrapForTesting; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.indices.recovery.RecoveryState; -import org.elasticsearch.node.Node; -import org.elasticsearch.test.BackgroundIndexer; -import org.elasticsearch.transport.TransportModule; - -import java.util.List; -import java.util.Random; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class ReplicaRecoveryBenchmark { - - private static final String INDEX_NAME = "index"; - private static final String TYPE_NAME = "type"; - - - static int DOC_COUNT = (int) SizeValue.parseSizeValue("40k").singles(); - static int CONCURRENT_INDEXERS = 2; - - public static void main(String[] args) throws Exception { - System.setProperty("es.logger.prefix", ""); - BootstrapForTesting.ensureInitialized(); - - Settings settings = settingsBuilder() - .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, "false") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(TransportModule.TRANSPORT_TYPE_KEY, "local") - .build(); - - String clusterName = ReplicaRecoveryBenchmark.class.getSimpleName(); - Node node1 = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings)) - .node(); - - final ESLogger logger = ESLoggerFactory.getLogger("benchmark"); - - final Client client1 = node1.client(); - client1.admin().cluster().prepareUpdateSettings().setPersistentSettings("logger.indices.recovery: TRACE").get(); - final BackgroundIndexer indexer = new BackgroundIndexer(INDEX_NAME, TYPE_NAME, client1, 0, CONCURRENT_INDEXERS, false, new Random()); - indexer.setMinFieldSize(10); - indexer.setMaxFieldSize(150); - try { - client1.admin().indices().prepareDelete(INDEX_NAME).get(); - } catch (IndexNotFoundException e) { - } - client1.admin().indices().prepareCreate(INDEX_NAME).get(); - indexer.start(DOC_COUNT / 2); - while (indexer.totalIndexedDocs() < DOC_COUNT / 2) { - Thread.sleep(5000); - logger.info("--> indexed {} of {}", indexer.totalIndexedDocs(), DOC_COUNT); - } - client1.admin().indices().prepareFlush().get(); - indexer.continueIndexing(DOC_COUNT / 2); - while (indexer.totalIndexedDocs() < DOC_COUNT) { - Thread.sleep(5000); - logger.info("--> indexed {} of {}", indexer.totalIndexedDocs(), DOC_COUNT); - } - - - logger.info("--> starting another node and allocating a shard on it"); - - Node node2 = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings)) - .node(); - - client1.admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(IndexMetaData.SETTING_NUMBER_OF_REPLICAS + ": 1").get(); - - final AtomicBoolean end = new AtomicBoolean(false); - - final Thread backgroundLogger = new Thread(new Runnable() { - - long lastTime = System.currentTimeMillis(); - long lastDocs = indexer.totalIndexedDocs(); - long lastBytes = 0; - long lastTranslogOps = 0; - - @Override - public void run() { - while (true) { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - - } - if (end.get()) { - return; - } - long currentTime = System.currentTimeMillis(); - long currentDocs = indexer.totalIndexedDocs(); - RecoveryResponse recoveryResponse = client1.admin().indices().prepareRecoveries(INDEX_NAME).setActiveOnly(true).get(); - List indexRecoveries = recoveryResponse.shardRecoveryStates().get(INDEX_NAME); - long translogOps; - long bytes; - if (indexRecoveries.size() > 0) { - translogOps = indexRecoveries.get(0).getTranslog().recoveredOperations(); - bytes = recoveryResponse.shardRecoveryStates().get(INDEX_NAME).get(0).getIndex().recoveredBytes(); - } else { - bytes = lastBytes = 0; - translogOps = lastTranslogOps = 0; - } - float seconds = (currentTime - lastTime) / 1000.0F; - logger.info("--> indexed [{}];[{}] doc/s, recovered [{}] MB/s , translog ops [{}]/s ", - currentDocs, (currentDocs - lastDocs) / seconds, - (bytes - lastBytes) / 1024.0F / 1024F / seconds, (translogOps - lastTranslogOps) / seconds); - lastBytes = bytes; - lastTranslogOps = translogOps; - lastTime = currentTime; - lastDocs = currentDocs; - } - } - }); - - backgroundLogger.start(); - - client1.admin().cluster().prepareHealth().setWaitForGreenStatus().get(); - - logger.info("--> green. starting relocation cycles"); - - long startDocIndexed = indexer.totalIndexedDocs(); - indexer.continueIndexing(DOC_COUNT * 50); - - long totalRecoveryTime = 0; - long startTime = System.currentTimeMillis(); - long[] recoveryTimes = new long[3]; - for (int iteration = 0; iteration < 3; iteration++) { - logger.info("--> removing replicas"); - client1.admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(IndexMetaData.SETTING_NUMBER_OF_REPLICAS + ": 0").get(); - logger.info("--> adding replica again"); - long recoveryStart = System.currentTimeMillis(); - client1.admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(IndexMetaData.SETTING_NUMBER_OF_REPLICAS + ": 1").get(); - client1.admin().cluster().prepareHealth(INDEX_NAME).setWaitForGreenStatus().setTimeout("15m").get(); - long recoveryTime = System.currentTimeMillis() - recoveryStart; - totalRecoveryTime += recoveryTime; - recoveryTimes[iteration] = recoveryTime; - logger.info("--> recovery done in [{}]", new TimeValue(recoveryTime)); - - // sleep some to let things clean up - Thread.sleep(10000); - } - - long endDocIndexed = indexer.totalIndexedDocs(); - long totalTime = System.currentTimeMillis() - startTime; - indexer.stop(); - - end.set(true); - - backgroundLogger.interrupt(); - - backgroundLogger.join(); - - logger.info("average doc/s [{}], average relocation time [{}], taking [{}], [{}], [{}]", (endDocIndexed - startDocIndexed) * 1000.0 / totalTime, new TimeValue(totalRecoveryTime / 3), - TimeValue.timeValueMillis(recoveryTimes[0]), TimeValue.timeValueMillis(recoveryTimes[1]), TimeValue.timeValueMillis(recoveryTimes[2]) - ); - - client1.close(); - node1.close(); - node2.close(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript1.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript1.java deleted file mode 100644 index 6f666b3f977..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript1.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.expression; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.Map; - -public class NativeScript1 extends AbstractSearchScript { - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativeScript1(); - } - - @Override - public boolean needsScores() { - return false; - } - } - - public static final String NATIVE_SCRIPT_1 = "native_1"; - - @Override - public Object run() { - return docFieldLongs("x").getValue(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript2.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript2.java deleted file mode 100644 index 585d7a57c3c..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript2.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.expression; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.Map; - -public class NativeScript2 extends AbstractSearchScript { - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativeScript2(); - } - - @Override - public boolean needsScores() { - return false; - } - } - - public static final String NATIVE_SCRIPT_2 = "native_2"; - - @Override - public Object run() { - return docFieldLongs("x").getValue() + docFieldDoubles("y").getValue(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript3.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript3.java deleted file mode 100644 index c2d50fed9cc..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript3.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.expression; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.Map; - -public class NativeScript3 extends AbstractSearchScript { - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativeScript3(); - } - - @Override - public boolean needsScores() { - return false; - } - } - - public static final String NATIVE_SCRIPT_3 = "native_3"; - - @Override - public Object run() { - return 1.2 * docFieldLongs("x").getValue() / docFieldDoubles("y").getValue(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript4.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript4.java deleted file mode 100644 index 2bda86e35c1..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript4.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.expression; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.Map; - -public class NativeScript4 extends AbstractSearchScript { - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativeScript4(); - } - - @Override - public boolean needsScores() { - return false; - } - } - - public static final String NATIVE_SCRIPT_4 = "native_4"; - - @Override - public Object run() { - return Math.sqrt(Math.abs(docFieldDoubles("z").getValue())) + Math.log(Math.abs(docFieldLongs("x").getValue() * docFieldDoubles("y").getValue())); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java deleted file mode 100644 index ce4cbf11577..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.expression; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.node.MockNode; -import org.elasticsearch.node.Node; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.search.sort.ScriptSortBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.joda.time.PeriodType; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Random; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -public class ScriptComparisonBenchmark { - - static final String clusterName = ScriptComparisonBenchmark.class.getSimpleName(); - static final String indexName = "test"; - - static String[] langs = { - "expression", - "native", - "groovy" - }; - static String[][] scripts = { - // the first value is the "reference" version (pure math) - { - "x", - "doc['x'].value", - NativeScript1.NATIVE_SCRIPT_1, - "doc['x'].value" - }, { - "x + y", - "doc['x'].value + doc['y'].value", - NativeScript2.NATIVE_SCRIPT_2, - "doc['x'].value + doc['y'].value", - }, { - "1.2 * x / y", - "1.2 * doc['x'].value / doc['y'].value", - NativeScript3.NATIVE_SCRIPT_3, - "1.2 * doc['x'].value / doc['y'].value", - }, { - "sqrt(abs(z)) + ln(abs(x * y))", - "sqrt(abs(doc['z'].value)) + ln(abs(doc['x'].value * doc['y'].value))", - NativeScript4.NATIVE_SCRIPT_4, - "sqrt(abs(doc['z'].value)) + log(abs(doc['x'].value * doc['y'].value))" - } - }; - - public static void main(String[] args) throws Exception { - int numDocs = 1000000; - int numQueries = 1000; - Client client = setupIndex(); - indexDocs(client, numDocs); - - for (int scriptNum = 0; scriptNum < scripts.length; ++scriptNum) { - runBenchmark(client, scriptNum, numQueries); - } - } - - static void runBenchmark(Client client, int scriptNum, int numQueries) { - System.out.println(""); - System.out.println("Script: " + scripts[scriptNum][0]); - System.out.println("--------------------------------"); - for (int langNum = 0; langNum < langs.length; ++langNum) { - String lang = langs[langNum]; - String script = scripts[scriptNum][langNum + 1]; - - timeQueries(client, lang, script, numQueries / 10); // warmup - TimeValue time = timeQueries(client, lang, script, numQueries); - printResults(lang, time, numQueries); - } - } - - static Client setupIndex() throws Exception { - // create cluster - Settings settings = settingsBuilder().put("name", "node1") - .put("cluster.name", clusterName).build(); - Collection> plugins = Collections.>singletonList(NativeScriptPlugin.class); - Node node1 = new MockNode(settings, Version.CURRENT, plugins); - node1.start(); - Client client = node1.client(); - client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - - // delete the index, if it exists - try { - client.admin().indices().prepareDelete(indexName).execute().actionGet(); - } catch (ElasticsearchException e) { - // ok if the index didn't exist - } - - // create mappings - IndicesAdminClient admin = client.admin().indices(); - admin.prepareCreate(indexName).addMapping("doc", "x", "type=long", "y", "type=double"); - - client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - return client; - } - - static void indexDocs(Client client, int numDocs) { - System.out.print("Indexing " + numDocs + " random docs..."); - BulkRequestBuilder bulkRequest = client.prepareBulk(); - Random r = new Random(1); - for (int i = 0; i < numDocs; i++) { - bulkRequest.add(client.prepareIndex("test", "doc", Integer.toString(i)) - .setSource("x", r.nextInt(), "y", r.nextDouble(), "z", r.nextDouble())); - - if (i % 1000 == 0) { - bulkRequest.execute().actionGet(); - bulkRequest = client.prepareBulk(); - } - } - bulkRequest.execute().actionGet(); - client.admin().indices().prepareRefresh("test").execute().actionGet(); - client.admin().indices().prepareFlush("test").execute().actionGet(); - System.out.println("done"); - } - - static TimeValue timeQueries(Client client, String lang, String script, int numQueries) { - ScriptSortBuilder sort = SortBuilders.scriptSort(new Script(script, ScriptType.INLINE, lang, null), "number"); - SearchRequestBuilder req = client.prepareSearch(indexName) - .setQuery(QueryBuilders.matchAllQuery()) - .addSort(sort); - - StopWatch timer = new StopWatch(); - timer.start(); - for (int i = 0; i < numQueries; ++i) { - req.get(); - } - timer.stop(); - return timer.totalTime(); - } - - static void printResults(String lang, TimeValue time, int numQueries) { - long avgReq = time.millis() / numQueries; - System.out.println(lang + ": " + time.format(PeriodType.seconds()) + " (" + avgReq + " msec per req)"); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/BasicScriptBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/BasicScriptBenchmark.java deleted file mode 100644 index 81d4a788ac4..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/BasicScriptBenchmark.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.score; - -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.lucene.search.function.CombineFunction; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.script.ScriptScoreFunctionBuilder; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.security.SecureRandom; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; - -import static org.elasticsearch.client.Requests.searchRequest; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; -import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; -import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; - -public class BasicScriptBenchmark { - - public static class RequestInfo { - public RequestInfo(SearchRequest source, int i) { - request = source; - numTerms = i; - } - - SearchRequest request; - int numTerms; - } - - public static class Results { - public static final String TIME_PER_DOCIN_MILLIS = "timePerDocinMillis"; - public static final String NUM_TERMS = "numTerms"; - public static final String NUM_DOCS = "numDocs"; - public static final String TIME_PER_QUERY_IN_SEC = "timePerQueryInSec"; - public static final String TOTAL_TIME_IN_SEC = "totalTimeInSec"; - Double[] resultSeconds; - Double[] resultMSPerQuery; - Long[] numDocs; - Integer[] numTerms; - Double[] timePerDoc; - String label; - String description; - public String lineStyle; - public String color; - - void init(int numVariations, String label, String description, String color, String lineStyle) { - resultSeconds = new Double[numVariations]; - resultMSPerQuery = new Double[numVariations]; - numDocs = new Long[numVariations]; - numTerms = new Integer[numVariations]; - timePerDoc = new Double[numVariations]; - this.label = label; - this.description = description; - this.color = color; - this.lineStyle = lineStyle; - } - - void set(SearchResponse searchResponse, StopWatch stopWatch, String message, int maxIter, int which, int numTerms) { - resultSeconds[which] = (double) ((double) stopWatch.lastTaskTime().getMillis() / (double) 1000); - resultMSPerQuery[which] = (double) ((double) stopWatch.lastTaskTime().secondsFrac() / (double) maxIter); - numDocs[which] = searchResponse.getHits().totalHits(); - this.numTerms[which] = numTerms; - timePerDoc[which] = resultMSPerQuery[which] / numDocs[which]; - } - - public void printResults(BufferedWriter writer) throws IOException { - String comma = (writer == null) ? "" : ";"; - String results = description + "\n" + Results.TOTAL_TIME_IN_SEC + " = " + getResultArray(resultSeconds) + comma + "\n" - + Results.TIME_PER_QUERY_IN_SEC + " = " + getResultArray(resultMSPerQuery) + comma + "\n" + Results.NUM_DOCS + " = " - + getResultArray(numDocs) + comma + "\n" + Results.NUM_TERMS + " = " + getResultArray(numTerms) + comma + "\n" - + Results.TIME_PER_DOCIN_MILLIS + " = " + getResultArray(timePerDoc) + comma + "\n"; - if (writer != null) { - writer.write(results); - } else { - System.out.println(results); - } - - } - - private String getResultArray(Object[] resultArray) { - String result = "["; - for (int i = 0; i < resultArray.length; i++) { - result += resultArray[i].toString(); - if (i != resultArray.length - 1) { - result += ","; - } - } - result += "]"; - return result; - } - } - - public BasicScriptBenchmark() { - } - - static List termsList = new ArrayList<>(); - - static void init(int numTerms) { - SecureRandom random = new SecureRandom(); - random.setSeed(1); - termsList.clear(); - for (int i = 0; i < numTerms; i++) { - String term = new BigInteger(512, random).toString(32); - termsList.add(term); - } - - } - - static String[] getTerms(int numTerms) { - String[] terms = new String[numTerms]; - for (int i = 0; i < numTerms; i++) { - terms[i] = termsList.get(i); - } - return terms; - } - - public static void writeHelperFunction() throws IOException { - try (BufferedWriter out = Files.newBufferedWriter(PathUtils.get("addToPlot.m"), StandardCharsets.UTF_8)) { - out.write("function handle = addToPlot(numTerms, perDoc, color, linestyle, linewidth)\n" + "handle = line(numTerms, perDoc);\n" - + "set(handle, 'color', color);\n" + "set(handle, 'linestyle',linestyle);\n" + "set(handle, 'LineWidth',linewidth);\n" - + "end\n"); - } - } - - public static void printOctaveScript(List allResults, String[] args) throws IOException { - if (args.length == 0) { - return; - } - try (BufferedWriter out = Files.newBufferedWriter(PathUtils.get(args[0]), StandardCharsets.UTF_8)) { - out.write("#! /usr/local/bin/octave -qf"); - out.write("\n\n\n\n"); - out.write("######################################\n"); - out.write("# Octave script for plotting results\n"); - String filename = "scriptScoreBenchmark" + new DateTime(DateTimeZone.UTC).toString(); - out.write("#Call '" + args[0] + "' from the command line. The plot is then in " + filename + "\n\n"); - - out.write("handleArray = [];\n tagArray = [];\n plot([]);\n hold on;\n"); - for (Results result : allResults) { - out.write("\n"); - out.write("# " + result.description); - result.printResults(out); - out.write("handleArray = [handleArray, addToPlot(" + Results.NUM_TERMS + ", " + Results.TIME_PER_DOCIN_MILLIS + ", '" - + result.color + "','" + result.lineStyle + "',5)];\n"); - out.write("tagArray = [tagArray; '" + result.label + "'];\n"); - out.write("\n"); - } - - out.write("xlabel(\'number of query terms');"); - out.write("ylabel(\'query time per document');"); - - out.write("legend(handleArray,tagArray);\n"); - - out.write("saveas(gcf,'" + filename + ".png','png')\n"); - out.write("hold off;\n\n"); - } catch (IOException e) { - System.err.println("Error: " + e.getMessage()); - } - writeHelperFunction(); - } - - static void printResult(SearchResponse searchResponse, StopWatch stopWatch, String queryInfo) { - System.out.println("--> Searching with " + queryInfo + " took " + stopWatch.lastTaskTime() + ", per query " - + (stopWatch.lastTaskTime().secondsFrac() / 100) + " for " + searchResponse.getHits().totalHits() + " docs"); - } - - static void indexData(long numDocs, Client client, boolean randomizeTerms) throws IOException { - try { - client.admin().indices().prepareDelete("test").execute().actionGet(); - } catch (Throwable t) { - // index might exist already, in this case we do nothing TODO: make - // saver in general - } - - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("text").field("type", "string").field("index_options", "offsets").field("analyzer", "payload_float") - .endObject().endObject().endObject().endObject(); - client.admin() - .indices() - .prepareCreate("test") - .addMapping("type1", mapping) - .setSettings( - Settings.settingsBuilder().put("index.analysis.analyzer.payload_float.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.payload_float.filter", "delimited_float") - .put("index.analysis.filter.delimited_float.delimiter", "|") - .put("index.analysis.filter.delimited_float.encoding", "float") - .put("index.analysis.filter.delimited_float.type", "delimited_payload_filter") - .put("index.number_of_replicas", 0).put("index.number_of_shards", 1)).execute().actionGet(); - client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - BulkRequestBuilder bulkRequest = client.prepareBulk(); - Random random = new Random(1); - for (int i = 0; i < numDocs; i++) { - - bulkRequest.add(client.prepareIndex().setType("type1").setIndex("test") - .setSource(jsonBuilder().startObject().field("text", randomText(random, randomizeTerms)).endObject())); - if (i % 1000 == 0) { - bulkRequest.execute().actionGet(); - bulkRequest = client.prepareBulk(); - } - } - bulkRequest.execute().actionGet(); - client.admin().indices().prepareRefresh("test").execute().actionGet(); - client.admin().indices().prepareFlush("test").execute().actionGet(); - System.out.println("Done indexing " + numDocs + " documents"); - - } - - private static String randomText(Random random, boolean randomizeTerms) { - String text = ""; - for (int i = 0; i < termsList.size(); i++) { - if (random.nextInt(5) == 3 || !randomizeTerms) { - text = text + " " + termsList.get(i) + "|1"; - } - } - return text; - } - - static void printTimings(SearchResponse searchResponse, StopWatch stopWatch, String message, int maxIter) { - System.out.println(message); - System.out.println(stopWatch.lastTaskTime() + ", " + (stopWatch.lastTaskTime().secondsFrac() / maxIter) + ", " - + searchResponse.getHits().totalHits() + ", " - + (stopWatch.lastTaskTime().secondsFrac() / (maxIter + searchResponse.getHits().totalHits()))); - } - - static List> initTermQueries(int minTerms, int maxTerms) { - List> termSearchRequests = new ArrayList<>(); - for (int nTerms = minTerms; nTerms < maxTerms; nTerms++) { - Map params = new HashMap<>(); - String[] terms = getTerms(nTerms + 1); - params.put("text", terms); - SearchRequest request = searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source( - searchSource().explain(false).size(0).query(QueryBuilders.termsQuery("text", terms))); - String infoString = "Results for term query with " + (nTerms + 1) + " terms:"; - termSearchRequests.add(new AbstractMap.SimpleEntry<>(infoString, new RequestInfo(request, nTerms + 1))); - } - return termSearchRequests; - } - - static List> initNativeSearchRequests(int minTerms, int maxTerms, String script, boolean langNative) { - List> nativeSearchRequests = new ArrayList<>(); - for (int nTerms = minTerms; nTerms < maxTerms; nTerms++) { - Map params = new HashMap<>(); - String[] terms = getTerms(nTerms + 1); - params.put("text", terms); - String infoString = "Results for native script with " + (nTerms + 1) + " terms:"; - ScriptScoreFunctionBuilder scriptFunction = (langNative == true) ? scriptFunction(new Script(script, ScriptType.INLINE, - "native", params)) : scriptFunction(new Script(script, ScriptType.INLINE, null, params)); - SearchRequest request = searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source( - searchSource() - .explain(false) - .size(0) - .query(functionScoreQuery(QueryBuilders.termsQuery("text", terms), scriptFunction).boostMode( - CombineFunction.REPLACE))); - nativeSearchRequests.add(new AbstractMap.SimpleEntry<>(infoString, new RequestInfo(request, nTerms + 1))); - } - return nativeSearchRequests; - } - - static List> initScriptMatchAllSearchRequests(String script, boolean langNative) { - List> nativeSearchRequests = new ArrayList<>(); - String infoString = "Results for constant score script:"; - ScriptScoreFunctionBuilder scriptFunction = (langNative == true) ? scriptFunction(new Script(script, ScriptType.INLINE, "native", - null)) : scriptFunction(new Script(script)); - SearchRequest request = searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source( - searchSource().explain(false).size(0) - .query(functionScoreQuery(QueryBuilders.matchAllQuery(), scriptFunction).boostMode(CombineFunction.REPLACE))); - nativeSearchRequests.add(new AbstractMap.SimpleEntry<>(infoString, new RequestInfo(request, 0))); - - return nativeSearchRequests; - } - - static void runBenchmark(Client client, int maxIter, Results results, List> nativeSearchRequests, - int minTerms, int warmerIter) throws IOException { - int counter = 0; - for (Entry entry : nativeSearchRequests) { - SearchResponse searchResponse = null; - // warm up - for (int i = 0; i < warmerIter; i++) { - searchResponse = client.search(entry.getValue().request).actionGet(); - } - System.gc(); - // run benchmark - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - for (int i = 0; i < maxIter; i++) { - searchResponse = client.search(entry.getValue().request).actionGet(); - } - stopWatch.stop(); - results.set(searchResponse, stopWatch, entry.getKey(), maxIter, counter, entry.getValue().numTerms); - counter++; - } - results.printResults(null); - } -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsConstantScoreBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsConstantScoreBenchmark.java deleted file mode 100644 index 53baf78d4c1..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsConstantScoreBenchmark.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.scripts.score; - -import org.elasticsearch.Version; -import org.elasticsearch.benchmark.scripts.score.plugin.NativeScriptExamplesPlugin; -import org.elasticsearch.benchmark.scripts.score.script.NativeConstantForLoopScoreScript; -import org.elasticsearch.benchmark.scripts.score.script.NativeConstantScoreScript; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.MockNode; -import org.elasticsearch.node.Node; -import org.elasticsearch.plugins.Plugin; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map.Entry; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class ScriptsConstantScoreBenchmark extends BasicScriptBenchmark { - - public static void main(String[] args) throws Exception { - - int minTerms = 49; - int maxTerms = 50; - int maxIter = 1000; - int warmerIter = 1000; - - init(maxTerms); - List allResults = new ArrayList<>(); - - String clusterName = ScriptsConstantScoreBenchmark.class.getSimpleName(); - Settings settings = settingsBuilder().put("name", "node1") - .put("cluster.name", clusterName).build(); - Collection> plugins = Collections.>singletonList(NativeScriptExamplesPlugin.class); - Node node1 = new MockNode(settings, Version.CURRENT, plugins); - node1.start(); - Client client = node1.client(); - client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - - indexData(10000, client, true); - client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - - Results results = new Results(); - - results.init(maxTerms - minTerms, "native const script score (log(2) 10X)", - "Results for native const script score with score = log(2) 10X:", "black", "-."); - // init script searches - List> searchRequests = initScriptMatchAllSearchRequests( - NativeConstantForLoopScoreScript.NATIVE_CONSTANT_FOR_LOOP_SCRIPT_SCORE, true); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - // init native script searches - results = new Results(); - results.init(maxTerms - minTerms, "mvel const (log(2) 10X)", "Results for mvel const score = log(2) 10X:", "red", "-."); - searchRequests = initScriptMatchAllSearchRequests("score = 0; for (int i=0; i<10;i++) {score = score + log(2);} return score", - false); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - results = new Results(); - results.init(maxTerms - minTerms, "native const script score (2)", "Results for native const script score with score = 2:", - "black", ":"); - // init native script searches - searchRequests = initScriptMatchAllSearchRequests(NativeConstantScoreScript.NATIVE_CONSTANT_SCRIPT_SCORE, true); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - results = new Results(); - results.init(maxTerms - minTerms, "mvel const (2)", "Results for mvel const score = 2:", "red", "--"); - // init native script searches - searchRequests = initScriptMatchAllSearchRequests("2", false); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - printOctaveScript(allResults, args); - - client.close(); - node1.close(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsScoreBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsScoreBenchmark.java deleted file mode 100644 index 53c34a2c88e..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/ScriptsScoreBenchmark.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.scripts.score; - -import org.elasticsearch.Version; -import org.elasticsearch.benchmark.scripts.score.plugin.NativeScriptExamplesPlugin; -import org.elasticsearch.benchmark.scripts.score.script.NativeNaiveTFIDFScoreScript; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.MockNode; -import org.elasticsearch.node.Node; -import org.elasticsearch.plugins.Plugin; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map.Entry; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class ScriptsScoreBenchmark extends BasicScriptBenchmark { - - public static void main(String[] args) throws Exception { - - int minTerms = 1; - int maxTerms = 50; - int maxIter = 100; - int warmerIter = 10; - - boolean runMVEL = false; - init(maxTerms); - List allResults = new ArrayList<>(); - String clusterName = ScriptsScoreBenchmark.class.getSimpleName(); - Settings settings = settingsBuilder().put("name", "node1") - .put("cluster.name", clusterName).build(); - Collection> plugins = Collections.>singletonList(NativeScriptExamplesPlugin.class); - Node node1 = new MockNode(settings, Version.CURRENT, plugins); - node1.start(); - Client client = node1.client(); - client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - - indexData(10000, client, false); - client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - - Results results = new Results(); - results.init(maxTerms - minTerms, "native tfidf script score dense posting list", - "Results for native script score with dense posting list:", "black", "--"); - // init native script searches - List> searchRequests = initNativeSearchRequests(minTerms, maxTerms, - NativeNaiveTFIDFScoreScript.NATIVE_NAIVE_TFIDF_SCRIPT_SCORE, true); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - results = new Results(); - - results.init(maxTerms - minTerms, "term query dense posting list", "Results for term query with dense posting lists:", "green", - "--"); - // init term queries - searchRequests = initTermQueries(minTerms, maxTerms); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - if (runMVEL) { - - results = new Results(); - results.init(maxTerms - minTerms, "mvel tfidf dense posting list", "Results for mvel score with dense posting list:", "red", - "--"); - // init native script searches - searchRequests = initNativeSearchRequests( - minTerms, - maxTerms, - "score = 0.0; fi= _terminfo[\"text\"]; for(i=0; i allResults = new ArrayList<>(); - String clusterName = ScriptsScoreBenchmark.class.getSimpleName(); - Settings settings = settingsBuilder().put("name", "node1") - .put("cluster.name", clusterName).build(); - Collection> plugins = Collections.>singletonList(NativeScriptExamplesPlugin.class); - Node node1 = new MockNode(settings, Version.CURRENT, plugins); - node1.start(); - Client client = node1.client(); - client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - - indexData(10000, client, false); - client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - - Results results = new Results(); - // init script searches - results.init(maxTerms - minTerms, "native payload sum script score", "Results for native script score:", "green", ":"); - List> searchRequests = initNativeSearchRequests(minTerms, maxTerms, - NativePayloadSumScoreScript.NATIVE_PAYLOAD_SUM_SCRIPT_SCORE, true); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - results = new Results(); - // init script searches - results.init(maxTerms - minTerms, "native payload sum script score no record", "Results for native script score:", "black", ":"); - searchRequests = initNativeSearchRequests(minTerms, maxTerms, - NativePayloadSumNoRecordScoreScript.NATIVE_PAYLOAD_SUM_NO_RECORD_SCRIPT_SCORE, true); - // run actual benchmark - runBenchmark(client, maxIter, results, searchRequests, minTerms, warmerIter); - allResults.add(results); - - printOctaveScript(allResults, args); - - client.close(); - node1.close(); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/plugin/NativeScriptExamplesPlugin.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/plugin/NativeScriptExamplesPlugin.java deleted file mode 100644 index 2a25f8f21a5..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/plugin/NativeScriptExamplesPlugin.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.scripts.score.plugin; - -import org.elasticsearch.benchmark.scripts.score.script.NativeConstantForLoopScoreScript; -import org.elasticsearch.benchmark.scripts.score.script.NativeConstantScoreScript; -import org.elasticsearch.benchmark.scripts.score.script.NativeNaiveTFIDFScoreScript; -import org.elasticsearch.benchmark.scripts.score.script.NativePayloadSumNoRecordScoreScript; -import org.elasticsearch.benchmark.scripts.score.script.NativePayloadSumScoreScript; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptModule; - -public class NativeScriptExamplesPlugin extends Plugin { - - - @Override - public String name() { - return "native-script-example"; - } - - @Override - public String description() { - return "Native script examples"; - } - - public void onModule(ScriptModule module) { - module.registerScript(NativeNaiveTFIDFScoreScript.NATIVE_NAIVE_TFIDF_SCRIPT_SCORE, NativeNaiveTFIDFScoreScript.Factory.class); - module.registerScript(NativeConstantForLoopScoreScript.NATIVE_CONSTANT_FOR_LOOP_SCRIPT_SCORE, NativeConstantForLoopScoreScript.Factory.class); - module.registerScript(NativeConstantScoreScript.NATIVE_CONSTANT_SCRIPT_SCORE, NativeConstantScoreScript.Factory.class); - module.registerScript(NativePayloadSumScoreScript.NATIVE_PAYLOAD_SUM_SCRIPT_SCORE, NativePayloadSumScoreScript.Factory.class); - module.registerScript(NativePayloadSumNoRecordScoreScript.NATIVE_PAYLOAD_SUM_NO_RECORD_SCRIPT_SCORE, NativePayloadSumNoRecordScoreScript.Factory.class); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeConstantForLoopScoreScript.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeConstantForLoopScoreScript.java deleted file mode 100644 index fee0a7ec4fc..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeConstantForLoopScoreScript.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.score.script; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.Map; - -public class NativeConstantForLoopScoreScript extends AbstractSearchScript { - - public static final String NATIVE_CONSTANT_FOR_LOOP_SCRIPT_SCORE = "native_constant_for_loop_script_score"; - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativeConstantForLoopScoreScript(params); - } - - @Override - public boolean needsScores() { - return false; - } - } - - private NativeConstantForLoopScoreScript(Map params) { - - } - - @Override - public Object run() { - float score = 0; - for (int i = 0; i < 10; i++) { - score += Math.log(2); - } - return score; - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeConstantScoreScript.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeConstantScoreScript.java deleted file mode 100644 index 17220cd4fbd..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeConstantScoreScript.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.score.script; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.Map; - -public class NativeConstantScoreScript extends AbstractSearchScript { - - public static final String NATIVE_CONSTANT_SCRIPT_SCORE = "native_constant_script_score"; - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativeConstantScoreScript(); - } - - @Override - public boolean needsScores() { - return false; - } - } - - private NativeConstantScoreScript() { - } - - @Override - public Object run() { - return 2; - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeNaiveTFIDFScoreScript.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeNaiveTFIDFScoreScript.java deleted file mode 100644 index 9d6a1cd2f07..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativeNaiveTFIDFScoreScript.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.score.script; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; -import org.elasticsearch.search.lookup.IndexFieldTerm; -import org.elasticsearch.search.lookup.IndexField; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Map; - -public class NativeNaiveTFIDFScoreScript extends AbstractSearchScript { - - public static final String NATIVE_NAIVE_TFIDF_SCRIPT_SCORE = "native_naive_tfidf_script_score"; - String field = null; - String[] terms = null; - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativeNaiveTFIDFScoreScript(params); - } - - @Override - public boolean needsScores() { - return false; - } - } - - private NativeNaiveTFIDFScoreScript(Map params) { - params.entrySet(); - terms = new String[params.size()]; - field = params.keySet().iterator().next(); - Object o = params.get(field); - ArrayList arrayList = (ArrayList) o; - terms = arrayList.toArray(new String[arrayList.size()]); - - } - - @Override - public Object run() { - float score = 0; - IndexField indexField = indexLookup().get(field); - for (int i = 0; i < terms.length; i++) { - IndexFieldTerm indexFieldTerm = indexField.get(terms[i]); - try { - if (indexFieldTerm.tf() != 0) { - score += indexFieldTerm.tf() * indexField.docCount() / indexFieldTerm.df(); - } - } catch (IOException e) { - throw new RuntimeException(); - } - } - return score; - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativePayloadSumNoRecordScoreScript.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativePayloadSumNoRecordScoreScript.java deleted file mode 100644 index 757042600d7..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativePayloadSumNoRecordScoreScript.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.score.script; - -import org.elasticsearch.search.lookup.IndexFieldTerm; -import org.elasticsearch.search.lookup.IndexField; -import org.elasticsearch.search.lookup.IndexLookup; -import org.elasticsearch.search.lookup.TermPosition; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.ArrayList; -import java.util.Map; - -public class NativePayloadSumNoRecordScoreScript extends AbstractSearchScript { - - public static final String NATIVE_PAYLOAD_SUM_NO_RECORD_SCRIPT_SCORE = "native_payload_sum_no_record_script_score"; - String field = null; - String[] terms = null; - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativePayloadSumNoRecordScoreScript(params); - } - - @Override - public boolean needsScores() { - return false; - } - } - - private NativePayloadSumNoRecordScoreScript(Map params) { - params.entrySet(); - terms = new String[params.size()]; - field = params.keySet().iterator().next(); - Object o = params.get(field); - ArrayList arrayList = (ArrayList) o; - terms = arrayList.toArray(new String[arrayList.size()]); - - } - - @Override - public Object run() { - float score = 0; - IndexField indexField = indexLookup().get(field); - for (int i = 0; i < terms.length; i++) { - IndexFieldTerm indexFieldTerm = indexField.get(terms[i], IndexLookup.FLAG_PAYLOADS); - for (TermPosition pos : indexFieldTerm) { - score += pos.payloadAsFloat(0); - } - } - return score; - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativePayloadSumScoreScript.java b/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativePayloadSumScoreScript.java deleted file mode 100644 index 1522b3a1ea1..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/score/script/NativePayloadSumScoreScript.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts.score.script; - -import org.elasticsearch.search.lookup.IndexFieldTerm; -import org.elasticsearch.search.lookup.IndexField; -import org.elasticsearch.search.lookup.IndexLookup; -import org.elasticsearch.search.lookup.TermPosition; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.script.AbstractSearchScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.NativeScriptFactory; - -import java.util.ArrayList; -import java.util.Map; - -public class NativePayloadSumScoreScript extends AbstractSearchScript { - - public static final String NATIVE_PAYLOAD_SUM_SCRIPT_SCORE = "native_payload_sum_script_score"; - String field = null; - String[] terms = null; - - public static class Factory implements NativeScriptFactory { - - @Override - public ExecutableScript newScript(@Nullable Map params) { - return new NativePayloadSumScoreScript(params); - } - - @Override - public boolean needsScores() { - return false; - } - } - - private NativePayloadSumScoreScript(Map params) { - params.entrySet(); - terms = new String[params.size()]; - field = params.keySet().iterator().next(); - Object o = params.get(field); - ArrayList arrayList = (ArrayList) o; - terms = arrayList.toArray(new String[arrayList.size()]); - - } - - @Override - public Object run() { - float score = 0; - IndexField indexField = indexLookup().get(field); - for (int i = 0; i < terms.length; i++) { - IndexFieldTerm indexFieldTerm = indexField.get(terms[i], IndexLookup.FLAG_PAYLOADS | IndexLookup.FLAG_CACHE); - for (TermPosition pos : indexFieldTerm) { - score += pos.payloadAsFloat(0); - } - } - return score; - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/SuggestSearchBenchMark.java b/core/src/test/java/org/elasticsearch/benchmark/search/SuggestSearchBenchMark.java deleted file mode 100644 index 213a522c80d..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/SuggestSearchBenchMark.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option; -import org.elasticsearch.search.suggest.SuggestBuilders; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - */ -public class SuggestSearchBenchMark { - - public static void main(String[] args) throws Exception { - int SEARCH_ITERS = 200; - - Settings settings = settingsBuilder() - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node[] nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node" + i)).node(); - } - - Client client = nodes[0].client(); - try { - client.admin().indices().prepareCreate("test").setSettings(settings).addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("_source").field("enabled", false).endObject() - .startObject("_all").field("enabled", false).endObject() - .startObject("_type").field("index", "no").endObject() - .startObject("_id").field("index", "no").endObject() - .startObject("properties") - .startObject("field").field("type", "string").field("index", "not_analyzed").field("omit_norms", true).endObject() - .endObject() - .endObject().endObject()).execute().actionGet(); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - - StopWatch stopWatch = new StopWatch().start(); - long COUNT = SizeValue.parseSizeValue("10m").singles(); - int BATCH = 100; - System.out.println("Indexing [" + COUNT + "] ..."); - long ITERS = COUNT / BATCH; - long i = 1; - char character = 'a'; - int idCounter = 0; - for (; i <= ITERS; i++) { - int termCounter = 0; - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(idCounter++)).source(source("prefix" + character + termCounter++))); - } - character++; - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("failures..."); - } - } - System.out.println("Indexing took " + stopWatch.totalTime()); - - client.admin().indices().prepareRefresh().execute().actionGet(); - System.out.println("Count: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount()); - } catch (Exception e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - client.admin().indices().prepareRefresh().execute().actionGet(); - System.out.println("Count: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount()); - } - - - System.out.println("Warming up..."); - char startChar = 'a'; - for (int i = 0; i <= 20; i++) { - String term = "prefix" + startChar; - SearchResponse response = client.prepareSearch() - .setQuery(prefixQuery("field", term)) - .addSuggestion(SuggestBuilders.termSuggestion("field").field("field").text(term).suggestMode("always")) - .execute().actionGet(); - if (response.getHits().totalHits() == 0) { - System.err.println("No hits"); - continue; - } - startChar++; - } - - - System.out.println("Starting benchmarking suggestions."); - startChar = 'a'; - long timeTaken = 0; - for (int i = 0; i <= SEARCH_ITERS; i++) { - String term = "prefix" + startChar; - SearchResponse response = client.prepareSearch() - .setQuery(matchQuery("field", term)) - .addSuggestion(SuggestBuilders.termSuggestion("field").text(term).field("field").suggestMode("always")) - .execute().actionGet(); - timeTaken += response.getTookInMillis(); - if (response.getSuggest() == null) { - System.err.println("No suggestions"); - continue; - } - List options = response.getSuggest().getSuggestion("field").getEntries().get(0).getOptions(); - if (options == null || options.isEmpty()) { - System.err.println("No suggestions"); - } - startChar++; - } - - System.out.println("Avg time taken without filter " + (timeTaken / SEARCH_ITERS)); - - client.close(); - for (Node node : nodes) { - node.close(); - } - } - - private static XContentBuilder source(String nameValue) throws IOException { - return jsonBuilder().startObject() - .field("field", nameValue) - .endObject(); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/CardinalityAggregationSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/CardinalityAggregationSearchBenchmark.java deleted file mode 100644 index 40e278159c5..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/CardinalityAggregationSearchBenchmark.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.aggregations; - -import com.carrotsearch.randomizedtesting.generators.RandomInts; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; - -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality; - -public class CardinalityAggregationSearchBenchmark { - - private static final Random R = new Random(); - private static final String CLUSTER_NAME = CardinalityAggregationSearchBenchmark.class.getSimpleName(); - private static final int NUM_DOCS = 10000000; - private static final int LOW_CARD = 1000; - private static final int HIGH_CARD = 1000000; - private static final int BATCH = 100; - private static final int WARM = 5; - private static final int RUNS = 10; - private static final int ITERS = 5; - - public static void main(String[] args) { - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 5) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node[] nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - Node clientNode = nodeBuilder() - .clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - - Client client = clientNode.client(); - - try { - client.admin().indices().create(createIndexRequest("index").settings(settings).mapping("type", - jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("low_card_str_value") - .field("type", "multi_field") - .startObject("fields") - .startObject("low_card_str_value") - .field("type", "string") - .endObject() - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() - .endObject() - .startObject("high_card_str_value") - .field("type", "multi_field") - .startObject("fields") - .startObject("high_card_str_value") - .field("type", "string") - .endObject() - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() - .endObject() - .startObject("low_card_num_value") - .field("type", "long") - .endObject() - .startObject("high_card_num_value") - .field("type", "long") - .endObject() - .endObject().endObject().endObject())).actionGet(); - - System.out.println("Indexing " + NUM_DOCS + " documents"); - - StopWatch stopWatch = new StopWatch().start(); - for (int i = 0; i < NUM_DOCS; ) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH && i < NUM_DOCS; ++j) { - final int lowCard = RandomInts.randomInt(R, LOW_CARD); - final int highCard = RandomInts.randomInt(R, HIGH_CARD); - request.add(client.prepareIndex("index", "type", Integer.toString(i)).setSource("low_card_str_value", "str" + lowCard, "high_card_str_value", "str" + highCard, "low_card_num_value", lowCard , "high_card_num_value", highCard)); - ++i; - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - System.err.println(response.buildFailureMessage()); - } - if ((i % 100000) == 0) { - System.out.println("--> Indexed " + i + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - - client.admin().indices().prepareRefresh("index").execute().actionGet(); - } catch (Exception e) { - System.out.println("Index already exists, skipping index creation"); - } - - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - - for (int i = 0; i < WARM + RUNS; ++i) { - if (i >= WARM) { - System.out.println("RUN " + (i - WARM)); - } - for (String field : new String[] {"low_card_str_value", "low_card_str_value.hash", "high_card_str_value", "high_card_str_value.hash", "low_card_num_value", "high_card_num_value"}) { - long start = System.nanoTime(); - SearchResponse resp = null; - for (int j = 0; j < ITERS; ++j) { - resp = client.prepareSearch("index").setSize(0).addAggregation(cardinality("cardinality").field(field)).execute().actionGet(); - } - long end = System.nanoTime(); - final long cardinality = ((Cardinality) resp.getAggregations().get("cardinality")).getValue(); - if (i >= WARM) { - System.out.println(field + "\t" + new TimeValue((end - start) / ITERS, TimeUnit.NANOSECONDS) + "\tcardinality=" + cardinality); - } - } - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/GlobalOrdinalsBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/GlobalOrdinalsBenchmark.java deleted file mode 100644 index c986dc41444..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/GlobalOrdinalsBenchmark.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.search.aggregations; - -import com.carrotsearch.hppc.IntIntHashMap; -import com.carrotsearch.hppc.ObjectHashSet; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.benchmark.search.aggregations.TermsAggregationSearchBenchmark.StatsResult; -import org.elasticsearch.bootstrap.BootstrapForTesting; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.Discovery; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.transport.TransportModule; - -import java.util.*; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class GlobalOrdinalsBenchmark { - - private static final String INDEX_NAME = "index"; - private static final String TYPE_NAME = "type"; - private static final int QUERY_WARMUP = 25; - private static final int QUERY_COUNT = 100; - private static final int FIELD_START = 1; - private static final int FIELD_LIMIT = 1 << 22; - private static final boolean USE_DOC_VALUES = false; - - static long COUNT = SizeValue.parseSizeValue("5m").singles(); - static Node node; - static Client client; - - public static void main(String[] args) throws Exception { - System.setProperty("es.logger.prefix", ""); - BootstrapForTesting.ensureInitialized(); - Random random = new Random(); - - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(TransportModule.TRANSPORT_TYPE_KEY, "local") - .build(); - - String clusterName = GlobalOrdinalsBenchmark.class.getSimpleName(); - node = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings)) - .node(); - - client = node.client(); - - try { - client.admin().indices().prepareCreate(INDEX_NAME) - .addMapping(TYPE_NAME, jsonBuilder().startObject().startObject(TYPE_NAME) - .startArray("dynamic_templates") - .startObject() - .startObject("default") - .field("match", "*") - .field("match_mapping_type", "string") - .startObject("mapping") - .field("type", "string") - .field("index", "not_analyzed") - .startObject("fields") - .startObject("doc_values") - .field("type", "string") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endArray() - .endObject().endObject()) - .get(); - ObjectHashSet uniqueTerms = new ObjectHashSet<>(); - for (int i = 0; i < FIELD_LIMIT; i++) { - boolean added; - do { - added = uniqueTerms.add(RandomStrings.randomAsciiOfLength(random, 16)); - } while (!added); - } - String[] sValues = uniqueTerms.toArray(String.class); - uniqueTerms = null; - - BulkRequestBuilder builder = client.prepareBulk(); - IntIntHashMap tracker = new IntIntHashMap(); - for (int i = 0; i < COUNT; i++) { - Map fieldValues = new HashMap<>(); - for (int fieldSuffix = 1; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) { - int index = tracker.putOrAdd(fieldSuffix, 0, 0); - if (index >= fieldSuffix) { - index = random.nextInt(fieldSuffix); - fieldValues.put("field_" + fieldSuffix, sValues[index]); - } else { - fieldValues.put("field_" + fieldSuffix, sValues[index]); - tracker.put(fieldSuffix, ++index); - } - } - builder.add( - client.prepareIndex(INDEX_NAME, TYPE_NAME, String.valueOf(i)) - .setSource(fieldValues) - ); - - if (builder.numberOfActions() >= 1000) { - builder.get(); - builder = client.prepareBulk(); - } - } - if (builder.numberOfActions() > 0) { - builder.get(); - } - } catch (IndexAlreadyExistsException e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - - client.admin().cluster().prepareUpdateSettings() - .setTransientSettings(Settings.builder().put("logger.index.fielddata.ordinals", "DEBUG")) - .get(); - - client.admin().indices().prepareRefresh(INDEX_NAME).execute().actionGet(); - COUNT = client.prepareCount(INDEX_NAME).setQuery(matchAllQuery()).execute().actionGet().getCount(); - System.out.println("--> Number of docs in index: " + COUNT); - - List stats = new ArrayList<>(); - for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) { - String fieldName = "field_" + fieldSuffix; - String name = "global_ordinals-" + fieldName; - if (USE_DOC_VALUES) { - fieldName = fieldName + ".doc_values"; - name = name + "_doc_values"; // can't have . in agg name - } - stats.add(terms(name, fieldName, "global_ordinals_low_cardinality")); - } - - for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) { - String fieldName = "field_" + fieldSuffix; - String name = "ordinals-" + fieldName; - if (USE_DOC_VALUES) { - fieldName = fieldName + ".doc_values"; - name = name + "_doc_values"; // can't have . in agg name - } - stats.add(terms(name, fieldName, "ordinals")); - } - - System.out.println("------------------ SUMMARY -----------------------------------------"); - System.out.format(Locale.ENGLISH, "%30s%10s%10s%15s\n", "name", "took", "millis", "fieldata size"); - for (StatsResult stat : stats) { - System.out.format(Locale.ENGLISH, "%30s%10s%10d%15s\n", stat.name, TimeValue.timeValueMillis(stat.took), (stat.took / QUERY_COUNT), stat.fieldDataMemoryUsed); - } - System.out.println("------------------ SUMMARY -----------------------------------------"); - - client.close(); - node.close(); - } - - private static StatsResult terms(String name, String field, String executionHint) { - long totalQueryTime;// LM VALUE - - client.admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet(); - System.gc(); - - System.out.println("--> Warmup (" + name + ")..."); - // run just the child query, warm up first - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = client.prepareSearch(INDEX_NAME) - .setSize(0) - .setQuery(matchAllQuery()) - .addAggregation(AggregationBuilders.terms(name).field(field).executionHint(executionHint)) - .get(); - if (j == 0) { - System.out.println("--> Loading (" + field + "): took: " + searchResponse.getTook()); - } - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - } - System.out.println("--> Warmup (" + name + ") DONE"); - - - System.out.println("--> Running (" + name + ")..."); - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(INDEX_NAME) - .setSize(0) - .setQuery(matchAllQuery()) - .addAggregation(AggregationBuilders.terms(name).field(field).executionHint(executionHint)) - .get(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Terms Agg (" + name + "): " + (totalQueryTime / QUERY_COUNT) + "ms"); - - String nodeId = node.injector().getInstance(Discovery.class).localNode().getId(); - ClusterStatsResponse clusterStateResponse = client.admin().cluster().prepareClusterStats().setNodesIds(nodeId).get(); - System.out.println("--> Heap used: " + clusterStateResponse.getNodesStats().getJvm().getHeapUsed()); - ByteSizeValue fieldDataMemoryUsed = clusterStateResponse.getIndicesStats().getFieldData().getMemorySize(); - System.out.println("--> Fielddata memory size: " + fieldDataMemoryUsed); - - return new StatsResult(name, totalQueryTime, fieldDataMemoryUsed); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/HDRPercentilesAggregationBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/HDRPercentilesAggregationBenchmark.java deleted file mode 100644 index af0eee65f08..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/HDRPercentilesAggregationBenchmark.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.aggregations; - -import com.carrotsearch.randomizedtesting.generators.RandomInts; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeUnit; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod; -import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles; -import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles; - -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles; - -public class HDRPercentilesAggregationBenchmark { - - private static final String TYPE_NAME = "type"; - private static final String INDEX_NAME = "index"; - private static final String HIGH_CARD_FIELD_NAME = "high_card"; - private static final String LOW_CARD_FIELD_NAME = "low_card"; - private static final String GAUSSIAN_FIELD_NAME = "gauss"; - private static final Random R = new Random(); - private static final String CLUSTER_NAME = HDRPercentilesAggregationBenchmark.class.getSimpleName(); - private static final int NUM_DOCS = 10000000; - private static final int LOW_CARD = 1000; - private static final int HIGH_CARD = 1000000; - private static final int BATCH = 100; - private static final int WARM = 5; - private static final int RUNS = 10; - private static final int ITERS = 5; - - public static void main(String[] args) { - long overallStartTime = System.currentTimeMillis(); - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 5) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node[] nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - Node clientNode = nodeBuilder() - .clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - - Client client = clientNode.client(); - - try { - client.admin().indices().prepareCreate(INDEX_NAME); - - System.out.println("Indexing " + NUM_DOCS + " documents"); - - StopWatch stopWatch = new StopWatch().start(); - for (int i = 0; i < NUM_DOCS; ) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH && i < NUM_DOCS; ++j) { - final int lowCard = RandomInts.randomInt(R, LOW_CARD); - final int highCard = RandomInts.randomInt(R, HIGH_CARD); - int gauss = -1; - while (gauss < 0) { - gauss = (int) (R.nextGaussian() * 1000) + 5000; // mean: 5 sec, std deviation: 1 sec - } - request.add(client.prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i)).setSource(LOW_CARD_FIELD_NAME, lowCard, - HIGH_CARD_FIELD_NAME, highCard, GAUSSIAN_FIELD_NAME, gauss)); - ++i; - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - System.err.println(response.buildFailureMessage()); - } - if ((i % 100000) == 0) { - System.out.println("--> Indexed " + i + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - - client.admin().indices().prepareRefresh(INDEX_NAME).execute().actionGet(); - } catch (Exception e) { - System.out.println("Index already exists, skipping index creation"); - } - - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - - System.out.println("Run\tField\tMethod\tAggregationTime\tEstimatedMemory"); - for (int i = 0; i < WARM + RUNS; ++i) { - for (String field : new String[] { LOW_CARD_FIELD_NAME, HIGH_CARD_FIELD_NAME, GAUSSIAN_FIELD_NAME }) { - for (PercentilesMethod method : new PercentilesMethod[] {PercentilesMethod.TDIGEST, PercentilesMethod.HDR}) { - long start = System.nanoTime(); - SearchResponse resp = null; - for (int j = 0; j < ITERS; ++j) { - resp = client.prepareSearch(INDEX_NAME).setSize(0).addAggregation(percentiles("percentiles").field(field).method(method)).execute().actionGet(); - } - long end = System.nanoTime(); - long memoryEstimate = 0; - switch (method) { - case TDIGEST: - memoryEstimate = ((InternalTDigestPercentiles) resp.getAggregations().get("percentiles")) - .getEstimatedMemoryFootprint(); - break; - case HDR: - memoryEstimate = ((InternalHDRPercentiles) resp.getAggregations().get("percentiles")).getEstimatedMemoryFootprint(); - break; - } - if (i >= WARM) { - System.out.println((i - WARM) + "\t" + field + "\t" + method + "\t" - + new TimeValue((end - start) / ITERS, TimeUnit.NANOSECONDS).millis() + "\t" - + new SizeValue(memoryEstimate, SizeUnit.SINGLE).singles()); - } - } - } - } - long overallEndTime = System.currentTimeMillis(); - System.out.println("Benchmark completed in " + ((overallEndTime - overallStartTime) / 1000) + " seconds"); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/HistogramAggregationSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/HistogramAggregationSearchBenchmark.java deleted file mode 100644 index 03fb38344d2..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/HistogramAggregationSearchBenchmark.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.aggregations; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.node.Node; - -import java.util.Date; -import java.util.Random; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; - -/** - * - */ -public class HistogramAggregationSearchBenchmark { - - static final long COUNT = SizeValue.parseSizeValue("20m").singles(); - static final int BATCH = 1000; - static final int QUERY_WARMUP = 5; - static final int QUERY_COUNT = 20; - static final int NUMBER_OF_TERMS = 1000; - - public static void main(String[] args) throws Exception { - Settings settings = settingsBuilder() - .put("refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = HistogramAggregationSearchBenchmark.class.getSimpleName(); - Node node1 = nodeBuilder() - .clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "node1")).node(); - - //Node clientNode = nodeBuilder().clusterName(clusterName).settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - - Client client = node1.client(); - - long[] lValues = new long[NUMBER_OF_TERMS]; - for (int i = 0; i < NUMBER_OF_TERMS; i++) { - lValues[i] = i; - } - - Random r = new Random(); - try { - client.admin().indices().prepareCreate("test") - .setSettings(settingsBuilder().put(settings)) - .addMapping("type1", jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("l_value") - .field("type", "long") - .endObject() - .startObject("i_value") - .field("type", "integer") - .endObject() - .startObject("s_value") - .field("type", "short") - .endObject() - .startObject("b_value") - .field("type", "byte") - .endObject() - .endObject() - .endObject() - .endObject()) - .execute().actionGet(); - - StopWatch stopWatch = new StopWatch().start(); - - System.out.println("--> Indexing [" + COUNT + "] ..."); - long iters = COUNT / BATCH; - long i = 1; - int counter = 0; - for (; i <= iters; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - counter++; - final long value = lValues[r.nextInt(lValues.length)]; - XContentBuilder source = jsonBuilder().startObject() - .field("id", Integer.valueOf(counter)) - .field("l_value", value) - .field("i_value", (int) value) - .field("s_value", (short) value) - .field("b_value", (byte) value) - .field("date", new Date()) - .endObject(); - request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter)) - .source(source)); - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * BATCH) % 10000) == 0) { - System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - client.admin().indices().prepareFlush("test").execute().actionGet(); - System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT)) / stopWatch.totalTime().secondsFrac())); - } catch (Exception e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - if (client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount() != COUNT) { - throw new Error(); - } - System.out.println("--> Number of docs in index: " + COUNT); - - System.out.println("--> Warmup..."); - // run just the child query, warm up first - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setQuery(matchAllQuery()) - .addAggregation(histogram("l_value").field("l_value").interval(4)) - .addAggregation(histogram("i_value").field("i_value").interval(4)) - .addAggregation(histogram("s_value").field("s_value").interval(4)) - .addAggregation(histogram("b_value").field("b_value").interval(4)) - .addAggregation(histogram("date").field("date").interval(1000)) - .execute().actionGet(); - if (j == 0) { - System.out.println("--> Warmup took: " + searchResponse.getTook()); - } - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - } - System.out.println("--> Warmup DONE"); - - long totalQueryTime = 0; - for (String field : new String[] {"b_value", "s_value", "i_value", "l_value"}) { - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setQuery(matchAllQuery()) - .addAggregation(histogram(field).field(field).interval(4)) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Histogram Aggregation (" + field + ") " + (totalQueryTime / QUERY_COUNT) + "ms"); - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setQuery(matchAllQuery()) - .addAggregation(histogram(field).field(field).subAggregation(stats(field).field(field)).interval(4)) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Histogram Aggregation (" + field + "/" + field + ") " + (totalQueryTime / QUERY_COUNT) + "ms"); - } - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setQuery(matchAllQuery()) - .addAggregation(dateHistogram("date").field("date").interval(1000)) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Histogram Aggregation (date) " + (totalQueryTime / QUERY_COUNT) + "ms"); - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setQuery(matchAllQuery()) - .addAggregation(dateHistogram("date").field("date").interval(1000).subAggregation(stats("stats").field("l_value"))) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Histogram Aggregation (date/l_value) " + (totalQueryTime / QUERY_COUNT) + "ms"); - - node1.close(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/IncludeExcludeAggregationSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/IncludeExcludeAggregationSearchBenchmark.java deleted file mode 100644 index 1bf8a33c095..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/IncludeExcludeAggregationSearchBenchmark.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.aggregations; - -import org.apache.lucene.util.TestUtil; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.node.Node; - -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; - -public class IncludeExcludeAggregationSearchBenchmark { - - private static final Random R = new Random(); - private static final String CLUSTER_NAME = IncludeExcludeAggregationSearchBenchmark.class.getSimpleName(); - private static final int NUM_DOCS = 10000000; - private static final int BATCH = 100; - private static final int WARM = 3; - private static final int RUNS = 10; - private static final int ITERS = 3; - - public static void main(String[] args) { - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node[] nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - Node clientNode = nodeBuilder() - .clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - - Client client = clientNode.client(); - - try { - client.admin().indices().create(createIndexRequest("index").settings(settings).mapping("type", - jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("str") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .endObject().endObject().endObject())).actionGet(); - - System.out.println("Indexing " + NUM_DOCS + " documents"); - - StopWatch stopWatch = new StopWatch().start(); - for (int i = 0; i < NUM_DOCS; ) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH && i < NUM_DOCS; ++j) { - request.add(client.prepareIndex("index", "type", Integer.toString(i)).setSource("str", TestUtil.randomSimpleString(R))); - ++i; - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - System.err.println(response.buildFailureMessage()); - } - if ((i % 100000) == 0) { - System.out.println("--> Indexed " + i + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - - client.admin().indices().prepareRefresh("index").execute().actionGet(); - } catch (Exception e) { - System.out.println("Index already exists, skipping index creation"); - } - - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - - for (int i = 0; i < WARM + RUNS; ++i) { - if (i >= WARM) { - System.out.println("RUN " + (i - WARM)); - } - long start = System.nanoTime(); - SearchResponse resp = null; - for (int j = 0; j < ITERS; ++j) { - resp = client.prepareSearch("index").setQuery(QueryBuilders.prefixQuery("str", "sf")).setSize(0).addAggregation(terms("t").field("str").include("s.*")).execute().actionGet(); - } - long end = System.nanoTime(); - if (i >= WARM) { - System.out.println(new TimeValue((end - start) / ITERS, TimeUnit.NANOSECONDS)); - } - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/PercentilesAggregationSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/PercentilesAggregationSearchBenchmark.java deleted file mode 100644 index f42f46a6508..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/PercentilesAggregationSearchBenchmark.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.aggregations; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; -import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; - -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.client.Requests.getRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles; - -public class PercentilesAggregationSearchBenchmark { - - private static final int AMPLITUDE = 10000; - private static final int NUM_DOCS = (int) SizeValue.parseSizeValue("1m").singles(); - private static final int BATCH = 100; - private static final String CLUSTER_NAME = PercentilesAggregationSearchBenchmark.class.getSimpleName(); - private static final double[] PERCENTILES = new double[] { 0, 0.01, 0.1, 1, 10, 25, 50, 75, 90, 99, 99.9, 99.99, 100}; - private static final int QUERY_WARMUP = 10; - private static final int QUERY_COUNT = 20; - - private static Random R = new Random(0); - - // we generate ints to not disadvantage qdigest which only works with integers - private enum Distribution { - UNIFORM { - @Override - int next() { - return (int) (R.nextDouble() * AMPLITUDE); - } - }, - GAUSS { - @Override - int next() { - return (int) (R.nextDouble() * AMPLITUDE); - } - }, - LOG_NORMAL { - @Override - int next() { - return (int) Math.exp(R.nextDouble() * Math.log(AMPLITUDE)); - } - }; - String indexName() { - return name().toLowerCase(Locale.ROOT); - } - abstract int next(); - } - - private static double accuratePercentile(double percentile, int[] sortedValues) { - final double index = percentile / 100 * (sortedValues.length - 1); - final int intIndex = (int) index; - final double delta = index - intIndex; - if (delta == 0) { - return sortedValues[intIndex]; - } else { - return sortedValues[intIndex] * (1 - delta) + sortedValues[intIndex + 1] * delta; - } - } - - public static void main(String[] args) throws Exception { - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 100) // to also test performance and accuracy of the reduce phase - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node[] nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - Node clientNode = nodeBuilder() - .clusterName(CLUSTER_NAME) - .settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - - Client client = clientNode.client(); - - for (Distribution d : Distribution.values()) { - try { -// client.admin().indices().prepareDelete(d.indexName()).execute().actionGet(); - client.admin().indices().create(createIndexRequest(d.indexName()).settings(settings)).actionGet(); - } catch (Exception e) { - System.out.println("Index " + d.indexName() + " already exists, skipping index creation"); - continue; - } - - final int[] values = new int[NUM_DOCS]; - for (int i = 0; i < NUM_DOCS; ++i) { - values[i] = d.next(); - } - System.out.println("Indexing " + NUM_DOCS + " documents into " + d.indexName()); - StopWatch stopWatch = new StopWatch().start(); - for (int i = 0; i < NUM_DOCS; ) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH && i < NUM_DOCS; ++j) { - request.add(client.prepareIndex(d.indexName(), "values", Integer.toString(i)).setSource("v", values[i])); - ++i; - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - System.err.println(response.buildFailureMessage()); - } - if ((i % 100000) == 0) { - System.out.println("--> Indexed " + i + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - Arrays.sort(values); - XContentBuilder builder = JsonXContent.contentBuilder().startObject(); - for (double percentile : PERCENTILES) { - builder.field(Double.toString(percentile), accuratePercentile(percentile, values)); - } - client.prepareIndex(d.indexName(), "values", "percentiles").setSource(builder.endObject()).execute().actionGet(); - client.admin().indices().prepareRefresh(d.indexName()).execute().actionGet(); - } - - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - - System.out.println("## Precision"); - for (Distribution d : Distribution.values()) { - System.out.println("#### " + d); - final long count = client.prepareCount(d.indexName()).setQuery(matchAllQuery()).execute().actionGet().getCount(); - if (count != NUM_DOCS + 1) { - throw new Error("Expected " + NUM_DOCS + " documents, got " + (count - 1)); - } - Map percentilesUnsorted = client.get(getRequest(d.indexName()).type("values").id("percentiles")).actionGet().getSourceAsMap(); - SortedMap percentiles = new TreeMap<>(); - for (Map.Entry entry : percentilesUnsorted.entrySet()) { - percentiles.put(Double.parseDouble(entry.getKey()), (Double) entry.getValue()); - } - System.out.println("Expected percentiles: " + percentiles); - System.out.println(); - SearchResponse resp = client.prepareSearch(d.indexName()).setSize(0).addAggregation(percentiles("pcts").field("v").percentiles(PERCENTILES)).execute().actionGet(); - Percentiles pcts = resp.getAggregations().get("pcts"); - Map asMap = new LinkedHashMap<>(); - double sumOfErrorSquares = 0; - for (Percentile percentile : pcts) { - asMap.put(percentile.getPercent(), percentile.getValue()); - double error = percentile.getValue() - percentiles.get(percentile.getPercent()); - sumOfErrorSquares += error * error; - } - System.out.println("Percentiles: " + asMap); - System.out.println("Sum of error squares: " + sumOfErrorSquares); - System.out.println(); - } - - System.out.println("## Performance"); - for (int i = 0; i < 3; ++i) { - for (Distribution d : Distribution.values()) { - System.out.println("#### " + d); - for (int j = 0; j < QUERY_WARMUP; ++j) { - client.prepareSearch(d.indexName()).setSize(0).addAggregation(percentiles("pcts").field("v").percentiles(PERCENTILES)).execute().actionGet(); - } - long start = System.nanoTime(); - for (int j = 0; j < QUERY_COUNT; ++j) { - client.prepareSearch(d.indexName()).setSize(0).addAggregation(percentiles("pcts").field("v").percentiles(PERCENTILES)).execute().actionGet(); - } - System.out.println(new TimeValue((System.nanoTime() - start) / QUERY_COUNT, TimeUnit.NANOSECONDS)); - } - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/QueryFilterAggregationSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/QueryFilterAggregationSearchBenchmark.java deleted file mode 100644 index 8e7d24697bb..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/QueryFilterAggregationSearchBenchmark.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.aggregations; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.AggregationBuilders; - -import java.util.concurrent.ThreadLocalRandom; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -public class QueryFilterAggregationSearchBenchmark { - - static final long COUNT = SizeValue.parseSizeValue("5m").singles(); - static final int BATCH = 1000; - static final int QUERY_COUNT = 200; - static final int NUMBER_OF_TERMS = 200; - - static Client client; - - public static void main(String[] args) throws Exception { - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 2) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = QueryFilterAggregationSearchBenchmark.class.getSimpleName(); - Node node1 = nodeBuilder() - .clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "node1")).node(); - client = node1.client(); - - long[] lValues = new long[NUMBER_OF_TERMS]; - for (int i = 0; i < NUMBER_OF_TERMS; i++) { - lValues[i] = ThreadLocalRandom.current().nextLong(); - } - - Thread.sleep(10000); - try { - client.admin().indices().create(createIndexRequest("test")).actionGet(); - - StopWatch stopWatch = new StopWatch().start(); - - System.out.println("--> Indexing [" + COUNT + "] ..."); - long ITERS = COUNT / BATCH; - long i = 1; - int counter = 0; - for (; i <= ITERS; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - counter++; - - XContentBuilder builder = jsonBuilder().startObject(); - builder.field("id", Integer.toString(counter)); - builder.field("l_value", lValues[ThreadLocalRandom.current().nextInt(NUMBER_OF_TERMS)]); - - builder.endObject(); - - request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter)) - .source(builder)); - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * BATCH) % 100000) == 0) { - System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT)) / stopWatch.totalTime().secondsFrac())); - } catch (Exception e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - if (client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount() != COUNT) { - throw new Error(); - } - System.out.println("--> Number of docs in index: " + COUNT); - - final long anyValue = ((Number) client.prepareSearch().execute().actionGet().getHits().hits()[0].sourceAsMap().get("l_value")).longValue(); - - long totalQueryTime = 0; - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setSize(0) - .setQuery(termQuery("l_value", anyValue)) - .execute().actionGet(); - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Simple Query on first l_value " + totalQueryTime + "ms"); - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setSize(0) - .setQuery(termQuery("l_value", anyValue)) - .addAggregation(AggregationBuilders.filter("filter").filter(QueryBuilders.termQuery("l_value", anyValue))) - .execute().actionGet(); - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Filter agg first l_value " + totalQueryTime + "ms"); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/SubAggregationSearchCollectModeBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/SubAggregationSearchCollectModeBenchmark.java deleted file mode 100644 index cd06b7b0579..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/SubAggregationSearchCollectModeBenchmark.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.search.aggregations; - -import com.carrotsearch.hppc.ObjectScatterSet; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.bootstrap.BootstrapForTesting; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.discovery.Discovery; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; - -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class SubAggregationSearchCollectModeBenchmark { - - static long COUNT = SizeValue.parseSizeValue("2m").singles(); - static int BATCH = 1000; - static int QUERY_WARMUP = 10; - static int QUERY_COUNT = 100; - static int NUMBER_OF_TERMS = 200; - static int NUMBER_OF_MULTI_VALUE_TERMS = 10; - static int STRING_TERM_SIZE = 5; - - static Client client; - static Node[] nodes; - - public static void main(String[] args) throws Exception { - BootstrapForTesting.ensureInitialized(); - Random random = new Random(); - - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = SubAggregationSearchCollectModeBenchmark.class.getSimpleName(); - nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - Node clientNode = nodeBuilder() - .clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - - client = clientNode.client(); - - Thread.sleep(10000); - try { - client.admin().indices().create(createIndexRequest("test").mapping("type1", jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("s_value_dv") - .field("type", "string") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .startObject("sm_value_dv") - .field("type", "string") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .startObject("l_value_dv") - .field("type", "long") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .startObject("lm_value_dv") - .field("type", "long") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject())).actionGet(); - - long[] lValues = new long[NUMBER_OF_TERMS]; - for (int i = 0; i < NUMBER_OF_TERMS; i++) { - lValues[i] = ThreadLocalRandom.current().nextLong(); - } - ObjectScatterSet uniqueTerms = new ObjectScatterSet<>(); - for (int i = 0; i < NUMBER_OF_TERMS; i++) { - boolean added; - do { - added = uniqueTerms.add(RandomStrings.randomAsciiOfLength(random, STRING_TERM_SIZE)); - } while (!added); - } - String[] sValues = uniqueTerms.toArray(String.class); - uniqueTerms = null; - - StopWatch stopWatch = new StopWatch().start(); - - System.out.println("--> Indexing [" + COUNT + "] ..."); - long ITERS = COUNT / BATCH; - long i = 1; - int counter = 0; - for (; i <= ITERS; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - counter++; - - XContentBuilder builder = jsonBuilder().startObject(); - builder.field("id", Integer.toString(counter)); - final String sValue = sValues[ThreadLocalRandom.current().nextInt(sValues.length)]; - final long lValue = lValues[ThreadLocalRandom.current().nextInt(lValues.length)]; - builder.field("s_value", sValue); - builder.field("l_value", lValue); - builder.field("s_value_dv", sValue); - builder.field("l_value_dv", lValue); - - for (String field : new String[] {"sm_value", "sm_value_dv"}) { - builder.startArray(field); - for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) { - builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]); - } - builder.endArray(); - } - - for (String field : new String[] {"lm_value", "lm_value_dv"}) { - builder.startArray(field); - for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) { - builder.value(lValues[ThreadLocalRandom.current().nextInt(sValues.length)]); - } - builder.endArray(); - } - - builder.endObject(); - - request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter)) - .source(builder)); - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * BATCH) % 10000) == 0) { - System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT)) / stopWatch.totalTime().secondsFrac())); - } catch (Exception e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - COUNT = client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(); - System.out.println("--> Number of docs in index: " + COUNT); - - List stats = new ArrayList<>(); - stats.add(runTest("0000", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("0001", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - stats.add(runTest("0010", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("0011", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - stats.add(runTest("0100", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("0101", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - stats.add(runTest("0110", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("0111", new SubAggCollectionMode[] {SubAggCollectionMode.DEPTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - stats.add(runTest("1000", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("1001", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - stats.add(runTest("1010", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("1011", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - stats.add(runTest("1100", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("1101", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - stats.add(runTest("1110", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.DEPTH_FIRST})); - stats.add(runTest("1111", new SubAggCollectionMode[] {SubAggCollectionMode.BREADTH_FIRST,SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST, SubAggCollectionMode.BREADTH_FIRST})); - - System.out.println("------------------ SUMMARY ----------------------------------------------"); - System.out.format(Locale.ENGLISH, "%35s%10s%10s%15s%15s\n", "name", "took", "millis", "fieldata size", "heap used"); - for (StatsResult stat : stats) { - System.out.format(Locale.ENGLISH, "%35s%10s%10d%15s%15s\n", stat.name, TimeValue.timeValueMillis(stat.took), (stat.took / QUERY_COUNT), stat.fieldDataMemoryUsed, stat.heapUsed); - } - System.out.println("------------------ SUMMARY ----------------------------------------------"); - - clientNode.close(); - - for (Node node : nodes) { - node.close(); - } - } - - public static class StatsResult { - final String name; - final long took; - final ByteSizeValue fieldDataMemoryUsed; - final ByteSizeValue heapUsed; - - public StatsResult(String name, long took, ByteSizeValue fieldDataMemoryUsed, ByteSizeValue heapUsed) { - this.name = name; - this.took = took; - this.fieldDataMemoryUsed = fieldDataMemoryUsed; - this.heapUsed = heapUsed; - } - } - - private static StatsResult runTest(String name, SubAggCollectionMode[] collectionModes) { - long totalQueryTime;// LM VALUE - - client.admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet(); - System.gc(); - - System.out.println("--> Warmup (" + name + ")..."); - // run just the child query, warm up first - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = client.prepareSearch("test") - .setSize(0) - .setQuery(matchAllQuery()) - .addAggregation(AggregationBuilders.terms(name + "s_value").field("s_value").collectMode(collectionModes[0]) - .subAggregation(AggregationBuilders.terms(name + "l_value").field("l_value").collectMode(collectionModes[1]) - .subAggregation(AggregationBuilders.terms(name + "s_value_dv").field("s_value_dv").collectMode(collectionModes[2]) - .subAggregation(AggregationBuilders.terms(name + "l_value_dv").field("l_value_dv").collectMode(collectionModes[3]))))) - .execute().actionGet(); - if (j == 0) { - System.out.println("--> Loading : took: " + searchResponse.getTook()); - } - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - } - System.out.println("--> Warmup (" + name + ") DONE"); - - - System.out.println("--> Running (" + name + ")..."); - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch("test") - .setSize(0) - .setQuery(matchAllQuery()) - .addAggregation(AggregationBuilders.terms(name + "s_value").field("s_value").collectMode(collectionModes[0]) - .subAggregation(AggregationBuilders.terms(name + "l_value").field("l_value").collectMode(collectionModes[1]) - .subAggregation(AggregationBuilders.terms(name + "s_value_dv").field("s_value_dv").collectMode(collectionModes[2]) - .subAggregation(AggregationBuilders.terms(name + "l_value_dv").field("l_value_dv").collectMode(collectionModes[3]))))) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Terms Agg (" + name + "): " + (totalQueryTime / QUERY_COUNT) + "ms"); - - String[] nodeIds = new String[nodes.length]; - for (int i = 0; i < nodeIds.length; i++) { - nodeIds[i] = nodes[i].injector().getInstance(Discovery.class).localNode().getId(); - } - - ClusterStatsResponse clusterStateResponse = client.admin().cluster().prepareClusterStats().setNodesIds(nodeIds).get(); - ByteSizeValue heapUsed = clusterStateResponse.getNodesStats().getJvm().getHeapUsed(); - System.out.println("--> Heap used: " + heapUsed); - ByteSizeValue fieldDataMemoryUsed = clusterStateResponse.getIndicesStats().getFieldData().getMemorySize(); - System.out.println("--> Fielddata memory size: " + fieldDataMemoryUsed); - - return new StatsResult(name, totalQueryTime, fieldDataMemoryUsed, heapUsed); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchAndIndexingBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchAndIndexingBenchmark.java deleted file mode 100644 index 0d11da04141..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchAndIndexingBenchmark.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.search.aggregations; - -import com.carrotsearch.hppc.ObjectScatterSet; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.bootstrap.BootstrapForTesting; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.discovery.Discovery; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.node.Node; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; - -import static org.elasticsearch.benchmark.search.aggregations.TermsAggregationSearchBenchmark.Method; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class TermsAggregationSearchAndIndexingBenchmark { - - static String indexName = "test"; - static String typeName = "type1"; - static Random random = new Random(); - - static long COUNT = SizeValue.parseSizeValue("2m").singles(); - static int BATCH = 1000; - static int NUMBER_OF_TERMS = (int) SizeValue.parseSizeValue("100k").singles(); - static int NUMBER_OF_MULTI_VALUE_TERMS = 10; - static int STRING_TERM_SIZE = 5; - - static Node[] nodes; - - public static void main(String[] args) throws Exception { - BootstrapForTesting.ensureInitialized(); - Settings settings = settingsBuilder() - .put("refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = TermsAggregationSearchAndIndexingBenchmark.class.getSimpleName(); - nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node1")) - .clusterName(clusterName) - .node(); - } - Client client = nodes[0].client(); - - client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - try { - client.admin().indices().prepareCreate(indexName) - .addMapping(typeName, generateMapping("eager", "lazy")) - .get(); - Thread.sleep(5000); - - long startTime = System.currentTimeMillis(); - ObjectScatterSet uniqueTerms = new ObjectScatterSet<>(); - for (int i = 0; i < NUMBER_OF_TERMS; i++) { - boolean added; - do { - added = uniqueTerms.add(RandomStrings.randomAsciiOfLength(random, STRING_TERM_SIZE)); - } while (!added); - } - String[] sValues = uniqueTerms.toArray(String.class); - long ITERS = COUNT / BATCH; - long i = 1; - int counter = 0; - for (; i <= ITERS; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - counter++; - - XContentBuilder builder = jsonBuilder().startObject(); - builder.field("id", Integer.toString(counter)); - final String sValue = sValues[counter % sValues.length]; - builder.field("s_value", sValue); - builder.field("s_value_dv", sValue); - - for (String field : new String[] {"sm_value", "sm_value_dv"}) { - builder.startArray(field); - for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) { - builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]); - } - builder.endArray(); - } - - request.add(Requests.indexRequest(indexName).type("type1").id(Integer.toString(counter)) - .source(builder)); - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * BATCH) % 10000) == 0) { - System.out.println("--> Indexed " + (i * BATCH)); - } - } - - System.out.println("--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds."); - } catch (IndexAlreadyExistsException e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().preparePutMapping(indexName) - .setType(typeName) - .setSource(generateMapping("lazy", "lazy")) - .get(); - client.admin().indices().prepareRefresh().execute().actionGet(); - System.out.println("--> Number of docs in index: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount()); - - - String[] nodeIds = new String[nodes.length]; - for (int i = 0; i < nodeIds.length; i++) { - nodeIds[i] = nodes[i].injector().getInstance(Discovery.class).localNode().getId(); - } - - List testRuns = new ArrayList<>(); - testRuns.add(new TestRun("Regular field ordinals", "eager", "lazy", "s_value", "ordinals")); - testRuns.add(new TestRun("Docvalues field ordinals", "lazy", "eager", "s_value_dv", "ordinals")); - testRuns.add(new TestRun("Regular field global ordinals", "eager_global_ordinals", "lazy", "s_value", null)); - testRuns.add(new TestRun("Docvalues field global", "lazy", "eager_global_ordinals", "s_value_dv", null)); - - List testResults = new ArrayList<>(); - for (TestRun testRun : testRuns) { - client.admin().indices().preparePutMapping(indexName).setType(typeName) - .setSource(generateMapping(testRun.indexedFieldEagerLoading, testRun.docValuesEagerLoading)).get(); - client.admin().indices().prepareClearCache(indexName).setFieldDataCache(true).get(); - SearchThread searchThread = new SearchThread(client, testRun.termsAggsField, testRun.termsAggsExecutionHint); - RefreshThread refreshThread = new RefreshThread(client); - System.out.println("--> Running '" + testRun.name + "' round..."); - new Thread(refreshThread).start(); - new Thread(searchThread).start(); - Thread.sleep(2 * 60 * 1000); - refreshThread.stop(); - searchThread.stop(); - - System.out.println("--> Avg refresh time: " + refreshThread.avgRefreshTime + " ms"); - System.out.println("--> Avg query time: " + searchThread.avgQueryTime + " ms"); - - ClusterStatsResponse clusterStateResponse = client.admin().cluster().prepareClusterStats().setNodesIds(nodeIds).get(); - System.out.println("--> Heap used: " + clusterStateResponse.getNodesStats().getJvm().getHeapUsed()); - ByteSizeValue fieldDataMemoryUsed = clusterStateResponse.getIndicesStats().getFieldData().getMemorySize(); - System.out.println("--> Fielddata memory size: " + fieldDataMemoryUsed); - testResults.add(new TestResult(testRun.name, refreshThread.avgRefreshTime, searchThread.avgQueryTime, fieldDataMemoryUsed)); - } - - System.out.println("----------------------------------------- SUMMARY ----------------------------------------------"); - System.out.format(Locale.ENGLISH, "%30s%18s%15s%15s\n", "name", "avg refresh time", "avg query time", "fieldata size"); - for (TestResult testResult : testResults) { - System.out.format(Locale.ENGLISH, "%30s%18s%15s%15s\n", testResult.name, testResult.avgRefreshTime, testResult.avgQueryTime, testResult.fieldDataSizeInMemory); - } - System.out.println("----------------------------------------- SUMMARY ----------------------------------------------"); - - client.close(); - for (Node node : nodes) { - node.close(); - } - } - - static class RefreshThread implements Runnable { - - private final Client client; - private volatile boolean run = true; - private volatile boolean stopped = false; - private volatile long avgRefreshTime = 0; - - RefreshThread(Client client) throws IOException { - this.client = client; - } - - @Override - public void run() { - long totalRefreshTime = 0; - int numExecutedRefreshed = 0; - while (run) { - long docIdLimit = COUNT; - for (long docId = 1; run && docId < docIdLimit;) { - try { - for (int j = 0; j < 8; j++) { - GetResponse getResponse = client - .prepareGet(indexName, "type1", String.valueOf(++docId)) - .get(); - client.prepareIndex(indexName, "type1", getResponse.getId()) - .setSource(getResponse.getSource()) - .get(); - } - long startTime = System.currentTimeMillis(); - client.admin().indices().prepareRefresh(indexName).execute().actionGet(); - totalRefreshTime += System.currentTimeMillis() - startTime; - numExecutedRefreshed++; - Thread.sleep(500); - } catch (Throwable e) { - e.printStackTrace(); - } - } - } - avgRefreshTime = totalRefreshTime / numExecutedRefreshed; - stopped = true; - } - - public void stop() throws InterruptedException { - run = false; - while (!stopped) { - Thread.sleep(100); - } - } - - } - - private static class TestRun { - - final String name; - final String indexedFieldEagerLoading; - final String docValuesEagerLoading; - final String termsAggsField; - final String termsAggsExecutionHint; - - private TestRun(String name, String indexedFieldEagerLoading, String docValuesEagerLoading, String termsAggsField, String termsAggsExecutionHint) { - this.name = name; - this.indexedFieldEagerLoading = indexedFieldEagerLoading; - this.docValuesEagerLoading = docValuesEagerLoading; - this.termsAggsField = termsAggsField; - this.termsAggsExecutionHint = termsAggsExecutionHint; - } - } - - private static class TestResult { - - final String name; - final TimeValue avgRefreshTime; - final TimeValue avgQueryTime; - final ByteSizeValue fieldDataSizeInMemory; - - private TestResult(String name, long avgRefreshTime, long avgQueryTime, ByteSizeValue fieldDataSizeInMemory) { - this.name = name; - this.avgRefreshTime = TimeValue.timeValueMillis(avgRefreshTime); - this.avgQueryTime = TimeValue.timeValueMillis(avgQueryTime); - this.fieldDataSizeInMemory = fieldDataSizeInMemory; - } - } - - static class SearchThread implements Runnable { - - private final Client client; - private final String field; - private final String executionHint; - private volatile boolean run = true; - private volatile boolean stopped = false; - private volatile long avgQueryTime = 0; - - SearchThread(Client client, String field, String executionHint) { - this.client = client; - this.field = field; - this.executionHint = executionHint; - } - - @Override - public void run() { - long totalQueryTime = 0; - int numExecutedQueries = 0; - while (run) { - try { - SearchResponse searchResponse = Method.AGGREGATION.addTermsAgg(client.prepareSearch() - .setSize(0) - .setQuery(matchAllQuery()), "test", field, executionHint) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - numExecutedQueries++; - } catch (Throwable e) { - e.printStackTrace(); - } - } - avgQueryTime = totalQueryTime / numExecutedQueries; - stopped = true; - } - - public void stop() throws InterruptedException { - run = false; - while (!stopped) { - Thread.sleep(100); - } - } - - } - - private static XContentBuilder generateMapping(String loading1, String loading2) throws IOException { - return jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("s_value") - .field("type", "string") - .field("index", "not_analyzed") - .startObject("fielddata") - .field("loading", loading1) - .endObject() - .endObject() - .startObject("s_value_dv") - .field("type", "string") - .field("index", "no") - .startObject("fielddata") - .field("loading", loading2) - .field("format", "doc_values") - .endObject() - .endObject() - .endObject().endObject().endObject(); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchBenchmark.java deleted file mode 100644 index 7240ee308f2..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchBenchmark.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.search.aggregations; - -import com.carrotsearch.hppc.ObjectScatterSet; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.bootstrap.BootstrapForTesting; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.discovery.Discovery; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; - -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class TermsAggregationSearchBenchmark { - - static long COUNT = SizeValue.parseSizeValue("2m").singles(); - static int BATCH = 1000; - static int QUERY_WARMUP = 10; - static int QUERY_COUNT = 100; - static int NUMBER_OF_TERMS = 200; - static int NUMBER_OF_MULTI_VALUE_TERMS = 10; - static int STRING_TERM_SIZE = 5; - - static Client client; - static Node[] nodes; - - public enum Method { - AGGREGATION { - @Override - SearchRequestBuilder addTermsAgg(SearchRequestBuilder builder, String name, String field, String executionHint) { - return builder.addAggregation(AggregationBuilders.terms(name).executionHint(executionHint).field(field)); - } - - @Override - SearchRequestBuilder addTermsStatsAgg(SearchRequestBuilder builder, String name, String keyField, String valueField) { - return builder.addAggregation(AggregationBuilders.terms(name).field(keyField).subAggregation(AggregationBuilders.stats("stats").field(valueField))); - } - }, - AGGREGATION_DEFERRED { - @Override - SearchRequestBuilder addTermsAgg(SearchRequestBuilder builder, String name, String field, String executionHint) { - return builder.addAggregation(AggregationBuilders.terms(name).executionHint(executionHint).field(field).collectMode(SubAggCollectionMode.BREADTH_FIRST)); - } - - @Override - SearchRequestBuilder addTermsStatsAgg(SearchRequestBuilder builder, String name, String keyField, String valueField) { - return builder.addAggregation(AggregationBuilders.terms(name).field(keyField).collectMode(SubAggCollectionMode.BREADTH_FIRST).subAggregation(AggregationBuilders.stats("stats").field(valueField))); - } - }; - abstract SearchRequestBuilder addTermsAgg(SearchRequestBuilder builder, String name, String field, String executionHint); - abstract SearchRequestBuilder addTermsStatsAgg(SearchRequestBuilder builder, String name, String keyField, String valueField); - } - - public static void main(String[] args) throws Exception { - BootstrapForTesting.ensureInitialized(); - Random random = new Random(); - - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = TermsAggregationSearchBenchmark.class.getSimpleName(); - nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("path.home", ".")) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - Node clientNode = nodeBuilder() - .clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "client")) - .settings(settingsBuilder().put(settings).put("path.home", ".")).client(true).node(); - - client = clientNode.client(); - - Thread.sleep(10000); - try { - client.admin().indices().create(createIndexRequest("test").mapping("type1", jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("s_value_dv") - .field("type", "string") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .startObject("sm_value_dv") - .field("type", "string") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .startObject("l_value_dv") - .field("type", "long") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .startObject("lm_value_dv") - .field("type", "long") - .field("index", "no") - .startObject("fielddata") - .field("format", "doc_values") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject())).actionGet(); - - ObjectScatterSet uniqueTerms = new ObjectScatterSet<>(); - for (int i = 0; i < NUMBER_OF_TERMS; i++) { - boolean added; - do { - added = uniqueTerms.add(RandomStrings.randomAsciiOfLength(random, STRING_TERM_SIZE)); - } while (!added); - } - String[] sValues = uniqueTerms.toArray(String.class); - uniqueTerms = null; - - StopWatch stopWatch = new StopWatch().start(); - - System.out.println("--> Indexing [" + COUNT + "] ..."); - long ITERS = COUNT / BATCH; - long i = 1; - int counter = 0; - for (; i <= ITERS; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - counter++; - - XContentBuilder builder = jsonBuilder().startObject(); - builder.field("id", Integer.toString(counter)); - final String sValue = sValues[ThreadLocalRandom.current().nextInt(sValues.length)]; - final long lValue = ThreadLocalRandom.current().nextInt(NUMBER_OF_TERMS); - builder.field("s_value", sValue); - builder.field("l_value", lValue); - builder.field("s_value_dv", sValue); - builder.field("l_value_dv", lValue); - - for (String field : new String[] {"sm_value", "sm_value_dv"}) { - builder.startArray(field); - for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) { - builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]); - } - builder.endArray(); - } - - for (String field : new String[] {"lm_value", "lm_value_dv"}) { - builder.startArray(field); - for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) { - builder.value(ThreadLocalRandom.current().nextInt(NUMBER_OF_TERMS)); - } - builder.endArray(); - } - - builder.endObject(); - - request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter)) - .source(builder)); - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * BATCH) % 10000) == 0) { - System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT)) / stopWatch.totalTime().secondsFrac())); - } catch (Exception e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForYellowStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - COUNT = client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(); - System.out.println("--> Number of docs in index: " + COUNT); - - - List stats = new ArrayList<>(); - stats.add(terms("terms_agg_s", Method.AGGREGATION, "s_value", null)); - stats.add(terms("terms_agg_s_dv", Method.AGGREGATION, "s_value_dv", null)); - stats.add(terms("terms_agg_map_s", Method.AGGREGATION, "s_value", "map")); - stats.add(terms("terms_agg_map_s_dv", Method.AGGREGATION, "s_value_dv", "map")); - stats.add(terms("terms_agg_def_s", Method.AGGREGATION_DEFERRED, "s_value", null)); - stats.add(terms("terms_agg_def_s_dv", Method.AGGREGATION_DEFERRED, "s_value_dv", null)); - stats.add(terms("terms_agg_def_map_s", Method.AGGREGATION_DEFERRED, "s_value", "map")); - stats.add(terms("terms_agg_def_map_s_dv", Method.AGGREGATION_DEFERRED, "s_value_dv", "map")); - stats.add(terms("terms_agg_l", Method.AGGREGATION, "l_value", null)); - stats.add(terms("terms_agg_l_dv", Method.AGGREGATION, "l_value_dv", null)); - stats.add(terms("terms_agg_def_l", Method.AGGREGATION_DEFERRED, "l_value", null)); - stats.add(terms("terms_agg_def_l_dv", Method.AGGREGATION_DEFERRED, "l_value_dv", null)); - stats.add(terms("terms_agg_sm", Method.AGGREGATION, "sm_value", null)); - stats.add(terms("terms_agg_sm_dv", Method.AGGREGATION, "sm_value_dv", null)); - stats.add(terms("terms_agg_map_sm", Method.AGGREGATION, "sm_value", "map")); - stats.add(terms("terms_agg_map_sm_dv", Method.AGGREGATION, "sm_value_dv", "map")); - stats.add(terms("terms_agg_def_sm", Method.AGGREGATION_DEFERRED, "sm_value", null)); - stats.add(terms("terms_agg_def_sm_dv", Method.AGGREGATION_DEFERRED, "sm_value_dv", null)); - stats.add(terms("terms_agg_def_map_sm", Method.AGGREGATION_DEFERRED, "sm_value", "map")); - stats.add(terms("terms_agg_def_map_sm_dv", Method.AGGREGATION_DEFERRED, "sm_value_dv", "map")); - stats.add(terms("terms_agg_lm", Method.AGGREGATION, "lm_value", null)); - stats.add(terms("terms_agg_lm_dv", Method.AGGREGATION, "lm_value_dv", null)); - stats.add(terms("terms_agg_def_lm", Method.AGGREGATION_DEFERRED, "lm_value", null)); - stats.add(terms("terms_agg_def_lm_dv", Method.AGGREGATION_DEFERRED, "lm_value_dv", null)); - - stats.add(termsStats("terms_stats_agg_s_l", Method.AGGREGATION, "s_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_s_l_dv", Method.AGGREGATION, "s_value_dv", "l_value_dv", null)); - stats.add(termsStats("terms_stats_agg_def_s_l", Method.AGGREGATION_DEFERRED, "s_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_def_s_l_dv", Method.AGGREGATION_DEFERRED, "s_value_dv", "l_value_dv", null)); - stats.add(termsStats("terms_stats_agg_s_lm", Method.AGGREGATION, "s_value", "lm_value", null)); - stats.add(termsStats("terms_stats_agg_s_lm_dv", Method.AGGREGATION, "s_value_dv", "lm_value_dv", null)); - stats.add(termsStats("terms_stats_agg_def_s_lm", Method.AGGREGATION_DEFERRED, "s_value", "lm_value", null)); - stats.add(termsStats("terms_stats_agg_def_s_lm_dv", Method.AGGREGATION_DEFERRED, "s_value_dv", "lm_value_dv", null)); - stats.add(termsStats("terms_stats_agg_sm_l", Method.AGGREGATION, "sm_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_sm_l_dv", Method.AGGREGATION, "sm_value_dv", "l_value_dv", null)); - stats.add(termsStats("terms_stats_agg_def_sm_l", Method.AGGREGATION_DEFERRED, "sm_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_def_sm_l_dv", Method.AGGREGATION_DEFERRED, "sm_value_dv", "l_value_dv", null)); - - stats.add(termsStats("terms_stats_agg_s_l", Method.AGGREGATION, "s_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_s_l_dv", Method.AGGREGATION, "s_value_dv", "l_value_dv", null)); - stats.add(termsStats("terms_stats_agg_def_s_l", Method.AGGREGATION_DEFERRED, "s_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_def_s_l_dv", Method.AGGREGATION_DEFERRED, "s_value_dv", "l_value_dv", null)); - stats.add(termsStats("terms_stats_agg_s_lm", Method.AGGREGATION, "s_value", "lm_value", null)); - stats.add(termsStats("terms_stats_agg_s_lm_dv", Method.AGGREGATION, "s_value_dv", "lm_value_dv", null)); - stats.add(termsStats("terms_stats_agg_def_s_lm", Method.AGGREGATION_DEFERRED, "s_value", "lm_value", null)); - stats.add(termsStats("terms_stats_agg_def_s_lm_dv", Method.AGGREGATION_DEFERRED, "s_value_dv", "lm_value_dv", null)); - stats.add(termsStats("terms_stats_agg_sm_l", Method.AGGREGATION, "sm_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_sm_l_dv", Method.AGGREGATION, "sm_value_dv", "l_value_dv", null)); - stats.add(termsStats("terms_stats_agg_def_sm_l", Method.AGGREGATION_DEFERRED, "sm_value", "l_value", null)); - stats.add(termsStats("terms_stats_agg_def_sm_l_dv", Method.AGGREGATION_DEFERRED, "sm_value_dv", "l_value_dv", null)); - - System.out.println("------------------ SUMMARY ----------------------------------------------"); - System.out.format(Locale.ENGLISH, "%35s%10s%10s%15s\n", "name", "took", "millis", "fieldata size"); - for (StatsResult stat : stats) { - System.out.format(Locale.ENGLISH, "%35s%10s%10d%15s\n", stat.name, TimeValue.timeValueMillis(stat.took), (stat.took / QUERY_COUNT), stat.fieldDataMemoryUsed); - } - System.out.println("------------------ SUMMARY ----------------------------------------------"); - - clientNode.close(); - - for (Node node : nodes) { - node.close(); - } - } - - public static class StatsResult { - final String name; - final long took; - final ByteSizeValue fieldDataMemoryUsed; - - public StatsResult(String name, long took, ByteSizeValue fieldDataMemoryUsed) { - this.name = name; - this.took = took; - this.fieldDataMemoryUsed = fieldDataMemoryUsed; - } - } - - private static StatsResult terms(String name, Method method, String field, String executionHint) { - long totalQueryTime;// LM VALUE - - client.admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet(); - System.gc(); - - System.out.println("--> Warmup (" + name + ")..."); - // run just the child query, warm up first - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = method.addTermsAgg(client.prepareSearch("test") - .setSize(0) - .setQuery(matchAllQuery()), name, field, executionHint) - .execute().actionGet(); - if (j == 0) { - System.out.println("--> Loading (" + field + "): took: " + searchResponse.getTook()); - } - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - } - System.out.println("--> Warmup (" + name + ") DONE"); - - - System.out.println("--> Running (" + name + ")..."); - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = method.addTermsAgg(client.prepareSearch() - .setSize(0) - .setQuery(matchAllQuery()), name, field, executionHint) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Terms Agg (" + name + "): " + (totalQueryTime / QUERY_COUNT) + "ms"); - - String[] nodeIds = new String[nodes.length]; - for (int i = 0; i < nodeIds.length; i++) { - nodeIds[i] = nodes[i].injector().getInstance(Discovery.class).localNode().getId(); - } - - ClusterStatsResponse clusterStateResponse = client.admin().cluster().prepareClusterStats().setNodesIds(nodeIds).get(); - System.out.println("--> Heap used: " + clusterStateResponse.getNodesStats().getJvm().getHeapUsed()); - ByteSizeValue fieldDataMemoryUsed = clusterStateResponse.getIndicesStats().getFieldData().getMemorySize(); - System.out.println("--> Fielddata memory size: " + fieldDataMemoryUsed); - - return new StatsResult(name, totalQueryTime, fieldDataMemoryUsed); - } - - private static StatsResult termsStats(String name, Method method, String keyField, String valueField, String executionHint) { - long totalQueryTime; - - client.admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet(); - System.gc(); - - System.out.println("--> Warmup (" + name + ")..."); - // run just the child query, warm up first - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = method.addTermsStatsAgg(client.prepareSearch() - .setSize(0) - .setQuery(matchAllQuery()), name, keyField, valueField) - .execute().actionGet(); - if (j == 0) { - System.out.println("--> Loading (" + name + "): took: " + searchResponse.getTook()); - } - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - } - System.out.println("--> Warmup (" + name + ") DONE"); - - - System.out.println("--> Running (" + name + ")..."); - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = method.addTermsStatsAgg(client.prepareSearch() - .setSize(0) - .setQuery(matchAllQuery()), name, keyField, valueField) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != COUNT) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Terms stats agg (" + name + "): " + (totalQueryTime / QUERY_COUNT) + "ms"); - return new StatsResult(name, totalQueryTime, ByteSizeValue.parseBytesSizeValue("0b", "StatsResult")); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TimeDataHistogramAggregationBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TimeDataHistogramAggregationBenchmark.java deleted file mode 100644 index 5b3984d19cb..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TimeDataHistogramAggregationBenchmark.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.aggregations; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.node.Node; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.search.aggregations.AggregationBuilders; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ThreadLocalRandom; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class TimeDataHistogramAggregationBenchmark { - - static long COUNT = SizeValue.parseSizeValue("5m").singles(); - static long TIME_PERIOD = 24 * 3600 * 1000; - static int BATCH = 100; - static int QUERY_WARMUP = 50; - static int QUERY_COUNT = 500; - static IndexFieldData.CommonSettings.MemoryStorageFormat MEMORY_FORMAT = IndexFieldData.CommonSettings.MemoryStorageFormat.PAGED; - static double ACCEPTABLE_OVERHEAD_RATIO = 0.5; - static float MATCH_PERCENTAGE = 0.1f; - - static Client client; - - public static void main(String[] args) throws Exception { - - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put("node.local", true) - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = TimeDataHistogramAggregationBenchmark.class.getSimpleName(); - Node[] nodes = new Node[1]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - client = nodes[0].client(); - - Thread.sleep(10000); - try { - client.admin().indices().create(createIndexRequest("test")).actionGet(); - - StopWatch stopWatch = new StopWatch().start(); - - System.out.println("--> Indexing [" + COUNT + "] ..."); - long ITERS = COUNT / BATCH; - long i = 1; - int counter = 0; - long[] currentTimeInMillis1 = new long[]{System.currentTimeMillis()}; - long[] currentTimeInMillis2 = new long[]{System.currentTimeMillis()}; - long startTimeInMillis = currentTimeInMillis1[0]; - long averageMillisChange = TIME_PERIOD / COUNT * 2; - long backwardSkew = Math.max(1, (long) (averageMillisChange * 0.1)); - long bigOutOfOrder = 1; - for (; i <= ITERS; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - counter++; - - XContentBuilder builder = jsonBuilder().startObject(); - builder.field("id", Integer.toString(counter)); - // move forward in time and sometimes a little bit back (delayed delivery) - long diff = ThreadLocalRandom.current().nextLong(2 * averageMillisChange + 2 * backwardSkew) - backwardSkew; - long[] currentTime = counter % 2 == 0 ? currentTimeInMillis1 : currentTimeInMillis2; - currentTime[0] += diff; - if (ThreadLocalRandom.current().nextLong(100) <= bigOutOfOrder) { - builder.field("l_value", currentTime[0] - 60000); // 1m delays - } else { - builder.field("l_value", currentTime[0]); - } - - builder.endObject(); - - request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter)) - .source(builder)); - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * BATCH) % 10000) == 0) { - System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT)) / stopWatch.totalTime().secondsFrac())); - System.out.println("Time range 1: " + (currentTimeInMillis1[0] - startTimeInMillis) / 1000.0 / 3600 + " hours"); - System.out.println("Time range 2: " + (currentTimeInMillis2[0] - startTimeInMillis) / 1000.0 / 3600 + " hours"); - System.out.println("--> optimizing index"); - client.admin().indices().prepareOptimize().setMaxNumSegments(1).get(); - } catch (IndexAlreadyExistsException e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - COUNT = client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(); - System.out.println("--> Number of docs in index: " + COUNT); - - // load with the reverse options to make sure jit doesn't optimize one away - setMapping(ACCEPTABLE_OVERHEAD_RATIO, MEMORY_FORMAT.equals(IndexFieldData.CommonSettings.MemoryStorageFormat.PACKED) ? IndexFieldData.CommonSettings.MemoryStorageFormat.PAGED : IndexFieldData.CommonSettings.MemoryStorageFormat.PACKED); - warmUp("hist_l", "l_value", MATCH_PERCENTAGE); - - setMapping(ACCEPTABLE_OVERHEAD_RATIO, MEMORY_FORMAT); - warmUp("hist_l", "l_value", MATCH_PERCENTAGE); - - List stats = new ArrayList<>(); - stats.add(measureAgg("hist_l", "l_value", MATCH_PERCENTAGE)); - - NodesStatsResponse nodeStats = client.admin().cluster().prepareNodesStats(nodes[0].settings().get("name")).clear() - .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.FieldData)).get(); - - - System.out.println("------------------ SUMMARY -------------------------------"); - - System.out.println("docs: " + COUNT); - System.out.println("match percentage: " + MATCH_PERCENTAGE); - System.out.println("memory format hint: " + MEMORY_FORMAT); - System.out.println("acceptable_overhead_ratio: " + ACCEPTABLE_OVERHEAD_RATIO); - System.out.println("field data: " + nodeStats.getNodes()[0].getIndices().getFieldData().getMemorySize()); - System.out.format(Locale.ROOT, "%25s%10s%10s\n", "name", "took", "millis"); - for (StatsResult stat : stats) { - System.out.format(Locale.ROOT, "%25s%10s%10d\n", stat.name, TimeValue.timeValueMillis(stat.took), (stat.took / QUERY_COUNT)); - } - System.out.println("------------------ SUMMARY -------------------------------"); - - for (Node node : nodes) { - node.close(); - } - } - - protected static void setMapping(double acceptableOverheadRatio, IndexFieldData.CommonSettings.MemoryStorageFormat fielddataStorageFormat) throws IOException { - XContentBuilder mapping = JsonXContent.contentBuilder(); - mapping.startObject().startObject("type1").startObject("properties").startObject("l_value") - .field("type", "long") - .startObject("fielddata") - .field("acceptable_transient_overhead_ratio", acceptableOverheadRatio) - .field("acceptable_overhead_ratio", acceptableOverheadRatio) - .field(IndexFieldData.CommonSettings.SETTING_MEMORY_STORAGE_HINT, fielddataStorageFormat.name().toLowerCase(Locale.ROOT)) - .endObject() - .endObject().endObject().endObject().endObject(); - client.admin().indices().preparePutMapping("test").setType("type1").setSource(mapping).get(); - } - - static class StatsResult { - final String name; - final long took; - - StatsResult(String name, long took) { - this.name = name; - this.took = took; - } - } - - private static SearchResponse doTermsAggsSearch(String name, String field, float matchPercentage) { - Map params = new HashMap<>(); - params.put("matchP", matchPercentage); - SearchResponse response = client.prepareSearch() - .setSize(0) - .setQuery( - QueryBuilders.constantScoreQuery(QueryBuilders.scriptQuery(new Script("random() warning - big deviation from expected count: " + response.getHits().totalHits() + " expected: " + COUNT * matchPercentage); - } - - return response; - } - - private static StatsResult measureAgg(String name, String field, float matchPercentage) { - long totalQueryTime;// LM VALUE - - System.out.println("--> Running (" + name + ")..."); - totalQueryTime = 0; - long previousCount = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = doTermsAggsSearch(name, field, matchPercentage); - if (previousCount == 0) { - previousCount = searchResponse.getHits().getTotalHits(); - } else if (searchResponse.getHits().totalHits() != previousCount) { - System.err.println("*** HIT COUNT CHANGE -> CACHE EXPIRED? ***"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Histogram aggregations (" + field + "): " + (totalQueryTime / QUERY_COUNT) + "ms"); - return new StatsResult(name, totalQueryTime); - } - - private static void warmUp(String name, String field, float matchPercentage) { - System.out.println("--> Warmup (" + name + ")..."); - client.admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet(); - - // run just the child query, warm up first - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = doTermsAggsSearch(name, field, matchPercentage); - if (j == 0) { - System.out.println("--> Loading (" + field + "): took: " + searchResponse.getTook()); - } - } - System.out.println("--> Warmup (" + name + ") DONE"); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchAndIndexingBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchAndIndexingBenchmark.java deleted file mode 100644 index fd49a35c965..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchAndIndexingBenchmark.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.search.child; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.node.Node; - -import java.util.Arrays; -import java.util.Random; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class ChildSearchAndIndexingBenchmark { - - static int PARENT_COUNT = (int) SizeValue.parseSizeValue("1m").singles(); - static int NUM_CHILDREN_PER_PARENT = 12; - static int QUERY_VALUE_RATIO_PER_PARENT = 3; - static int QUERY_COUNT = 50; - static String indexName = "test"; - static Random random = new Random(); - - public static void main(String[] args) throws Exception { - Settings settings = settingsBuilder() - .put("refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = ChildSearchAndIndexingBenchmark.class.getSimpleName(); - Node node1 = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node1")) - .clusterName(clusterName) - .node(); - Client client = node1.client(); - - client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - try { - client.admin().indices().create(createIndexRequest(indexName)).actionGet(); - client.admin().indices().preparePutMapping(indexName).setType("child").setSource(XContentFactory.jsonBuilder().startObject().startObject("child") - .startObject("_parent").field("type", "parent").endObject() - .endObject().endObject()).execute().actionGet(); - Thread.sleep(5000); - - long startTime = System.currentTimeMillis(); - ParentChildIndexGenerator generator = new ParentChildIndexGenerator(client, PARENT_COUNT, NUM_CHILDREN_PER_PARENT, QUERY_VALUE_RATIO_PER_PARENT); - generator.index(); - System.out.println("--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds."); - } catch (IndexAlreadyExistsException e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - System.out.println("--> Number of docs in index: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount()); - - SearchThread searchThread = new SearchThread(client); - new Thread(searchThread).start(); - IndexThread indexThread = new IndexThread(client); - new Thread(indexThread).start(); - - System.in.read(); - - indexThread.stop(); - searchThread.stop(); - client.close(); - node1.close(); - } - - static class IndexThread implements Runnable { - - private final Client client; - private volatile boolean run = true; - - IndexThread(Client client) { - this.client = client; - } - - @Override - public void run() { - while (run) { - int childIdLimit = PARENT_COUNT * NUM_CHILDREN_PER_PARENT; - for (int childId = 1; run && childId < childIdLimit;) { - try { - for (int j = 0; j < 8; j++) { - GetResponse getResponse = client - .prepareGet(indexName, "child", String.valueOf(++childId)) - .setFields("_source", "_parent") - .setRouting("1") // Doesn't matter what value, since there is only one shard - .get(); - client.prepareIndex(indexName, "child", Integer.toString(childId) + "_" + j) - .setParent(getResponse.getField("_parent").getValue().toString()) - .setSource(getResponse.getSource()) - .get(); - } - client.admin().indices().prepareRefresh(indexName).execute().actionGet(); - Thread.sleep(1000); - if (childId % 500 == 0) { - NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats() - .clear().setIndices(true).execute().actionGet(); - System.out.println("Deleted docs: " + statsResponse.getAt(0).getIndices().getDocs().getDeleted()); - } - } catch (Throwable e) { - e.printStackTrace(); - } - } - } - } - - public void stop() { - run = false; - } - - } - - static class SearchThread implements Runnable { - - private final Client client; - private final int numValues; - private volatile boolean run = true; - - SearchThread(Client client) { - this.client = client; - this.numValues = NUM_CHILDREN_PER_PARENT / NUM_CHILDREN_PER_PARENT; - } - - @Override - public void run() { - while (run) { - try { - long totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasChildQuery("child", termQuery("field2", "value" + random.nextInt(numValues))) - ) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child filter with term filter Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - totalQueryTime = 0; - for (int j = 1; j <= QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasChildQuery("child", matchAllQuery())) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child filter with match_all child query, Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).execute().actionGet(); - System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - Thread.sleep(1000); - } catch (Throwable e) { - e.printStackTrace(); - } - } - } - - public void stop() { - run = false; - } - - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java deleted file mode 100644 index 3d22f07a52d..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.search.child; - -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.children.Children; - -import java.util.Arrays; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class ChildSearchBenchmark { - - /* - Run: MAVEN_OPTS=-Xmx4g mvn test-compile exec:java -Dexec.mainClass="org.elasticsearch.benchmark.search.child.ChildSearchBenchmark" -Dexec.classpathScope="test" -Dexec.args="bwc false" - */ - - public static void main(String[] args) throws Exception { - boolean bwcMode = false; - int numParents = (int) SizeValue.parseSizeValue("2m").singles();; - - if (args.length % 2 != 0) { - throw new IllegalArgumentException("Uneven number of arguments"); - } - for (int i = 0; i < args.length; i += 2) { - String value = args[i + 1]; - if ("--bwc_mode".equals(args[i])) { - bwcMode = Boolean.valueOf(value); - } else if ("--num_parents".equals(args[i])) { - numParents = Integer.valueOf(value); - } - } - - - Settings.Builder settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0); - - // enable bwc parent child mode: - if (bwcMode) { - settings.put("tests.mock.version", Version.V_1_6_0); - } - - String clusterName = ChildSearchBenchmark.class.getSimpleName(); - Node node1 = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings.build()).put("name", "node1")).node(); - Client client = node1.client(); - - int CHILD_COUNT = 15; - int QUERY_VALUE_RATIO = 3; - int QUERY_WARMUP = 10; - int QUERY_COUNT = 20; - String indexName = "test"; - - ParentChildIndexGenerator parentChildIndexGenerator = new ParentChildIndexGenerator(client, numParents, CHILD_COUNT, QUERY_VALUE_RATIO); - client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - try { - client.admin().indices().create(createIndexRequest(indexName)).actionGet(); - client.admin().indices().preparePutMapping(indexName).setType("child").setSource(XContentFactory.jsonBuilder().startObject().startObject("child") - .startObject("_parent").field("type", "parent").endObject() - .endObject().endObject()).execute().actionGet(); - Thread.sleep(5000); - long startTime = System.currentTimeMillis(); - parentChildIndexGenerator.index(); - System.out.println("--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds."); - } catch (IndexAlreadyExistsException e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - System.out.println("--> Number of docs in index: " + client.prepareCount(indexName).setQuery(matchAllQuery()).execute().actionGet().getCount()); - - System.out.println("--> Running just child query"); - // run just the child query, warm up first - for (int j = 0; j < QUERY_WARMUP; j++) { - client.prepareSearch(indexName).setQuery(termQuery("child.tag", "tag1")).execute().actionGet(); - } - - long totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(termQuery("child.tag", "tag1")).execute().actionGet(); - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Just Child Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).execute().actionGet(); - System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - - // run parent child constant query - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasChildQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue()))) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - } - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasChildQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue()))) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child filter Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - System.out.println("--> Running has_child filter with match_all child query"); - totalQueryTime = 0; - for (int j = 1; j <= QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasChildQuery("child", matchAllQuery())) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child filter with match_all child query, Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - - System.out.println("--> Running children agg"); - totalQueryTime = 0; - for (int j = 1; j <= QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery(matchQuery("field1", parentChildIndexGenerator.getQueryValue())) - .addAggregation( - AggregationBuilders.children("to-child").childType("child") - ) - .execute().actionGet(); - totalQueryTime += searchResponse.getTookInMillis(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - Children children = searchResponse.getAggregations().get("to-child"); - if (j % 10 == 0) { - System.out.println("--> children doc count [" + j + "], got [" + children.getDocCount() + "]"); - } - } - System.out.println("--> children agg, Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - System.out.println("--> Running children agg with match_all"); - totalQueryTime = 0; - for (int j = 1; j <= QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .addAggregation( - AggregationBuilders.children("to-child").childType("child") - ) - .execute().actionGet(); - totalQueryTime += searchResponse.getTookInMillis(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - Children children = searchResponse.getAggregations().get("to-child"); - if (j % 10 == 0) { - System.out.println("--> children doc count [" + j + "], got [" + children.getDocCount() + "]"); - } - } - System.out.println("--> children agg, Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - // run parent child constant query - for (int j = 0; j < QUERY_WARMUP; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasParentQuery("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue()))) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - } - - totalQueryTime = 0; - for (int j = 1; j <= QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasParentQuery("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue()))) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_parent filter Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - System.out.println("--> Running has_parent filter with match_all parent query "); - totalQueryTime = 0; - for (int j = 1; j <= QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - boolQuery() - .must(matchAllQuery()) - .filter(hasParentQuery("parent", matchAllQuery())) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_parent filter with match_all parent query, Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).setIndices(true).execute().actionGet(); - - System.out.println("--> Field data size: " + statsResponse.getNodes()[0].getIndices().getFieldData().getMemorySize()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - - System.out.println("--> Running has_child query with score type"); - // run parent child score query - for (int j = 0; j < QUERY_WARMUP; j++) { - client.prepareSearch(indexName).setQuery(hasChildQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue())).scoreMode(ScoreMode.Max)).execute().actionGet(); - } - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue())).scoreMode(ScoreMode.Max)).execute().actionGet(); - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - totalQueryTime = 0; - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", matchAllQuery()).scoreMode(ScoreMode.Max)).execute().actionGet(); - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child query with match_all Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - System.out.println("--> Running has_parent query with score type"); - // run parent child score query - for (int j = 0; j < QUERY_WARMUP; j++) { - client.prepareSearch(indexName).setQuery(hasParentQuery("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue())).score(true)).execute().actionGet(); - } - - totalQueryTime = 0; - for (int j = 1; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasParentQuery("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue())).score(true)).execute().actionGet(); - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_parent Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - totalQueryTime = 0; - for (int j = 1; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasParentQuery("parent", matchAllQuery()).score(true)).execute().actionGet(); - if (j % 10 == 0) { - System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_parent query with match_all Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - - System.gc(); - statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).setIndices(true).execute().actionGet(); - - System.out.println("--> Field data size: " + statsResponse.getNodes()[0].getIndices().getFieldData().getMemorySize()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - - client.close(); - node1.close(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchShortCircuitBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchShortCircuitBenchmark.java deleted file mode 100644 index 388bf954822..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchShortCircuitBenchmark.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.search.child; - -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.node.Node; - -import java.io.IOException; -import java.util.Arrays; - -import static org.elasticsearch.client.Requests.createIndexRequest; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - * - */ -public class ChildSearchShortCircuitBenchmark { - - public static void main(String[] args) throws Exception { - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - String clusterName = ChildSearchShortCircuitBenchmark.class.getSimpleName(); - Node node1 = nodeBuilder().clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "node1")) - .node(); - Client client = node1.client(); - - long PARENT_COUNT = SizeValue.parseSizeValue("10M").singles(); - int BATCH = 100; - int QUERY_WARMUP = 5; - int QUERY_COUNT = 25; - String indexName = "test"; - - client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); - try { - client.admin().indices().create(createIndexRequest(indexName)).actionGet(); - client.admin().indices().preparePutMapping(indexName).setType("child").setSource(XContentFactory.jsonBuilder().startObject().startObject("child") - .startObject("_parent").field("type", "parent").endObject() - .endObject().endObject()).execute().actionGet(); - Thread.sleep(5000); - - StopWatch stopWatch = new StopWatch().start(); - - System.out.println("--> Indexing [" + PARENT_COUNT + "] parent document and some child documents"); - long ITERS = PARENT_COUNT / BATCH; - int i = 1; - int counter = 0; - for (; i <= ITERS; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < BATCH; j++) { - counter++; - request.add(Requests.indexRequest(indexName).type("parent").id(Integer.toString(counter)) - .source(parentSource(counter))); - - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * BATCH) % 10000) == 0) { - System.out.println("--> Indexed " + (i * BATCH) + "parent docs; took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - - int id = 0; - for (i = 1; i <= PARENT_COUNT; i *= 2) { - int parentId = 1; - for (int j = 0; j < i; j++) { - client.prepareIndex(indexName, "child", Integer.toString(id++)) - .setParent(Integer.toString(parentId++)) - .setSource(childSource(i)) - .execute().actionGet(); - } - } - - System.out.println("--> Indexing took " + stopWatch.totalTime()); - } catch (Exception e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - System.out.println("--> Number of docs in index: " + client.prepareCount(indexName).setQuery(matchAllQuery()).execute().actionGet().getCount()); - - System.out.println("--> Running just child query"); - // run just the child query, warm up first - for (int i = 1; i <= 10000; i *= 2) { - SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(matchQuery("child.field2", i)).execute().actionGet(); - System.out.println("--> Warmup took["+ i +"]: " + searchResponse.getTook()); - if (searchResponse.getHits().totalHits() != i) { - System.err.println("--> mismatch on hits"); - } - } - - NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).execute().actionGet(); - System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - - // run parent child constant query - for (int j = 1; j < QUERY_WARMUP; j *= 2) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery( - hasChildQuery("child", matchQuery("field2", j)) - ) - .execute().actionGet(); - if (searchResponse.getFailedShards() > 0) { - System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures())); - } - if (searchResponse.getHits().totalHits() != j) { - System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + PARENT_COUNT + "]"); - } - } - - long totalQueryTime = 0; - for (int i = 1; i < PARENT_COUNT; i *= 2) { - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery(boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", matchQuery("field2", i)))) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != i) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child filter " + i +" Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - } - - statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).setIndices(true).execute().actionGet(); - - System.out.println("--> Field data size: " + statsResponse.getNodes()[0].getIndices().getFieldData().getMemorySize()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - - totalQueryTime = 0; - for (int i = 1; i < PARENT_COUNT; i *= 2) { - for (int j = 0; j < QUERY_COUNT; j++) { - SearchResponse searchResponse = client.prepareSearch(indexName) - .setQuery(hasChildQuery("child", matchQuery("field2", i)).scoreMode(ScoreMode.Max)) - .execute().actionGet(); - if (searchResponse.getHits().totalHits() != i) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> has_child query " + i +" Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - } - - System.gc(); - statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).setIndices(true).execute().actionGet(); - - System.out.println("--> Field data size: " + statsResponse.getNodes()[0].getIndices().getFieldData().getMemorySize()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - - client.close(); - node1.close(); - } - - private static XContentBuilder parentSource(int val) throws IOException { - return jsonBuilder().startObject().field("field1", Integer.toString(val)).endObject(); - } - - private static XContentBuilder childSource(int val) throws IOException { - return jsonBuilder().startObject().field("field2", Integer.toString(val)).endObject(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/child/ParentChildIndexGenerator.java b/core/src/test/java/org/elasticsearch/benchmark/search/child/ParentChildIndexGenerator.java deleted file mode 100644 index 1d02a1f5f3e..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/child/ParentChildIndexGenerator.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.child; - -import com.carrotsearch.hppc.ObjectArrayList; -import com.carrotsearch.hppc.ObjectHashSet; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; - -import java.util.Random; - -/** - */ -public class ParentChildIndexGenerator { - - private final static Random RANDOM = new Random(); - - private final Client client; - private final int numParents; - private final int numChildrenPerParent; - private final int queryValueRatio; - - public ParentChildIndexGenerator(Client client, int numParents, int numChildrenPerParent, int queryValueRatio) { - this.client = client; - this.numParents = numParents; - this.numChildrenPerParent = numChildrenPerParent; - this.queryValueRatio = queryValueRatio; - } - - public void index() { - // Memory intensive... - ObjectHashSet usedParentIds = new ObjectHashSet<>(numParents, 0.5d); - ObjectArrayList parents = new ObjectArrayList<>(numParents); - - for (int i = 0; i < numParents; i++) { - String parentId; - do { - parentId = RandomStrings.randomAsciiOfLength(RANDOM, 10); - } while (!usedParentIds.add(parentId)); - String[] queryValues = new String[numChildrenPerParent]; - for (int j = 0; j < numChildrenPerParent; j++) { - queryValues[j] = getQueryValue(); - } - parents.add(new ParentDocument(parentId, queryValues)); - } - - int indexCounter = 0; - int childIdCounter = 0; - while (!parents.isEmpty()) { - BulkRequestBuilder request = client.prepareBulk(); - for (int i = 0; !parents.isEmpty() && i < 100; i++) { - int index = RANDOM.nextInt(parents.size()); - ParentDocument parentDocument = parents.get(index); - - if (parentDocument.indexCounter == -1) { - request.add(Requests.indexRequest("test").type("parent") - .id(parentDocument.parentId) - .source("field1", getQueryValue())); - } else { - request.add(Requests.indexRequest("test").type("child") - .parent(parentDocument.parentId) - .id(String.valueOf(++childIdCounter)) - .source("field2", parentDocument.queryValues[parentDocument.indexCounter])); - } - - if (++parentDocument.indexCounter == parentDocument.queryValues.length) { - parents.remove(index); - } - } - - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - - indexCounter += response.getItems().length; - if (indexCounter % 100000 == 0) { - System.out.println("--> Indexed " + indexCounter + " documents"); - } - } - } - - public String getQueryValue() { - return "value" + RANDOM.nextInt(numChildrenPerParent / queryValueRatio); - } - - class ParentDocument { - - final String parentId; - final String[] queryValues; - int indexCounter; - - ParentDocument(String parentId, String[] queryValues) { - this.parentId = parentId; - this.queryValues = queryValues; - this.indexCounter = -1; - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/geo/GeoDistanceSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/geo/GeoDistanceSearchBenchmark.java deleted file mode 100644 index eada2ebeee4..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/geo/GeoDistanceSearchBenchmark.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.geo; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.geo.GeoDistance; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; - -/** - */ -public class GeoDistanceSearchBenchmark { - - public static void main(String[] args) throws Exception { - - Node node = NodeBuilder.nodeBuilder().clusterName(GeoDistanceSearchBenchmark.class.getSimpleName()).node(); - Client client = node.client(); - - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("Failed to wait for green status, bailing"); - System.exit(1); - } - - final long NUM_DOCS = SizeValue.parseSizeValue("1m").singles(); - final long NUM_WARM = 50; - final long NUM_RUNS = 100; - - if (client.admin().indices().prepareExists("test").execute().actionGet().isExists()) { - System.out.println("Found an index, count: " + client.prepareCount("test").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount()); - } else { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject().string(); - client.admin().indices().prepareCreate("test") - .setSettings(Settings.settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) - .addMapping("type1", mapping) - .execute().actionGet(); - - System.err.println("--> Indexing [" + NUM_DOCS + "]"); - for (long i = 0; i < NUM_DOCS; ) { - client.prepareIndex("test", "type1", Long.toString(i++)).setSource(jsonBuilder().startObject() - .field("name", "New York") - .startObject("location").field("lat", 40.7143528).field("lon", -74.0059731).endObject() - .endObject()).execute().actionGet(); - - // to NY: 5.286 km - client.prepareIndex("test", "type1", Long.toString(i++)).setSource(jsonBuilder().startObject() - .field("name", "Times Square") - .startObject("location").field("lat", 40.759011).field("lon", -73.9844722).endObject() - .endObject()).execute().actionGet(); - - // to NY: 0.4621 km - client.prepareIndex("test", "type1", Long.toString(i++)).setSource(jsonBuilder().startObject() - .field("name", "Tribeca") - .startObject("location").field("lat", 40.718266).field("lon", -74.007819).endObject() - .endObject()).execute().actionGet(); - - // to NY: 1.258 km - client.prepareIndex("test", "type1", Long.toString(i++)).setSource(jsonBuilder().startObject() - .field("name", "Soho") - .startObject("location").field("lat", 40.7247222).field("lon", -74).endObject() - .endObject()).execute().actionGet(); - - // to NY: 8.572 km - client.prepareIndex("test", "type1", Long.toString(i++)).setSource(jsonBuilder().startObject() - .field("name", "Brooklyn") - .startObject("location").field("lat", 40.65).field("lon", -73.95).endObject() - .endObject()).execute().actionGet(); - - if ((i % 10000) == 0) { - System.err.println("--> indexed " + i); - } - } - System.err.println("Done indexed"); - client.admin().indices().prepareFlush("test").execute().actionGet(); - client.admin().indices().prepareRefresh().execute().actionGet(); - } - - System.err.println("--> Warming up (ARC) - optimize_bbox"); - long start = System.currentTimeMillis(); - for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.ARC, "memory"); - } - long totalTime = System.currentTimeMillis() - start; - System.err.println("--> Warmup (ARC) - optimize_bbox (memory) " + (totalTime / NUM_WARM) + "ms"); - - System.err.println("--> Perf (ARC) - optimize_bbox (memory)"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.ARC, "memory"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Perf (ARC) - optimize_bbox " + (totalTime / NUM_RUNS) + "ms"); - - System.err.println("--> Warming up (ARC) - optimize_bbox (indexed)"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.ARC, "indexed"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Warmup (ARC) - optimize_bbox (indexed) " + (totalTime / NUM_WARM) + "ms"); - - System.err.println("--> Perf (ARC) - optimize_bbox (indexed)"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.ARC, "indexed"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Perf (ARC) - optimize_bbox (indexed) " + (totalTime / NUM_RUNS) + "ms"); - - - System.err.println("--> Warming up (ARC) - no optimize_bbox"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.ARC, "none"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Warmup (ARC) - no optimize_bbox " + (totalTime / NUM_WARM) + "ms"); - - System.err.println("--> Perf (ARC) - no optimize_bbox"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.ARC, "none"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Perf (ARC) - no optimize_bbox " + (totalTime / NUM_RUNS) + "ms"); - - System.err.println("--> Warming up (SLOPPY_ARC)"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.SLOPPY_ARC, "memory"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Warmup (SLOPPY_ARC) " + (totalTime / NUM_WARM) + "ms"); - - System.err.println("--> Perf (SLOPPY_ARC)"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.SLOPPY_ARC, "memory"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Perf (SLOPPY_ARC) " + (totalTime / NUM_RUNS) + "ms"); - - System.err.println("--> Warming up (PLANE)"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.PLANE, "memory"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Warmup (PLANE) " + (totalTime / NUM_WARM) + "ms"); - - System.err.println("--> Perf (PLANE)"); - start = System.currentTimeMillis(); - for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.PLANE, "memory"); - } - totalTime = System.currentTimeMillis() - start; - System.err.println("--> Perf (PLANE) " + (totalTime / NUM_RUNS) + "ms"); - - node.close(); - } - - public static void run(Client client, GeoDistance geoDistance, String optimizeBbox) { - client.prepareSearch() // from NY - .setSize(0) - .setQuery(boolQuery().must(matchAllQuery()).filter(geoDistanceQuery("location") - .distance("2km") - .optimizeBbox(optimizeBbox) - .geoDistance(geoDistance) - .point(40.7143528, -74.0059731))) - .execute().actionGet(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/nested/NestedSearchBenchMark.java b/core/src/test/java/org/elasticsearch/benchmark/search/nested/NestedSearchBenchMark.java deleted file mode 100644 index c01ee21e784..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/nested/NestedSearchBenchMark.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.nested; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - */ -public class NestedSearchBenchMark { - - public static void main(String[] args) throws Exception { - Settings settings = settingsBuilder() - .put("index.refresh_interval", "-1") - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node node1 = nodeBuilder() - .settings(settingsBuilder().put(settings).put("name", "node1")) - .node(); - Client client = node1.client(); - - int count = (int) SizeValue.parseSizeValue("1m").singles(); - int nestedCount = 10; - int rootDocs = count / nestedCount; - int batch = 100; - int queryWarmup = 5; - int queryCount = 500; - String indexName = "test"; - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth() - .setWaitForGreenStatus().execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - - try { - client.admin().indices().prepareCreate(indexName) - .addMapping("type", XContentFactory.jsonBuilder() - .startObject() - .startObject("type") - .startObject("properties") - .startObject("field1") - .field("type", "integer") - .endObject() - .startObject("field2") - .field("type", "nested") - .startObject("properties") - .startObject("field3") - .field("type", "integer") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ).execute().actionGet(); - clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - - StopWatch stopWatch = new StopWatch().start(); - - System.out.println("--> Indexing [" + rootDocs + "] root documents and [" + (rootDocs * nestedCount) + "] nested objects"); - long ITERS = rootDocs / batch; - long i = 1; - int counter = 0; - for (; i <= ITERS; i++) { - BulkRequestBuilder request = client.prepareBulk(); - for (int j = 0; j < batch; j++) { - counter++; - XContentBuilder doc = XContentFactory.jsonBuilder().startObject() - .field("field1", counter) - .startArray("field2"); - for (int k = 0; k < nestedCount; k++) { - doc = doc.startObject() - .field("field3", k) - .endObject(); - } - doc = doc.endArray(); - request.add( - Requests.indexRequest(indexName).type("type").id(Integer.toString(counter)).source(doc) - ); - } - BulkResponse response = request.execute().actionGet(); - if (response.hasFailures()) { - System.err.println("--> failures..."); - } - if (((i * batch) % 10000) == 0) { - System.out.println("--> Indexed " + (i * batch) + " took " + stopWatch.stop().lastTaskTime()); - stopWatch.start(); - } - } - System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (count * (1 + nestedCount))) / stopWatch.totalTime().secondsFrac())); - } catch (Exception e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - client.admin().indices().prepareRefresh().execute().actionGet(); - System.out.println("--> Number of docs in index: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount()); - - NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).execute().actionGet(); - System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - - System.out.println("--> Running match_all with sorting on nested field"); - // run just the child query, warm up first - for (int j = 0; j < queryWarmup; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setQuery(matchAllQuery()) - .addSort( - SortBuilders.fieldSort("field2.field3") - .setNestedPath("field2") - .sortMode("avg") - .order(SortOrder.ASC) - ) - .execute().actionGet(); - if (j == 0) { - System.out.println("--> Warmup took: " + searchResponse.getTook()); - } - if (searchResponse.getHits().totalHits() != rootDocs) { - System.err.println("--> mismatch on hits"); - } - } - - long totalQueryTime = 0; - for (int j = 0; j < queryCount; j++) { - SearchResponse searchResponse = client.prepareSearch() - .setQuery(matchAllQuery()) - .addSort( - SortBuilders.fieldSort("field2.field3") - .setNestedPath("field2") - .sortMode("avg") - .order(j % 2 == 0 ? SortOrder.ASC : SortOrder.DESC) - ) - .execute().actionGet(); - - if (searchResponse.getHits().totalHits() != rootDocs) { - System.err.println("--> mismatch on hits"); - } - totalQueryTime += searchResponse.getTookInMillis(); - } - System.out.println("--> Sorting by nested fields took: " + (totalQueryTime / queryCount) + "ms"); - - statsResponse = client.admin().cluster().prepareNodesStats() - .setJvm(true).execute().actionGet(); - System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted()); - System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/search/scroll/ScrollSearchBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/search/scroll/ScrollSearchBenchmark.java deleted file mode 100644 index a6909a36df8..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/search/scroll/ScrollSearchBenchmark.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.search.scroll; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.SizeValue; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.monitor.jvm.JvmStats; -import org.elasticsearch.node.Node; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.sort.SortOrder; - -import java.util.Locale; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -/** - */ -public class ScrollSearchBenchmark { - - // Run with: -Xms1G -Xms1G - public static void main(String[] args) { - String indexName = "test"; - String typeName = "type"; - String clusterName = ScrollSearchBenchmark.class.getSimpleName(); - long numDocs = SizeValue.parseSizeValue("300k").singles(); - int requestSize = 50; - - Settings settings = settingsBuilder() - .put(SETTING_NUMBER_OF_SHARDS, 3) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - - Node[] nodes = new Node[3]; - for (int i = 0; i < nodes.length; i++) { - nodes[i] = nodeBuilder() - .clusterName(clusterName) - .settings(settingsBuilder().put(settings).put("name", "node" + i)) - .node(); - } - - Client client = nodes[0].client(); - - try { - client.admin().indices().prepareCreate(indexName).get(); - for (int counter = 1; counter <= numDocs;) { - BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); - for (int bulkCounter = 0; bulkCounter < 100; bulkCounter++) { - if (counter > numDocs) { - break; - } - bulkRequestBuilder.add( - client.prepareIndex(indexName, typeName, String.valueOf(counter)) - .setSource("field1", counter++) - ); - } - int indexedDocs = counter - 1; - if (indexedDocs % 100000 == 0) { - System.out.printf(Locale.ENGLISH, "--> Indexed %d so far\n", indexedDocs); - } - bulkRequestBuilder.get(); - } - } catch (IndexAlreadyExistsException e) { - System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); - if (clusterHealthResponse.isTimedOut()) { - System.err.println("--> Timed out waiting for cluster health"); - } - } - - client.admin().indices().prepareRefresh(indexName).get(); - System.out.printf(Locale.ENGLISH, "--> Number of docs in index: %d\n", client.prepareCount().get().getCount()); - - Long counter = numDocs; - SearchResponse searchResponse = client.prepareSearch(indexName) - .addSort("field1", SortOrder.DESC) - .setSize(requestSize) - .setScroll("10m").get(); - - if (searchResponse.getHits().getTotalHits() != numDocs) { - System.err.printf(Locale.ENGLISH, "Expected total hits [%d] but got [%d]\n", numDocs, searchResponse.getHits().getTotalHits()); - } - - if (searchResponse.getHits().hits().length != requestSize) { - System.err.printf(Locale.ENGLISH, "Expected hits length [%d] but got [%d]\n", requestSize, searchResponse.getHits().hits().length); - } - - for (SearchHit hit : searchResponse.getHits()) { - if (!hit.sortValues()[0].equals(counter--)) { - System.err.printf(Locale.ENGLISH, "Expected sort value [%d] but got [%s]\n", counter + 1, hit.sortValues()[0]); - } - } - String scrollId = searchResponse.getScrollId(); - int scrollRequestCounter = 0; - long sumTimeSpent = 0; - while (true) { - long timeSpent = System.currentTimeMillis(); - searchResponse = client.prepareSearchScroll(scrollId).setScroll("10m").get(); - sumTimeSpent += (System.currentTimeMillis() - timeSpent); - scrollRequestCounter++; - if (searchResponse.getHits().getTotalHits() != numDocs) { - System.err.printf(Locale.ENGLISH, "Expected total hits [%d] but got [%d]\n", numDocs, searchResponse.getHits().getTotalHits()); - } - if (scrollRequestCounter % 20 == 0) { - long avgTimeSpent = sumTimeSpent / 20; - JvmStats.Mem mem = JvmStats.jvmStats().getMem(); - System.out.printf(Locale.ENGLISH, "Cursor location=%d, avg time spent=%d ms\n", (requestSize * scrollRequestCounter), (avgTimeSpent)); - System.out.printf(Locale.ENGLISH, "heap max=%s, used=%s, percentage=%d\n", mem.getHeapMax(), mem.getHeapUsed(), mem.getHeapUsedPercent()); - sumTimeSpent = 0; - } - if (searchResponse.getHits().hits().length == 0) { - break; - } - if (searchResponse.getHits().hits().length != requestSize) { - System.err.printf(Locale.ENGLISH, "Expected hits length [%d] but got [%d]\n", requestSize, searchResponse.getHits().hits().length); - } - for (SearchHit hit : searchResponse.getHits()) { - if (!hit.sortValues()[0].equals(counter--)) { - System.err.printf(Locale.ENGLISH, "Expected sort value [%d] but got [%s]\n", counter + 1, hit.sortValues()[0]); - } - } - scrollId = searchResponse.getScrollId(); - } - if (counter != 0) { - System.err.printf(Locale.ENGLISH, "Counter should be 0 because scroll has been consumed\n"); - } - - for (Node node : nodes) { - node.close(); - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/time/SimpleTimeBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/time/SimpleTimeBenchmark.java deleted file mode 100644 index 37b20bce574..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/time/SimpleTimeBenchmark.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.benchmark.time; - -import org.elasticsearch.common.StopWatch; - -import java.util.concurrent.CountDownLatch; - -/** - * - */ -public class SimpleTimeBenchmark { - - private static boolean USE_NANO_TIME = false; - private static long NUMBER_OF_ITERATIONS = 1000000; - private static int NUMBER_OF_THREADS = 100; - - public static void main(String[] args) throws Exception { - StopWatch stopWatch = new StopWatch().start(); - System.out.println("Running " + NUMBER_OF_ITERATIONS); - for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) { - System.currentTimeMillis(); - } - System.out.println("Took " + stopWatch.stop().totalTime() + " TP Millis " + (NUMBER_OF_ITERATIONS / stopWatch.totalTime().millisFrac())); - - System.out.println("Running using " + NUMBER_OF_THREADS + " threads with " + NUMBER_OF_ITERATIONS + " iterations"); - final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS); - Thread[] threads = new Thread[NUMBER_OF_THREADS]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(new Runnable() { - @Override - public void run() { - if (USE_NANO_TIME) { - for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) { - System.nanoTime(); - } - } else { - for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) { - System.currentTimeMillis(); - } - } - latch.countDown(); - } - }); - } - stopWatch = new StopWatch().start(); - for (Thread thread : threads) { - thread.start(); - } - latch.await(); - stopWatch.stop(); - System.out.println("Took " + stopWatch.totalTime() + " TP Millis " + ((NUMBER_OF_ITERATIONS * NUMBER_OF_THREADS) / stopWatch.totalTime().millisFrac())); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkMessageRequest.java b/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkMessageRequest.java deleted file mode 100644 index 2978c5c4175..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkMessageRequest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.transport; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.transport.TransportRequest; - -import java.io.IOException; - -/** - * - */ -public class BenchmarkMessageRequest extends TransportRequest { - - long id; - byte[] payload; - - public BenchmarkMessageRequest(long id, byte[] payload) { - this.id = id; - this.payload = payload; - } - - public BenchmarkMessageRequest() { - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - id = in.readLong(); - payload = new byte[in.readVInt()]; - in.readFully(payload); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeLong(id); - out.writeVInt(payload.length); - out.writeBytes(payload); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkMessageResponse.java b/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkMessageResponse.java deleted file mode 100644 index 7a7e3d9ab99..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkMessageResponse.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.transport; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.transport.TransportResponse; - -import java.io.IOException; - -/** - * - */ -public class BenchmarkMessageResponse extends TransportResponse { - - long id; - byte[] payload; - - public BenchmarkMessageResponse(BenchmarkMessageRequest request) { - this.id = request.id; - this.payload = request.payload; - } - - public BenchmarkMessageResponse(long id, byte[] payload) { - this.id = id; - this.payload = payload; - } - - public BenchmarkMessageResponse() { - } - - public long id() { - return id; - } - - public byte[] payload() { - return payload; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - id = in.readLong(); - payload = new byte[in.readVInt()]; - in.readFully(payload); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeLong(id); - out.writeVInt(payload.length); - out.writeBytes(payload); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkNettyLargeMessages.java b/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkNettyLargeMessages.java deleted file mode 100644 index d8a518e3ea0..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkNettyLargeMessages.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.transport; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.settings.DynamicSettings; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.node.settings.NodeSettingsService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.*; -import org.elasticsearch.transport.netty.NettyTransport; - -import java.net.InetAddress; -import java.util.concurrent.CountDownLatch; - -import static org.elasticsearch.transport.TransportRequestOptions.options; - -/** - * - */ -public class BenchmarkNettyLargeMessages { - - public static void main(String[] args) throws Exception { - final ByteSizeValue payloadSize = new ByteSizeValue(10, ByteSizeUnit.MB); - final int NUMBER_OF_ITERATIONS = 100000; - final int NUMBER_OF_CLIENTS = 5; - final byte[] payload = new byte[(int) payloadSize.bytes()]; - - Settings settings = Settings.settingsBuilder() - .build(); - - NetworkService networkService = new NetworkService(settings); - - final ThreadPool threadPool = new ThreadPool("BenchmarkNettyLargeMessages"); - final TransportService transportServiceServer = new TransportService( - new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, new NamedWriteableRegistry()), threadPool - ).start(); - final TransportService transportServiceClient = new TransportService( - new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, new NamedWriteableRegistry()), threadPool - ).start(); - - final DiscoveryNode bigNode = new DiscoveryNode("big", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300), Version.CURRENT); -// final DiscoveryNode smallNode = new DiscoveryNode("small", new InetSocketTransportAddress("localhost", 9300)); - final DiscoveryNode smallNode = bigNode; - - transportServiceClient.connectToNode(bigNode); - transportServiceClient.connectToNode(smallNode); - - transportServiceServer.registerRequestHandler("benchmark", BenchmarkMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { - @Override - public void messageReceived(BenchmarkMessageRequest request, TransportChannel channel) throws Exception { - channel.sendResponse(new BenchmarkMessageResponse(request)); - } - }); - - final CountDownLatch latch = new CountDownLatch(NUMBER_OF_CLIENTS); - for (int i = 0; i < NUMBER_OF_CLIENTS; i++) { - new Thread(new Runnable() { - @Override - public void run() { - for (int i = 0; i < NUMBER_OF_ITERATIONS; i++) { - BenchmarkMessageRequest message = new BenchmarkMessageRequest(1, payload); - transportServiceClient.submitRequest(bigNode, "benchmark", message, options().withType(TransportRequestOptions.Type.BULK), new BaseTransportResponseHandler() { - @Override - public BenchmarkMessageResponse newInstance() { - return new BenchmarkMessageResponse(); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - - @Override - public void handleResponse(BenchmarkMessageResponse response) { - } - - @Override - public void handleException(TransportException exp) { - exp.printStackTrace(); - } - }).txGet(); - } - latch.countDown(); - } - }).start(); - } - - new Thread(new Runnable() { - @Override - public void run() { - for (int i = 0; i < 1; i++) { - BenchmarkMessageRequest message = new BenchmarkMessageRequest(2, BytesRef.EMPTY_BYTES); - long start = System.currentTimeMillis(); - transportServiceClient.submitRequest(smallNode, "benchmark", message, options().withType(TransportRequestOptions.Type.STATE), new BaseTransportResponseHandler() { - @Override - public BenchmarkMessageResponse newInstance() { - return new BenchmarkMessageResponse(); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - - @Override - public void handleResponse(BenchmarkMessageResponse response) { - } - - @Override - public void handleException(TransportException exp) { - exp.printStackTrace(); - } - }).txGet(); - long took = System.currentTimeMillis() - start; - System.out.println("Took " + took + "ms"); - } - } - }).start(); - - latch.await(); - } -} diff --git a/core/src/test/java/org/elasticsearch/benchmark/transport/TransportBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/transport/TransportBenchmark.java deleted file mode 100644 index 5ccc264399b..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/transport/TransportBenchmark.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.transport; - -import org.elasticsearch.Version; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.*; -import org.elasticsearch.transport.local.LocalTransport; -import org.elasticsearch.transport.netty.NettyTransport; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicLong; - -/** - * - */ -public class TransportBenchmark { - - static enum Type { - LOCAL { - @Override - public Transport newTransport(Settings settings, ThreadPool threadPool) { - return new LocalTransport(settings, threadPool, Version.CURRENT, new NamedWriteableRegistry()); - } - }, - NETTY { - @Override - public Transport newTransport(Settings settings, ThreadPool threadPool) { - return new NettyTransport(settings, threadPool, new NetworkService(Settings.EMPTY), BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, new NamedWriteableRegistry()); - } - }; - - public abstract Transport newTransport(Settings settings, ThreadPool threadPool); - } - - public static void main(String[] args) { - final String executor = ThreadPool.Names.GENERIC; - final boolean waitForRequest = true; - final ByteSizeValue payloadSize = new ByteSizeValue(100, ByteSizeUnit.BYTES); - final int NUMBER_OF_CLIENTS = 10; - final int NUMBER_OF_ITERATIONS = 100000; - final byte[] payload = new byte[(int) payloadSize.bytes()]; - final AtomicLong idGenerator = new AtomicLong(); - final Type type = Type.NETTY; - - - Settings settings = Settings.settingsBuilder() - .build(); - - final ThreadPool serverThreadPool = new ThreadPool("server"); - final TransportService serverTransportService = new TransportService(type.newTransport(settings, serverThreadPool), serverThreadPool).start(); - - final ThreadPool clientThreadPool = new ThreadPool("client"); - final TransportService clientTransportService = new TransportService(type.newTransport(settings, clientThreadPool), clientThreadPool).start(); - - final DiscoveryNode node = new DiscoveryNode("server", serverTransportService.boundAddress().publishAddress(), Version.CURRENT); - - serverTransportService.registerRequestHandler("benchmark", BenchmarkMessageRequest::new, executor, new TransportRequestHandler() { - @Override - public void messageReceived(BenchmarkMessageRequest request, TransportChannel channel) throws Exception { - channel.sendResponse(new BenchmarkMessageResponse(request)); - } - }); - - clientTransportService.connectToNode(node); - - for (int i = 0; i < 10000; i++) { - BenchmarkMessageRequest message = new BenchmarkMessageRequest(1, payload); - clientTransportService.submitRequest(node, "benchmark", message, new BaseTransportResponseHandler() { - @Override - public BenchmarkMessageResponse newInstance() { - return new BenchmarkMessageResponse(); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - - @Override - public void handleResponse(BenchmarkMessageResponse response) { - } - - @Override - public void handleException(TransportException exp) { - exp.printStackTrace(); - } - }).txGet(); - } - - - Thread[] clients = new Thread[NUMBER_OF_CLIENTS]; - final CountDownLatch latch = new CountDownLatch(NUMBER_OF_CLIENTS * NUMBER_OF_ITERATIONS); - for (int i = 0; i < NUMBER_OF_CLIENTS; i++) { - clients[i] = new Thread(new Runnable() { - @Override - public void run() { - for (int j = 0; j < NUMBER_OF_ITERATIONS; j++) { - final long id = idGenerator.incrementAndGet(); - BenchmarkMessageRequest request = new BenchmarkMessageRequest(id, payload); - BaseTransportResponseHandler handler = new BaseTransportResponseHandler() { - @Override - public BenchmarkMessageResponse newInstance() { - return new BenchmarkMessageResponse(); - } - - @Override - public String executor() { - return executor; - } - - @Override - public void handleResponse(BenchmarkMessageResponse response) { - if (response.id() != id) { - System.out.println("NO ID MATCH [" + response.id() + "] and [" + id + "]"); - } - latch.countDown(); - } - - @Override - public void handleException(TransportException exp) { - exp.printStackTrace(); - latch.countDown(); - } - }; - - if (waitForRequest) { - clientTransportService.submitRequest(node, "benchmark", request, handler).txGet(); - } else { - clientTransportService.sendRequest(node, "benchmark", request, handler); - } - } - } - }); - } - - StopWatch stopWatch = new StopWatch().start(); - for (int i = 0; i < NUMBER_OF_CLIENTS; i++) { - clients[i].start(); - } - - try { - latch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - stopWatch.stop(); - - System.out.println("Ran [" + NUMBER_OF_CLIENTS + "], each with [" + NUMBER_OF_ITERATIONS + "] iterations, payload [" + payloadSize + "]: took [" + stopWatch.totalTime() + "], TPS: " + (NUMBER_OF_CLIENTS * NUMBER_OF_ITERATIONS) / stopWatch.totalTime().secondsFrac()); - - clientTransportService.close(); - clientThreadPool.shutdownNow(); - - serverTransportService.close(); - serverThreadPool.shutdownNow(); - } -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/benchmark/transport/netty/NettyEchoBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/transport/netty/NettyEchoBenchmark.java deleted file mode 100644 index fd76504f2cb..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/transport/netty/NettyEchoBenchmark.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.transport.netty; - -import org.jboss.netty.bootstrap.ClientBootstrap; -import org.jboss.netty.bootstrap.ServerBootstrap; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.*; -import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; -import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; - -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; - -public class NettyEchoBenchmark { - - public static void main(String[] args) throws Exception { - final int payloadSize = 100; - int CYCLE_SIZE = 50000; - final long NUMBER_OF_ITERATIONS = 500000; - - ChannelBuffer message = ChannelBuffers.buffer(100); - for (int i = 0; i < message.capacity(); i++) { - message.writeByte((byte) i); - } - - // Configure the server. - ServerBootstrap serverBootstrap = new ServerBootstrap( - new NioServerSocketChannelFactory( - Executors.newCachedThreadPool(), - Executors.newCachedThreadPool())); - - // Set up the pipeline factory. - serverBootstrap.setPipelineFactory(new ChannelPipelineFactory() { - @Override - public ChannelPipeline getPipeline() throws Exception { - return Channels.pipeline(new EchoServerHandler()); - } - }); - - // Bind and start to accept incoming connections. - serverBootstrap.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 9000)); - - ClientBootstrap clientBootstrap = new ClientBootstrap( - new NioClientSocketChannelFactory( - Executors.newCachedThreadPool(), - Executors.newCachedThreadPool())); - -// ClientBootstrap clientBootstrap = new ClientBootstrap( -// new OioClientSocketChannelFactory(Executors.newCachedThreadPool())); - - // Set up the pipeline factory. - final EchoClientHandler clientHandler = new EchoClientHandler(); - clientBootstrap.setPipelineFactory(new ChannelPipelineFactory() { - @Override - public ChannelPipeline getPipeline() throws Exception { - return Channels.pipeline(clientHandler); - } - }); - - // Start the connection attempt. - ChannelFuture future = clientBootstrap.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), 9000)); - future.awaitUninterruptibly(); - Channel clientChannel = future.getChannel(); - - System.out.println("Warming up..."); - for (long i = 0; i < 10000; i++) { - clientHandler.latch = new CountDownLatch(1); - clientChannel.write(message); - try { - clientHandler.latch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - System.out.println("Warmed up"); - - - long start = System.currentTimeMillis(); - long cycleStart = System.currentTimeMillis(); - for (long i = 1; i < NUMBER_OF_ITERATIONS; i++) { - clientHandler.latch = new CountDownLatch(1); - clientChannel.write(message); - try { - clientHandler.latch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if ((i % CYCLE_SIZE) == 0) { - long cycleEnd = System.currentTimeMillis(); - System.out.println("Ran 50000, TPS " + (CYCLE_SIZE / ((double) (cycleEnd - cycleStart) / 1000))); - cycleStart = cycleEnd; - } - } - long end = System.currentTimeMillis(); - long seconds = (end - start) / 1000; - System.out.println("Ran [" + NUMBER_OF_ITERATIONS + "] iterations, payload [" + payloadSize + "]: took [" + seconds + "], TPS: " + ((double) NUMBER_OF_ITERATIONS) / seconds); - - clientChannel.close().awaitUninterruptibly(); - clientBootstrap.releaseExternalResources(); - serverBootstrap.releaseExternalResources(); - } - - public static class EchoClientHandler extends SimpleChannelUpstreamHandler { - - public volatile CountDownLatch latch; - - public EchoClientHandler() { - } - - @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { - latch.countDown(); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) { - e.getCause().printStackTrace(); - e.getChannel().close(); - } - } - - - public static class EchoServerHandler extends SimpleChannelUpstreamHandler { - - @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { - e.getChannel().write(e.getMessage()); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) { - // Close the connection when an exception is raised. - e.getCause().printStackTrace(); - e.getChannel().close(); - } - } -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/benchmark/uuid/SimpleUuidBenchmark.java b/core/src/test/java/org/elasticsearch/benchmark/uuid/SimpleUuidBenchmark.java deleted file mode 100644 index d9995e1a209..00000000000 --- a/core/src/test/java/org/elasticsearch/benchmark/uuid/SimpleUuidBenchmark.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.uuid; - -import org.elasticsearch.common.StopWatch; - -import java.util.UUID; -import java.util.concurrent.CountDownLatch; - -/** - * - */ -public class SimpleUuidBenchmark { - - private static long NUMBER_OF_ITERATIONS = 10000; - private static int NUMBER_OF_THREADS = 100; - - public static void main(String[] args) throws Exception { - StopWatch stopWatch = new StopWatch().start(); - System.out.println("Running " + NUMBER_OF_ITERATIONS); - for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) { - UUID.randomUUID().toString(); - } - System.out.println("Generated in " + stopWatch.stop().totalTime() + " TP Millis " + (NUMBER_OF_ITERATIONS / stopWatch.totalTime().millisFrac())); - - System.out.println("Generating using " + NUMBER_OF_THREADS + " threads with " + NUMBER_OF_ITERATIONS + " iterations"); - final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS); - Thread[] threads = new Thread[NUMBER_OF_THREADS]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(new Runnable() { - @Override - public void run() { - for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) { - UUID.randomUUID().toString(); - } - latch.countDown(); - } - }); - } - stopWatch = new StopWatch().start(); - for (Thread thread : threads) { - thread.start(); - } - latch.await(); - stopWatch.stop(); - System.out.println("Generate in " + stopWatch.totalTime() + " TP Millis " + ((NUMBER_OF_ITERATIONS * NUMBER_OF_THREADS) / stopWatch.totalTime().millisFrac())); - } -} diff --git a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java index d256c6a2c98..f7270ca2c3d 100644 --- a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.HashMap; @@ -37,9 +36,7 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class SimpleBlocksIT extends ESIntegTestCase { - - @Test - public void verifyIndexAndClusterReadOnly() throws Exception { + public void testVerifyIndexAndClusterReadOnly() throws Exception { // cluster.read_only = null: write and metadata not blocked canCreateIndex("test1"); canIndexDocument("test1"); @@ -82,7 +79,6 @@ public class SimpleBlocksIT extends ESIntegTestCase { canIndexExists("ro"); } - @Test public void testIndexReadWriteMetaDataBlocks() { canCreateIndex("test1"); canIndexDocument("test1"); diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ESPolicyTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ESPolicyTests.java index 565507d1369..cc1134e594a 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/ESPolicyTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/ESPolicyTests.java @@ -21,58 +21,18 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.test.ESTestCase; -import java.io.FilePermission; import java.security.AccessControlContext; import java.security.AccessController; -import java.security.AllPermission; -import java.security.CodeSource; -import java.security.Permission; import java.security.PermissionCollection; import java.security.Permissions; import java.security.PrivilegedAction; import java.security.ProtectionDomain; -import java.security.cert.Certificate; -import java.util.Collections; /** * Tests for ESPolicy - *

    - * Most unit tests won't run under security manager, since we don't allow - * access to the policy (you cannot construct it) */ public class ESPolicyTests extends ESTestCase { - /** - * Test policy with null codesource. - *

    - * This can happen when restricting privileges with doPrivileged, - * even though ProtectionDomain's ctor javadocs might make you think - * that the policy won't be consulted. - */ - public void testNullCodeSource() throws Exception { - assumeTrue("test cannot run with security manager", System.getSecurityManager() == null); - // create a policy with AllPermission - Permission all = new AllPermission(); - PermissionCollection allCollection = all.newPermissionCollection(); - allCollection.add(all); - ESPolicy policy = new ESPolicy(allCollection, Collections.emptyMap()); - // restrict ourselves to NoPermission - PermissionCollection noPermissions = new Permissions(); - assertFalse(policy.implies(new ProtectionDomain(null, noPermissions), new FilePermission("foo", "read"))); - } - - /** - * test with null location - *

    - * its unclear when/if this happens, see https://bugs.openjdk.java.net/browse/JDK-8129972 - */ - public void testNullLocation() throws Exception { - assumeTrue("test cannot run with security manager", System.getSecurityManager() == null); - PermissionCollection noPermissions = new Permissions(); - ESPolicy policy = new ESPolicy(noPermissions, Collections.emptyMap()); - assertFalse(policy.implies(new ProtectionDomain(new CodeSource(null, (Certificate[])null), noPermissions), new FilePermission("foo", "read"))); - } - /** * test restricting privileges to no permissions actually works */ diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JNANativesTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JNANativesTests.java index 8497f91cdca..47683702324 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/JNANativesTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/JNANativesTests.java @@ -21,20 +21,16 @@ package org.elasticsearch.bootstrap; import org.apache.lucene.util.Constants; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; public class JNANativesTests extends ESTestCase { - - @Test public void testMlockall() { if (Constants.MAC_OS_X) { assertFalse("Memory locking is not available on OS X platforms", JNANatives.LOCAL_MLOCKALL); } } - - @Test + public void testConsoleCtrlHandler() { if (Constants.WINDOWS) { assertNotNull(JNAKernel32Library.getInstance()); diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java index 14a66128eda..984b9846b15 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java @@ -74,18 +74,6 @@ public class JarHellTests extends ESTestCase { } } - public void testBootclasspathLeniency() throws Exception { - Path dir = createTempDir(); - String previousJavaHome = System.getProperty("java.home"); - System.setProperty("java.home", dir.toString()); - URL[] jars = {makeJar(dir, "foo.jar", null, "DuplicateClass.class"), makeJar(dir, "bar.jar", null, "DuplicateClass.class")}; - try { - JarHell.checkJarHell(jars); - } finally { - System.setProperty("java.home", previousJavaHome); - } - } - public void testDuplicateClasspathLeniency() throws Exception { Path dir = createTempDir(); URL jar = makeJar(dir, "foo.jar", null, "Foo.class"); @@ -179,40 +167,6 @@ public class JarHellTests extends ESTestCase { } } - public void testRequiredJDKVersionIsOK() throws Exception { - Path dir = createTempDir(); - String previousJavaVersion = System.getProperty("java.specification.version"); - System.setProperty("java.specification.version", "1.7"); - - Manifest manifest = new Manifest(); - Attributes attributes = manifest.getMainAttributes(); - attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0"); - attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7"); - URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")}; - try { - JarHell.checkJarHell(jars); - } finally { - System.setProperty("java.specification.version", previousJavaVersion); - } - } - - public void testBadJDKVersionProperty() throws Exception { - Path dir = createTempDir(); - String previousJavaVersion = System.getProperty("java.specification.version"); - System.setProperty("java.specification.version", "bogus"); - - Manifest manifest = new Manifest(); - Attributes attributes = manifest.getMainAttributes(); - attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0"); - attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7"); - URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")}; - try { - JarHell.checkJarHell(jars); - } finally { - System.setProperty("java.specification.version", previousJavaVersion); - } - } - public void testBadJDKVersionInJar() throws Exception { Path dir = createTempDir(); Manifest manifest = new Manifest(); diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java index 851e0fd9f85..21bfa05c1d4 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java @@ -20,14 +20,12 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.List; import static org.hamcrest.CoreMatchers.is; public class JavaVersionTests extends ESTestCase { - @Test public void testParse() { JavaVersion javaVersion = JavaVersion.parse("1.7.0"); List version = javaVersion.getVersion(); @@ -37,13 +35,11 @@ public class JavaVersionTests extends ESTestCase { assertThat(0, is(version.get(2))); } - @Test public void testToString() { JavaVersion javaVersion = JavaVersion.parse("1.7.0"); assertThat("1.7.0", is(javaVersion.toString())); } - @Test public void testCompare() { JavaVersion onePointSix = JavaVersion.parse("1.6"); JavaVersion onePointSeven = JavaVersion.parse("1.7"); @@ -61,7 +57,6 @@ public class JavaVersionTests extends ESTestCase { assertTrue(onePointSevenPointTwo.compareTo(onePointSevenPointTwoPointOne) < 0); } - @Test public void testValidVersions() { String[] versions = new String[]{"1.7", "1.7.0", "0.1.7", "1.7.0.80"}; for (String version : versions) { @@ -69,7 +64,6 @@ public class JavaVersionTests extends ESTestCase { } } - @Test public void testInvalidVersions() { String[] versions = new String[]{"", "1.7.0_80", "1.7."}; for (String version : versions) { diff --git a/core/src/test/java/org/elasticsearch/bootstrap/MockPluginPolicy.java b/core/src/test/java/org/elasticsearch/bootstrap/MockPluginPolicy.java deleted file mode 100644 index 91ed11cce63..00000000000 --- a/core/src/test/java/org/elasticsearch/bootstrap/MockPluginPolicy.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.bootstrap; - -import com.carrotsearch.randomizedtesting.RandomizedRunner; - -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.common.logging.Loggers; -import org.junit.Assert; - -import java.net.URL; -import java.security.CodeSource; -import java.security.Permission; -import java.security.PermissionCollection; -import java.security.Policy; -import java.security.ProtectionDomain; -import java.security.cert.Certificate; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -/** - * Simulates in unit tests per-plugin permissions. - * Unit tests for plugins do not have a proper plugin structure, - * so we don't know which codebases to apply the permission to. - *

    - * As an approximation, we just exclude es/test/framework classes, - * because they will be present in stacks and fail tests for the - * simple case where an AccessController block is missing, because - * java security checks every codebase in the stacktrace, and we - * are sure to pollute it. - */ -final class MockPluginPolicy extends Policy { - final ESPolicy standardPolicy; - final PermissionCollection extraPermissions; - final Set excludedSources; - - /** - * Create a new MockPluginPolicy with dynamic {@code permissions} and - * adding the extra plugin permissions from {@code insecurePluginProp} to - * all code except test classes. - */ - MockPluginPolicy(PermissionCollection standard, PermissionCollection extra) throws Exception { - // the hack begins! - - this.standardPolicy = new ESPolicy(standard, Collections.emptyMap()); - this.extraPermissions = extra; - - excludedSources = new HashSet(); - // exclude some obvious places - // es core - excludedSources.add(Bootstrap.class.getProtectionDomain().getCodeSource()); - // es test framework - excludedSources.add(getClass().getProtectionDomain().getCodeSource()); - // lucene test framework - excludedSources.add(LuceneTestCase.class.getProtectionDomain().getCodeSource()); - // test runner - excludedSources.add(RandomizedRunner.class.getProtectionDomain().getCodeSource()); - // junit library - excludedSources.add(Assert.class.getProtectionDomain().getCodeSource()); - // scripts - excludedSources.add(new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[])null)); - - Loggers.getLogger(getClass()).debug("Apply extra permissions [{}] excluding codebases [{}]", extraPermissions, excludedSources); - } - - @Override - public boolean implies(ProtectionDomain domain, Permission permission) { - CodeSource codeSource = domain.getCodeSource(); - // codesource can be null when reducing privileges via doPrivileged() - if (codeSource == null) { - return false; - } - - if (standardPolicy.implies(domain, permission)) { - return true; - } else if (excludedSources.contains(codeSource) == false && - codeSource.toString().contains("test-classes") == false) { - return extraPermissions.implies(permission); - } else { - return false; - } - } -} diff --git a/core/src/test/java/org/elasticsearch/bootstrap/SecurityTests.java b/core/src/test/java/org/elasticsearch/bootstrap/SecurityTests.java index d0685b1a6ce..132e46c7b90 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/SecurityTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/SecurityTests.java @@ -19,119 +19,14 @@ package org.elasticsearch.bootstrap; -import org.apache.lucene.util.Constants; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; -import java.io.FilePermission; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.security.PermissionCollection; -import java.security.Permissions; -import java.util.Set; public class SecurityTests extends ESTestCase { - /** test generated permissions */ - public void testGeneratedPermissions() throws Exception { - Path path = createTempDir(); - // make a fake ES home and ensure we only grant permissions to that. - Path esHome = path.resolve("esHome"); - Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put("path.home", esHome.toString()); - Settings settings = settingsBuilder.build(); - - Path fakeTmpDir = createTempDir(); - String realTmpDir = System.getProperty("java.io.tmpdir"); - Permissions permissions; - try { - System.setProperty("java.io.tmpdir", fakeTmpDir.toString()); - Environment environment = new Environment(settings); - permissions = Security.createPermissions(environment); - } finally { - System.setProperty("java.io.tmpdir", realTmpDir); - } - - // the fake es home - assertNoPermissions(esHome, permissions); - // its parent - assertNoPermissions(esHome.getParent(), permissions); - // some other sibling - assertNoPermissions(esHome.getParent().resolve("other"), permissions); - // double check we overwrote java.io.tmpdir correctly for the test - assertNoPermissions(PathUtils.get(realTmpDir), permissions); - } - - /** test generated permissions for all configured paths */ - public void testEnvironmentPaths() throws Exception { - Path path = createTempDir(); - // make a fake ES home and ensure we only grant permissions to that. - Path esHome = path.resolve("esHome"); - - Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put("path.home", esHome.resolve("home").toString()); - settingsBuilder.put("path.conf", esHome.resolve("conf").toString()); - settingsBuilder.put("path.scripts", esHome.resolve("scripts").toString()); - settingsBuilder.put("path.plugins", esHome.resolve("plugins").toString()); - settingsBuilder.putArray("path.data", esHome.resolve("data1").toString(), esHome.resolve("data2").toString()); - settingsBuilder.put("path.shared_data", esHome.resolve("custom").toString()); - settingsBuilder.put("path.logs", esHome.resolve("logs").toString()); - settingsBuilder.put("pidfile", esHome.resolve("test.pid").toString()); - Settings settings = settingsBuilder.build(); - - Path fakeTmpDir = createTempDir(); - String realTmpDir = System.getProperty("java.io.tmpdir"); - Permissions permissions; - Environment environment; - try { - System.setProperty("java.io.tmpdir", fakeTmpDir.toString()); - environment = new Environment(settings); - permissions = Security.createPermissions(environment); - } finally { - System.setProperty("java.io.tmpdir", realTmpDir); - } - - // the fake es home - assertNoPermissions(esHome, permissions); - // its parent - assertNoPermissions(esHome.getParent(), permissions); - // some other sibling - assertNoPermissions(esHome.getParent().resolve("other"), permissions); - // double check we overwrote java.io.tmpdir correctly for the test - assertNoPermissions(PathUtils.get(realTmpDir), permissions); - - // check that all directories got permissions: - - // bin file: ro - assertExactPermissions(new FilePermission(environment.binFile().toString(), "read,readlink"), permissions); - // lib file: ro - assertExactPermissions(new FilePermission(environment.libFile().toString(), "read,readlink"), permissions); - // config file: ro - assertExactPermissions(new FilePermission(environment.configFile().toString(), "read,readlink"), permissions); - // scripts file: ro - assertExactPermissions(new FilePermission(environment.scriptsFile().toString(), "read,readlink"), permissions); - // plugins: ro - assertExactPermissions(new FilePermission(environment.pluginsFile().toString(), "read,readlink"), permissions); - - // data paths: r/w - for (Path dataPath : environment.dataFiles()) { - assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions); - } - for (Path dataPath : environment.dataWithClusterFiles()) { - assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions); - } - assertExactPermissions(new FilePermission(environment.sharedDataFile().toString(), "read,readlink,write,delete"), permissions); - // logs: r/w - assertExactPermissions(new FilePermission(environment.logsFile().toString(), "read,readlink,write,delete"), permissions); - // temp dir: r/w - assertExactPermissions(new FilePermission(fakeTmpDir.toString(), "read,readlink,write,delete"), permissions); - // PID file: delete only (for the shutdown hook) - assertExactPermissions(new FilePermission(environment.pidFile().toString(), "delete"), permissions); - } - public void testEnsureExists() throws IOException { Path p = createTempDir(); @@ -163,43 +58,6 @@ public class SecurityTests extends ESTestCase { } catch (IOException expected) {} } - public void testEnsureSymlink() throws IOException { - Path p = createTempDir(); - - Path exists = p.resolve("exists"); - Files.createDirectory(exists); - - // symlink - Path linkExists = p.resolve("linkExists"); - try { - Files.createSymbolicLink(linkExists, exists); - } catch (UnsupportedOperationException | IOException e) { - assumeNoException("test requires filesystem that supports symbolic links", e); - } catch (SecurityException e) { - assumeNoException("test cannot create symbolic links with security manager enabled", e); - } - Security.ensureDirectoryExists(linkExists); - Files.createTempFile(linkExists, null, null); - } - - public void testEnsureBrokenSymlink() throws IOException { - Path p = createTempDir(); - - // broken symlink - Path brokenLink = p.resolve("brokenLink"); - try { - Files.createSymbolicLink(brokenLink, p.resolve("nonexistent")); - } catch (UnsupportedOperationException | IOException e) { - assumeNoException("test requires filesystem that supports symbolic links", e); - } catch (SecurityException e) { - assumeNoException("test cannot create symbolic links with security manager enabled", e); - } - try { - Security.ensureDirectoryExists(brokenLink); - fail("didn't get expected exception"); - } catch (IOException expected) {} - } - /** can't execute processes */ public void testProcessExecution() throws Exception { assumeTrue("test requires security manager", System.getSecurityManager() != null); @@ -208,61 +66,4 @@ public class SecurityTests extends ESTestCase { fail("didn't get expected exception"); } catch (SecurityException expected) {} } - - /** When a configured dir is a symlink, test that permissions work on link target */ - public void testSymlinkPermissions() throws IOException { - // see https://github.com/elastic/elasticsearch/issues/12170 - assumeFalse("windows does not automatically grant permission to the target of symlinks", Constants.WINDOWS); - Path dir = createTempDir(); - - Path target = dir.resolve("target"); - Files.createDirectory(target); - - // symlink - Path link = dir.resolve("link"); - try { - Files.createSymbolicLink(link, target); - } catch (UnsupportedOperationException | IOException e) { - assumeNoException("test requires filesystem that supports symbolic links", e); - } catch (SecurityException e) { - assumeNoException("test cannot create symbolic links with security manager enabled", e); - } - Permissions permissions = new Permissions(); - Security.addPath(permissions, "testing", link, "read"); - assertExactPermissions(new FilePermission(link.toString(), "read"), permissions); - assertExactPermissions(new FilePermission(link.resolve("foo").toString(), "read"), permissions); - assertExactPermissions(new FilePermission(target.toString(), "read"), permissions); - assertExactPermissions(new FilePermission(target.resolve("foo").toString(), "read"), permissions); - } - - /** - * checks exact file permissions, meaning those and only those for that path. - */ - static void assertExactPermissions(FilePermission expected, PermissionCollection actual) { - String target = expected.getName(); // see javadocs - Set permissionSet = asSet(expected.getActions().split(",")); - boolean read = permissionSet.remove("read"); - boolean readlink = permissionSet.remove("readlink"); - boolean write = permissionSet.remove("write"); - boolean delete = permissionSet.remove("delete"); - boolean execute = permissionSet.remove("execute"); - assertTrue("unrecognized permission: " + permissionSet, permissionSet.isEmpty()); - assertEquals(read, actual.implies(new FilePermission(target, "read"))); - assertEquals(readlink, actual.implies(new FilePermission(target, "readlink"))); - assertEquals(write, actual.implies(new FilePermission(target, "write"))); - assertEquals(delete, actual.implies(new FilePermission(target, "delete"))); - assertEquals(execute, actual.implies(new FilePermission(target, "execute"))); - } - - /** - * checks that this path has no permissions - */ - static void assertNoPermissions(Path path, PermissionCollection actual) { - String target = path.toString(); - assertFalse(actual.implies(new FilePermission(target, "read"))); - assertFalse(actual.implies(new FilePermission(target, "readlink"))); - assertFalse(actual.implies(new FilePermission(target, "write"))); - assertFalse(actual.implies(new FilePermission(target, "delete"))); - assertFalse(actual.implies(new FilePermission(target, "execute"))); - } } diff --git a/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java b/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java index e2da7024582..18c1572e865 100644 --- a/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java +++ b/core/src/test/java/org/elasticsearch/broadcast/BroadcastActionsIT.java @@ -19,17 +19,13 @@ package org.elasticsearch.broadcast; -import java.nio.charset.StandardCharsets; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; -import static org.elasticsearch.client.Requests.countRequest; import static org.elasticsearch.client.Requests.indexRequest; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -42,7 +38,6 @@ public class BroadcastActionsIT extends ESIntegTestCase { return 1; } - @Test public void testBroadcastOperations() throws IOException { assertAcked(prepareCreate("test", 1).execute().actionGet(5000)); @@ -60,23 +55,14 @@ public class BroadcastActionsIT extends ESIntegTestCase { // check count for (int i = 0; i < 5; i++) { // test successful - CountResponse countResponse = client().prepareCount("test") + SearchResponse countResponse = client().prepareSearch("test").setSize(0) .setQuery(termQuery("_type", "type1")) .get(); - assertThat(countResponse.getCount(), equalTo(2l)); + assertThat(countResponse.getHits().totalHits(), equalTo(2l)); assertThat(countResponse.getTotalShards(), equalTo(numShards.numPrimaries)); assertThat(countResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(countResponse.getFailedShards(), equalTo(0)); } - - for (int i = 0; i < 5; i++) { - // test failed (simply query that can't be parsed) - try { - client().count(countRequest("test").source("{ term : { _type : \"type1 } }".getBytes(StandardCharsets.UTF_8))).actionGet(); - } catch(SearchPhaseExecutionException e) { - assertThat(e.shardFailures().length, equalTo(numShards.numPrimaries)); - } - } } private XContentBuilder source(String id, String nameValue) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java index 3a58d710510..07d59b820aa 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.bwcompat; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.util.TestUtil; import org.elasticsearch.Version; @@ -26,7 +27,6 @@ import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.test.ESBackcompatTestCase; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.List; @@ -48,10 +48,8 @@ public class BasicAnalysisBackwardCompatibilityIT extends ESBackcompatTestCase { * Simple upgrade test for analyzers to make sure they analyze to the same tokens after upgrade * TODO we need this for random tokenizers / tokenfilters as well */ - @Test public void testAnalyzerTokensAfterUpgrade() throws IOException, ExecutionException, InterruptedException { int numFields = randomIntBetween(PreBuiltAnalyzers.values().length, PreBuiltAnalyzers.values().length * 10); - StringBuilder builder = new StringBuilder(); String[] fields = new String[numFields * 2]; int fieldId = 0; for (int i = 0; i < fields.length; i++) { diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java index 5d65bf4556a..95735b8648f 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java @@ -19,23 +19,27 @@ package org.elasticsearch.bwcompat; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.index.Fields; import org.apache.lucene.util.English; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.explain.ExplainResponse; -import org.elasticsearch.action.get.*; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.action.update.UpdateRequestBuilder; @@ -58,7 +62,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -66,11 +69,19 @@ import java.util.List; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.existsQuery; -import static org.elasticsearch.index.query.QueryBuilders.missingQuery; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; /** */ @@ -79,7 +90,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { /** * Basic test using Index & Realtime Get with external versioning. This test ensures routing works correctly across versions. */ - @Test public void testExternalVersion() throws Exception { createIndex("test"); final boolean routing = randomBoolean(); @@ -103,7 +113,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { /** * Basic test using Index & Realtime Get with internal versioning. This test ensures routing works correctly across versions. */ - @Test public void testInternalVersion() throws Exception { createIndex("test"); final boolean routing = randomBoolean(); @@ -127,7 +136,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { /** * Very basic bw compat test with a mixed version cluster random indexing and lookup by ID via term query */ - @Test public void testIndexAndSearch() throws Exception { createIndex("test"); int numDocs = randomIntBetween(10, 20); @@ -144,7 +152,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertVersionCreated(compatibilityVersion(), "test"); } - @Test public void testRecoverFromPreviousVersion() throws ExecutionException, InterruptedException { if (backwardsCluster().numNewDataNodes() == 0) { backwardsCluster().startNewNode(); @@ -162,7 +169,7 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { docs[i] = client().prepareIndex("test", "type1", id).setSource("field1", English.intToEnglish(i)); } indexRandom(true, docs); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); if (randomBoolean()) { @@ -201,7 +208,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { /** * Test that ensures that we will never recover from a newer to an older version (we are not forward compatible) */ - @Test public void testNoRecoveryFromNewNodes() throws ExecutionException, InterruptedException { assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.routing.allocation.exclude._name", backwardsCluster().backwardsNodePattern()).put(indexSettings()))); if (backwardsCluster().numNewDataNodes() == 0) { @@ -223,11 +229,11 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { backwardsCluster().startNewNode(); } assertAllShardsOnNodes("test", backwardsCluster().newNodePattern()); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); final int numIters = randomIntBetween(10, 20); for (int i = 0; i < numIters; i++) { - countResponse = client().prepareCount().get(); + countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); assertSimpleSort("num_double", "num_int"); } @@ -269,7 +275,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { /** * Upgrades a single node to the current version */ - @Test public void testIndexUpgradeSingleNode() throws Exception { assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.routing.allocation.exclude._name", backwardsCluster().newNodePattern()).put(indexSettings()))); ensureYellow(); @@ -283,7 +288,7 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertAllShardsOnNodes("test", backwardsCluster().backwardsNodePattern()); disableAllocation("test"); backwardsCluster().allowOnAllNodes("test"); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); backwardsCluster().upgradeOneNode(); ensureYellow(); @@ -297,7 +302,7 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { ensureYellow(); final int numIters = randomIntBetween(1, 20); for (int i = 0; i < numIters; i++) { - assertHitCount(client().prepareCount().get(), numDocs); + assertHitCount(client().prepareSearch().setSize(0).get(), numDocs); assertSimpleSort("num_double", "num_int"); } assertVersionCreated(compatibilityVersion(), "test"); @@ -308,7 +313,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { * one node after another is shut down and restarted from a newer version and we verify * that all documents are still around after each nodes upgrade. */ - @Test public void testIndexRollingUpgrade() throws Exception { String[] indices = new String[randomIntBetween(1, 3)]; for (int i = 0; i < indices.length; i++) { @@ -332,12 +336,12 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { boolean upgraded; do { logClusterState(); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); assertSimpleSort("num_double", "num_int"); upgraded = backwardsCluster().upgradeOneNode(); ensureYellow(); - countResponse = client().prepareCount().get(); + countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex(indexForDoc[i], "type1", String.valueOf(i)).setSource("field1", English.intToEnglish(i), "num_int", randomInt(), "num_double", randomDouble()); @@ -346,7 +350,7 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { } while (upgraded); enableAllocation(indices); ensureYellow(); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); assertSimpleSort("num_double", "num_int"); @@ -371,7 +375,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { } - @Test public void testUnsupportedFeatures() throws IOException { XContentBuilder mapping = XContentBuilder.builder(JsonXContent.jsonXContent) .startObject() @@ -399,7 +402,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { * This filter had a major upgrade in 1.3 where we started to index the field names. Lets see if they still work as expected... * this test is basically copied from SimpleQueryTests... */ - @Test public void testExistsFilter() throws IOException, ExecutionException, InterruptedException { int indexId = 0; String indexName; @@ -414,48 +416,32 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { client().prepareIndex(indexName, "type1", "3").setSource(jsonBuilder().startObject().startObject("obj2").field("obj2_val", "1").endObject().field("y1", "y_1").field("field2", "value2_3").endObject()), client().prepareIndex(indexName, "type1", "4").setSource(jsonBuilder().startObject().startObject("obj2").field("obj2_val", "1").endObject().field("y2", "y_2").field("field3", "value3_4").endObject())); - CountResponse countResponse = client().prepareCount().setQuery(existsQuery("field1")).get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).setQuery(existsQuery("field1")).get(); assertHitCount(countResponse, 2l); - countResponse = client().prepareCount().setQuery(constantScoreQuery(existsQuery("field1"))).get(); + countResponse = client().prepareSearch().setSize(0).setQuery(constantScoreQuery(existsQuery("field1"))).get(); assertHitCount(countResponse, 2l); - countResponse = client().prepareCount().setQuery(queryStringQuery("_exists_:field1")).get(); + countResponse = client().prepareSearch().setSize(0).setQuery(queryStringQuery("_exists_:field1")).get(); assertHitCount(countResponse, 2l); - countResponse = client().prepareCount().setQuery(existsQuery("field2")).get(); + countResponse = client().prepareSearch().setSize(0).setQuery(existsQuery("field2")).get(); assertHitCount(countResponse, 2l); - countResponse = client().prepareCount().setQuery(existsQuery("field3")).get(); + countResponse = client().prepareSearch().setSize(0).setQuery(existsQuery("field3")).get(); assertHitCount(countResponse, 1l); // wildcard check - countResponse = client().prepareCount().setQuery(existsQuery("x*")).get(); + countResponse = client().prepareSearch().setSize(0).setQuery(existsQuery("x*")).get(); assertHitCount(countResponse, 2l); // object check - countResponse = client().prepareCount().setQuery(existsQuery("obj1")).get(); + countResponse = client().prepareSearch().setSize(0).setQuery(existsQuery("obj1")).get(); assertHitCount(countResponse, 2l); - countResponse = client().prepareCount().setQuery(missingQuery("field1")).get(); + countResponse = client().prepareSearch().setSize(0).setQuery(queryStringQuery("_missing_:field1")).get(); assertHitCount(countResponse, 2l); - countResponse = client().prepareCount().setQuery(missingQuery("field1")).get(); - assertHitCount(countResponse, 2l); - - countResponse = client().prepareCount().setQuery(constantScoreQuery(missingQuery("field1"))).get(); - assertHitCount(countResponse, 2l); - - countResponse = client().prepareCount().setQuery(queryStringQuery("_missing_:field1")).get(); - assertHitCount(countResponse, 2l); - - // wildcard check - countResponse = client().prepareCount().setQuery(missingQuery("x*")).get(); - assertHitCount(countResponse, 2l); - - // object check - countResponse = client().prepareCount().setQuery(missingQuery("obj1")).get(); - assertHitCount(countResponse, 2l); if (!backwardsCluster().upgradeOneNode()) { break; } @@ -472,7 +458,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { return client().admin().cluster().prepareState().get().getState().nodes().masterNode().getVersion(); } - @Test public void testDeleteRoutingRequired() throws ExecutionException, InterruptedException, IOException { createIndexWithAlias(); assertAcked(client().admin().indices().preparePutMapping("test").setType("test").setSource( @@ -509,7 +494,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1)); } - @Test public void testIndexGetAndDelete() throws ExecutionException, InterruptedException { createIndexWithAlias(); ensureYellow("test"); @@ -546,7 +530,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1)); } - @Test public void testUpdate() { createIndexWithAlias(); ensureYellow("test"); @@ -577,7 +560,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(getResponse.getSourceAsMap().containsKey("field2"), equalTo(true)); } - @Test public void testAnalyze() { createIndexWithAlias(); assertAcked(client().admin().indices().preparePutMapping("test").setType("test").setSource("field", "type=string,analyzer=keyword")); @@ -587,7 +569,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test")); } - @Test public void testExplain() { createIndexWithAlias(); ensureYellow("test"); @@ -604,7 +585,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(response.getExplanation().getDetails().length, equalTo(1)); } - @Test public void testGetTermVector() throws IOException { createIndexWithAlias(); assertAcked(client().admin().indices().preparePutMapping("test").setType("type1").setSource("field", "type=string,term_vector=with_positions_offsets_payloads").get()); @@ -622,7 +602,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(fields.terms("field").size(), equalTo(8l)); } - @Test public void testIndicesStats() { createIndex("test"); ensureYellow("test"); @@ -632,7 +611,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(indicesStatsResponse.getIndices().containsKey("test"), equalTo(true)); } - @Test public void testMultiGet() throws ExecutionException, InterruptedException { createIndexWithAlias(); ensureYellow("test"); @@ -665,7 +643,6 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase { } - @Test public void testScroll() throws ExecutionException, InterruptedException { createIndex("test"); ensureYellow("test"); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java index 1fe1dbb0637..665aa5217ac 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java @@ -30,17 +30,16 @@ import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; import static org.hamcrest.Matchers.equalTo; public class ClusterStateBackwardsCompatIT extends ESBackcompatTestCase { - - @Test public void testClusterState() throws Exception { createIndex("test"); @@ -57,7 +56,6 @@ public class ClusterStateBackwardsCompatIT extends ESBackcompatTestCase { } } - @Test public void testClusterStateWithBlocks() { createIndex("test-blocks"); @@ -87,7 +85,7 @@ public class ClusterStateBackwardsCompatIT extends ESBackcompatTestCase { IndexMetaData indexMetaData = response.getState().getMetaData().getIndices().get("test-blocks"); assertNotNull(indexMetaData); - assertTrue(indexMetaData.settings().getAsBoolean(block.getKey(), null)); + assertTrue(indexMetaData.getSettings().getAsBoolean(block.getKey(), null)); } } } finally { diff --git a/core/src/test/java/org/elasticsearch/bwcompat/GetIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/GetIndexBackwardsCompatibilityIT.java index a7e9380d02a..9a87c888747 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/GetIndexBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/GetIndexBackwardsCompatibilityIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.warmer.IndexWarmersMetaData.Entry; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; import java.util.List; @@ -40,8 +39,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; public class GetIndexBackwardsCompatibilityIT extends ESBackcompatTestCase { - - @Test public void testGetAliases() throws Exception { CreateIndexResponse createIndexResponse = prepareCreate("test").addAlias(new Alias("testAlias")).execute().actionGet(); assertAcked(createIndexResponse); @@ -58,7 +55,6 @@ public class GetIndexBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(alias.alias(), equalTo("testAlias")); } - @Test public void testGetMappings() throws Exception { CreateIndexResponse createIndexResponse = prepareCreate("test").addMapping("type1", "{\"type1\":{}}").execute().actionGet(); assertAcked(createIndexResponse); @@ -79,7 +75,6 @@ public class GetIndexBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(mapping.type(), equalTo("type1")); } - @Test public void testGetSettings() throws Exception { CreateIndexResponse createIndexResponse = prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 1)).execute().actionGet(); assertAcked(createIndexResponse); @@ -93,7 +88,6 @@ public class GetIndexBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(settings.get("index.number_of_shards"), equalTo("1")); } - @Test public void testGetWarmers() throws Exception { createIndex("test"); ensureSearchable("test"); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java index 8cb9fc7566b..39ed23b9779 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java @@ -25,17 +25,14 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilde import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase; import java.lang.reflect.Method; @ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE, numClientNodes = 0) public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase { - - @Test public void testNodeStatsSetIndices() throws Exception { createIndex("test"); @@ -54,7 +51,6 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase { } } - @Test public void testNodeStatsSetRandom() throws Exception { createIndex("test"); @@ -73,7 +69,7 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase { NodesStatsRequestBuilder nsBuilder = tc.admin().cluster().prepareNodesStats(); Class c = nsBuilder.getClass(); - for (Method method : c.getDeclaredMethods()) { + for (Method method : c.getMethods()) { if (method.getName().startsWith("set")) { if (method.getParameterTypes().length == 1 && method.getParameterTypes()[0] == boolean.class) { method.invoke(nsBuilder, randomBoolean()); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java index 990c399d8ee..97e77a00e38 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java @@ -24,12 +24,14 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.segments.IndexSegments; +import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; +import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.upgrade.UpgradeIT; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; @@ -38,6 +40,7 @@ import org.elasticsearch.common.util.MultiDataPathUpgrader; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.mapper.string.StringFieldMapperPositionIncrementGapTests; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.MergePolicyConfig; @@ -54,28 +57,15 @@ import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.hamcrest.Matchers; import org.junit.AfterClass; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.io.InputStream; -import java.nio.file.DirectoryStream; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; +import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.Future; +import java.util.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.greaterThanOrEqualTo; // needs at least 2 nodes since it bumps replicas to 1 @@ -268,7 +258,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase { public void testOldIndexes() throws Exception { setupCluster(); - Collections.shuffle(indexes, getRandom()); + Collections.shuffle(indexes, random()); for (String index : indexes) { long startTime = System.currentTimeMillis(); logger.info("--> Testing old index " + index); @@ -277,52 +267,11 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase { } } - @Test - public void testHandlingOfUnsupportedDanglingIndexes() throws Exception { - setupCluster(); - Collections.shuffle(unsupportedIndexes, getRandom()); - for (String index : unsupportedIndexes) { - assertUnsupportedIndexHandling(index); - } - } - - /** - * Waits for the index to show up in the cluster state in closed state - */ - void ensureClosed(final String index) throws InterruptedException { - assertTrue(awaitBusy(() -> { - ClusterState state = client().admin().cluster().prepareState().get().getState(); - return state.metaData().hasIndex(index) && state.metaData().index(index).getState() == IndexMetaData.State.CLOSE; - } - ) - ); - } - - /** - * Checks that the given index cannot be opened due to incompatible version - */ - void assertUnsupportedIndexHandling(String index) throws Exception { - long startTime = System.currentTimeMillis(); - logger.info("--> Testing old index " + index); - String indexName = loadIndex(index); - // force reloading dangling indices with a cluster state republish - client().admin().cluster().prepareReroute().get(); - ensureClosed(indexName); - try { - client().admin().indices().prepareOpen(indexName).get(); - fail("Shouldn't be able to open an old index"); - } catch (IllegalStateException ex) { - assertThat(ex.getMessage(), containsString("was created before v2.0.0.beta1 and wasn't upgraded")); - } - unloadIndex(indexName); - logger.info("--> Done testing " + index + ", took " + ((System.currentTimeMillis() - startTime) / 1000.0) + " seconds"); - } - void assertOldIndexWorks(String index) throws Exception { Version version = extractVersion(index); String indexName = loadIndex(index); importIndex(indexName); - assertIndexSanity(indexName); + assertIndexSanity(indexName, version); assertBasicSearchWorks(indexName); assertBasicAggregationWorks(indexName); assertRealtimeGetWorks(indexName); @@ -342,11 +291,22 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase { version.luceneVersion.minor == Version.CURRENT.luceneVersion.minor; } - void assertIndexSanity(String indexName) { + void assertIndexSanity(String indexName, Version indexCreated) { GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().addIndices(indexName).get(); assertEquals(1, getIndexResponse.indices().length); assertEquals(indexName, getIndexResponse.indices()[0]); + Version actualVersionCreated = Version.indexCreated(getIndexResponse.getSettings().get(indexName)); + assertEquals(indexCreated, actualVersionCreated); ensureYellow(indexName); + IndicesSegmentResponse segmentsResponse = client().admin().indices().prepareSegments(indexName).get(); + IndexSegments segments = segmentsResponse.getIndices().get(indexName); + for (IndexShardSegments indexShardSegments : segments) { + for (ShardSegments shardSegments : indexShardSegments) { + for (Segment segment : shardSegments) { + assertEquals(indexCreated.luceneVersion, segment.version); + } + } + } SearchResponse test = client().prepareSearch(indexName).get(); assertThat(test.getHits().getTotalHits(), greaterThanOrEqualTo(1l)); } @@ -368,13 +328,6 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase { searchRsp = searchReq.get(); ElasticsearchAssertions.assertNoFailures(searchRsp); assertEquals(numDocs, searchRsp.getHits().getTotalHits()); - - logger.info("--> testing missing filter"); - // the field for the missing filter here needs to be different than the exists filter above, to avoid being found in the cache - searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.missingQuery("long_sort")); - searchRsp = searchReq.get(); - ElasticsearchAssertions.assertNoFailures(searchRsp); - assertEquals(0, searchRsp.getHits().getTotalHits()); } void assertBasicAggregationWorks(String indexName) { diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java index 895748514d4..a573a8374ef 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java @@ -20,13 +20,10 @@ package org.elasticsearch.bwcompat; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; -import org.junit.Test; import static org.hamcrest.Matchers.containsString; public class RecoveryWithUnsupportedIndicesIT extends StaticIndexBackwardCompatibilityIT { - - @Test public void testUpgradeStartClusterOn_0_20_6() throws Exception { String indexName = "unsupported-0.20.6"; diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index 740b185e745..228f1a65121 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotRestoreException; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.io.IOException; import java.lang.reflect.Modifier; @@ -51,7 +50,10 @@ import java.util.TreeSet; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; @ClusterScope(scope = Scope.TEST) public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { @@ -79,8 +81,7 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { } } - @Test - public void restoreOldSnapshots() throws Exception { + public void testRestoreOldSnapshots() throws Exception { String repo = "test_repo"; String snapshot = "test_1"; List repoVersions = repoVersions(); @@ -91,7 +92,7 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { } SortedSet expectedVersions = new TreeSet<>(); - for (java.lang.reflect.Field field : Version.class.getDeclaredFields()) { + for (java.lang.reflect.Field field : Version.class.getFields()) { if (Modifier.isStatic(field.getModifiers()) && field.getType() == Version.class) { Version v = (Version) field.get(Version.class); if (v.snapshot()) continue; @@ -115,7 +116,6 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { } } - @Test public void testRestoreUnsupportedSnapshots() throws Exception { String repo = "test_repo"; String snapshot = "test_1"; diff --git a/core/src/test/java/org/elasticsearch/bwcompat/TransportClientBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/TransportClientBackwardsCompatibilityIT.java index e071c17b948..0e2f94e3cf1 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/TransportClientBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/TransportClientBackwardsCompatibilityIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.CompositeTestCluster; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; import java.util.concurrent.ExecutionException; @@ -38,10 +37,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSear import static org.hamcrest.CoreMatchers.equalTo; public class TransportClientBackwardsCompatibilityIT extends ESBackcompatTestCase { - - @Test public void testSniffMode() throws ExecutionException, InterruptedException { - Settings settings = Settings.builder().put(requiredSettings()).put("client.transport.nodes_sampler_interval", "1s") .put("name", "transport_client_sniff_mode").put(ClusterName.SETTING, cluster().getClusterName()) .put("client.transport.sniff", true).build(); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/UnicastBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/UnicastBackwardsCompatibilityIT.java index 59dd6695218..ab7e95812fe 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/UnicastBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/UnicastBackwardsCompatibilityIT.java @@ -22,12 +22,10 @@ package org.elasticsearch.bwcompat; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; public class UnicastBackwardsCompatibilityIT extends ESBackcompatTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -46,7 +44,6 @@ public class UnicastBackwardsCompatibilityIT extends ESBackcompatTestCase { .build(); } - @Test public void testUnicastDiscovery() { ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().get(); assertThat(healthResponse.getNumberOfDataNodes(), equalTo(cluster().numDataNodes())); diff --git a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java index b00b677a4b8..b814cff520c 100644 --- a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java +++ b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java @@ -53,7 +53,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportMessage; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -72,7 +71,6 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase { .put(Headers.PREFIX + ".key2", "val 2") .build(); - @SuppressWarnings("unchecked") private static final GenericAction[] ACTIONS = new GenericAction[] { // client actions GetAction.INSTANCE, SearchAction.INSTANCE, DeleteAction.INSTANCE, DeleteIndexedScriptAction.INSTANCE, @@ -107,7 +105,6 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase { protected abstract Client buildClient(Settings headersSettings, GenericAction[] testedActions); - @Test public void testActions() { // TODO this is a really shitty way to test it, we need to figure out a way to test all the client methods @@ -134,7 +131,6 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase { client.admin().indices().prepareFlush().execute().addListener(new AssertingActionListener(FlushAction.NAME)); } - @Test public void testOverideHeader() throws Exception { String key1Val = randomAsciiOfLength(5); Map expected = new HashMap<>(); diff --git a/core/src/test/java/org/elasticsearch/client/node/NodeClientIT.java b/core/src/test/java/org/elasticsearch/client/node/NodeClientIT.java index 499998096b1..966553b8f3d 100644 --- a/core/src/test/java/org/elasticsearch/client/node/NodeClientIT.java +++ b/core/src/test/java/org/elasticsearch/client/node/NodeClientIT.java @@ -21,11 +21,10 @@ package org.elasticsearch.client.node; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.Matchers.is; /** @@ -33,17 +32,14 @@ import static org.hamcrest.Matchers.is; */ @ClusterScope(scope = Scope.SUITE) public class NodeClientIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put(Client.CLIENT_TYPE_SETTING, "anything").build(); } - @Test public void testThatClientTypeSettingCannotBeChanged() { for (Settings settings : internalCluster().getInstances(Settings.class)) { assertThat(settings.get(Client.CLIENT_TYPE_SETTING), is("node")); } } - } diff --git a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index 95456fda901..9f16ade87e8 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -189,6 +189,6 @@ abstract class FailAndRetryMockTransport imp @Override public Map profileBoundAddresses() { - return Collections.EMPTY_MAP; + return Collections.emptyMap(); } } diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java index 22d5ba20e11..f4b29768b91 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java +++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; -import org.junit.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -75,7 +74,6 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase { return client; } - @Test public void testWithSniffing() throws Exception { TransportClient client = TransportClient.builder() .settings(Settings.builder() diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java index 144d79db35b..f01fdffd147 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java +++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java @@ -28,12 +28,10 @@ import org.elasticsearch.node.Node; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.transport.TransportService; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; @@ -41,8 +39,6 @@ import static org.hamcrest.Matchers.startsWith; @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 1.0) public class TransportClientIT extends ESIntegTestCase { - - @Test public void testPickingUpChangesInDiscoveryNode() { String nodeName = internalCluster().startNode(Settings.builder().put("node.data", false)); @@ -51,17 +47,18 @@ public class TransportClientIT extends ESIntegTestCase { } - @Test public void testNodeVersionIsUpdated() { TransportClient client = (TransportClient) internalCluster().client(); TransportClientNodesService nodeService = client.nodeService(); - Node node = nodeBuilder().data(false).settings(Settings.builder() + Node node = new Node(Settings.builder() .put(internalCluster().getDefaultSettings()) .put("path.home", createTempDir()) .put("node.name", "testNodeVersionIsUpdated") .put("http.enabled", false) + .put("node.data", false) + .put("cluster.name", "foobar") .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) // make sure we get what we set :) - .build()).clusterName("foobar").build(); + .build()); node.start(); try { TransportAddress transportAddress = node.injector().getInstance(TransportService.class).boundAddress().publishAddress(); @@ -85,14 +82,12 @@ public class TransportClientIT extends ESIntegTestCase { } } - @Test public void testThatTransportClientSettingIsSet() { TransportClient client = (TransportClient) internalCluster().client(); Settings settings = client.injector.getInstance(Settings.class); assertThat(settings.get(Client.CLIENT_TYPE_SETTING), is("transport")); } - @Test public void testThatTransportClientSettingCannotBeChanged() { Settings baseSettings = settingsBuilder().put(Client.CLIENT_TYPE_SETTING, "anything").put("path.home", createTempDir()).build(); try (TransportClient client = TransportClient.builder().settings(baseSettings).build()) { diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 98cef9783fc..093e46186b3 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -28,8 +28,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.*; -import org.junit.Test; +import org.elasticsearch.transport.BaseTransportResponseHandler; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportService; import java.io.Closeable; import java.util.Collections; @@ -39,7 +43,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; @@ -57,7 +63,7 @@ public class TransportClientNodesServiceTests extends ESTestCase { transport = new FailAndRetryMockTransport(getRandom()) { @Override public List getLocalAddresses() { - return Collections.EMPTY_LIST; + return Collections.emptyList(); } @Override @@ -89,9 +95,7 @@ public class TransportClientNodesServiceTests extends ESTestCase { } } - @Test public void testListenerFailures() throws InterruptedException { - int iters = iterations(10, 100); for (int i = 0; i () { + iteration.transportService.sendRequest(node, "action", new TestRequest(), TransportRequestOptions.EMPTY, new BaseTransportResponseHandler() { @Override public TestResponse newInstance() { return new TestResponse(); diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java index 020106ab867..b28fdba8c77 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java +++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java @@ -29,26 +29,22 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.transport.TransportService; -import org.junit.Test; import java.io.IOException; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @ClusterScope(scope = Scope.TEST, numClientNodes = 0) @TestLogging("discovery.zen:TRACE") public class TransportClientRetryIT extends ESIntegTestCase { - - @Test public void testRetry() throws IOException, ExecutionException, InterruptedException { - Iterable instances = internalCluster().getInstances(TransportService.class); TransportAddress[] addresses = new TransportAddress[internalCluster().size()]; int i = 0; diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java index f34f693af95..c4a3ecba839 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java @@ -20,18 +20,14 @@ package org.elasticsearch.cluster; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; public class ClusterHealthIT extends ESIntegTestCase { - - - @Test - public void simpleLocalHealthTest() { + public void testSimpleLocalHealth() { createIndex("test"); ensureGreen(); // master should thing it's green now. @@ -43,7 +39,6 @@ public class ClusterHealthIT extends ESIntegTestCase { } } - @Test public void testHealth() { logger.info("--> running cluster health on an index that does not exists"); ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth("test1").setWaitForYellowStatus().setTimeout("1s").execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index f938a78c76f..5ed45620a03 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -52,7 +52,6 @@ import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.Collection; @@ -137,7 +136,6 @@ public class ClusterInfoServiceIT extends ESIntegTestCase { MockTransportService.TestPlugin.class); } - @Test public void testClusterInfoServiceCollectsInformation() throws Exception { internalCluster().startNodesAsync(2, Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, "200ms").build()) @@ -187,7 +185,6 @@ public class ClusterInfoServiceIT extends ESIntegTestCase { } - @Test public void testClusterInfoServiceInformationClearOnError() throws InterruptedException, ExecutionException { internalCluster().startNodesAsync(2, // manually control publishing @@ -210,7 +207,7 @@ public class ClusterInfoServiceIT extends ESIntegTestCase { final Set blockedActions = newHashSet(NodesStatsAction.NAME, NodesStatsAction.NAME + "[n]", IndicesStatsAction.NAME, IndicesStatsAction.NAME + "[n]"); // drop all outgoing stats requests to force a timeout. for (DiscoveryNode node : internalTestCluster.clusterService().state().getNodes()) { - mockTransportService.addDelegate(node, new MockTransportService.DelegateTransport(mockTransportService.original()) { + mockTransportService.addDelegate(internalTestCluster.getInstance(TransportService.class, node.getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) { @Override public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java index aab7e89dfc9..9e842a38722 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java @@ -37,19 +37,21 @@ import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockLogAppender; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.ThreadPool; -import org.junit.Test; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.*; @@ -65,7 +67,6 @@ public class ClusterServiceIT extends ESIntegTestCase { return pluginList(TestPlugin.class); } - @Test public void testTimeoutUpdateTask() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") @@ -134,7 +135,6 @@ public class ClusterServiceIT extends ESIntegTestCase { assertThat(executeCalled.get(), equalTo(false)); } - @Test public void testAckedUpdateTask() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") @@ -211,7 +211,6 @@ public class ClusterServiceIT extends ESIntegTestCase { assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true)); } - @Test public void testAckedUpdateTaskSameClusterState() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") @@ -283,7 +282,6 @@ public class ClusterServiceIT extends ESIntegTestCase { assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true)); } - @Test public void testMasterAwareExecution() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") @@ -340,7 +338,6 @@ public class ClusterServiceIT extends ESIntegTestCase { assertFalse("non-master cluster state update task was not executed", taskFailed[0]); } - @Test public void testAckedUpdateTaskNoAckExpected() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") @@ -413,7 +410,6 @@ public class ClusterServiceIT extends ESIntegTestCase { assertThat(onFailure.get(), equalTo(false)); } - @Test public void testAckedUpdateTaskTimeoutZero() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") @@ -490,7 +486,6 @@ public class ClusterServiceIT extends ESIntegTestCase { assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true)); } - @Test public void testPendingUpdateTask() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") @@ -626,7 +621,6 @@ public class ClusterServiceIT extends ESIntegTestCase { block2.countDown(); } - @Test public void testLocalNodeMasterListenerCallbacks() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "zen") @@ -705,40 +699,154 @@ public class ClusterServiceIT extends ESIntegTestCase { /** * Note, this test can only work as long as we have a single thread executor executing the state update tasks! */ - @Test public void testPrioritizedTasks() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "local") .build(); internalCluster().startNode(settings); ClusterService clusterService = internalCluster().getInstance(ClusterService.class); - BlockingTask block = new BlockingTask(); - clusterService.submitStateUpdateTask("test", Priority.IMMEDIATE, block); + BlockingTask block = new BlockingTask(Priority.IMMEDIATE); + clusterService.submitStateUpdateTask("test", block); int taskCount = randomIntBetween(5, 20); Priority[] priorities = Priority.values(); // will hold all the tasks in the order in which they were executed - List tasks = new ArrayList<>(taskCount); + List tasks = new ArrayList<>(taskCount); CountDownLatch latch = new CountDownLatch(taskCount); for (int i = 0; i < taskCount; i++) { Priority priority = priorities[randomIntBetween(0, priorities.length - 1)]; - clusterService.submitStateUpdateTask("test", priority, new PrioritiezedTask(priority, latch, tasks)); + clusterService.submitStateUpdateTask("test", new PrioritizedTask(priority, latch, tasks)); } block.release(); latch.await(); Priority prevPriority = null; - for (PrioritiezedTask task : tasks) { + for (PrioritizedTask task : tasks) { if (prevPriority == null) { - prevPriority = task.priority; + prevPriority = task.priority(); } else { - assertThat(task.priority.sameOrAfter(prevPriority), is(true)); + assertThat(task.priority().sameOrAfter(prevPriority), is(true)); } } } - @Test + public void testClusterStateBatchedUpdates() throws InterruptedException { + Settings settings = settingsBuilder() + .put("discovery.type", "local") + .build(); + internalCluster().startNode(settings); + ClusterService clusterService = internalCluster().getInstance(ClusterService.class); + + AtomicInteger counter = new AtomicInteger(); + class Task { + private AtomicBoolean state = new AtomicBoolean(); + + public void execute() { + if (!state.compareAndSet(false, true)) { + throw new IllegalStateException(); + } else { + counter.incrementAndGet(); + } + } + } + + class TaskExecutor implements ClusterStateTaskExecutor { + private AtomicInteger counter = new AtomicInteger(); + + @Override + public BatchResult execute(ClusterState currentState, List tasks) throws Exception { + tasks.forEach(task -> task.execute()); + counter.addAndGet(tasks.size()); + return BatchResult.builder().successes(tasks).build(currentState); + } + + @Override + public boolean runOnlyOnMaster() { + return false; + } + } + int numberOfThreads = randomIntBetween(2, 8); + int tasksSubmittedPerThread = randomIntBetween(1, 1024); + + ConcurrentMap counters = new ConcurrentHashMap<>(); + CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread); + ClusterStateTaskListener listener = new ClusterStateTaskListener() { + @Override + public void onFailure(String source, Throwable t) { + assert false; + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + counters.computeIfAbsent(source, key -> new AtomicInteger()).incrementAndGet(); + updateLatch.countDown(); + } + }; + + int numberOfExecutors = Math.max(1, numberOfThreads / 4); + List executors = new ArrayList<>(); + for (int i = 0; i < numberOfExecutors; i++) { + executors.add(new TaskExecutor()); + } + + // randomly assign tasks to executors + List assignments = new ArrayList<>(); + for (int i = 0; i < numberOfThreads; i++) { + for (int j = 0; j < tasksSubmittedPerThread; j++) { + assignments.add(randomFrom(executors)); + } + } + + Map counts = new HashMap<>(); + for (TaskExecutor executor : assignments) { + counts.merge(executor, 1, (previous, one) -> previous + one); + } + + CountDownLatch startingGun = new CountDownLatch(1 + numberOfThreads); + List threads = new ArrayList<>(); + for (int i = 0; i < numberOfThreads; i++) { + final int index = i; + Thread thread = new Thread(() -> { + startingGun.countDown(); + for (int j = 0; j < tasksSubmittedPerThread; j++) { + ClusterStateTaskExecutor executor = assignments.get(index * tasksSubmittedPerThread + j); + clusterService.submitStateUpdateTask( + Thread.currentThread().getName(), + new Task(), + ClusterStateTaskConfig.build(randomFrom(Priority.values())), + executor, + listener); + } + }); + threads.add(thread); + thread.start(); + } + + startingGun.countDown(); + for (Thread thread : threads) { + thread.join(); + } + + // wait until all the cluster state updates have been processed + updateLatch.await(); + + // assert the number of executed tasks is correct + assertEquals(numberOfThreads * tasksSubmittedPerThread, counter.get()); + + // assert each executor executed the correct number of tasks + for (TaskExecutor executor : executors) { + if (counts.containsKey(executor)) { + assertEquals((int) counts.get(executor), executor.counter.get()); + } + } + + // assert the correct number of clusterStateProcessed events were triggered + for (Map.Entry entry : counters.entrySet()) { + assertEquals(entry.getValue().get(), tasksSubmittedPerThread); + } + } + @TestLogging("cluster:TRACE") // To ensure that we log cluster state events on TRACE level public void testClusterStateUpdateLogging() throws Exception { Settings settings = settingsBuilder() @@ -828,7 +936,6 @@ public class ClusterServiceIT extends ESIntegTestCase { mockAppender.assertAllExpectationsMatched(); } - @Test @TestLogging("cluster:WARN") // To ensure that we log cluster state events on WARN level public void testLongClusterStateUpdateLogging() throws Exception { Settings settings = settingsBuilder() @@ -950,6 +1057,10 @@ public class ClusterServiceIT extends ESIntegTestCase { private static class BlockingTask extends ClusterStateUpdateTask { private final CountDownLatch latch = new CountDownLatch(1); + public BlockingTask(Priority priority) { + super(priority); + } + @Override public ClusterState execute(ClusterState currentState) throws Exception { latch.await(); @@ -966,14 +1077,13 @@ public class ClusterServiceIT extends ESIntegTestCase { } - private static class PrioritiezedTask extends ClusterStateUpdateTask { + private static class PrioritizedTask extends ClusterStateUpdateTask { - private final Priority priority; private final CountDownLatch latch; - private final List tasks; + private final List tasks; - private PrioritiezedTask(Priority priority, CountDownLatch latch, List tasks) { - this.priority = priority; + private PrioritizedTask(Priority priority, CountDownLatch latch, List tasks) { + super(priority); this.latch = latch; this.tasks = tasks; } diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index ac2182c14db..8d4540aad3b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -20,25 +20,13 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.elasticsearch.Version; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.RepositoriesMetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.cluster.metadata.*; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -52,7 +40,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collections; import java.util.List; @@ -70,8 +57,6 @@ import static org.hamcrest.Matchers.is; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0) public class ClusterStateDiffIT extends ESIntegTestCase { - - @Test public void testClusterStateDiffSerialization() throws Exception { DiscoveryNode masterNode = new DiscoveryNode("master", new LocalTransportAddress("master"), Version.CURRENT); DiscoveryNode otherNode = new DiscoveryNode("other", new LocalTransportAddress("other"), Version.CURRENT); @@ -510,17 +495,17 @@ public class ClusterStateDiffIT extends ESIntegTestCase { IndexMetaData.Builder builder = IndexMetaData.builder(part); switch (randomIntBetween(0, 3)) { case 0: - builder.settings(Settings.builder().put(part.settings()).put(randomSettings(Settings.EMPTY))); + builder.settings(Settings.builder().put(part.getSettings()).put(randomSettings(Settings.EMPTY))); break; case 1: - if (randomBoolean() && part.aliases().isEmpty() == false) { - builder.removeAlias(randomFrom(part.aliases().keys().toArray(String.class))); + if (randomBoolean() && part.getAliases().isEmpty() == false) { + builder.removeAlias(randomFrom(part.getAliases().keys().toArray(String.class))); } else { builder.putAlias(AliasMetaData.builder(randomAsciiOfLength(10))); } break; case 2: - builder.settings(Settings.builder().put(part.settings()).put(IndexMetaData.SETTING_INDEX_UUID, Strings.randomBase64UUID())); + builder.settings(Settings.builder().put(part.getSettings()).put(IndexMetaData.SETTING_INDEX_UUID, Strings.randomBase64UUID())); break; case 3: builder.putCustom(IndexWarmersMetaData.TYPE, randomWarmers()); @@ -543,7 +528,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase { randomName("warm"), new String[]{randomName("type")}, randomBoolean(), - new BytesArray(randomAsciiOfLength(1000))) + new IndexWarmersMetaData.SearchSource(new BytesArray(randomAsciiOfLength(1000)))) ); } else { return new IndexWarmersMetaData(); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 19f90f2962c..9ee095e6106 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.cluster; -import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index 595dbc9a94f..98eea13e673 100644 --- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -33,16 +33,13 @@ import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.nio.file.Path; import static org.hamcrest.Matchers.equalTo; public class DiskUsageTests extends ESTestCase { - - @Test - public void diskUsageCalcTest() { + public void testDiskUsageCalc() { DiskUsage du = new DiskUsage("node1", "n1", "random", 100, 40); assertThat(du.getFreeDiskAsPercentage(), equalTo(40.0)); assertThat(du.getUsedDiskAsPercentage(), equalTo(100.0 - 40.0)); @@ -71,8 +68,7 @@ public class DiskUsageTests extends ESTestCase { assertThat(du4.getTotalBytes(), equalTo(0L)); } - @Test - public void randomDiskUsageTest() { + public void testRandomDiskUsage() { int iters = scaledRandomIntBetween(1000, 10000); for (int i = 1; i < iters; i++) { long total = between(Integer.MIN_VALUE, Integer.MAX_VALUE); @@ -145,11 +141,11 @@ public class DiskUsageTests extends ESTestCase { }; NodeStats[] nodeStats = new NodeStats[] { new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null), + null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null), new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null), + null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null), new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null) + null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null) }; InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages); DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1"); diff --git a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index 9ca3f175670..648356be173 100644 --- a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -34,10 +34,10 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.disruption.NetworkDelaysPartition; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; -import org.junit.Test; import java.util.*; import java.util.concurrent.CountDownLatch; @@ -47,7 +47,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.*; @@ -62,9 +61,8 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { return classes; } - @Test @TestLogging("cluster.service:TRACE,discovery.zen:TRACE,gateway:TRACE,transport.tracer:TRACE") - public void simpleMinimumMasterNodes() throws Exception { + public void testSimpleMinimumMasterNodes() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "zen") @@ -109,7 +107,7 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { logger.info("--> verify we the data back"); for (int i = 0; i < 10; i++) { - assertThat(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(100l)); + assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(100l)); } internalCluster().stopCurrentMasterNode(); @@ -140,7 +138,7 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { logger.info("--> verify we the data back after cluster reform"); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); + assertHitCount(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); } internalCluster().stopRandomNonMasterNode(); @@ -173,12 +171,11 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { logger.info("--> verify we the data back"); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); + assertHitCount(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); } } - @Test - public void multipleNodesShutdownNonMasterNodes() throws Exception { + public void testMultipleNodesShutdownNonMasterNodes() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "zen") .put("discovery.zen.minimum_master_nodes", 3) @@ -226,7 +223,7 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { refresh(); logger.info("--> verify we the data back"); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); + assertHitCount(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); } internalCluster().stopRandomNonMasterNode(); @@ -250,12 +247,11 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { logger.info("--> verify we the data back"); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); + assertHitCount(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 100); } } - @Test - public void dynamicUpdateMinimumMasterNodes() throws Exception { + public void testDynamicUpdateMinimumMasterNodes() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "zen") .put(ZenDiscovery.SETTING_PING_TIMEOUT, "400ms") @@ -312,7 +308,6 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { ); } - @Test public void testCanNotBringClusterDown() throws ExecutionException, InterruptedException { int nodeCount = scaledRandomIntBetween(1, 5); Settings.Builder settings = settingsBuilder() @@ -360,6 +355,7 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { .put(DiscoverySettings.COMMIT_TIMEOUT, "100ms") // speed things up .build(); internalCluster().startNodesAsync(3, settings).get(); + ensureGreen(); // ensure cluster state is recovered before we disrupt things final String master = internalCluster().getMasterName(); Set otherNodes = new HashSet<>(Arrays.asList(internalCluster().getNodeNames())); diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java index fd8d6d6d155..e0f8b2cb840 100644 --- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.cluster; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.percolate.PercolateSourceBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -40,22 +39,23 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.util.HashMap; import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; /** */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0) @ESIntegTestCase.SuppressLocalMode public class NoMasterNodeIT extends ESIntegTestCase { - - @Test public void testNoMasterActions() throws Exception { // note, sometimes, we want to check with the fact that an index gets created, sometimes not... boolean autoCreateIndex = randomBoolean(); @@ -127,11 +127,11 @@ public class NoMasterNodeIT extends ESIntegTestCase { ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE ); - assertThrows(client().prepareCount("test"), + assertThrows(client().prepareSearch("test").setSize(0), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE ); - assertThrows(client().prepareCount("no_index"), + assertThrows(client().prepareSearch("no_index").setSize(0), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE ); @@ -212,8 +212,7 @@ public class NoMasterNodeIT extends ESIntegTestCase { } } - @Test - public void testNoMasterActions_writeMasterBlock() throws Exception { + public void testNoMasterActionsWriteMasterBlock() throws Exception { Settings settings = settingsBuilder() .put("discovery.type", "zen") .put("action.auto_create_index", false) @@ -248,13 +247,13 @@ public class NoMasterNodeIT extends ESIntegTestCase { GetResponse getResponse = client().prepareGet("test1", "type1", "1").get(); assertExists(getResponse); - CountResponse countResponse = client().prepareCount("test1").get(); + SearchResponse countResponse = client().prepareSearch("test1").setSize(0).get(); assertHitCount(countResponse, 1l); SearchResponse searchResponse = client().prepareSearch("test1").get(); assertHitCount(searchResponse, 1l); - countResponse = client().prepareCount("test2").get(); + countResponse = client().prepareSearch("test2").setSize(0).get(); assertThat(countResponse.getTotalShards(), equalTo(2)); assertThat(countResponse.getSuccessfulShards(), equalTo(1)); diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index 4088b693468..d78356cbf64 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -34,11 +34,12 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.CollectionAssertions; import org.junit.Before; -import org.junit.Test; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertIndexTemplateExists; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; /** * Checking simple filtering capabilites of the cluster state @@ -54,7 +55,6 @@ public class SimpleClusterStateIT extends ESIntegTestCase { refresh(); } - @Test public void testRoutingTable() throws Exception { ClusterStateResponse clusterStateResponseUnfiltered = client().admin().cluster().prepareState().clear().setRoutingTable(true).get(); assertThat(clusterStateResponseUnfiltered.getState().routingTable().hasIndex("foo"), is(true)); @@ -69,7 +69,6 @@ public class SimpleClusterStateIT extends ESIntegTestCase { assertThat(clusterStateResponse.getState().routingTable().hasIndex("non-existent"), is(false)); } - @Test public void testNodes() throws Exception { ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().clear().setNodes(true).get(); assertThat(clusterStateResponse.getState().nodes().nodes().size(), is(cluster().size())); @@ -78,7 +77,6 @@ public class SimpleClusterStateIT extends ESIntegTestCase { assertThat(clusterStateResponseFiltered.getState().nodes().nodes().size(), is(0)); } - @Test public void testMetadata() throws Exception { ClusterStateResponse clusterStateResponseUnfiltered = client().admin().cluster().prepareState().clear().setMetaData(true).get(); assertThat(clusterStateResponseUnfiltered.getState().metaData().indices().size(), is(3)); @@ -87,7 +85,6 @@ public class SimpleClusterStateIT extends ESIntegTestCase { assertThat(clusterStateResponse.getState().metaData().indices().size(), is(0)); } - @Test public void testIndexTemplates() throws Exception { client().admin().indices().preparePutTemplate("foo_template") .setTemplate("te*") @@ -113,7 +110,6 @@ public class SimpleClusterStateIT extends ESIntegTestCase { assertIndexTemplateExists(getIndexTemplatesResponse, "foo_template"); } - @Test public void testThatFilteringByIndexWorksForMetadataAndRoutingTable() throws Exception { ClusterStateResponse clusterStateResponseFiltered = client().admin().cluster().prepareState().clear() .setMetaData(true).setRoutingTable(true).setIndices("foo", "fuu", "non-existent").get(); @@ -129,7 +125,6 @@ public class SimpleClusterStateIT extends ESIntegTestCase { assertThat(clusterStateResponseFiltered.getState().routingTable().hasIndex("baz"), is(false)); } - @Test public void testLargeClusterStatePublishing() throws Exception { int estimatedBytesSize = scaledRandomIntBetween(ByteSizeValue.parseBytesSizeValue("10k", "estimatedBytesSize").bytesAsInt(), ByteSizeValue.parseBytesSizeValue("256k", "estimatedBytesSize").bytesAsInt()); @@ -162,7 +157,6 @@ public class SimpleClusterStateIT extends ESIntegTestCase { } } - @Test public void testIndicesOptions() throws Exception { ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("f*") .get(); @@ -172,14 +166,14 @@ public class SimpleClusterStateIT extends ESIntegTestCase { client().admin().indices().close(Requests.closeIndexRequest("fuu")).get(); clusterStateResponse = client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("f*").get(); assertThat(clusterStateResponse.getState().metaData().indices().size(), is(1)); - assertThat(clusterStateResponse.getState().metaData().index("foo").state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(clusterStateResponse.getState().metaData().index("foo").getState(), equalTo(IndexMetaData.State.OPEN)); // expand_wildcards_closed should toggle return only closed index fuu IndicesOptions expandCloseOptions = IndicesOptions.fromOptions(false, true, false, true); clusterStateResponse = client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("f*") .setIndicesOptions(expandCloseOptions).get(); assertThat(clusterStateResponse.getState().metaData().indices().size(), is(1)); - assertThat(clusterStateResponse.getState().metaData().index("fuu").state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(clusterStateResponse.getState().metaData().index("fuu").getState(), equalTo(IndexMetaData.State.CLOSE)); // ignore_unavailable set to true should not raise exception on fzzbzz IndicesOptions ignoreUnavailabe = IndicesOptions.fromOptions(true, true, true, false); @@ -195,17 +189,25 @@ public class SimpleClusterStateIT extends ESIntegTestCase { assertThat(clusterStateResponse.getState().metaData().indices().isEmpty(), is(true)); } - @Test(expected=IndexNotFoundException.class) public void testIndicesOptionsOnAllowNoIndicesFalse() throws Exception { // empty wildcard expansion throws exception when allowNoIndices is turned off IndicesOptions allowNoIndices = IndicesOptions.fromOptions(false, false, true, false); - client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("a*").setIndicesOptions(allowNoIndices).get(); + try { + client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("a*").setIndicesOptions(allowNoIndices).get(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test(expected=IndexNotFoundException.class) public void testIndicesIgnoreUnavailableFalse() throws Exception { // ignore_unavailable set to false throws exception when allowNoIndices is turned off IndicesOptions allowNoIndices = IndicesOptions.fromOptions(false, true, true, false); - client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("fzzbzz").setIndicesOptions(allowNoIndices).get(); + try { + client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("fzzbzz").setIndicesOptions(allowNoIndices).get(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } } diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java index 2979b1eac85..bc3aea4ac76 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java @@ -25,12 +25,11 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.common.Priority; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.client.Requests.createIndexRequest; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -import static org.elasticsearch.test.ESIntegTestCase.*; import static org.hamcrest.Matchers.equalTo; /** @@ -38,8 +37,6 @@ import static org.hamcrest.Matchers.equalTo; */ @ClusterScope(scope= Scope.TEST, numDataNodes =0) public class SimpleDataNodesIT extends ESIntegTestCase { - - @Test public void testDataNodes() throws Exception { internalCluster().startNode(settingsBuilder().put("node.data", false).build()); client().admin().indices().create(createIndexRequest("test")).actionGet(); diff --git a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index 43b403d306c..90c39d7bbec 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -25,24 +25,23 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.io.IOException; -import static org.elasticsearch.test.ESIntegTestCase.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) @ESIntegTestCase.SuppressLocalMode public class SpecificMasterNodesIT extends ESIntegTestCase { - protected final Settings.Builder settingsBuilder() { return Settings.builder().put("discovery.type", "zen"); } - @Test - public void simpleOnlyMasterNodeElection() throws IOException { + public void testSimpleOnlyMasterNodeElection() throws IOException { logger.info("--> start data node / non master node"); internalCluster().startNode(settingsBuilder().put("node.data", true).put("node.master", false).put("discovery.initial_state_timeout", "1s")); try { @@ -72,8 +71,7 @@ public class SpecificMasterNodesIT extends ESIntegTestCase { assertThat(internalCluster().masterClient().admin().cluster().prepareState().execute().actionGet().getState().nodes().masterNode().name(), equalTo(nextMasterEligibleNodeName)); } - @Test - public void electOnlyBetweenMasterNodes() throws IOException { + public void testElectOnlyBetweenMasterNodes() throws IOException { logger.info("--> start data node / non master node"); internalCluster().startNode(settingsBuilder().put("node.data", true).put("node.master", false).put("discovery.initial_state_timeout", "1s")); try { @@ -103,7 +101,6 @@ public class SpecificMasterNodesIT extends ESIntegTestCase { * Tests that putting custom default mapping and then putting a type mapping will have the default mapping merged * to the type mapping. */ - @Test public void testCustomDefaultMapping() throws Exception { logger.info("--> start master node / non data"); internalCluster().startNode(settingsBuilder().put("node.data", false).put("node.master", true)); @@ -124,7 +121,6 @@ public class SpecificMasterNodesIT extends ESIntegTestCase { assertThat(type1Mapping.getSourceAsMap().get("_timestamp"), notNullValue()); } - @Test public void testAliasFilterValidation() throws Exception { logger.info("--> start master node / non data"); internalCluster().startNode(settingsBuilder().put("node.data", false).put("node.master", true)); diff --git a/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java b/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java index c91915bd9cf..526f64a8b4e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java @@ -23,20 +23,17 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.common.Priority; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.util.List; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.*; import static org.hamcrest.Matchers.equalTo; /** */ @ClusterScope(scope= Scope.TEST, numDataNodes =0) public class UpdateSettingsValidationIT extends ESIntegTestCase { - - @Test public void testUpdateSettingsValidation() throws Exception { List nodes = internalCluster().startNodesAsync( settingsBuilder().put("node.data", false).build(), diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java index d6e4abb020e..81de8b1a43c 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java @@ -29,14 +29,14 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -51,6 +51,7 @@ public class AckClusterUpdateSettingsIT extends ESIntegTestCase { //make sure that enough concurrent reroutes can happen at the same time //we have a minimum of 2 nodes, and a maximum of 10 shards, thus 5 should be enough .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES, 5) + .put(ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE, 10) .build(); } @@ -71,7 +72,6 @@ public class AckClusterUpdateSettingsIT extends ESIntegTestCase { assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT, "0"))); } - @Test public void testClusterUpdateSettingsAcknowledgement() { createIndex("test"); ensureGreen(); @@ -112,7 +112,6 @@ public class AckClusterUpdateSettingsIT extends ESIntegTestCase { } } - @Test public void testClusterUpdateSettingsNoAcknowledgement() { client().admin().indices().prepareCreate("test") .setSettings(settingsBuilder() @@ -143,7 +142,6 @@ public class AckClusterUpdateSettingsIT extends ESIntegTestCase { return client.admin().cluster().prepareState().setLocal(true).get().getState(); } - @Test public void testOpenIndexNoAcknowledgement() { createIndex("test"); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java index dd482174b90..47517a753af 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.ack; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; @@ -35,6 +36,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetaData.State; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -44,16 +46,18 @@ import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import java.util.List; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; @ClusterScope(minNumDataNodes = 2) public class AckIT extends ESIntegTestCase { @@ -66,7 +70,6 @@ public class AckIT extends ESIntegTestCase { .put(DiscoverySettings.PUBLISH_TIMEOUT, 0).build(); } - @Test public void testUpdateSettingsAcknowledgement() { createIndex("test"); @@ -74,12 +77,11 @@ public class AckIT extends ESIntegTestCase { .setSettings(Settings.builder().put("refresh_interval", 9999, TimeUnit.MILLISECONDS))); for (Client client : clients()) { - String refreshInterval = getLocalClusterState(client).metaData().index("test").settings().get("index.refresh_interval"); + String refreshInterval = getLocalClusterState(client).metaData().index("test").getSettings().get("index.refresh_interval"); assertThat(refreshInterval, equalTo("9999ms")); } } - @Test public void testUpdateSettingsNoAcknowledgement() { createIndex("test"); UpdateSettingsResponse updateSettingsResponse = client().admin().indices().prepareUpdateSettings("test").setTimeout("0s") @@ -87,7 +89,6 @@ public class AckIT extends ESIntegTestCase { assertThat(updateSettingsResponse.isAcknowledged(), equalTo(false)); } - @Test public void testPutWarmerAcknowledgement() { createIndex("test"); // make sure one shard is started so the search during put warmer will not fail @@ -106,7 +107,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testPutWarmerNoAcknowledgement() throws InterruptedException { createIndex("test"); // make sure one shard is started so the search during put warmer will not fail @@ -131,7 +131,6 @@ public class AckIT extends ESIntegTestCase { assertAcked(client().admin().indices().prepareDeleteWarmer().setIndices("test").setNames("custom_warmer")); } - @Test public void testDeleteWarmerAcknowledgement() { createIndex("test"); index("test", "type", "1", "f", 1); @@ -147,7 +146,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testDeleteWarmerNoAcknowledgement() throws InterruptedException { createIndex("test"); index("test", "type", "1", "f", 1); @@ -168,7 +166,6 @@ public class AckIT extends ESIntegTestCase { })); } - @Test public void testClusterRerouteAcknowledgement() throws InterruptedException { assertAcked(prepareCreate("test").setSettings(Settings.builder() .put(indexSettings()) @@ -203,7 +200,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testClusterRerouteNoAcknowledgement() throws InterruptedException { client().admin().indices().prepareCreate("test") .setSettings(settingsBuilder() @@ -217,7 +213,6 @@ public class AckIT extends ESIntegTestCase { assertThat(clusterRerouteResponse.isAcknowledged(), equalTo(false)); } - @Test public void testClusterRerouteAcknowledgementDryRun() throws InterruptedException { client().admin().indices().prepareCreate("test") .setSettings(settingsBuilder() @@ -250,7 +245,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testClusterRerouteNoAcknowledgementDryRun() throws InterruptedException { client().admin().indices().prepareCreate("test") .setSettings(settingsBuilder() @@ -293,7 +287,6 @@ public class AckIT extends ESIntegTestCase { return new MoveAllocationCommand(shardToBeMoved.shardId(), fromNodeId, toNodeId); } - @Test public void testIndicesAliasesAcknowledgement() { createIndex("test"); @@ -310,7 +303,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testIndicesAliasesNoAcknowledgement() { createIndex("test"); @@ -330,7 +322,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testCloseIndexNoAcknowledgement() { createIndex("test"); ensureGreen(); @@ -339,7 +330,6 @@ public class AckIT extends ESIntegTestCase { assertThat(closeIndexResponse.isAcknowledged(), equalTo(false)); } - @Test public void testOpenIndexAcknowledgement() { createIndex("test"); ensureGreen(); @@ -354,7 +344,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testPutMappingAcknowledgement() { createIndex("test"); ensureGreen(); @@ -366,7 +355,6 @@ public class AckIT extends ESIntegTestCase { } } - @Test public void testPutMappingNoAcknowledgement() { createIndex("test"); ensureGreen(); @@ -375,7 +363,6 @@ public class AckIT extends ESIntegTestCase { assertThat(putMappingResponse.isAcknowledged(), equalTo(false)); } - @Test public void testCreateIndexAcknowledgement() { createIndex("test"); @@ -388,7 +375,6 @@ public class AckIT extends ESIntegTestCase { ensureGreen(); } - @Test public void testCreateIndexNoAcknowledgement() { CreateIndexResponse createIndexResponse = client().admin().indices().prepareCreate("test").setTimeout("0s").get(); assertThat(createIndexResponse.isAcknowledged(), equalTo(false)); diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java new file mode 100644 index 00000000000..96eea881e9e --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -0,0 +1,190 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.action.shard; + +import org.apache.lucene.index.CorruptIndexException; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardsIterator; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.cluster.TestClusterService; +import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ReceiveTimeoutTransportException; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportService; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithStartedPrimary; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ShardStateActionTests extends ESTestCase { + private static ThreadPool THREAD_POOL; + + private ShardStateAction shardStateAction; + private CapturingTransport transport; + private TransportService transportService; + private TestClusterService clusterService; + + @BeforeClass + public static void startThreadPool() { + THREAD_POOL = new ThreadPool("ShardStateActionTest"); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + this.transport = new CapturingTransport(); + clusterService = new TestClusterService(THREAD_POOL); + transportService = new TransportService(transport, THREAD_POOL); + transportService.start(); + shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null); + } + + @Override + @After + public void tearDown() throws Exception { + transportService.stop(); + super.tearDown(); + } + + @AfterClass + public static void stopThreadPool() { + ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS); + THREAD_POOL = null; + } + + public void testNoMaster() { + final String index = "test"; + + clusterService.setState(stateWithStartedPrimary(index, true, randomInt(5))); + + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(clusterService.state().nodes()); + builder.masterNodeId(null); + clusterService.setState(ClusterState.builder(clusterService.state()).nodes(builder)); + + String indexUUID = clusterService.state().metaData().index(index).getIndexUUID(); + + AtomicBoolean noMaster = new AtomicBoolean(); + assert !noMaster.get(); + + shardStateAction.shardFailed(getRandomShardRouting(index), indexUUID, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + @Override + public void onShardFailedNoMaster() { + noMaster.set(true); + } + + @Override + public void onShardFailedFailure(DiscoveryNode master, TransportException e) { + + } + }); + + assertTrue(noMaster.get()); + } + + public void testFailure() { + final String index = "test"; + + clusterService.setState(stateWithStartedPrimary(index, true, randomInt(5))); + + String indexUUID = clusterService.state().metaData().index(index).getIndexUUID(); + + AtomicBoolean failure = new AtomicBoolean(); + assert !failure.get(); + + shardStateAction.shardFailed(getRandomShardRouting(index), indexUUID, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + @Override + public void onShardFailedNoMaster() { + + } + + @Override + public void onShardFailedFailure(DiscoveryNode master, TransportException e) { + failure.set(true); + } + }); + + final CapturingTransport.CapturedRequest[] capturedRequests = transport.capturedRequests(); + transport.clear(); + assertThat(capturedRequests.length, equalTo(1)); + assert !failure.get(); + transport.handleResponse(capturedRequests[0].requestId, new TransportException("simulated")); + + assertTrue(failure.get()); + } + + public void testTimeout() throws InterruptedException { + final String index = "test"; + + clusterService.setState(stateWithStartedPrimary(index, true, randomInt(5))); + + String indexUUID = clusterService.state().metaData().index(index).getIndexUUID(); + + AtomicBoolean progress = new AtomicBoolean(); + AtomicBoolean timedOut = new AtomicBoolean(); + + TimeValue timeout = new TimeValue(1, TimeUnit.MILLISECONDS); + CountDownLatch latch = new CountDownLatch(1); + shardStateAction.shardFailed(getRandomShardRouting(index), indexUUID, "test", getSimulatedFailure(), timeout, new ShardStateAction.Listener() { + @Override + public void onShardFailedFailure(DiscoveryNode master, TransportException e) { + if (e instanceof ReceiveTimeoutTransportException) { + assertFalse(progress.get()); + timedOut.set(true); + } + latch.countDown(); + } + }); + + latch.await(); + progress.set(true); + assertTrue(timedOut.get()); + + final CapturingTransport.CapturedRequest[] capturedRequests = transport.capturedRequests(); + transport.clear(); + assertThat(capturedRequests.length, equalTo(1)); + } + + private ShardRouting getRandomShardRouting(String index) { + IndexRoutingTable indexRoutingTable = clusterService.state().routingTable().index(index); + ShardsIterator shardsIterator = indexRoutingTable.randomAllActiveShardsIt(); + ShardRouting shardRouting = shardsIterator.nextOrNull(); + assert shardRouting != null; + return shardRouting; + } + + private Throwable getSimulatedFailure() { + return new CorruptIndexException("simulated", (String) null); + } +} diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java index 8dbad73ff45..f9151628b8a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.allocation; import com.carrotsearch.hppc.ObjectIntHashMap; + import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -34,7 +35,6 @@ import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.List; import java.util.concurrent.TimeUnit; @@ -55,7 +55,6 @@ public class AwarenessAllocationIT extends ESIntegTestCase { return 1; } - @Test public void testSimpleAwareness() throws Exception { Settings commonSettings = Settings.settingsBuilder() .put("cluster.routing.allocation.awareness.attributes", "rack_id") @@ -104,8 +103,7 @@ public class AwarenessAllocationIT extends ESIntegTestCase { TimeUnit.SECONDS ), equalTo(true)); } - - @Test + public void testAwarenessZones() throws Exception { Settings commonSettings = Settings.settingsBuilder() .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP + "zone.values", "a,b") @@ -153,8 +151,7 @@ public class AwarenessAllocationIT extends ESIntegTestCase { assertThat(counts.get(A_0), anyOf(equalTo(2),equalTo(3))); assertThat(counts.get(B_0), anyOf(equalTo(2),equalTo(3))); } - - @Test + public void testAwarenessZonesIncrementalNodes() throws Exception { Settings commonSettings = Settings.settingsBuilder() .put("cluster.routing.allocation.awareness.force.zone.values", "a,b") @@ -208,7 +205,7 @@ public class AwarenessAllocationIT extends ESIntegTestCase { assertThat(counts.get(A_0), equalTo(5)); assertThat(counts.get(B_0), equalTo(3)); assertThat(counts.get(B_1), equalTo(2)); - + String noZoneNode = internalCluster().startNode(); health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("4").execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); @@ -227,7 +224,7 @@ public class AwarenessAllocationIT extends ESIntegTestCase { } } } - + assertThat(counts.get(A_0), equalTo(5)); assertThat(counts.get(B_0), equalTo(3)); assertThat(counts.get(B_1), equalTo(2)); @@ -248,7 +245,7 @@ public class AwarenessAllocationIT extends ESIntegTestCase { } } } - + assertThat(counts.get(A_0), equalTo(3)); assertThat(counts.get(B_0), equalTo(3)); assertThat(counts.get(B_1), equalTo(2)); diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java index 2df79586b86..1605e70637e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java @@ -21,7 +21,7 @@ package org.elasticsearch.cluster.allocation; import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -45,17 +45,19 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; -import org.junit.Test; import java.nio.file.Path; import java.util.Arrays; import java.util.List; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; @@ -65,11 +67,9 @@ import static org.hamcrest.Matchers.hasSize; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class ClusterRerouteIT extends ESIntegTestCase { - private final ESLogger logger = Loggers.getLogger(ClusterRerouteIT.class); - @Test - public void rerouteWithCommands_disableAllocationSettings() throws Exception { + public void testRerouteWithCommands_disableAllocationSettings() throws Exception { Settings commonSettings = settingsBuilder() .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none") .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, "none") @@ -77,8 +77,7 @@ public class ClusterRerouteIT extends ESIntegTestCase { rerouteWithCommands(commonSettings); } - @Test - public void rerouteWithCommands_enableAllocationSettings() throws Exception { + public void testRerouteWithCommands_enableAllocationSettings() throws Exception { Settings commonSettings = settingsBuilder() .put(CLUSTER_ROUTING_ALLOCATION_ENABLE, Allocation.NONE.name()) .build(); @@ -146,8 +145,7 @@ public class ClusterRerouteIT extends ESIntegTestCase { assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_2).id()).get(0).state(), equalTo(ShardRoutingState.STARTED)); } - @Test - public void rerouteWithAllocateLocalGateway_disableAllocationSettings() throws Exception { + public void testRerouteWithAllocateLocalGateway_disableAllocationSettings() throws Exception { Settings commonSettings = settingsBuilder() .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none") .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, "none") @@ -155,15 +153,13 @@ public class ClusterRerouteIT extends ESIntegTestCase { rerouteWithAllocateLocalGateway(commonSettings); } - @Test - public void rerouteWithAllocateLocalGateway_enableAllocationSettings() throws Exception { + public void testRerouteWithAllocateLocalGateway_enableAllocationSettings() throws Exception { Settings commonSettings = settingsBuilder() .put(CLUSTER_ROUTING_ALLOCATION_ENABLE, Allocation.NONE.name()) .build(); rerouteWithAllocateLocalGateway(commonSettings); } - @Test public void testDelayWithALargeAmountOfShards() throws Exception { Settings commonSettings = settingsBuilder() .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CONCURRENT_RECOVERIES, 1) @@ -264,8 +260,7 @@ public class ClusterRerouteIT extends ESIntegTestCase { } - @Test - public void rerouteExplain() { + public void testRerouteExplain() { Settings commonSettings = settingsBuilder().build(); logger.info("--> starting a node"); @@ -307,7 +302,6 @@ public class ClusterRerouteIT extends ESIntegTestCase { assertThat(explanation.decisions().type(), equalTo(Decision.Type.YES)); } - @Test public void testClusterRerouteWithBlocks() throws Exception { List nodesIds = internalCluster().startNodesAsync(2).get(); diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java index 80eaf296558..0c3eed1d532 100644 --- a/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java @@ -29,12 +29,11 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.util.List; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.*; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope= Scope.TEST, numDataNodes =0) @@ -42,14 +41,13 @@ public class FilteringAllocationIT extends ESIntegTestCase { private final ESLogger logger = Loggers.getLogger(FilteringAllocationIT.class); - @Test public void testDecommissionNodeNoReplicas() throws Exception { logger.info("--> starting 2 nodes"); List nodesIds = internalCluster().startNodesAsync(2).get(); final String node_0 = nodesIds.get(0); final String node_1 = nodesIds.get(1); assertThat(cluster().size(), equalTo(2)); - + logger.info("--> creating an index with no replicas"); client().admin().indices().prepareCreate("test") .setSettings(settingsBuilder().put("index.number_of_replicas", 0)) @@ -60,7 +58,7 @@ public class FilteringAllocationIT extends ESIntegTestCase { client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(100l)); + assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(100l)); logger.info("--> decommission the second node"); client().admin().cluster().prepareUpdateSettings() @@ -79,10 +77,9 @@ public class FilteringAllocationIT extends ESIntegTestCase { } client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(100l)); + assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(100l)); } - @Test public void testDisablingAllocationFiltering() throws Exception { logger.info("--> starting 2 nodes"); List nodesIds = internalCluster().startNodesAsync(2).get(); @@ -102,7 +99,7 @@ public class FilteringAllocationIT extends ESIntegTestCase { client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(100l)); + assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(100l)); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); IndexRoutingTable indexRoutingTable = clusterState.routingTable().index("test"); int numShardsOnNode1 = 0; @@ -118,7 +115,7 @@ public class FilteringAllocationIT extends ESIntegTestCase { client().admin().cluster().prepareUpdateSettings() .setTransientSettings(settingsBuilder().put("cluster.routing.allocation.node_concurrent_recoveries", numShardsOnNode1)).execute().actionGet(); // make sure we can recover all the nodes at once otherwise we might run into a state where one of the shards has not yet started relocating - // but we already fired up the request to wait for 0 relocating shards. + // but we already fired up the request to wait for 0 relocating shards. } logger.info("--> remove index from the first node"); client().admin().indices().prepareUpdateSettings("test") diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java index 1360937a013..565104b08eb 100644 --- a/core/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java @@ -18,16 +18,11 @@ */ package org.elasticsearch.cluster.allocation; -import org.elasticsearch.cluster.ClusterInfoService; -import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -45,10 +40,9 @@ public class SimpleAllocationIT extends ESIntegTestCase { } /** - * Test for + * Test for * https://groups.google.com/d/msg/elasticsearch/y-SY_HyoB-8/EZdfNt9VO44J */ - @Test public void testSaneAllocation() { assertAcked(prepareCreate("test", 3)); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java index 0cc7b70796a..daf715f385e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java @@ -24,16 +24,15 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.EnumSet; import static org.elasticsearch.test.VersionUtils.randomVersion; +import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; public class ClusterBlockTests extends ESTestCase { - - @Test public void testSerialization() throws Exception { int iterations = randomIntBetween(10, 100); for (int i = 0; i < iterations; i++) { @@ -66,4 +65,15 @@ public class ClusterBlockTests extends ESTestCase { assertArrayEquals(result.levels().toArray(), clusterBlock.levels().toArray()); } } + + public void testToStringDanglingComma() { + EnumSet levels = EnumSet.noneOf(ClusterBlockLevel.class); + int nbLevels = randomIntBetween(1, ClusterBlockLevel.values().length); + for (int j = 0; j < nbLevels; j++) { + levels.add(randomFrom(ClusterBlockLevel.values())); + } + ClusterBlock clusterBlock = new ClusterBlock(randomInt(), "cluster block #" + randomInt(), randomBoolean(), + randomBoolean(), randomFrom(RestStatus.values()), levels); + assertThat(clusterBlock.toString(), not(endsWith(","))); + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java new file mode 100644 index 00000000000..63e42d9078d --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.health; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.empty; + +public class ClusterIndexHealthTests extends ESTestCase { + public void testClusterIndexHealth() { + RoutingTableGenerator routingTableGenerator = new RoutingTableGenerator(); + int numberOfShards = randomInt(3) + 1; + int numberOfReplicas = randomInt(4); + IndexMetaData indexMetaData = IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas(numberOfReplicas).build(); + RoutingTableGenerator.ShardCounter counter = new RoutingTableGenerator.ShardCounter(); + IndexRoutingTable indexRoutingTable = routingTableGenerator.genIndexRoutingTable(indexMetaData, counter); + + ClusterIndexHealth indexHealth = new ClusterIndexHealth(indexMetaData, indexRoutingTable); + logger.info("index status: {}, expected {}", indexHealth.getStatus(), counter.status()); + assertIndexHealth(indexHealth, counter, indexMetaData); + } + + + private void assertIndexHealth(ClusterIndexHealth indexHealth, RoutingTableGenerator.ShardCounter counter, IndexMetaData indexMetaData) { + assertThat(indexHealth.getStatus(), equalTo(counter.status())); + assertThat(indexHealth.getNumberOfShards(), equalTo(indexMetaData.getNumberOfShards())); + assertThat(indexHealth.getNumberOfReplicas(), equalTo(indexMetaData.getNumberOfReplicas())); + assertThat(indexHealth.getActiveShards(), equalTo(counter.active)); + assertThat(indexHealth.getRelocatingShards(), equalTo(counter.relocating)); + assertThat(indexHealth.getInitializingShards(), equalTo(counter.initializing)); + assertThat(indexHealth.getUnassignedShards(), equalTo(counter.unassigned)); + assertThat(indexHealth.getShards().size(), equalTo(indexMetaData.getNumberOfShards())); + assertThat(indexHealth.getValidationFailures(), empty()); + int totalShards = 0; + for (ClusterShardHealth shardHealth : indexHealth.getShards().values()) { + totalShards += shardHealth.getActiveShards() + shardHealth.getInitializingShards() + shardHealth.getUnassignedShards(); + } + + assertThat(totalShards, equalTo(indexMetaData.getNumberOfShards() * (1 + indexMetaData.getNumberOfReplicas()))); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java new file mode 100644 index 00000000000..a4d5a6f4a75 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.health; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.*; + +public class ClusterStateHealthTests extends ESTestCase { + private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); + + public void testClusterHealth() throws IOException { + RoutingTableGenerator routingTableGenerator = new RoutingTableGenerator(); + RoutingTableGenerator.ShardCounter counter = new RoutingTableGenerator.ShardCounter(); + RoutingTable.Builder routingTable = RoutingTable.builder(); + MetaData.Builder metaData = MetaData.builder(); + for (int i = randomInt(4); i >= 0; i--) { + int numberOfShards = randomInt(3) + 1; + int numberOfReplicas = randomInt(4); + IndexMetaData indexMetaData = IndexMetaData + .builder("test_" + Integer.toString(i)) + .settings(settings(Version.CURRENT)) + .numberOfShards(numberOfShards) + .numberOfReplicas(numberOfReplicas) + .build(); + IndexRoutingTable indexRoutingTable = routingTableGenerator.genIndexRoutingTable(indexMetaData, counter); + metaData.put(indexMetaData, true); + routingTable.add(indexRoutingTable); + } + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable.build()).build(); + String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), (String[]) null); + ClusterStateHealth clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices); + logger.info("cluster status: {}, expected {}", clusterStateHealth.getStatus(), counter.status()); + clusterStateHealth = maybeSerialize(clusterStateHealth); + assertClusterHealth(clusterStateHealth, counter); + } + + public void testValidations() throws IOException { + RoutingTableGenerator routingTableGenerator = new RoutingTableGenerator(); + IndexMetaData indexMetaData = IndexMetaData + .builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(2) + .numberOfReplicas(2) + .build(); + RoutingTableGenerator.ShardCounter counter = new RoutingTableGenerator.ShardCounter(); + IndexRoutingTable indexRoutingTable = routingTableGenerator.genIndexRoutingTable(indexMetaData, counter); + indexMetaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(3).build(); + + ClusterIndexHealth indexHealth = new ClusterIndexHealth(indexMetaData, indexRoutingTable); + assertThat(indexHealth.getValidationFailures(), Matchers.hasSize(2)); + + RoutingTable.Builder routingTable = RoutingTable.builder(); + MetaData.Builder metaData = MetaData.builder(); + metaData.put(indexMetaData, true); + routingTable.add(indexRoutingTable); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable.build()).build(); + String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), (String[]) null); + ClusterStateHealth clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices); + clusterStateHealth = maybeSerialize(clusterStateHealth); + // currently we have no cluster level validation failures as index validation issues are reported per index. + assertThat(clusterStateHealth.getValidationFailures(), Matchers.hasSize(0)); + } + + + ClusterStateHealth maybeSerialize(ClusterStateHealth clusterStateHealth) throws IOException { + if (randomBoolean()) { + BytesStreamOutput out = new BytesStreamOutput(); + clusterStateHealth.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes()); + clusterStateHealth = ClusterStateHealth.readClusterHealth(in); + } + return clusterStateHealth; + } + + private void assertClusterHealth(ClusterStateHealth clusterStateHealth, RoutingTableGenerator.ShardCounter counter) { + assertThat(clusterStateHealth.getStatus(), equalTo(counter.status())); + assertThat(clusterStateHealth.getActiveShards(), equalTo(counter.active)); + assertThat(clusterStateHealth.getActivePrimaryShards(), equalTo(counter.primaryActive)); + assertThat(clusterStateHealth.getInitializingShards(), equalTo(counter.initializing)); + assertThat(clusterStateHealth.getRelocatingShards(), equalTo(counter.relocating)); + assertThat(clusterStateHealth.getUnassignedShards(), equalTo(counter.unassigned)); + assertThat(clusterStateHealth.getValidationFailures(), empty()); + assertThat(clusterStateHealth.getActiveShardsPercent(), is(allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0)))); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/cluster/health/RoutingTableGenerator.java b/core/src/test/java/org/elasticsearch/cluster/health/RoutingTableGenerator.java new file mode 100644 index 00000000000..730763372ca --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/health/RoutingTableGenerator.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.health; + +import com.carrotsearch.randomizedtesting.RandomizedContext; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.index.shard.ShardId; + +class RoutingTableGenerator { + private static int node_id = 1; + + private ShardRouting genShardRouting(String index, int shardId, boolean primary) { + + ShardRoutingState state; + + int stateRandomizer = RandomizedContext.current().getRandom().nextInt(40); + if (stateRandomizer > 5) { + state = ShardRoutingState.STARTED; + } else if (stateRandomizer > 3) { + state = ShardRoutingState.RELOCATING; + } else { + state = ShardRoutingState.INITIALIZING; + } + + switch (state) { + case STARTED: + return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, primary, ShardRoutingState.STARTED, 1); + case INITIALIZING: + return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, primary, ShardRoutingState.INITIALIZING, 1); + case RELOCATING: + return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), "node_" + Integer.toString(node_id++), null, primary, ShardRoutingState.RELOCATING, 1); + default: + throw new ElasticsearchException("Unknown state: " + state.name()); + } + + } + + public IndexShardRoutingTable genShardRoutingTable(String index, int shardId, int replicas, ShardCounter counter) { + IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(new ShardId(index, shardId)); + ShardRouting shardRouting = genShardRouting(index, shardId, true); + counter.update(shardRouting); + builder.addShard(shardRouting); + for (; replicas > 0; replicas--) { + shardRouting = genShardRouting(index, shardId, false); + counter.update(shardRouting); + builder.addShard(shardRouting); + } + + return builder.build(); + } + + public IndexRoutingTable genIndexRoutingTable(IndexMetaData indexMetaData, ShardCounter counter) { + IndexRoutingTable.Builder builder = IndexRoutingTable.builder(indexMetaData.getIndex()); + for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) { + builder.addIndexShard(genShardRoutingTable(indexMetaData.getIndex(), shard, indexMetaData.getNumberOfReplicas(), counter)); + } + return builder.build(); + } + + static class ShardCounter { + public int active; + public int relocating; + public int initializing; + public int unassigned; + public int primaryActive; + public int primaryInactive; + + public ClusterHealthStatus status() { + if (primaryInactive > 0) { + return ClusterHealthStatus.RED; + } + if (unassigned > 0 || initializing > 0) { + return ClusterHealthStatus.YELLOW; + } + return ClusterHealthStatus.GREEN; + } + + public void update(ShardRouting shardRouting) { + if (shardRouting.active()) { + active++; + if (shardRouting.primary()) { + primaryActive++; + } + if (shardRouting.relocating()) { + relocating++; + } + return; + } + + if (shardRouting.primary()) { + primaryInactive++; + } + if (shardRouting.initializing()) { + initializing++; + } else { + unassigned++; + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java index d63319386b4..0215f94d4da 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java @@ -30,13 +30,13 @@ import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.joda.time.DateTimeZone.UTC; @@ -176,24 +176,44 @@ public class DateMathExpressionResolverTests extends ESTestCase { assertThat(results.get(0), equalTo(".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.withZone(timeZone)))); } - @Test(expected = ElasticsearchParseException.class) - public void testExpression_Invalid_Unescaped() throws Exception { - expressionResolver.resolve(context, Arrays.asList("<.mar}vel-{now/d}>")); + public void testExpressionInvalidUnescaped() throws Exception { + try { + expressionResolver.resolve(context, Arrays.asList("<.mar}vel-{now/d}>")); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("invalid dynamic name expression")); + assertThat(e.getMessage(), containsString("invalid character at position [")); + } } - @Test(expected = ElasticsearchParseException.class) - public void testExpression_Invalid_DateMathFormat() throws Exception { - expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}>")); + public void testExpressionInvalidDateMathFormat() throws Exception { + try { + expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}>")); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("invalid dynamic name expression")); + assertThat(e.getMessage(), containsString("date math placeholder is open ended")); + } } - @Test(expected = ElasticsearchParseException.class) - public void testExpression_Invalid_EmptyDateMathFormat() throws Exception { - expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}}>")); + public void testExpressionInvalidEmptyDateMathFormat() throws Exception { + try { + expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}}>")); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("invalid dynamic name expression")); + assertThat(e.getMessage(), containsString("missing date format")); + } } - @Test(expected = ElasticsearchParseException.class) - public void testExpression_Invalid_OpenEnded() throws Exception { - expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d>")); + public void testExpressionInvalidOpenEnded() throws Exception { + try { + expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d>")); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("invalid dynamic name expression")); + assertThat(e.getMessage(), containsString("date math placeholder is open ended")); + } } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java index 4163ad05d60..9be087e0e5d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java @@ -24,13 +24,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.junit.Test; import static org.elasticsearch.test.VersionUtils.randomVersion; public class HumanReadableIndexSettingsTests extends ESTestCase { - - @Test public void testHumanReadableSettings() { Version versionCreated = randomVersion(random()); Version versionUpgraded = randomVersion(random()); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 3bae8e158d8..7b8eb2ebc51 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.Collections; @@ -42,16 +41,15 @@ import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; /** */ public class IndexNameExpressionResolverTests extends ESTestCase { - private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); - @Test - public void testIndexOptions_strict() { + public void testIndexOptionsStrict() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("foo").putAlias(AliasMetaData.builder("foofoobar"))) .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("foofoobar"))) @@ -79,7 +77,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(results, arrayContainingInAnyOrder("foofoo", "foobar")); results = indexNameExpressionResolver.concreteIndices(context, "foofoobar"); - assertEquals(new HashSet<>(Arrays.asList("foo", "foobar")), + assertEquals(new HashSet<>(Arrays.asList("foo", "foobar")), new HashSet<>(Arrays.asList(results))); try { @@ -140,8 +138,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo", "foofoo-closed")); } - @Test - public void testIndexOptions_lenient() { + public void testIndexOptionsLenient() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("foo").putAlias(AliasMetaData.builder("foofoobar"))) .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("foofoobar"))) @@ -166,7 +163,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { results = indexNameExpressionResolver.concreteIndices(context, "foofoobar"); assertEquals(2, results.length); - assertEquals(new HashSet<>(Arrays.asList("foo", "foobar")), + assertEquals(new HashSet<>(Arrays.asList("foo", "foobar")), new HashSet<>(Arrays.asList(results))); results = indexNameExpressionResolver.concreteIndices(context, "foo", "bar"); @@ -208,8 +205,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo", "foofoo-closed")); } - @Test - public void testIndexOptions_allowUnavailableDisallowEmpty() { + public void testIndexOptionsAllowUnavailableDisallowEmpty() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("foo")) .put(indexBuilder("foobar")) @@ -258,8 +254,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertEquals(4, results.length); } - @Test - public void testIndexOptions_wildcardExpansion() { + public void testIndexOptionsWildcardExpansion() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("foo").state(IndexMetaData.State.CLOSE)) .put(indexBuilder("bar")) @@ -329,8 +324,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { } } - @Test - public void testIndexOptions_noExpandWildcards() { + public void testIndexOptionsNoExpandWildcards() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("foo").putAlias(AliasMetaData.builder("foofoobar"))) .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("foofoobar"))) @@ -423,8 +417,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { } } - @Test - public void testIndexOptions_singleIndexNoExpandWildcards() { + public void testIndexOptionsSingleIndexNoExpandWildcards() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("foo").putAlias(AliasMetaData.builder("foofoobar"))) .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("foofoobar"))) @@ -481,8 +474,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(results, arrayContainingInAnyOrder("foo", "foofoo")); } - @Test - public void testIndexOptions_emptyCluster() { + public void testIndexOptionsEmptyCluster() { ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(MetaData.builder().build()).build(); IndicesOptions options = IndicesOptions.strictExpandOpen(); @@ -527,7 +519,6 @@ public class IndexNameExpressionResolverTests extends ESTestCase { return IndexMetaData.builder(index).settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); } - @Test(expected = IndexNotFoundException.class) public void testConcreteIndicesIgnoreIndicesOneMissingIndex() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX")) @@ -535,10 +526,14 @@ public class IndexNameExpressionResolverTests extends ESTestCase { ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen()); - indexNameExpressionResolver.concreteIndices(context, "testZZZ"); + try { + indexNameExpressionResolver.concreteIndices(context, "testZZZ"); + fail("Expected IndexNotFoundException"); + } catch(IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test public void testConcreteIndicesIgnoreIndicesOneMissingIndexOtherFound() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX")) @@ -549,7 +544,6 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testXXX", "testZZZ")), equalTo(newHashSet("testXXX"))); } - @Test(expected = IndexNotFoundException.class) public void testConcreteIndicesIgnoreIndicesAllMissing() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX")) @@ -557,10 +551,14 @@ public class IndexNameExpressionResolverTests extends ESTestCase { ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen()); - assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testMo", "testMahdy")), equalTo(newHashSet("testXXX"))); + try { + indexNameExpressionResolver.concreteIndices(context, "testMo", "testMahdy"); + fail("Expected IndexNotFoundException"); + } catch(IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test public void testConcreteIndicesIgnoreIndicesEmptyRequest() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX")) @@ -570,7 +568,6 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, new String[]{})), equalTo(newHashSet("kuku", "testXXX"))); } - @Test public void testConcreteIndicesWildcardExpansion() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX").state(State.OPEN)) @@ -593,7 +590,6 @@ public class IndexNameExpressionResolverTests extends ESTestCase { /** * test resolving _all pattern (null, empty array or "_all") for random IndicesOptions */ - @Test public void testConcreteIndicesAllPatternRandom() { for (int i = 0; i < 10; i++) { String[] allIndices = null; @@ -660,7 +656,6 @@ public class IndexNameExpressionResolverTests extends ESTestCase { /** * test resolving wildcard pattern that matches no index of alias for random IndicesOptions */ - @Test public void testConcreteIndicesWildcardNoMatch() { for (int i = 0; i < 10; i++) { IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); @@ -687,92 +682,76 @@ public class IndexNameExpressionResolverTests extends ESTestCase { } } - @Test - public void testIsAllIndices_null() throws Exception { + public void testIsAllIndicesNull() throws Exception { assertThat(IndexNameExpressionResolver.isAllIndices(null), equalTo(true)); } - @Test - public void testIsAllIndices_empty() throws Exception { + public void testIsAllIndicesEmpty() throws Exception { assertThat(IndexNameExpressionResolver.isAllIndices(Collections.emptyList()), equalTo(true)); } - @Test - public void testIsAllIndices_explicitAll() throws Exception { + public void testIsAllIndicesExplicitAll() throws Exception { assertThat(IndexNameExpressionResolver.isAllIndices(Arrays.asList("_all")), equalTo(true)); } - @Test - public void testIsAllIndices_explicitAllPlusOther() throws Exception { + public void testIsAllIndicesExplicitAllPlusOther() throws Exception { assertThat(IndexNameExpressionResolver.isAllIndices(Arrays.asList("_all", "other")), equalTo(false)); } - @Test - public void testIsAllIndices_normalIndexes() throws Exception { + public void testIsAllIndicesNormalIndexes() throws Exception { assertThat(IndexNameExpressionResolver.isAllIndices(Arrays.asList("index1", "index2", "index3")), equalTo(false)); } - @Test - public void testIsAllIndices_wildcard() throws Exception { + public void testIsAllIndicesWildcard() throws Exception { assertThat(IndexNameExpressionResolver.isAllIndices(Arrays.asList("*")), equalTo(false)); } - @Test - public void testIsExplicitAllIndices_null() throws Exception { + public void testIsExplicitAllIndicesNull() throws Exception { assertThat(IndexNameExpressionResolver.isExplicitAllPattern(null), equalTo(false)); } - @Test - public void testIsExplicitAllIndices_empty() throws Exception { + public void testIsExplicitAllIndicesEmpty() throws Exception { assertThat(IndexNameExpressionResolver.isExplicitAllPattern(Collections.emptyList()), equalTo(false)); } - @Test - public void testIsExplicitAllIndices_explicitAll() throws Exception { + public void testIsExplicitAllIndicesExplicitAll() throws Exception { assertThat(IndexNameExpressionResolver.isExplicitAllPattern(Arrays.asList("_all")), equalTo(true)); } - @Test - public void testIsExplicitAllIndices_explicitAllPlusOther() throws Exception { + public void testIsExplicitAllIndicesExplicitAllPlusOther() throws Exception { assertThat(IndexNameExpressionResolver.isExplicitAllPattern(Arrays.asList("_all", "other")), equalTo(false)); } - @Test - public void testIsExplicitAllIndices_normalIndexes() throws Exception { + public void testIsExplicitAllIndicesNormalIndexes() throws Exception { assertThat(IndexNameExpressionResolver.isExplicitAllPattern(Arrays.asList("index1", "index2", "index3")), equalTo(false)); } - @Test - public void testIsExplicitAllIndices_wildcard() throws Exception { + public void testIsExplicitAllIndicesWildcard() throws Exception { assertThat(IndexNameExpressionResolver.isExplicitAllPattern(Arrays.asList("*")), equalTo(false)); } - @Test - public void testIsPatternMatchingAllIndices_explicitList() throws Exception { + public void testIsPatternMatchingAllIndicesExplicitList() throws Exception { //even though it does identify all indices, it's not a pattern but just an explicit list of them String[] concreteIndices = new String[]{"index1", "index2", "index3"}; MetaData metaData = metaDataBuilder(concreteIndices); assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, concreteIndices, concreteIndices), equalTo(false)); } - @Test - public void testIsPatternMatchingAllIndices_onlyWildcard() throws Exception { + public void testIsPatternMatchingAllIndicesOnlyWildcard() throws Exception { String[] indicesOrAliases = new String[]{"*"}; String[] concreteIndices = new String[]{"index1", "index2", "index3"}; MetaData metaData = metaDataBuilder(concreteIndices); assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true)); } - @Test - public void testIsPatternMatchingAllIndices_matchingTrailingWildcard() throws Exception { + public void testIsPatternMatchingAllIndicesMatchingTrailingWildcard() throws Exception { String[] indicesOrAliases = new String[]{"index*"}; String[] concreteIndices = new String[]{"index1", "index2", "index3"}; MetaData metaData = metaDataBuilder(concreteIndices); assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true)); } - @Test - public void testIsPatternMatchingAllIndices_nonMatchingTrailingWildcard() throws Exception { + public void testIsPatternMatchingAllIndicesNonMatchingTrailingWildcard() throws Exception { String[] indicesOrAliases = new String[]{"index*"}; String[] concreteIndices = new String[]{"index1", "index2", "index3"}; String[] allConcreteIndices = new String[]{"index1", "index2", "index3", "a", "b"}; @@ -780,16 +759,14 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(false)); } - @Test - public void testIsPatternMatchingAllIndices_matchingSingleExclusion() throws Exception { + public void testIsPatternMatchingAllIndicesMatchingSingleExclusion() throws Exception { String[] indicesOrAliases = new String[]{"-index1", "+index1"}; String[] concreteIndices = new String[]{"index1", "index2", "index3"}; MetaData metaData = metaDataBuilder(concreteIndices); assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true)); } - @Test - public void testIsPatternMatchingAllIndices_nonMatchingSingleExclusion() throws Exception { + public void testIsPatternMatchingAllIndicesNonMatchingSingleExclusion() throws Exception { String[] indicesOrAliases = new String[]{"-index1"}; String[] concreteIndices = new String[]{"index2", "index3"}; String[] allConcreteIndices = new String[]{"index1", "index2", "index3"}; @@ -797,16 +774,14 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(false)); } - @Test - public void testIsPatternMatchingAllIndices_matchingTrailingWildcardAndExclusion() throws Exception { + public void testIsPatternMatchingAllIndicesMatchingTrailingWildcardAndExclusion() throws Exception { String[] indicesOrAliases = new String[]{"index*", "-index1", "+index1"}; String[] concreteIndices = new String[]{"index1", "index2", "index3"}; MetaData metaData = metaDataBuilder(concreteIndices); assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true)); } - @Test - public void testIsPatternMatchingAllIndices_nonMatchingTrailingWildcardAndExclusion() throws Exception { + public void testIsPatternMatchingAllIndicesNonMatchingTrailingWildcardAndExclusion() throws Exception { String[] indicesOrAliases = new String[]{"index*", "-index1"}; String[] concreteIndices = new String[]{"index2", "index3"}; String[] allConcreteIndices = new String[]{"index1", "index2", "index3"}; @@ -814,8 +789,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(false)); } - @Test - public void testIndexOptions_failClosedIndicesAndAliases() { + public void testIndexOptionsFailClosedIndicesAndAliases() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("foo1-closed").state(IndexMetaData.State.CLOSE).putAlias(AliasMetaData.builder("foobar1-closed")).putAlias(AliasMetaData.builder("foobar2-closed"))) .put(indexBuilder("foo2-closed").state(IndexMetaData.State.CLOSE).putAlias(AliasMetaData.builder("foobar2-closed"))) @@ -873,6 +847,19 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(results, arrayContainingInAnyOrder("foo1-closed", "foo2-closed", "foo3")); } + public void testDedupConcreteIndices() { + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("index1").putAlias(AliasMetaData.builder("alias1"))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.strictExpandOpen(), IndicesOptions.strictExpand(), + IndicesOptions.lenientExpandOpen(), IndicesOptions.strictExpandOpenAndForbidClosed()}; + for (IndicesOptions options : indicesOptions) { + IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options); + String[] results = indexNameExpressionResolver.concreteIndices(context, "index1", "index1", "alias1"); + assertThat(results, equalTo(new String[]{"index1"})); + } + } + private MetaData metaDataBuilder(String... indices) { MetaData.Builder mdBuilder = MetaData.builder(); for (String concreteIndex : indices) { diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java index 2211b20ad95..e4462007c9a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java @@ -24,15 +24,12 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public class MappingMetaDataParserTests extends ESTestCase { - - @Test public void testParseIdAlone() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("id"), @@ -49,8 +46,7 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), nullValue()); assertThat(parseContext.timestampResolved(), equalTo(false)); } - - @Test + public void testFailIfIdIsNoValue() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("id"), @@ -65,7 +61,7 @@ public class MappingMetaDataParserTests extends ESTestCase { } catch (MapperParsingException ex) { // bogus its an array } - + bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("id").field("x", "id").endObject().field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes(); parseContext = md.createParseContext(null, "routing_value", "1"); @@ -77,7 +73,6 @@ public class MappingMetaDataParserTests extends ESTestCase { } } - @Test public void testParseRoutingAlone() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("id"), @@ -95,7 +90,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestampResolved(), equalTo(false)); } - @Test public void testParseTimestampAlone() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("id"), @@ -113,7 +107,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestampResolved(), equalTo(true)); } - @Test public void testParseTimestampEquals() throws Exception { MappingMetaData md1 = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("id"), @@ -126,7 +119,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(md1, equalTo(md2)); } - @Test public void testParseIdAndRoutingAndTimestamp() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("id"), @@ -141,7 +133,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), equalTo("1")); } - @Test public void testParseIdAndRoutingAndTimestampWithPath() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("obj1.id"), @@ -159,7 +150,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), equalTo("1")); } - @Test public void testParseIdWithPath() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("obj1.id"), @@ -180,7 +170,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestampResolved(), equalTo(false)); } - @Test public void testParseRoutingWithPath() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("obj1.id"), @@ -201,7 +190,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestampResolved(), equalTo(false)); } - @Test public void testParseTimestampWithPath() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("obj1.id"), @@ -222,7 +210,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestampResolved(), equalTo(true)); } - @Test public void testParseIdAndRoutingAndTimestampWithinSamePath() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("obj1.id"), @@ -240,7 +227,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), equalTo("1")); } - @Test public void testParseIdAndRoutingAndTimestampWithinSamePathAndMoreLevels() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("obj1.obj0.id"), @@ -268,8 +254,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), equalTo("1")); } - - @Test public void testParseIdAndRoutingAndTimestampWithSameRepeatedObject() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("obj1.id"), @@ -288,8 +272,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), equalTo("1")); } - // - @Test public void testParseIdRoutingTimestampWithRepeatedField() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("field1"), @@ -312,7 +294,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), equalTo("foo")); } - @Test public void testParseNoIdRoutingWithRepeatedFieldAndObject() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id("id"), @@ -335,7 +316,6 @@ public class MappingMetaDataParserTests extends ESTestCase { assertThat(parseContext.timestamp(), equalTo("foo")); } - @Test public void testParseRoutingWithRepeatedFieldAndValidRouting() throws Exception { MappingMetaData md = new MappingMetaData("type1", new CompressedXContent("{}"), new MappingMetaData.Id(null), diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataService.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java similarity index 51% rename from core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataService.java rename to core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index ca482ea604f..59116859322 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataService.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -19,30 +19,26 @@ package org.elasticsearch.cluster.metadata; -import org.elasticsearch.cluster.routing.Murmur3HashFunction; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.math.MathUtils; +import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; -import java.util.concurrent.Semaphore; +import static org.hamcrest.Matchers.equalTo; -/** - */ -public class MetaDataService extends AbstractComponent { +public class MetaDataTests extends ESTestCase { - private final Semaphore[] indexMdLocks; - - @Inject - public MetaDataService(Settings settings) { - super(settings); - indexMdLocks = new Semaphore[500]; - for (int i = 0; i < indexMdLocks.length; i++) { - indexMdLocks[i] = new Semaphore(1); + public void testIndexAndAliasWithSameName() { + IndexMetaData.Builder builder = IndexMetaData.builder("index") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder("index").build()); + try { + MetaData.builder().put(builder).build(); + fail("expection should have been thrown"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), equalTo("index and alias names need to be unique, but alias [index] and index [index] have the same name")); } } - public Semaphore indexMetaDataLock(String index) { - return indexMdLocks[MathUtils.mod(Murmur3HashFunction.hash(index), indexMdLocks.length)]; - } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index 4f8d98ed083..cbb5b7dfbdb 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -23,20 +23,19 @@ import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.cluster.metadata.AliasMetaData.newAliasMetaDataBuilder; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; /** * */ public class ToAndFromJsonMetaDataTests extends ESTestCase { - - @Test public void testSimpleJsonFromAndTo() throws IOException { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test1") @@ -151,142 +150,142 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { MetaData parsedMetaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(metaDataSource)); IndexMetaData indexMetaData = parsedMetaData.index("test1"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(-1l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(3)); - assertThat(indexMetaData.mappings().size(), equalTo(0)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(3)); + assertThat(indexMetaData.getMappings().size(), equalTo(0)); indexMetaData = parsedMetaData.index("test2"); - assertThat(indexMetaData.numberOfShards(), equalTo(2)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(3)); - assertThat(indexMetaData.creationDate(), equalTo(-1l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(0)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(2)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(3)); + assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(0)); indexMetaData = parsedMetaData.index("test3"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(-1l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(3)); - assertThat(indexMetaData.mappings().size(), equalTo(1)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(3)); + assertThat(indexMetaData.getMappings().size(), equalTo(1)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); indexMetaData = parsedMetaData.index("test4"); - assertThat(indexMetaData.creationDate(), equalTo(2l)); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4)); - assertThat(indexMetaData.mappings().size(), equalTo(0)); + assertThat(indexMetaData.getCreationDate(), equalTo(2l)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(4)); + assertThat(indexMetaData.getMappings().size(), equalTo(0)); indexMetaData = parsedMetaData.index("test5"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(-1l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(2)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); - assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(2)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); indexMetaData = parsedMetaData.index("test6"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(2l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(6)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(0)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(2l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(0)); indexMetaData = parsedMetaData.index("test7"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(2l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4)); - assertThat(indexMetaData.mappings().size(), equalTo(2)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); - assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(2l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(4)); + assertThat(indexMetaData.getMappings().size(), equalTo(2)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); indexMetaData = parsedMetaData.index("test8"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(-1l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(2)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); - assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); - assertThat(indexMetaData.aliases().size(), equalTo(2)); - assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1")); - assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2")); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(2)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); + assertThat(indexMetaData.getAliases().size(), equalTo(2)); + assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1")); + assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2")); indexMetaData = parsedMetaData.index("test9"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(2l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(6)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(2)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); - assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); - assertThat(indexMetaData.aliases().size(), equalTo(2)); - assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1")); - assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2")); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(2l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(2)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); + assertThat(indexMetaData.getAliases().size(), equalTo(2)); + assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1")); + assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2")); indexMetaData = parsedMetaData.index("test10"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(-1l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(2)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); - assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); - assertThat(indexMetaData.aliases().size(), equalTo(2)); - assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1")); - assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2")); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(2)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); + assertThat(indexMetaData.getAliases().size(), equalTo(2)); + assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1")); + assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2")); indexMetaData = parsedMetaData.index("test11"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(-1l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(5)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(2)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); - assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); - assertThat(indexMetaData.aliases().size(), equalTo(3)); - assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1")); - assertThat(indexMetaData.aliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1)); - assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2")); - assertThat(indexMetaData.aliases().get("alias2").filter(), nullValue()); - assertThat(indexMetaData.aliases().get("alias4").alias(), equalTo("alias4")); - assertThat(indexMetaData.aliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(2)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); + assertThat(indexMetaData.getAliases().size(), equalTo(3)); + assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1")); + assertThat(indexMetaData.getAliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1)); + assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2")); + assertThat(indexMetaData.getAliases().get("alias2").filter(), nullValue()); + assertThat(indexMetaData.getAliases().get("alias4").alias(), equalTo("alias4")); + assertThat(indexMetaData.getAliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2)); indexMetaData = parsedMetaData.index("test12"); - assertThat(indexMetaData.numberOfShards(), equalTo(1)); - assertThat(indexMetaData.numberOfReplicas(), equalTo(2)); - assertThat(indexMetaData.creationDate(), equalTo(2l)); - assertThat(indexMetaData.settings().getAsMap().size(), equalTo(6)); - assertThat(indexMetaData.settings().get("setting1"), equalTo("value1")); - assertThat(indexMetaData.settings().get("setting2"), equalTo("value2")); - assertThat(indexMetaData.mappings().size(), equalTo(2)); - assertThat(indexMetaData.mappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); - assertThat(indexMetaData.mappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); - assertThat(indexMetaData.aliases().size(), equalTo(3)); - assertThat(indexMetaData.aliases().get("alias1").alias(), equalTo("alias1")); - assertThat(indexMetaData.aliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1)); - assertThat(indexMetaData.aliases().get("alias2").alias(), equalTo("alias2")); - assertThat(indexMetaData.aliases().get("alias2").filter(), nullValue()); - assertThat(indexMetaData.aliases().get("alias4").alias(), equalTo("alias4")); - assertThat(indexMetaData.aliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2)); + assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); + assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); + assertThat(indexMetaData.getCreationDate(), equalTo(2l)); + assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6)); + assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); + assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2")); + assertThat(indexMetaData.getMappings().size(), equalTo(2)); + assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1)); + assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2)); + assertThat(indexMetaData.getAliases().size(), equalTo(3)); + assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1")); + assertThat(indexMetaData.getAliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1)); + assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2")); + assertThat(indexMetaData.getAliases().get("alias2").filter(), nullValue()); + assertThat(indexMetaData.getAliases().get("alias4").alias(), equalTo("alias4")); + assertThat(indexMetaData.getAliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2)); // templates assertThat(parsedMetaData.templates().get("foo").name(), is("foo")); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index fa6217e3d1f..324086a0448 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Arrays; @@ -32,8 +31,6 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.hamcrest.Matchers.equalTo; public class WildcardExpressionResolverTests extends ESTestCase { - - @Test public void testConvertWildcardsJustIndicesTests() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX")) @@ -52,7 +49,6 @@ public class WildcardExpressionResolverTests extends ESTestCase { assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testX*", "kuku"))), equalTo(newHashSet("testXXX", "testXYY", "kuku"))); } - @Test public void testConvertWildcardsTests() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX").putAlias(AliasMetaData.builder("alias1")).putAlias(AliasMetaData.builder("alias2"))) @@ -70,7 +66,6 @@ public class WildcardExpressionResolverTests extends ESTestCase { assertThat(newHashSet(resolver.resolve(context, Arrays.asList("+testYYY", "+testX*"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY"))); } - @Test public void testConvertWildcardsOpenClosedIndicesTests() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX").state(IndexMetaData.State.OPEN)) diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java index 2136d1e0a5c..9a91e1cd562 100644 --- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java @@ -26,15 +26,10 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Test; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -58,8 +53,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { localAddress = null; } - @Test - public void nameMatch() { + public void testNameMatch() { Settings settings = Settings.settingsBuilder() .put("xxx.name", "name1") .build(); @@ -72,8 +66,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(false)); } - @Test - public void idMatch() { + public void testIdMatch() { Settings settings = Settings.settingsBuilder() .put("xxx._id", "id1") .build(); @@ -86,8 +79,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(false)); } - @Test - public void idOrNameMatch() { + public void testIdOrNameMatch() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx._id", "id1,blah") .put("xxx.name", "blah,name2") @@ -104,8 +96,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(false)); } - @Test - public void tagAndGroupMatch() { + public void testTagAndGroupMatch() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "A") .put("xxx.group", "B") @@ -139,8 +130,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(false)); } - @Test - public void starMatch() { + public void testStarMatch() { Settings settings = Settings.settingsBuilder() .put("xxx.name", "*") .build(); @@ -150,8 +140,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(true)); } - @Test - public void ipBindFilteringMatchingAnd() { + public void testIpBindFilteringMatchingAnd() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "A") .put("xxx." + randomFrom("_ip", "_host_ip", "_publish_ip"), "192.1.1.54") @@ -162,8 +151,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(true)); } - @Test - public void ipBindFilteringNotMatching() { + public void testIpBindFilteringNotMatching() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "B") .put("xxx." + randomFrom("_ip", "_host_ip", "_publish_ip"), "192.1.1.54") @@ -174,8 +162,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(false)); } - @Test - public void ipBindFilteringNotMatchingAnd() { + public void testIpBindFilteringNotMatchingAnd() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "A") .put("xxx." + randomFrom("_ip", "_host_ip", "_publish_ip"), "8.8.8.8") @@ -186,8 +173,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(false)); } - @Test - public void ipBindFilteringMatchingOr() { + public void testIpBindFilteringMatchingOr() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx." + randomFrom("_ip", "_host_ip", "_publish_ip"), "192.1.1.54") .put("xxx.tag", "A") @@ -198,8 +184,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(true)); } - @Test - public void ipBindFilteringNotMatchingOr() { + public void testIpBindFilteringNotMatchingOr() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "A") .put("xxx." + randomFrom("_ip", "_host_ip", "_publish_ip"), "8.8.8.8") @@ -210,8 +195,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(true)); } - @Test - public void ipPublishFilteringMatchingAnd() { + public void testIpPublishFilteringMatchingAnd() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "A") .put("xxx._publish_ip", "192.1.1.54") @@ -222,8 +206,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(true)); } - @Test - public void ipPublishFilteringNotMatchingAnd() { + public void testIpPublishFilteringNotMatchingAnd() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "A") .put("xxx._publish_ip", "8.8.8.8") @@ -234,8 +217,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(false)); } - @Test - public void ipPublishFilteringMatchingOr() { + public void testIpPublishFilteringMatchingOr() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx._publish_ip", "192.1.1.54") .put("xxx.tag", "A") @@ -246,8 +228,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { assertThat(filters.match(node), equalTo(true)); } - @Test - public void ipPublishFilteringNotMatchingOr() { + public void testIpPublishFilteringNotMatchingOr() { Settings settings = shuffleSettings(Settings.settingsBuilder() .put("xxx.tag", "A") .put("xxx._publish_ip", "8.8.8.8") @@ -260,13 +241,11 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { private Settings shuffleSettings(Settings source) { Settings.Builder settings = Settings.settingsBuilder(); - List keys = new ArrayList(source.getAsMap().keySet()); - Collections.shuffle(keys, getRandom()); + List keys = new ArrayList<>(source.getAsMap().keySet()); + Collections.shuffle(keys, random()); for (String o : keys) { settings.put(o, source.getAsMap().get(o)); } return settings.build(); } - - } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java index 262b3db630b..8f7ae0c822b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java @@ -19,16 +19,22 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** */ public class AllocationIdTests extends ESTestCase { - - @Test public void testShardToStarted() { logger.info("-- create unassigned shard"); ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); @@ -49,7 +55,6 @@ public class AllocationIdTests extends ESTestCase { assertThat(allocationId.getRelocationId(), nullValue()); } - @Test public void testSuccessfulRelocation() { logger.info("-- build started shard"); ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); @@ -73,7 +78,6 @@ public class AllocationIdTests extends ESTestCase { assertThat(target.allocationId().getRelocationId(), nullValue()); } - @Test public void testCancelRelocation() { logger.info("-- build started shard"); ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); @@ -94,7 +98,6 @@ public class AllocationIdTests extends ESTestCase { assertThat(shard.allocationId().getRelocationId(), nullValue()); } - @Test public void testMoveToUnassigned() { logger.info("-- build started shard"); ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); @@ -106,7 +109,6 @@ public class AllocationIdTests extends ESTestCase { assertThat(shard.allocationId(), nullValue()); } - @Test public void testReinitializing() { logger.info("-- build started shard"); ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); @@ -120,4 +122,14 @@ public class AllocationIdTests extends ESTestCase { assertThat(shard.allocationId().getRelocationId(), nullValue()); assertThat(shard.allocationId().getId(), not(equalTo(allocationId.getId()))); } + + public void testSerialization() throws IOException { + AllocationId allocationId = AllocationId.newInitializing(); + if (randomBoolean()) { + allocationId = AllocationId.newRelocation(allocationId); + } + BytesReference bytes = allocationId.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).bytes(); + AllocationId parsedAllocationId = AllocationId.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(bytes)); + assertEquals(allocationId, parsedAllocationId); + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java index c6e9f0906ea..2d704380ae0 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; -import org.junit.Test; import java.util.Collections; import java.util.List; @@ -43,7 +42,6 @@ public class DelayedAllocationIT extends ESIntegTestCase { * Verifies that when there is no delay timeout, a 1/1 index shard will immediately * get allocated to a free node when the node hosting it leaves the cluster. */ - @Test public void testNoDelayedTimeout() throws Exception { internalCluster().startNodesAsync(3).get(); prepareCreate("test").setSettings(Settings.builder() @@ -63,7 +61,6 @@ public class DelayedAllocationIT extends ESIntegTestCase { * get allocated. Once we bring the node back, it gets allocated since it existed * on it before. */ - @Test public void testDelayedAllocationNodeLeavesAndComesBack() throws Exception { internalCluster().startNodesAsync(3).get(); prepareCreate("test").setSettings(Settings.builder() @@ -76,7 +73,7 @@ public class DelayedAllocationIT extends ESIntegTestCase { assertBusy(new Runnable() { @Override public void run() { - assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().hasUnassigned(), equalTo(true)); + assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true)); } }); assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(1)); @@ -88,7 +85,6 @@ public class DelayedAllocationIT extends ESIntegTestCase { * With a very small delay timeout, verify that it expires and we get to green even * though the node hosting the shard is not coming back. */ - @Test public void testDelayedAllocationTimesOut() throws Exception { internalCluster().startNodesAsync(3).get(); prepareCreate("test").setSettings(Settings.builder() @@ -111,7 +107,6 @@ public class DelayedAllocationIT extends ESIntegTestCase { * allocation to a very small value, it kicks the allocation of the unassigned shard * even though the node it was hosted on will not come back. */ - @Test public void testDelayedAllocationChangeWithSettingTo100ms() throws Exception { internalCluster().startNodesAsync(3).get(); prepareCreate("test").setSettings(Settings.builder() @@ -124,7 +119,7 @@ public class DelayedAllocationIT extends ESIntegTestCase { assertBusy(new Runnable() { @Override public void run() { - assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().hasUnassigned(), equalTo(true)); + assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true)); } }); assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(1)); @@ -138,7 +133,6 @@ public class DelayedAllocationIT extends ESIntegTestCase { * allocation to 0, it kicks the allocation of the unassigned shard * even though the node it was hosted on will not come back. */ - @Test public void testDelayedAllocationChangeWithSettingTo0() throws Exception { internalCluster().startNodesAsync(3).get(); prepareCreate("test").setSettings(Settings.builder() @@ -151,7 +145,7 @@ public class DelayedAllocationIT extends ESIntegTestCase { assertBusy(new Runnable() { @Override public void run() { - assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().hasUnassigned(), equalTo(true)); + assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true)); } }); assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(1)); @@ -176,7 +170,7 @@ public class DelayedAllocationIT extends ESIntegTestCase { private String findNodeWithShard() { ClusterState state = client().admin().cluster().prepareState().get().getState(); List startedShards = state.routingTable().shardsWithState(ShardRoutingState.STARTED); - Collections.shuffle(startedShards, getRandom()); + Collections.shuffle(startedShards,random()); return state.nodes().get(startedShards.get(0).currentNodeId()).getName(); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java index b451183826b..47ae3e68580 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java @@ -19,9 +19,7 @@ package org.elasticsearch.cluster.routing; -import static org.elasticsearch.test.ESTestCase.randomAsciiOfLength; -import static org.elasticsearch.test.ESTestCase.randomFrom; -import static org.elasticsearch.test.ESTestCase.randomInt; +import static org.elasticsearch.test.ESTestCase.*; /** * Utility class the makes random modifications to ShardRouting diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java index 29281e256f6..e8be4e34ae0 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java @@ -26,13 +26,11 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.Node; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import java.io.BufferedReader; import java.io.InputStreamReader; -import java.nio.file.Path; import java.util.Arrays; public class RoutingBackwardCompatibilityTests extends ESTestCase { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java index 9309fa71470..1711b0c33a8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java @@ -30,16 +30,18 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESAllocationTestCase; -import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.test.cluster.TestClusterService; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.hamcrest.Matchers.equalTo; /** @@ -58,16 +60,14 @@ public class RoutingServiceTests extends ESAllocationTestCase { routingService.shutdown(); } - @Test public void testReroute() { assertThat(routingService.hasReroutedAndClear(), equalTo(false)); routingService.reroute("test"); assertThat(routingService.hasReroutedAndClear(), equalTo(true)); } - @Test public void testNoDelayedUnassigned() throws Exception { - AllocationService allocation = createAllocationService(); + AllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "0")) .numberOfShards(1).numberOfReplicas(1)) @@ -76,28 +76,26 @@ public class RoutingServiceTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); // starting primaries clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); // starting replicas clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(false)); + assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false)); // remove node2 and reroute ClusterState prevState = clusterState; clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); ClusterState newState = clusterState; - assertThat(routingService.getRegisteredNextDelaySetting(), equalTo(Long.MAX_VALUE)); + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE)); routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState)); - assertThat(routingService.getRegisteredNextDelaySetting(), equalTo(Long.MAX_VALUE)); + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE)); assertThat(routingService.hasReroutedAndClear(), equalTo(false)); } - @Test - @TestLogging("_root:DEBUG") public void testDelayedUnassignedScheduleReroute() throws Exception { - AllocationService allocation = createAllocationService(); + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "100ms")) .numberOfShards(1).numberOfReplicas(1)) @@ -106,12 +104,12 @@ public class RoutingServiceTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); // starting primaries clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); // starting replicas clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().hasUnassigned()); + assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0); String nodeId = null; final List allShards = clusterState.getRoutingNodes().routingTable().allShards("test"); // we need to find the node with the replica otherwise we will not reroute @@ -122,62 +120,114 @@ public class RoutingServiceTests extends ESAllocationTestCase { } } assertNotNull(nodeId); - // remove node2 and reroute + // remove nodeId and reroute ClusterState prevState = clusterState; clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(nodeId)).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); - // We need to update the routing service's last attempted run to - // signal that the GatewayAllocator tried to allocated it but - // it was delayed - RoutingNodes.UnassignedShards unassigned = clusterState.getRoutingNodes().unassigned(); - assertEquals(1, unassigned.size()); - ShardRouting next = unassigned.iterator().next(); - routingService.setUnassignedShardsAllocatedTimestamp(next.unassignedInfo().getTimestampInMillis() + randomIntBetween(0, 99)); + // make sure the replica is marked as delayed (i.e. not reallocated) + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); + assertEquals(1, clusterState.getRoutingNodes().unassigned().size()); ClusterState newState = clusterState; routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState)); assertBusy(() -> assertTrue("routing service should have run a reroute", routingService.hasReroutedAndClear())); // verify the registration has been reset - assertThat(routingService.getRegisteredNextDelaySetting(), equalTo(Long.MAX_VALUE)); + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE)); } - @Test - public void testDelayedUnassignedDoesNotRerouteForNegativeDelays() throws Exception { - AllocationService allocation = createAllocationService(); - MetaData metaData = MetaData.builder() - .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "100ms")) - .numberOfShards(1).numberOfReplicas(1)) - .build(); - ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) - .metaData(metaData) - .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build(); - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); - // starting primaries - clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - // starting replicas - clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(false)); - // remove node2 and reroute - ClusterState prevState = clusterState; - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); - // Set it in the future so the delay will be negative - routingService.setUnassignedShardsAllocatedTimestamp(System.currentTimeMillis() + TimeValue.timeValueMinutes(1).millis()); + /** + * This tests that a new delayed reroute is scheduled right after a delayed reroute was run + */ + public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exception { + final ThreadPool testThreadPool = new ThreadPool(getTestName()); - ClusterState newState = clusterState; + try { + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("short_delay").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "100ms")) + .numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetaData.builder("long_delay").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10s")) + .numberOfShards(1).numberOfReplicas(1)) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData) + .routingTable(RoutingTable.builder().addAsNew(metaData.index("short_delay")).addAsNew(metaData.index("long_delay")).build()) + .nodes(DiscoveryNodes.builder() + .put(newNode("node0", singletonMap("data", Boolean.FALSE.toString()))).localNodeId("node0").masterNodeId("node0") + .put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build(); + // allocate shards + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); + // start primaries + clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); + // start replicas + clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); + assertThat("all shards should be started", clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4)); - routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState)); - assertBusy(new Runnable() { - @Override - public void run() { - assertThat(routingService.hasReroutedAndClear(), equalTo(false)); - - // verify the registration has been updated - assertThat(routingService.getRegisteredNextDelaySetting(), equalTo(100L)); + // find replica of short_delay + ShardRouting shortDelayReplica = null; + for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("short_delay")) { + if (shardRouting.primary() == false) { + shortDelayReplica = shardRouting; + break; + } } - }); + assertNotNull(shortDelayReplica); + + // find replica of long_delay + ShardRouting longDelayReplica = null; + for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("long_delay")) { + if (shardRouting.primary() == false) { + longDelayReplica = shardRouting; + break; + } + } + assertNotNull(longDelayReplica); + + final long baseTime = System.nanoTime(); + + // remove node of shortDelayReplica and node of longDelayReplica and reroute + ClusterState prevState = clusterState; + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(shortDelayReplica.currentNodeId()).remove(longDelayReplica.currentNodeId())).build(); + // make sure both replicas are marked as delayed (i.e. not reallocated) + allocation.setNanoTimeOverride(baseTime); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); + + // check that shortDelayReplica and longDelayReplica have been marked unassigned + RoutingNodes.UnassignedShards unassigned = clusterState.getRoutingNodes().unassigned(); + assertEquals(2, unassigned.size()); + // update shortDelayReplica and longDelayReplica variables with new shard routing + ShardRouting shortDelayUnassignedReplica = null; + ShardRouting longDelayUnassignedReplica = null; + for (ShardRouting shr : unassigned) { + if (shr.getIndex().equals("short_delay")) { + shortDelayUnassignedReplica = shr; + } else { + longDelayUnassignedReplica = shr; + } + } + assertTrue(shortDelayReplica.isSameShard(shortDelayUnassignedReplica)); + assertTrue(longDelayReplica.isSameShard(longDelayUnassignedReplica)); + + // manually trigger a clusterChanged event on routingService + ClusterState newState = clusterState; + // create fake cluster service + TestClusterService clusterService = new TestClusterService(newState, testThreadPool); + // create routing service, also registers listener on cluster service + RoutingService routingService = new RoutingService(Settings.EMPTY, testThreadPool, clusterService, allocation); + routingService.start(); // just so performReroute does not prematurely return + // next (delayed) reroute should only delay longDelayReplica/longDelayUnassignedReplica, simulate that we are now 1 second after shards became unassigned + allocation.setNanoTimeOverride(baseTime + TimeValue.timeValueSeconds(1).nanos()); + // register listener on cluster state so we know when cluster state has been changed + CountDownLatch latch = new CountDownLatch(1); + clusterService.addLast(event -> latch.countDown()); + // instead of clusterService calling clusterChanged, we call it directly here + routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState)); + // cluster service should have updated state and called routingService with clusterChanged + latch.await(); + // verify the registration has been set to the delay of longDelayReplica/longDelayUnassignedReplica + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(TimeValue.timeValueSeconds(10).nanos())); + } finally { + terminate(testThreadPool); + } } private class TestRoutingService extends RoutingService { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index 2a7ed6a4a93..d69264a1e3a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESAllocationTestCase; import org.junit.Before; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -88,7 +87,7 @@ public class RoutingTableTests extends ESAllocationTestCase { discoBuilder = discoBuilder.put(newNode("node" + i)); } this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build(); - RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.reroute(clusterState); + RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.reroute(clusterState, "reroute"); this.testRoutingTable = rerouteResult.routingTable(); assertThat(rerouteResult.changed(), is(true)); this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); @@ -109,7 +108,6 @@ public class RoutingTableTests extends ESAllocationTestCase { .numberOfShards(this.numberOfShards); } - @Test public void testAllShards() { assertThat(this.emptyRoutingTable.allShards().size(), is(0)); assertThat(this.testRoutingTable.allShards().size(), is(this.totalNumberOfShards)); @@ -123,26 +121,22 @@ public class RoutingTableTests extends ESAllocationTestCase { } } - @Test public void testHasIndex() { assertThat(this.testRoutingTable.hasIndex(TEST_INDEX_1), is(true)); assertThat(this.testRoutingTable.hasIndex("foobar"), is(false)); } - @Test public void testIndex() { assertThat(this.testRoutingTable.index(TEST_INDEX_1).getIndex(), is(TEST_INDEX_1)); assertThat(this.testRoutingTable.index("foobar"), is(nullValue())); } - @Test public void testIndicesRouting() { assertThat(this.testRoutingTable.indicesRouting().size(), is(2)); assertThat(this.testRoutingTable.getIndicesRouting().size(), is(2)); assertSame(this.testRoutingTable.getIndicesRouting(), this.testRoutingTable.indicesRouting()); } - @Test public void testShardsWithState() { assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards)); @@ -168,7 +162,6 @@ public class RoutingTableTests extends ESAllocationTestCase { assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(this.totalNumberOfShards)); } - @Test public void testActivePrimaryShardsGrouped() { assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], true).size(), is(0)); assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], false).size(), is(0)); @@ -198,7 +191,6 @@ public class RoutingTableTests extends ESAllocationTestCase { } } - @Test public void testAllActiveShardsGrouped() { assertThat(this.emptyRoutingTable.allActiveShardsGrouped(new String[0], true).size(), is(0)); assertThat(this.emptyRoutingTable.allActiveShardsGrouped(new String[0], false).size(), is(0)); @@ -227,7 +219,6 @@ public class RoutingTableTests extends ESAllocationTestCase { } } - @Test public void testAllAssignedShardsGrouped() { assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, false).size(), is(0)); assertThat(this.testRoutingTable.allAssignedShardsGrouped(new String[]{TEST_INDEX_1}, true).size(), is(this.shardsPerIndex)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java index 146e80c7665..54e39cc227d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java @@ -99,12 +99,18 @@ public class ShardRoutingTests extends ESTestCase { ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "node1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); ShardRouting initializingShard1 = TestShardRouting.newShardRouting("test", 1, "node1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); ShardRouting startedShard0 = new ShardRouting(initializingShard0); + assertFalse(startedShard0.isRelocationTarget()); startedShard0.moveToStarted(); + assertFalse(startedShard0.isRelocationTarget()); ShardRouting startedShard1 = new ShardRouting(initializingShard1); + assertFalse(startedShard1.isRelocationTarget()); startedShard1.moveToStarted(); + assertFalse(startedShard1.isRelocationTarget()); ShardRouting sourceShard0a = new ShardRouting(startedShard0); sourceShard0a.relocate("node2", -1); + assertFalse(sourceShard0a.isRelocationTarget()); ShardRouting targetShard0a = sourceShard0a.buildTargetRelocatingShard(); + assertTrue(targetShard0a.isRelocationTarget()); ShardRouting sourceShard0b = new ShardRouting(startedShard0); sourceShard0b.relocate("node2", -1); ShardRouting sourceShard1 = new ShardRouting(startedShard1); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java index fc7ce64c082..3288b92cb8e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing; import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -36,26 +35,16 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.util.Collections; import java.util.EnumSet; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; -import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.hamcrest.Matchers.*; /** */ public class UnassignedInfoTests extends ESAllocationTestCase { - - @Test public void testReasonOrdinalOrder() { UnassignedInfo.Reason[] order = new UnassignedInfo.Reason[]{ UnassignedInfo.Reason.INDEX_CREATED, @@ -76,7 +65,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { assertThat(UnassignedInfo.Reason.values().length, equalTo(order.length)); } - @Test public void testSerialization() throws Exception { UnassignedInfo meta = new UnassignedInfo(RandomPicks.randomFrom(getRandom(), UnassignedInfo.Reason.values()), randomBoolean() ? randomAsciiOfLength(4) : null); BytesStreamOutput out = new BytesStreamOutput(); @@ -85,12 +73,11 @@ public class UnassignedInfoTests extends ESAllocationTestCase { UnassignedInfo read = new UnassignedInfo(StreamInput.wrap(out.bytes())); assertThat(read.getReason(), equalTo(meta.getReason())); - assertThat(read.getTimestampInMillis(), equalTo(meta.getTimestampInMillis())); + assertThat(read.getUnassignedTimeInMillis(), equalTo(meta.getUnassignedTimeInMillis())); assertThat(read.getMessage(), equalTo(meta.getMessage())); assertThat(read.getDetails(), equalTo(meta.getDetails())); } - @Test public void testIndexCreated() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3))) @@ -103,7 +90,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { } } - @Test public void testClusterRecovered() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3))) @@ -116,7 +102,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { } } - @Test public void testIndexReopened() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3))) @@ -129,7 +114,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { } } - @Test public void testNewIndexRestored() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3))) @@ -142,7 +126,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { } } - @Test public void testExistingIndexRestored() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3))) @@ -155,7 +138,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { } } - @Test public void testDanglingIndexImported() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3))) @@ -168,7 +150,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { } } - @Test public void testReplicaAdded() { AllocationService allocation = createAllocationService(); MetaData metaData = MetaData.builder() @@ -178,7 +159,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); // starting primaries clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); IndexRoutingTable.Builder builder = IndexRoutingTable.builder("test"); @@ -195,7 +176,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { /** * The unassigned meta is kept when a shard goes to INITIALIZING, but cleared when it moves to STARTED. */ - @Test public void testStateTransitionMetaHandling() { ShardRouting shard = TestShardRouting.newShardRouting("test", 1, null, null, null, true, ShardRoutingState.UNASSIGNED, 1, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); ShardRouting mutable = new ShardRouting(shard); @@ -211,7 +191,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase { /** * Tests that during reroute when a node is detected as leaving the cluster, the right unassigned meta is set */ - @Test public void testNodeLeave() { AllocationService allocation = createAllocationService(); MetaData metaData = MetaData.builder() @@ -221,27 +200,26 @@ public class UnassignedInfoTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); // starting primaries clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); // starting replicas clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(false)); + assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false)); // remove node2 and reroute clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); // verify that NODE_LEAVE is the reason for meta - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(true)); + assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(true)); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1)); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo(), notNullValue()); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.NODE_LEFT)); - assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getTimestampInMillis(), greaterThan(0l)); + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getUnassignedTimeInMillis(), greaterThan(0l)); } /** * Verifies that when a shard fails, reason is properly set and details are preserved. */ - @Test public void testFailedShard() { AllocationService allocation = createAllocationService(); MetaData metaData = MetaData.builder() @@ -251,62 +229,77 @@ public class UnassignedInfoTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); // starting primaries clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); // starting replicas clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(false)); + assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false)); // fail shard ShardRouting shardToFail = clusterState.getRoutingNodes().shardsWithState(STARTED).get(0); clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyFailedShards(clusterState, Collections.singletonList(new FailedRerouteAllocation.FailedShard(shardToFail, "test fail", null)))).build(); // verify the reason and details - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(true)); + assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(true)); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1)); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo(), notNullValue()); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED)); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getMessage(), equalTo("test fail")); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getDetails(), equalTo("test fail")); - assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getTimestampInMillis(), greaterThan(0l)); + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0).unassignedInfo().getUnassignedTimeInMillis(), greaterThan(0l)); } /** * Verifies that delayed allocation calculation are correct. */ - @Test public void testUnassignedDelayedOnlyOnNodeLeft() throws Exception { final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, null); - long delay = unassignedInfo.getAllocationDelayTimeoutSetting(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), Settings.EMPTY); - assertThat(delay, equalTo(TimeValue.timeValueHours(10).millis())); - assertBusy(new Runnable() { - @Override - public void run() { - long delay = unassignedInfo.getDelayAllocationExpirationIn(System.currentTimeMillis(), - Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), Settings.EMPTY); - assertThat(delay, greaterThan(0l)); - assertThat(delay, lessThan(TimeValue.timeValueHours(10).millis())); - } - }); + long delay = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay + Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), Settings.EMPTY); + long cachedDelay = unassignedInfo.getLastComputedLeftDelayNanos(); + assertThat(delay, equalTo(cachedDelay)); + assertThat(delay, equalTo(TimeValue.timeValueHours(10).nanos() - 1)); } /** * Verifies that delayed allocation is only computed when the reason is NODE_LEFT. */ - @Test public void testUnassignedDelayOnlyNodeLeftNonNodeLeftReason() throws Exception { EnumSet reasons = EnumSet.allOf(UnassignedInfo.Reason.class); reasons.remove(UnassignedInfo.Reason.NODE_LEFT); UnassignedInfo unassignedInfo = new UnassignedInfo(RandomPicks.randomFrom(getRandom(), reasons), null); - long delay = unassignedInfo.getAllocationDelayTimeoutSetting(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), Settings.EMPTY); - assertThat(delay, equalTo(0l)); - delay = unassignedInfo.getDelayAllocationExpirationIn(System.currentTimeMillis(), + long delay = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), Settings.EMPTY); assertThat(delay, equalTo(0l)); + delay = unassignedInfo.getLastComputedLeftDelayNanos(); + assertThat(delay, equalTo(0l)); } - @Test + /** + * Verifies that delayed allocation calculation are correct. + */ + public void testLeftDelayCalculation() throws Exception { + final long baseTime = System.nanoTime(); + final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, baseTime, System.currentTimeMillis()); + final long totalDelayNanos = TimeValue.timeValueMillis(10).nanos(); + final Settings settings = Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueNanos(totalDelayNanos)).build(); + long delay = unassignedInfo.updateDelay(baseTime, settings, Settings.EMPTY); + assertThat(delay, equalTo(totalDelayNanos)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + long delta1 = randomIntBetween(1, (int) (totalDelayNanos - 1)); + delay = unassignedInfo.updateDelay(baseTime + delta1, settings, Settings.EMPTY); + assertThat(delay, equalTo(totalDelayNanos - delta1)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + delay = unassignedInfo.updateDelay(baseTime + totalDelayNanos, settings, Settings.EMPTY); + assertThat(delay, equalTo(0L)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + delay = unassignedInfo.updateDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), settings, Settings.EMPTY); + assertThat(delay, equalTo(0L)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + } + + public void testNumberOfDelayedUnassigned() throws Exception { - AllocationService allocation = createAllocationService(); + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) @@ -315,50 +308,58 @@ public class UnassignedInfoTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build()).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); - assertThat(UnassignedInfo.getNumberOfDelayedUnassigned(System.currentTimeMillis(), - Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), clusterState), equalTo(0)); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); + assertThat(UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(0)); // starting primaries clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); // starting replicas clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(false)); + assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false)); // remove node2 and reroute clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); - assertThat(clusterState.prettyPrint(), UnassignedInfo.getNumberOfDelayedUnassigned(System.currentTimeMillis(), - Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), clusterState), equalTo(2)); + // make sure both replicas are marked as delayed (i.e. not reallocated) + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); + assertThat(clusterState.prettyPrint(), UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(2)); } - @Test public void testFindNextDelayedAllocation() { - AllocationService allocation = createAllocationService(); + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); + final long baseTime = System.nanoTime(); + allocation.setNanoTimeOverride(baseTime); + final TimeValue delayTest1 = TimeValue.timeValueMillis(randomIntBetween(1, 200)); + final TimeValue delayTest2 = TimeValue.timeValueMillis(randomIntBetween(1, 200)); + final long expectMinDelaySettingsNanos = Math.min(delayTest1.nanos(), delayTest2.nanos()); + MetaData metaData = MetaData.builder() - .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) - .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, delayTest1)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, delayTest2)).numberOfShards(1).numberOfReplicas(1)) .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) .metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build()).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); - assertThat(UnassignedInfo.getNumberOfDelayedUnassigned(System.currentTimeMillis(), - Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), clusterState), equalTo(0)); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); + assertThat(UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(0)); // starting primaries clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); // starting replicas clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertThat(clusterState.getRoutingNodes().hasUnassigned(), equalTo(false)); + assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false)); // remove node2 and reroute clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState)).build(); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); - long nextDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSetting(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), clusterState); - assertThat(nextDelaySetting, equalTo(TimeValue.timeValueHours(10).millis())); + final long delta = randomBoolean() ? 0 : randomInt((int) expectMinDelaySettingsNanos); - long nextDelay = UnassignedInfo.findNextDelayedAllocationIn(System.currentTimeMillis(), - Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), clusterState); - assertThat(nextDelay, greaterThan(TimeValue.timeValueHours(9).millis())); - assertThat(nextDelay, lessThanOrEqualTo(TimeValue.timeValueHours(10).millis())); + if (delta > 0) { + allocation.setNanoTimeOverride(baseTime + delta); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "time moved")).build(); + } + + long minDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSettingNanos(Settings.EMPTY, clusterState); + assertThat(minDelaySetting, equalTo(expectMinDelaySettingsNanos)); + + long nextDelay = UnassignedInfo.findNextDelayedAllocationIn(clusterState); + assertThat(nextDelay, equalTo(expectMinDelaySettingsNanos - delta)); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java new file mode 100644 index 00000000000..7a7f4722e97 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java @@ -0,0 +1,81 @@ +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.test.ESAllocationTestCase; + +import java.util.Arrays; +import java.util.HashSet; + +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; +import static org.hamcrest.Matchers.equalTo; + +public class ActiveAllocationIdTests extends ESAllocationTestCase { + + public void testActiveAllocationIdsUpdated() { + AllocationService allocation = createAllocationService(); + + logger.info("creating an index with 1 shard, 2 replicas"); + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2)) + // add index metadata where we have no routing nodes to check that allocation ids are not removed + .put(IndexMetaData.builder("test-old").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2) + .putActiveAllocationIds(0, new HashSet<>(Arrays.asList("x", "y")))) + .build(); + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test")) + .build(); + ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); + + logger.info("adding three nodes and performing rerouting"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put( + newNode("node1")).put(newNode("node2")).put(newNode("node3"))).build(); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + + assertThat(clusterState.metaData().index("test").activeAllocationIds(0).size(), equalTo(0)); + assertThat(clusterState.metaData().index("test-old").activeAllocationIds(0), equalTo(new HashSet<>(Arrays.asList("x", "y")))); + + logger.info("start primary shard"); + rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + + assertThat(clusterState.getRoutingTable().shardsWithState(STARTED).size(), equalTo(1)); + assertThat(clusterState.metaData().index("test").activeAllocationIds(0).size(), equalTo(1)); + assertThat(clusterState.getRoutingTable().shardsWithState(STARTED).get(0).allocationId().getId(), + equalTo(clusterState.metaData().index("test").activeAllocationIds(0).iterator().next())); + assertThat(clusterState.metaData().index("test-old").activeAllocationIds(0), equalTo(new HashSet<>(Arrays.asList("x", "y")))); + + logger.info("start replica shards"); + rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + + assertThat(clusterState.metaData().index("test").activeAllocationIds(0).size(), equalTo(3)); + + logger.info("remove a node"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) + .remove("node1")) + .build(); + rerouteResult = allocation.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + + assertThat(clusterState.metaData().index("test").activeAllocationIds(0).size(), equalTo(2)); + + logger.info("remove all remaining nodes"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) + .remove("node2").remove("node3")) + .build(); + rerouteResult = allocation.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + + // active allocation ids should not be updated + assertThat(clusterState.getRoutingTable().shardsWithState(UNASSIGNED).size(), equalTo(3)); + assertThat(clusterState.metaData().index("test").activeAllocationIds(0).size(), equalTo(2)); + } +} diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java index 836422f251a..ee8bd067008 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.cluster.routing.allocation; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -38,7 +37,6 @@ import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.test.ESAllocationTestCase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Collections; @@ -50,7 +48,6 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; public class AddIncrementallyTests extends ESAllocationTestCase { private final ESLogger logger = Loggers.getLogger(AddIncrementallyTests.class); - @Test public void testAddNodesAndIndices() { Settings.Builder settings = settingsBuilder(); settings.put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()); @@ -94,7 +91,6 @@ public class AddIncrementallyTests extends ESAllocationTestCase { logger.debug("ClusterState: {}", clusterState.getRoutingNodes().prettyPrint()); } - @Test public void testMinimalRelocations() { Settings.Builder settings = settingsBuilder(); settings.put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()) @@ -116,7 +112,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase { nodes.put(newNode("node2")); clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build(); - RoutingTable routingTable = service.reroute(clusterState).routingTable(); + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -163,7 +159,6 @@ public class AddIncrementallyTests extends ESAllocationTestCase { logger.debug("ClusterState: {}", clusterState.getRoutingNodes().prettyPrint()); } - @Test public void testMinimalRelocationsNoLimit() { Settings.Builder settings = settingsBuilder(); settings.put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()) @@ -186,7 +181,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase { nodes.put(newNode("node2")); clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build(); - RoutingTable routingTable = service.reroute(clusterState).routingTable(); + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -267,7 +262,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build(); - RoutingTable routingTable = service.reroute(clusterState).routingTable(); + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -312,7 +307,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase { nodes.put(newNode("node" + i)); } ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build(); - routingTable = service.reroute(clusterState).routingTable(); + routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -357,7 +352,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase { MetaData metaData = metaDataBuilder.build(); RoutingTable routingTable = routingTableBuilder.build(); clusterState = ClusterState.builder(clusterState).metaData(metaData).routingTable(routingTable).build(); - routingTable = service.reroute(clusterState).routingTable(); + routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -392,7 +387,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase { logger.info("Removing [{}] nodes", numNodes); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); ArrayList discoveryNodes = CollectionUtils.iterableAsArrayList(clusterState.nodes()); - Collections.shuffle(discoveryNodes, getRandom()); + Collections.shuffle(discoveryNodes, random()); for (DiscoveryNode node : discoveryNodes) { nodes.remove(node.id()); numNodes--; @@ -415,7 +410,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase { routingNodes = clusterState.getRoutingNodes(); logger.info("rebalancing"); - routingTable = service.reroute(clusterState).routingTable(); + routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java index 96cba278fe4..6ac2b7df9ca 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java @@ -40,7 +40,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; @@ -52,11 +51,9 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class AllocationCommandsTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(AllocationCommandsTests.class); - @Test - public void moveShardCommand() { + public void testMoveShardCommand() { AllocationService allocation = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); logger.info("creating an index with 1 shard, no replica"); @@ -70,7 +67,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { logger.info("adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); logger.info("start primary shard"); @@ -99,8 +96,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node(toNodeId).get(0).state(), equalTo(ShardRoutingState.STARTED)); } - @Test - public void allocateCommand() { + public void testAllocateCommand() { AllocationService allocation = createAllocationService(settingsBuilder() .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none") .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, "none") @@ -122,7 +118,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { .put(newNode("node3")) .put(newNode("node4", singletonMap("data", Boolean.FALSE.toString()))) ).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); @@ -188,8 +184,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { } } - @Test - public void cancelCommand() { + public void testCancelCommand() { AllocationService allocation = createAllocationService(settingsBuilder() .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none") .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, "none") @@ -210,7 +205,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { .put(newNode("node2")) .put(newNode("node3")) ).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); @@ -338,8 +333,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0)); } - @Test - public void serialization() throws Exception { + public void testSerialization() throws Exception { AllocationCommands commands = new AllocationCommands( new AllocateAllocationCommand(new ShardId("test", 1), "node1", true), new MoveAllocationCommand(new ShardId("test", 3), "node2", "node3"), @@ -363,8 +357,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { assertThat(((CancelAllocationCommand) (sCommands.commands().get(2))).allowPrimary(), equalTo(true)); } - @Test - public void xContent() throws Exception { + public void testXContent() throws Exception { String commands = "{\n" + " \"commands\" : [\n" + " {\"allocate\" : {\"index\" : \"test\", \"shard\" : 1, \"node\" : \"node1\", \"allow_primary\" : true}}\n" + diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java index 6fbc92a66ad..d7a049d1b92 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java @@ -67,10 +67,10 @@ public class AllocationPriorityTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); - routingTable = allocation.reroute(clusterState).routingTable(); + routingTable = allocation.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size()); assertEquals(highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).index()); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java index e17ac302125..7be6037cf79 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; @@ -53,8 +52,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { private final ESLogger logger = Loggers.getLogger(AwarenessAllocationTests.class); - @Test - public void moveShardOnceNewNodeWithAttributeAdded1() { + public void testMoveShardOnceNewNodeWithAttributeAdded1() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -78,7 +76,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node1", singletonMap("rack_id", "1"))) .put(newNode("node2", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); @@ -96,7 +94,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); @@ -110,20 +108,19 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, make sure nothing moves"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(routingTable, sameInstance(clusterState.routingTable())); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2)); } - @Test - public void moveShardOnceNewNodeWithAttributeAdded2() { + public void testMoveShardOnceNewNodeWithAttributeAdded2() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -148,7 +145,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node2", singletonMap("rack_id", "1"))) .put(newNode("node3", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); @@ -166,7 +163,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); @@ -180,20 +177,19 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, make sure nothing moves"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node5", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(routingTable, sameInstance(clusterState.routingTable())); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2)); } - @Test - public void moveShardOnceNewNodeWithAttributeAdded3() { + public void testMoveShardOnceNewNodeWithAttributeAdded3() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) @@ -222,7 +218,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node1", singletonMap("rack_id", "1"))) .put(newNode("node2", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(INITIALIZING)) { @@ -254,7 +250,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(5)); @@ -273,13 +269,13 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, some more relocation should happen"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), greaterThan(0)); @@ -290,11 +286,10 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); } - @Test - public void moveShardOnceNewNodeWithAttributeAdded4() { + public void testMoveShardOnceNewNodeWithAttributeAdded4() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) @@ -322,7 +317,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node1", singletonMap("rack_id", "1"))) .put(newNode("node2", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(10)); @@ -340,7 +335,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); @@ -363,13 +358,13 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(5)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, some more relocation should happen"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), greaterThan(0)); @@ -386,11 +381,10 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(5)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); } - @Test - public void moveShardOnceNewNodeWithAttributeAdded5() { + public void testMoveShardOnceNewNodeWithAttributeAdded5() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -414,7 +408,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node1", singletonMap("rack_id", "1"))) .put(newNode("node2", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); @@ -432,7 +426,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); @@ -446,13 +440,13 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, we will have another relocation"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2)); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1)); @@ -465,11 +459,10 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3)); logger.info("--> make sure another reroute does not move things"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); } - @Test - public void moveShardOnceNewNodeWithAttributeAdded6() { + public void testMoveShardOnceNewNodeWithAttributeAdded6() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -495,7 +488,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node3", singletonMap("rack_id", "1"))) .put(newNode("node4", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); @@ -513,7 +506,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node5", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3)); @@ -527,13 +520,13 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, we will have another relocation"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node6", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(3)); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1)); @@ -546,11 +539,10 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4)); logger.info("--> make sure another reroute does not move things"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); } - @Test - public void fullAwareness1() { + public void testFullAwareness1() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -575,7 +567,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node1", singletonMap("rack_id", "1"))) .put(newNode("node2", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); @@ -591,7 +583,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); @@ -605,20 +597,19 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, make sure nothing moves"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(routingTable, sameInstance(clusterState.routingTable())); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2)); } - @Test - public void fullAwareness2() { + public void testFullAwareness2() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -644,7 +635,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node2", singletonMap("rack_id", "1"))) .put(newNode("node3", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); @@ -660,7 +651,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); @@ -674,20 +665,19 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, make sure nothing moves"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node5", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(routingTable, sameInstance(clusterState.routingTable())); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2)); } - @Test - public void fullAwareness3() { + public void testFullAwareness3() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) @@ -719,7 +709,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("node1", singletonMap("rack_id", "1"))) .put(newNode("node2", singletonMap("rack_id", "1"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(10)); @@ -733,7 +723,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3", singletonMap("rack_id", "2"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); @@ -751,13 +741,13 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); logger.info("--> add another node with a new rack, some more relocation should happen"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4", singletonMap("rack_id", "3"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), greaterThan(0)); @@ -768,10 +758,9 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20)); logger.info("--> do another reroute, make sure nothing moves"); - assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable())); } - @Test public void testUnbalancedZones() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.awareness.force.zone.values", "a,b") @@ -799,7 +788,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("A-0", singletonMap("zone", "a"))) .put(newNode("B-0", singletonMap("zone", "b"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0)); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(5)); @@ -820,7 +809,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("A-1", singletonMap("zone", "a"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(8)); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(2)); @@ -836,7 +825,6 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node("B-0").size(), equalTo(5)); } - @Test public void testUnassignedShardsWithUnbalancedZones() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -865,7 +853,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase { .put(newNode("A-4", singletonMap("zone", "a"))) .put(newNode("B-0", singletonMap("zone", "b"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0)); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java index 2bd18f814dc..1092b2ede19 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java @@ -44,7 +44,6 @@ import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.ESAllocationTestCase; import org.elasticsearch.test.gateway.NoopGatewayAllocator; import org.hamcrest.Matchers; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -59,7 +58,6 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { final int numberOfShards = 2; final int numberOfReplicas = 2; - @Test public void testIndexBalance() { /* Tests balance over indices only */ final float indexBalance = 1.0f; @@ -85,7 +83,6 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { } - @Test public void testReplicaBalance() { /* Tests balance over replicas only */ final float indexBalance = 0.0f; @@ -135,7 +132,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { nodes.put(newNode("node" + i)); } ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -171,7 +168,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { .put(newNode("node" + numberOfNodes))) .build(); - RoutingTable routingTable = strategy.reroute(clusterState).routingTable(); + RoutingTable routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -212,7 +209,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { routingNodes = clusterState.getRoutingNodes(); logger.info("rebalancing"); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -280,7 +277,6 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { } } - @Test public void testPersistedSettings() { Settings.Builder settings = settingsBuilder(); settings.put(BalancedShardsAllocator.SETTING_INDEX_BALANCE_FACTOR, 0.2); @@ -318,7 +314,6 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { assertThat(allocator.getThreshold(), Matchers.equalTo(3.0f)); } - @Test public void testNoRebalanceOnPrimaryOverload() { Settings.Builder settings = settingsBuilder(); AllocationService strategy = new AllocationService(settings.build(), randomAllocationDeciders(settings.build(), @@ -370,7 +365,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { public boolean allocateUnassigned(RoutingAllocation allocation) { RoutingNodes.UnassignedShards unassigned = allocation.routingNodes().unassigned(); boolean changed = !unassigned.isEmpty(); - for (ShardRouting sr : unassigned) { + for (ShardRouting sr : unassigned.drain()) { switch (sr.id()) { case 0: if (sr.primary()) { @@ -410,7 +405,6 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { } } - unassigned.clear(); return changed; } }), EmptyClusterInfoService.INSTANCE); @@ -430,7 +424,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { } ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); @@ -464,7 +458,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase { } logger.info("rebalancing"); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java index d0e3d727c10..fa8afc84876 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java @@ -65,7 +65,7 @@ public class BalanceUnbalancedClusterTests extends CatAllocationTestCase { .build(); ClusterState clusterState = ClusterState.builder(state).metaData(metaData).routingTable(routingTable).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); while (true) { if (routingTable.shardsWithState(INITIALIZING).isEmpty()) { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java index 6253febaec8..8ac6c4fcedc 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java @@ -19,26 +19,35 @@ package org.elasticsearch.cluster.routing.allocation; -import java.nio.charset.StandardCharsets; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.io.BufferedReader; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.common.settings.Settings.settingsBuilder; /** @@ -50,11 +59,9 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; * This can be used to debug cluster allocation decisions. */ public abstract class CatAllocationTestCase extends ESAllocationTestCase { - protected abstract Path getCatPath() throws IOException; - @Test - public void run() throws IOException { + public void testRun() throws IOException { Set nodes = new HashSet<>(); Map indices = new HashMap<>(); try (BufferedReader reader = Files.newBufferedReader(getCatPath(), StandardCharsets.UTF_8)) { @@ -130,7 +137,7 @@ public abstract class CatAllocationTestCase extends ESAllocationTestCase { private ClusterState rebalance(ClusterState clusterState) { RoutingTable routingTable;AllocationService strategy = createAllocationService(settingsBuilder() .build()); - RoutingAllocation.Result reroute = strategy.reroute(clusterState); + RoutingAllocation.Result reroute = strategy.reroute(clusterState, "reroute"); routingTable = reroute.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingTable = clusterState.routingTable(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java index 59e7529861c..8dad41db2f8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java @@ -26,11 +26,16 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; +import org.elasticsearch.test.gateway.NoopGatewayAllocator; + +import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.cluster.routing.ShardRoutingState.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -38,10 +43,8 @@ import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ClusterRebalanceRoutingTests.class); - @Test public void testAlways() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()).build()); @@ -61,7 +64,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { @@ -119,7 +122,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { .put(newNode("node3"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -128,7 +131,6 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { } - @Test public void testClusterPrimariesActive1() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_PRIMARIES_ACTIVE.toString()).build()); @@ -148,7 +150,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { @@ -225,7 +227,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { .put(newNode("node3"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -233,7 +235,6 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").get(0).shardId().index().name(), equalTo("test1")); } - @Test public void testClusterPrimariesActive2() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_PRIMARIES_ACTIVE.toString()).build()); @@ -253,7 +254,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { @@ -311,14 +312,13 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { .put(newNode("node3"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); assertThat(routingNodes.node("node3").isEmpty(), equalTo(true)); } - @Test public void testClusterAllActive1() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_ALL_ACTIVE.toString()).build()); @@ -338,7 +338,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { @@ -434,7 +434,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { .put(newNode("node3"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -442,7 +442,6 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").get(0).shardId().index().name(), anyOf(equalTo("test1"), equalTo("test2"))); } - @Test public void testClusterAllActive2() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_ALL_ACTIVE.toString()).build()); @@ -462,7 +461,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { @@ -520,14 +519,13 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { .put(newNode("node3"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); assertThat(routingNodes.node("node3").isEmpty(), equalTo(true)); } - @Test public void testClusterAllActive3() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_ALL_ACTIVE.toString()).build()); @@ -547,7 +545,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { @@ -624,10 +622,205 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase { .put(newNode("node3"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); assertThat(routingNodes.node("node3").isEmpty(), equalTo(true)); } + + public void testRebalanceWithIgnoredUnassignedShards() { + final AtomicBoolean allocateTest1 = new AtomicBoolean(false); + + AllocationService strategy = createAllocationService(Settings.EMPTY, new NoopGatewayAllocator() { + @Override + public boolean allocateUnassigned(RoutingAllocation allocation) { + if (allocateTest1.get() == false) { + RoutingNodes.UnassignedShards unassigned = allocation.routingNodes().unassigned(); + RoutingNodes.UnassignedShards.UnassignedIterator iterator = unassigned.iterator(); + while (iterator.hasNext()) { + ShardRouting next = iterator.next(); + if ("test1".equals(next.index())) { + iterator.removeAndIgnore(); + } + + } + } + return super.allocateUnassigned(allocation); + } + }); + + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .build(); + + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test")) + .addAsNew(metaData.index("test1")) + .build(); + + ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); + + logger.info("start two nodes"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(INITIALIZING)); + } + + logger.debug("start all the primary shards for test"); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test", INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED)); + } + + logger.debug("now, start 1 more node, check that rebalancing will not happen since we unassigned shards"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) + .put(newNode("node2"))) + .build(); + logger.debug("reroute and check that nothing has changed"); + RoutingAllocation.Result reroute = strategy.reroute(clusterState, "reroute"); + assertFalse(reroute.changed()); + routingTable = reroute.routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED)); + } + for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { + assertThat(routingTable.index("test1").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test1").shard(i).primaryShard().state(), equalTo(UNASSIGNED)); + } + logger.debug("now set allocateTest1 to true and reroute we should see the [test1] index initializing"); + allocateTest1.set(true); + reroute = strategy.reroute(clusterState, "reroute"); + assertTrue(reroute.changed()); + routingTable = reroute.routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { + assertThat(routingTable.index("test1").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test1").shard(i).primaryShard().state(), equalTo(INITIALIZING)); + } + + logger.debug("now start initializing shards and expect exactly one rebalance from node1 to node 2 sicne index [test] is all on node1"); + + routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING)).routingTable(); + + for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { + assertThat(routingTable.index("test1").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test1").shard(i).primaryShard().state(), equalTo(STARTED)); + } + int numStarted = 0; + int numRelocating = 0; + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + if (routingTable.index("test").shard(i).primaryShard().state() == STARTED) { + numStarted++; + } else if (routingTable.index("test").shard(i).primaryShard().state() == RELOCATING) { + numRelocating++; + } + } + assertEquals(numStarted, 1); + assertEquals(numRelocating, 1); + + } + + public void testRebalanceWhileShardFetching() { + final AtomicBoolean hasFetches = new AtomicBoolean(true); + AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, + ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()).build(), new NoopGatewayAllocator() { + @Override + public boolean allocateUnassigned(RoutingAllocation allocation) { + if (hasFetches.get()) { + allocation.setHasPendingAsyncFetch(); + } + return super.allocateUnassigned(allocation); + } + }); + + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "_id", "node1,node2")).numberOfShards(2).numberOfReplicas(0)) + .build(); + + // we use a second index here (test1) that never gets assigned otherwise allocateUnassinged is never called if we don't have unassigned shards. + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test")) + .addAsNew(metaData.index("test1")) + .build(); + + ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); + + logger.info("start two nodes"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(INITIALIZING)); + } + + logger.debug("start all the primary shards for test"); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test", INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED)); + } + + logger.debug("now, start 1 more node, check that rebalancing will not happen since we have shard sync going on"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) + .put(newNode("node2"))) + .build(); + logger.debug("reroute and check that nothing has changed"); + RoutingAllocation.Result reroute = strategy.reroute(clusterState, "reroute"); + assertFalse(reroute.changed()); + routingTable = reroute.routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED)); + } + for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { + assertThat(routingTable.index("test1").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test1").shard(i).primaryShard().state(), equalTo(UNASSIGNED)); + } + logger.debug("now set hasFetches to true and reroute we should now see exactly one relocating shard"); + hasFetches.set(false); + reroute = strategy.reroute(clusterState, "reroute"); + assertTrue(reroute.changed()); + routingTable = reroute.routingTable(); + int numStarted = 0; + int numRelocating = 0; + for (int i = 0; i < routingTable.index("test").shards().size(); i++) { + + assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(1)); + if (routingTable.index("test").shard(i).primaryShard().state() == STARTED) { + numStarted++; + } else if (routingTable.index("test").shard(i).primaryShard().state() == RELOCATING) { + numRelocating++; + } + } + for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { + assertThat(routingTable.index("test1").shard(i).shards().size(), equalTo(1)); + assertThat(routingTable.index("test1").shard(i).primaryShard().state(), equalTo(UNASSIGNED)); + } + assertEquals(numStarted, 1); + assertEquals(numRelocating, 1); + + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java index 7fe26d07255..34d78ae3099 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java @@ -29,18 +29,18 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ConcurrentRebalanceRoutingTests.class); - @Test public void testClusterConcurrentRebalance() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -70,8 +70,7 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start two nodes and fully start the shards"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test").shards().size(); i++) { @@ -82,7 +81,6 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start all the primary shards, replicas will start initializing"); RoutingNodes routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -97,8 +95,7 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3")).put(newNode("node4")).put(newNode("node5")).put(newNode("node6")).put(newNode("node7")).put(newNode("node8")).put(newNode("node9")).put(newNode("node10"))) .build(); - prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -110,7 +107,6 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { logger.info("start the replica shards, rebalancing should start, but, only 3 should be rebalancing"); routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -121,7 +117,6 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { logger.info("finalize this session relocation, 3 more should relocate now"); routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -132,7 +127,6 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { logger.info("finalize this session relocation, 2 more should relocate now"); routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -143,7 +137,6 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase { logger.info("finalize this session relocation, no more relocation"); routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java index 43f8479efec..e16e7cc2cec 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java @@ -31,20 +31,19 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; /** */ public class DeadNodesAllocationTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(DeadNodesAllocationTests.class); - @Test - public void simpleDeadNodeOnStartedPrimaryShard() { + public void testSimpleDeadNodeOnStartedPrimaryShard() { AllocationService allocation = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -65,7 +64,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { .put(newNode("node2")) ).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); // starting primaries @@ -88,15 +87,14 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { .put(newNode(nodeIdRemaining)) ).build(); - rerouteResult = allocation.reroute(clusterState); + rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).get(0).primary(), equalTo(true)); assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).get(0).state(), equalTo(STARTED)); } - @Test - public void deadNodeWhileRelocatingOnToNode() { + public void testDeadNodeWhileRelocatingOnToNode() { AllocationService allocation = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -117,7 +115,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { .put(newNode("node2")) ).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); // starting primaries @@ -137,7 +135,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3")) ).build(); - rerouteResult = allocation.reroute(clusterState); + rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1)); @@ -163,15 +161,14 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { .put(newNode(origPrimaryNodeId)) .put(newNode(origReplicaNodeId)) ).build(); - rerouteResult = allocation.reroute(clusterState); + rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0).state(), equalTo(STARTED)); assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).get(0).state(), equalTo(STARTED)); } - @Test - public void deadNodeWhileRelocatingOnFromNode() { + public void testDeadNodeWhileRelocatingOnFromNode() { AllocationService allocation = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -192,7 +189,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { .put(newNode("node2")) ).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); // starting primaries @@ -212,7 +209,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3")) ).build(); - rerouteResult = allocation.reroute(clusterState); + rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1)); @@ -238,7 +235,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase { .put(newNode("node3")) .put(newNode(origReplicaNodeId)) ).build(); - rerouteResult = allocation.reroute(clusterState); + rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).get(0).state(), equalTo(STARTED)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java index d2ece376345..e7c956c4ccd 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -41,10 +40,8 @@ import static org.hamcrest.Matchers.equalTo; * */ public class ElectReplicaAsPrimaryDuringRelocationTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ElectReplicaAsPrimaryDuringRelocationTests.class); - @Test public void testElectReplicaAsPrimaryDuringRelocation() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -63,7 +60,7 @@ public class ElectReplicaAsPrimaryDuringRelocationTests extends ESAllocationTest logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start the primary shards"); @@ -87,7 +84,7 @@ public class ElectReplicaAsPrimaryDuringRelocationTests extends ESAllocationTest logger.info("Start another node and perform rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("find the replica shard that gets relocated"); @@ -103,7 +100,7 @@ public class ElectReplicaAsPrimaryDuringRelocationTests extends ESAllocationTest logger.info("kill the node [{}] of the primary shard for the relocating replica", indexShardRoutingTable.primaryShard().currentNodeId()); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(indexShardRoutingTable.primaryShard().currentNodeId())).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("make sure all the primary shards are active"); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java index e6a0ec4bc97..8807816d2e8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java @@ -32,28 +32,20 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; -import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import java.util.Collections; - -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; -import static org.elasticsearch.cluster.routing.allocation.RoutingNodesUtils.numberOfShardsOfType; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.hamcrest.Matchers.equalTo; /** */ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ExpectedShardSizeAllocationTests.class); - @Test public void testInitializingHasExpectedSize() { final long byteSize = randomIntBetween(0, Integer.MAX_VALUE); AllocationService strategy = createAllocationService(Settings.EMPTY, new ClusterInfoService() { @@ -90,7 +82,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); logger.info("Adding one node and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(1, clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING)); @@ -105,14 +97,13 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase { logger.info("Add another one node and reroute"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(1, clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING)); assertEquals(byteSize, clusterState.getRoutingNodes().getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize()); } - @Test public void testExpectedSizeOnMove() { final long byteSize = randomIntBetween(0, Integer.MAX_VALUE); final AllocationService allocation = createAllocationService(Settings.EMPTY, new ClusterInfoService() { @@ -144,7 +135,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase { logger.info("adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); logger.info("start primary shard"); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java index 70b056b6298..3b242d8676f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -39,11 +38,9 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; public class FailedNodeRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(FailedNodeRoutingTests.class); - @Test - public void simpleFailedNodeTest() { + public void testSimpleFailedNodeTest() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()).build()); @@ -62,7 +59,7 @@ public class FailedNodeRoutingTests extends ESAllocationTestCase { logger.info("start 4 nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("start all the primary shards, replicas will start initializing"); @@ -93,7 +90,7 @@ public class FailedNodeRoutingTests extends ESAllocationTestCase { ) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java index ff2ae1051ed..9bfaf7e9997 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java @@ -24,30 +24,36 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Collections; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; /** * */ public class FailedShardsRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(FailedShardsRoutingTests.class); - @Test public void testFailedShardPrimaryRelocatingToAndFrom() { AllocationService allocation = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -69,7 +75,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { .put(newNode("node2")) ).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); // starting primaries @@ -89,7 +95,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3")) ).build(); - rerouteResult = allocation.reroute(clusterState); + rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1)); @@ -136,8 +142,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), anyOf(equalTo(origPrimaryNodeId), equalTo("node3"))); } - @Test - public void failPrimaryStartedCheckReplicaElected() { + public void testFailPrimaryStartedCheckReplicaElected() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -158,7 +163,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start the shards (primaries)"); @@ -218,8 +223,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { assertThat(strategy.applyFailedShard(clusterState, shardToFail).changed(), equalTo(false)); } - @Test - public void firstAllocationFailureSingleNode() { + public void testFirstAllocationFailureSingleNode() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -240,7 +244,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { logger.info("Adding single node and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -275,8 +279,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { assertThat(strategy.applyFailedShard(clusterState, firstShard).changed(), equalTo(false)); } - @Test - public void singleShardMultipleAllocationFailures() { + public void testSingleShardMultipleAllocationFailures() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -308,7 +311,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { ) .build(); // and assign more unassigned - clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState).routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build(); } int shardsToFail = randomIntBetween(1, numberOfReplicas); @@ -332,8 +335,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { } } - @Test - public void firstAllocationFailureTwoNodes() { + public void testFirstAllocationFailureTwoNodes() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -354,7 +356,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); final String nodeHoldingPrimary = routingTable.index("test").shard(0).primaryShard().currentNodeId(); @@ -393,8 +395,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { assertThat(strategy.applyFailedShard(clusterState, firstShard).changed(), equalTo(false)); } - @Test - public void rebalanceFailure() { + public void testRebalanceFailure() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") @@ -415,7 +416,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start the shards (primaries)"); @@ -457,7 +458,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { logger.info("Adding third node and reroute"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -489,7 +490,6 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").get(0).shardId(), not(equalTo(shardToFail.shardId()))); } - @Test public void testFailAllReplicasInitializingOnPrimaryFail() { AllocationService allocation = createAllocationService(settingsBuilder() .build()); @@ -506,7 +506,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { // add 4 nodes clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build(); - clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState).routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState, "reroute").routingTable()).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(2)); // start primary shards @@ -536,7 +536,6 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { assertThat(routingResult.changed(), equalTo(false)); } - @Test public void testFailAllReplicasInitializingOnPrimaryFailWhileHavingAReplicaToElect() { AllocationService allocation = createAllocationService(settingsBuilder() .build()); @@ -553,7 +552,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase { // add 4 nodes clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build(); - clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState).routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState, "reroute").routingTable()).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(2)); // start primary shards diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java index eb4d62a707f..6b55d743204 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -31,22 +32,20 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.List; import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; /** */ public class FilterRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(FilterRoutingTests.class); - @Test public void testClusterFilters() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.include.tag1", "value1,value2") @@ -72,7 +71,7 @@ public class FilterRoutingTests extends ESAllocationTestCase { .put(newNode("node3", singletonMap("tag1", "value3"))) .put(newNode("node4", singletonMap("tag1", "value4"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2)); @@ -92,7 +91,6 @@ public class FilterRoutingTests extends ESAllocationTestCase { } } - @Test public void testIndexFilters() { AllocationService strategy = createAllocationService(settingsBuilder() .build()); @@ -121,7 +119,7 @@ public class FilterRoutingTests extends ESAllocationTestCase { .put(newNode("node3", singletonMap("tag1", "value3"))) .put(newNode("node4", singletonMap("tag1", "value4"))) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2)); @@ -151,7 +149,7 @@ public class FilterRoutingTests extends ESAllocationTestCase { .build())) .build(); clusterState = ClusterState.builder(clusterState).metaData(metaData).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(2)); @@ -166,4 +164,69 @@ public class FilterRoutingTests extends ESAllocationTestCase { assertThat(startedShard.currentNodeId(), Matchers.anyOf(equalTo("node1"), equalTo("node4"))); } } + + public void testRebalanceAfterShardsCannotRemainOnNode() { + AllocationService strategy = createAllocationService(settingsBuilder().build()); + + logger.info("Building initial routing table"); + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .build(); + + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test1")) + .addAsNew(metaData.index("test2")) + .build(); + + ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes and performing rerouting"); + DiscoveryNode node1 = newNode("node1", singletonMap("tag1", "value1")); + DiscoveryNode node2 = newNode("node2", singletonMap("tag1", "value2")); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(node1).put(node2)).build(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.getRoutingNodes().node(node1.getId()).numberOfShardsWithState(INITIALIZING), equalTo(2)); + assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(INITIALIZING), equalTo(2)); + + logger.info("--> start the shards (only primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + logger.info("--> make sure all shards are started"); + assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4)); + + logger.info("--> disable allocation for node1 and reroute"); + strategy = createAllocationService(settingsBuilder() + .put("cluster.routing.allocation.cluster_concurrent_rebalance", "1") + .put("cluster.routing.allocation.exclude.tag1", "value1") + .build()); + + logger.info("--> move shards from node1 to node2"); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + logger.info("--> check that concurrent rebalance only allows 1 shard to move"); + assertThat(clusterState.getRoutingNodes().node(node1.getId()).numberOfShardsWithState(STARTED), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(INITIALIZING), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(STARTED), equalTo(2)); + + logger.info("--> start the shards (only primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + logger.info("--> move second shard from node1 to node2"); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(INITIALIZING), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(STARTED), equalTo(3)); + + logger.info("--> start the shards (only primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.getRoutingNodes().node(node2.getId()).numberOfShardsWithState(STARTED), equalTo(4)); + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java index aece576a782..aa6fdef828a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java @@ -30,9 +30,10 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -41,10 +42,8 @@ import static org.hamcrest.Matchers.nullValue; * */ public class IndexBalanceTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(IndexBalanceTests.class); - @Test public void testBalanceAllNodesStarted() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -86,7 +85,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -103,7 +102,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { logger.info("Another round of rebalancing"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable == routingTable, equalTo(true)); @@ -128,7 +127,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the more shards"); @@ -175,7 +174,6 @@ public class IndexBalanceTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2)); } - @Test public void testBalanceIncrementallyStartNodes() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -216,7 +214,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -234,7 +232,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState) .nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable == routingTable, equalTo(true)); @@ -260,7 +258,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the backup shard"); @@ -298,14 +296,14 @@ public class IndexBalanceTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState) .nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the backup shard"); @@ -338,7 +336,6 @@ public class IndexBalanceTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2)); } - @Test public void testBalanceAllNodesStartedAddIndex() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -369,7 +366,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -386,7 +383,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { logger.info("Another round of rebalancing"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable == routingTable, equalTo(true)); @@ -411,7 +408,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the more shards"); @@ -461,7 +458,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { assertThat(routingTable.index("test1").shards().size(), equalTo(3)); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -478,7 +475,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { logger.info("Another round of rebalancing"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable == routingTable, equalTo(true)); @@ -503,7 +500,7 @@ public class IndexBalanceTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the more shards"); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java index 344d6b909e4..2fe1d85b1f4 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java @@ -25,16 +25,15 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; import org.elasticsearch.test.VersionUtils; -import org.junit.Test; import java.util.ArrayList; import java.util.Collections; @@ -49,10 +48,8 @@ import static org.hamcrest.Matchers.*; * */ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(NodeVersionAllocationDeciderTests.class); - @Test public void testDoNotAllocateFromPrimary() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -86,7 +83,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { logger.info("start two nodes and fully start the shards"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test").shards().size(); i++) { @@ -127,7 +124,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { .put(newNode("node3", VersionUtils.getPreviousVersion()))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -143,7 +140,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { .put(newNode("node4"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -167,8 +164,6 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { } } - - @Test public void testRandom() { AllocationService service = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -199,7 +194,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); int numNodes = between(1, 20); if (nodes.size() > numNodes) { - Collections.shuffle(nodes, getRandom()); + Collections.shuffle(nodes, random()); nodes = nodes.subList(0, numNodes); } else { for (int j = nodes.size(); j < numNodes; j++) { @@ -218,7 +213,6 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { } } - @Test public void testRollingRestart() { AllocationService service = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -289,7 +283,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { private ClusterState stabilize(ClusterState clusterState, AllocationService service) { logger.trace("RoutingNodes: {}", clusterState.getRoutingNodes().prettyPrint()); - RoutingTable routingTable = service.reroute(clusterState).routingTable(); + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); RoutingNodes routingNodes = clusterState.getRoutingNodes(); assertRecoveryNodeVersions(routingNodes); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java index 616949ec72f..2e37a3a11b9 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; @@ -39,7 +38,6 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class PreferLocalPrimariesToRelocatingPrimariesTests extends ESAllocationTestCase { - @Test public void testPreferLocalPrimaryAllocationOverFiltered() { int concurrentRecoveries = randomIntBetween(1, 10); int primaryRecoveries = randomIntBetween(1, 10); @@ -71,7 +69,7 @@ public class PreferLocalPrimariesToRelocatingPrimariesTests extends ESAllocation .put(newNode("node1", singletonMap("tag1", "value1"))) .put(newNode("node2", singletonMap("tag1", "value2")))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); while (!clusterState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty()) { @@ -94,7 +92,7 @@ public class PreferLocalPrimariesToRelocatingPrimariesTests extends ESAllocation .build())) .build(); clusterState = ClusterState.builder(clusterState).metaData(metaData).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("[{}] primaries should be still started but [{}] other primaries should be unassigned", numberOfShards, numberOfShards); @@ -105,7 +103,7 @@ public class PreferLocalPrimariesToRelocatingPrimariesTests extends ESAllocation logger.info("start node back up"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node1", singletonMap("tag1", "value1")))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); while (clusterState.getRoutingNodes().shardsWithState(STARTED).size() < totalNumberOfShards) { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java index 108281eae7f..0ac98d4f92b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -37,10 +36,8 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class PreferPrimaryAllocationTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(PreferPrimaryAllocationTests.class); - @Test public void testPreferPrimaryAllocationOverReplicas() { logger.info("create an allocation with 1 initial recoveries"); AllocationService strategy = createAllocationService(settingsBuilder() @@ -64,7 +61,7 @@ public class PreferPrimaryAllocationTests extends ESAllocationTestCase { logger.info("adding two nodes and performing rerouting till all are allocated"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); while (!clusterState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty()) { @@ -77,7 +74,7 @@ public class PreferPrimaryAllocationTests extends ESAllocationTestCase { metaData = MetaData.builder(clusterState.metaData()).updateNumberOfReplicas(1).build(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaData).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("2 replicas should be initializing now for the existing indices (we throttle to 1)"); @@ -95,7 +92,7 @@ public class PreferPrimaryAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).metaData(metaData).routingTable(routingTable).build(); logger.info("reroute, verify that primaries for the new index primary shards are allocated"); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.routingTable().index("new_index").shardsWithState(INITIALIZING).size(), equalTo(2)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java index 69824f1a248..e994c885629 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -41,10 +40,8 @@ import static org.hamcrest.Matchers.nullValue; * */ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(PrimaryElectionRoutingTests.class); - @Test public void testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -63,7 +60,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start the primary shard (on node1)"); @@ -81,7 +78,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { logger.info("Adding third node and reroute and kill first node"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3")).remove("node1")).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -95,7 +92,6 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { assertThat(routingTable.index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node3")); } - @Test public void testRemovingInitializingReplicasIfPrimariesFails() { AllocationService allocation = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -113,7 +109,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); + RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); logger.info("Start the primary shards"); @@ -132,7 +128,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder() .put(newNode(nodeIdRemaining)) ).build(); - rerouteResult = allocation.reroute(clusterState); + rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java index c295b4f70e8..12ff9fd3f7d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -40,11 +39,8 @@ import static org.hamcrest.Matchers.equalTo; * */ public class PrimaryNotRelocatedWhileBeingRecoveredTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(PrimaryNotRelocatedWhileBeingRecoveredTests.class); - - @Test public void testPrimaryNotRelocatedWhileBeingRecoveredFrom() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -65,7 +61,7 @@ public class PrimaryNotRelocatedWhileBeingRecoveredTests extends ESAllocationTes logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start the primary shard (on node1)"); @@ -77,7 +73,7 @@ public class PrimaryNotRelocatedWhileBeingRecoveredTests extends ESAllocationTes logger.info("start another node, replica will start recovering form primary"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(5)); @@ -85,7 +81,7 @@ public class PrimaryNotRelocatedWhileBeingRecoveredTests extends ESAllocationTes logger.info("start another node, make sure the primary is not relocated"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(5)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java index 4c828603baf..4d5f4d07ea1 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESAllocationTestCase; import org.elasticsearch.test.gateway.NoopGatewayAllocator; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.Arrays; import java.util.HashSet; @@ -50,13 +49,11 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; public class RandomAllocationDeciderTests extends ESAllocationTestCase { - /* This test will make random allocation decision on a growing and shrinking * cluster leading to a random distribution of the shards. After a certain * amount of iterations the test allows allocation unless the same shard is * already allocated on a node and balances the cluster to gain optimal * balance.*/ - @Test public void testRandomDecisions() { RandomAllocationDecider randomAllocationDecider = new RandomAllocationDecider(getRandom()); AllocationService strategy = new AllocationService(settingsBuilder().build(), new AllocationDeciders(Settings.EMPTY, @@ -108,7 +105,7 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase { stateBuilder.nodes(newNodesBuilder.build()); clusterState = stateBuilder.build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); if (clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size() > 0) { routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)) @@ -134,7 +131,7 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase { int iterations = 0; do { iterations++; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); if (clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size() > 0) { routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)) diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java index 4dd88501ec2..fbc742573e9 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java @@ -33,13 +33,12 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import java.util.Collections; - -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -48,10 +47,8 @@ import static org.hamcrest.Matchers.nullValue; * */ public class RebalanceAfterActiveTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(RebalanceAfterActiveTests.class); - @Test public void testRebalanceOnlyAfterAllShardsAreActive() { final long[] sizes = new long[5]; for (int i =0; i < sizes.length; i++) { @@ -104,7 +101,7 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase { logger.info("start two nodes and fully start the shards"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test").shards().size(); i++) { @@ -132,7 +129,7 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase { .put(newNode("node3")).put(newNode("node4")).put(newNode("node5")).put(newNode("node6")).put(newNode("node7")).put(newNode("node8")).put(newNode("node9")).put(newNode("node10"))) .build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java index 54440581b79..0d33b5ecd46 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java @@ -29,20 +29,21 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; /** * */ public class ReplicaAllocatedAfterPrimaryTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ReplicaAllocatedAfterPrimaryTests.class); - @Test public void testBackupIsAllocatedAfterPrimary() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -70,7 +71,7 @@ public class ReplicaAllocatedAfterPrimaryTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); final String nodeHoldingPrimary = routingTable.index("test").shard(0).primaryShard().currentNodeId(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java index 98009665f4d..eca2a227f8f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -42,10 +41,8 @@ import static org.hamcrest.Matchers.equalTo; * */ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(IndexBalanceTests.class); - @Test public void testBalanceAllNodesStarted() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -75,7 +72,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(true)); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -87,7 +84,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { logger.info("Another round of rebalancing"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -98,7 +95,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); logger.info("Start the more shards"); routingNodes = clusterState.getRoutingNodes(); @@ -118,7 +115,6 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { } - @Test public void testBalanceIncrementallyStartNodes() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -139,14 +135,14 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Add another node and perform rerouting, nothing will happen since primary not started"); clusterState = ClusterState.builder(clusterState) .nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start the primary shard"); @@ -157,7 +153,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); logger.info("Start the backup shard"); routingNodes = clusterState.getRoutingNodes(); @@ -174,12 +170,12 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState) .nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); logger.info("Start the backup shard"); routingNodes = clusterState.getRoutingNodes(); @@ -211,7 +207,6 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2)); } - @Test public void testBalanceAllNodesStartedAddIndex() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 1) @@ -238,7 +233,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(true)); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -250,7 +245,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { logger.info("Another round of rebalancing"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable == routingTable, equalTo(true)); @@ -275,7 +270,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the more shards"); @@ -321,13 +316,13 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { assertThat(routingTable.index("test1").shards().size(), equalTo(3)); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Reroute, assign"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -374,7 +369,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase { logger.info("kill one node"); IndexShardRoutingTable indexShardRoutingTable = routingTable.index("test").shard(0); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(indexShardRoutingTable.primaryShard().currentNodeId())).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java index 86369b967ab..94ad80ecac3 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; @@ -44,11 +43,9 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class SameShardRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(SameShardRoutingTests.class); - @Test - public void sameHost() { + public void testSameHost() { AllocationService strategy = createAllocationService(settingsBuilder().put(SameShardAllocationDecider.SAME_HOST_SETTING, true).build()); MetaData metaData = MetaData.builder() @@ -64,7 +61,7 @@ public class SameShardRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder() .put(new DiscoveryNode("node1", "node1", "test1", "test1", DummyTransportAddress.INSTANCE, emptyMap(), Version.CURRENT)) .put(new DiscoveryNode("node2", "node2", "test1", "test1", DummyTransportAddress.INSTANCE, emptyMap(), Version.CURRENT))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), ShardRoutingState.INITIALIZING), equalTo(2)); @@ -79,7 +76,7 @@ public class SameShardRoutingTests extends ESAllocationTestCase { logger.info("--> add another node, with a different host, replicas will be allocating"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(new DiscoveryNode("node3", "node3", "test2", "test2", DummyTransportAddress.INSTANCE, emptyMap(), Version.CURRENT))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(numberOfShardsOfType(clusterState.getRoutingNodes(), ShardRoutingState.STARTED), equalTo(2)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java index 1074f8b4e17..f096ab0b13d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java @@ -30,18 +30,17 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; public class ShardVersioningTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ShardVersioningTests.class); - @Test - public void simple() { + public void testSimple() { AllocationService strategy = createAllocationService(settingsBuilder().put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()).build()); @@ -59,8 +58,7 @@ public class ShardVersioningTests extends ESAllocationTestCase { logger.info("start two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); for (int i = 0; i < routingTable.index("test1").shards().size(); i++) { @@ -79,7 +77,6 @@ public class ShardVersioningTests extends ESAllocationTestCase { logger.info("start all the primary shards for test1, replicas will start initializing"); RoutingNodes routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState("test1", INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java index c59e90f793d..11d41a6a336 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java @@ -24,17 +24,18 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.elasticsearch.cluster.routing.allocation.RoutingNodesUtils.numberOfShardsOfType; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; @@ -42,11 +43,9 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class ShardsLimitAllocationTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ShardsLimitAllocationTests.class); - @Test - public void indexLevelShardsLimitAllocate() { + public void testIndexLevelShardsLimitAllocate() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); logger.info("Building initial routing table"); @@ -65,7 +64,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); logger.info("Adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(2)); @@ -88,8 +87,65 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); } - @Test - public void indexLevelShardsLimitRemain() { + public void testClusterLevelShardsLimitAllocate() { + AllocationService strategy = createAllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE, 1) + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))) + .build(); + + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test")) + .build(); + + ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); + logger.info("Adding two nodes and performing rerouting"); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + + logger.info("Start the primary shards"); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1)); + assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(2)); + + // Bump the cluster total shards to 2 + strategy = createAllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE, 2) + .build()); + + logger.info("Do another reroute, make sure shards are now allocated"); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1)); + + routingNodes = clusterState.getRoutingNodes(); + routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2)); + assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2)); + assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0)); + } + + public void testIndexLevelShardsLimitRemain() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) @@ -115,7 +171,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); logger.info("Adding one node and reroute"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start the primary shards"); @@ -140,7 +196,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase { logger.info("Add another one node and reroute"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); @@ -169,7 +225,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).metaData(metaData).build(); logger.info("reroute after setting"); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(3)); @@ -181,7 +237,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase { routingNodes = clusterState.getRoutingNodes(); routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); - // now we are done compared to EvenShardCountAllocator since the Balancer is not soely based on the average + // now we are done compared to EvenShardCountAllocator since the Balancer is not soely based on the average assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(5)); assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(5)); } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java index e197dbd49c3..ed44b84a886 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.HashSet; @@ -57,10 +56,8 @@ import static org.hamcrest.Matchers.nullValue; * */ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(SingleShardNoReplicasRoutingTests.class); - @Test public void testSingleIndexStartedShard() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -85,7 +82,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { logger.info("Adding one node and performing rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable.index("test").shards().size(), equalTo(1)); @@ -97,7 +94,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { logger.info("Rerouting again, nothing should change"); prevRoutingTable = routingTable; clusterState = ClusterState.builder(clusterState).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(routingTable == prevRoutingTable, equalTo(true)); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); @@ -117,7 +114,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { logger.info("Starting another node and making sure nothing changed"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable == prevRoutingTable, equalTo(true)); @@ -131,7 +128,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable != prevRoutingTable, equalTo(true)); @@ -144,7 +141,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { logger.info("Start another node, make sure that things remain the same (shard is in node2 and initializing)"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable == prevRoutingTable, equalTo(true)); @@ -162,7 +159,6 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { assertThat(routingTable.index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node2")); } - @Test public void testSingleIndexShardFailed() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -187,7 +183,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { logger.info("Adding one node and rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -212,7 +208,6 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { assertThat(routingTable.index("test").shard(0).shards().get(0).currentNodeId(), nullValue()); } - @Test public void testMultiIndexEvenDistribution() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -253,7 +248,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { } RoutingTable prevRoutingTable = routingTable; clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -291,7 +286,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { } prevRoutingTable = routingTable; clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(false)); @@ -325,7 +320,6 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { assertThat(numberOfStartedShards, equalTo(25)); } - @Test public void testMultiIndexUnevenNodes() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -357,7 +351,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3"))) .build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -380,7 +374,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java index 7ea7e1cae23..ff442852017 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java @@ -29,9 +29,10 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -40,10 +41,8 @@ import static org.hamcrest.Matchers.nullValue; * */ public class SingleShardOneReplicaRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(SingleShardOneReplicaRoutingTests.class); - @Test public void testSingleIndexFirstStartPrimaryThenBackups() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -71,7 +70,7 @@ public class SingleShardOneReplicaRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -87,7 +86,7 @@ public class SingleShardOneReplicaRoutingTests extends ESAllocationTestCase { logger.info("Add another node and perform rerouting, nothing will happen since primary shards not started"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable == routingTable, equalTo(true)); @@ -112,7 +111,7 @@ public class SingleShardOneReplicaRoutingTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the backup shard"); @@ -135,7 +134,7 @@ public class SingleShardOneReplicaRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -153,7 +152,7 @@ public class SingleShardOneReplicaRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java index 1e8a5fbc1ec..28033915abe 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.util.Arrays; @@ -40,9 +39,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public class StartedShardsRoutingTests extends ESAllocationTestCase { - - @Test - public void tesStartedShardsMatching() { + public void testStartedShardsMatching() { AllocationService allocation = createAllocationService(); logger.info("--> building initial cluster state"); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java index d6e7c86277d..671cce007c9 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java @@ -30,20 +30,22 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.nullValue; /** * */ public class TenShardsOneReplicaRoutingTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(TenShardsOneReplicaRoutingTests.class); - @Test public void testSingleIndexFirstStartPrimaryThenBackups() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 10) @@ -81,7 +83,7 @@ public class TenShardsOneReplicaRoutingTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -99,7 +101,7 @@ public class TenShardsOneReplicaRoutingTests extends ESAllocationTestCase { logger.info("Add another node and perform rerouting, nothing will happen since primary not started"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable == routingTable, equalTo(true)); @@ -125,7 +127,7 @@ public class TenShardsOneReplicaRoutingTests extends ESAllocationTestCase { logger.info("Reroute, nothing should change"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); assertThat(prevRoutingTable == routingTable, equalTo(true)); logger.info("Start the backup shard"); @@ -152,7 +154,7 @@ public class TenShardsOneReplicaRoutingTests extends ESAllocationTestCase { logger.info("Add another node and perform rerouting"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingNodes = clusterState.getRoutingNodes(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java index c156aea3bd3..223da88192b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -28,9 +28,10 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; @@ -38,10 +39,8 @@ import static org.hamcrest.Matchers.equalTo; * */ public class ThrottlingAllocationTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(ThrottlingAllocationTests.class); - @Test public void testPrimaryRecoveryThrottling() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.node_concurrent_recoveries", 3) @@ -62,7 +61,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase { logger.info("start one node, do reroute, only 3 should initialize"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(0)); @@ -102,7 +101,6 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase { assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(10)); } - @Test public void testReplicaAndPrimaryRecoveryThrottling() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 3) @@ -123,7 +121,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase { logger.info("start one node, do reroute, only 3 should initialize"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(0)); @@ -148,7 +146,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase { logger.info("start another node, replicas should start being allocated"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(5)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java index 67b81be90ee..7fa27e7050c 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java @@ -29,20 +29,22 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; /** * */ public class UpdateNumberOfReplicasTests extends ESAllocationTestCase { - private final ESLogger logger = Loggers.getLogger(UpdateNumberOfReplicasTests.class); - @Test public void testUpdateNumberOfReplicas() { AllocationService strategy = createAllocationService(settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); @@ -71,7 +73,7 @@ public class UpdateNumberOfReplicasTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("Start all the primary shards"); @@ -106,7 +108,7 @@ public class UpdateNumberOfReplicasTests extends ESAllocationTestCase { metaData = MetaData.builder(clusterState.metaData()).updateNumberOfReplicas(2).build(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaData).build(); - assertThat(clusterState.metaData().index("test").numberOfReplicas(), equalTo(2)); + assertThat(clusterState.metaData().index("test").getNumberOfReplicas(), equalTo(2)); assertThat(prevRoutingTable != routingTable, equalTo(true)); assertThat(routingTable.index("test").shards().size(), equalTo(1)); @@ -121,7 +123,7 @@ public class UpdateNumberOfReplicasTests extends ESAllocationTestCase { logger.info("Add another node and start the added replica"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build(); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(true)); @@ -157,7 +159,7 @@ public class UpdateNumberOfReplicasTests extends ESAllocationTestCase { metaData = MetaData.builder(clusterState.metaData()).updateNumberOfReplicas(1).build(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaData).build(); - assertThat(clusterState.metaData().index("test").numberOfReplicas(), equalTo(1)); + assertThat(clusterState.metaData().index("test").getNumberOfReplicas(), equalTo(1)); assertThat(prevRoutingTable != routingTable, equalTo(true)); assertThat(routingTable.index("test").shards().size(), equalTo(1)); @@ -170,7 +172,7 @@ public class UpdateNumberOfReplicasTests extends ESAllocationTestCase { logger.info("do a reroute, should remain the same"); prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(prevRoutingTable != routingTable, equalTo(false)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 525c4465363..a739f30856a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -29,14 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators; @@ -49,21 +42,15 @@ import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; import org.elasticsearch.test.gateway.NoopGatewayAllocator; -import org.junit.Test; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; -import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class DiskThresholdDeciderTests extends ESAllocationTestCase { @@ -71,8 +58,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { return new ShardsAllocators(NoopGatewayAllocator.INSTANCE); } - @Test - public void diskThresholdTest() { + public void testDiskThreshold() { Settings diskSettings = settingsBuilder() .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true) .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, 0.7) @@ -129,7 +115,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -159,7 +145,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -199,7 +185,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) .build(), deciders, makeShardsAllocators(), cis); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -230,7 +216,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) .build(), deciders, makeShardsAllocators(), cis); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -246,7 +232,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -266,8 +252,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node("node4").size(), equalTo(1)); } - @Test - public void diskThresholdWithAbsoluteSizesTest() { + public void testDiskThresholdWithAbsoluteSizes() { Settings diskSettings = settingsBuilder() .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true) .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, "30b") @@ -327,7 +312,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -368,7 +353,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) .build(), deciders, makeShardsAllocators(), cis); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -395,7 +380,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node3")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -435,7 +420,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) .build(), deciders, makeShardsAllocators(), cis); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -466,7 +451,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) .build(), deciders, makeShardsAllocators(), cis); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -482,7 +467,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node4")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -509,7 +494,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(newNode("node5")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -535,8 +520,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node("node5").size(), equalTo(1)); } - @Test - public void diskThresholdWithShardSizes() { + public void testDiskThresholdWithShardSizes() { Settings diskSettings = settingsBuilder() .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true) .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, 0.7) @@ -590,7 +574,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) // node2 is added because DiskThresholdDecider automatically ignore single-node clusters ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logger.info("--> start the shards (primaries)"); routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); @@ -603,8 +587,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0)); } - @Test - public void unknownDiskUsageTest() { + public void testUnknownDiskUsage() { Settings diskSettings = settingsBuilder() .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true) .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, 0.7) @@ -659,7 +642,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node3")) // node3 is added because DiskThresholdDecider automatically ignore single-node clusters ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); // Shard can be allocated to node1, even though it only has 25% free, @@ -678,8 +661,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1)); } - @Test - public void averageUsageUnitTest() { + public void testAverageUsage() { RoutingNode rn = new RoutingNode("node1", newNode("node1")); DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY); @@ -692,8 +674,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { assertThat(node1Usage.getFreeBytes(), equalTo(25L)); } - @Test - public void freeDiskPercentageAfterShardAssignedUnitTest() { + public void testFreeDiskPercentageAfterShardAssigned() { RoutingNode rn = new RoutingNode("node1", newNode("node1")); DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY); @@ -705,7 +686,6 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { assertThat(after, equalTo(19.0)); } - @Test public void testShardRelocationsTakenIntoAccount() { Settings diskSettings = settingsBuilder() .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true) @@ -768,7 +748,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); logShardStates(clusterState); @@ -812,7 +792,6 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { } - @Test public void testCanRemainWithShardRelocatingAway() { Settings diskSettings = settingsBuilder() .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true) @@ -868,7 +847,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ) ); ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build(); - RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); assertThat(decision.type(), equalTo(Decision.Type.NO)); @@ -888,7 +867,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ) ); clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build(); - routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); @@ -915,7 +894,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .build(), deciders, makeShardsAllocators(), cis); // Ensure that the reroute call doesn't alter the routing table, since the first primary is relocating away // and therefor we will have sufficient disk space on node1. - RoutingAllocation.Result result = strategy.reroute(clusterState); + RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute"); assertThat(result.changed(), is(false)); assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED)); assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().currentNodeId(), equalTo("node1")); @@ -987,7 +966,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ) ); ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build(); - RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); // Two shards should start happily @@ -1013,7 +992,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always") .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) .build(), deciders, makeShardsAllocators(), cis); - RoutingAllocation.Result result = strategy.reroute(clusterState); + RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute"); assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED)); assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().currentNodeId(), equalTo("node2")); @@ -1044,11 +1023,11 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ); clusterState = ClusterState.builder(updateClusterState).routingTable(builder.build()).build(); - routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); - result = strategy.reroute(clusterState); + result = strategy.reroute(clusterState, "reroute"); assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED)); assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().currentNodeId(), equalTo("node2")); assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().relocatingNodeId(), nullValue()); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 5417a9b85f3..a386883ad1b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -20,21 +20,13 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterInfo; -import org.elasticsearch.cluster.ClusterInfoService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.DiskUsage; -import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.MockInternalClusterInfoService.DevNullClusterInfo; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingHelper; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -43,7 +35,6 @@ import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Arrays; @@ -53,8 +44,6 @@ import static org.hamcrest.CoreMatchers.equalTo; * Unit tests for the DiskThresholdDecider */ public class DiskThresholdDeciderUnitTests extends ESTestCase { - - @Test public void testDynamicSettings() { NodeSettingsService nss = new NodeSettingsService(Settings.EMPTY); @@ -132,7 +121,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase { ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder(); shardSizes.put("[test][0][p]", 10L); // 10 bytes final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of()); - RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo); + RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo, System.nanoTime()); assertEquals(mostAvailableUsage.toString(), Decision.YES, decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation)); assertEquals(mostAvailableUsage.toString(), Decision.NO, decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation)); } @@ -197,7 +186,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase { shardSizes.put("[test][2][p]", 10L); final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), shardRoutingMap.build()); - RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo); + RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo, System.nanoTime()); assertEquals(Decision.YES, decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation)); assertEquals(Decision.NO, decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation)); try { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java index 3823893977d..0049a120777 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java @@ -20,14 +20,15 @@ package org.elasticsearch.cluster.routing.allocation.decider; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.Allocation; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.Rebalance; @@ -36,7 +37,6 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.util.EnumSet; import java.util.List; @@ -56,7 +56,6 @@ public class EnableAllocationTests extends ESAllocationTestCase { private final ESLogger logger = Loggers.getLogger(EnableAllocationTests.class); - @Test public void testClusterEnableNone() { AllocationService strategy = createAllocationService(settingsBuilder() .put(CLUSTER_ROUTING_ALLOCATION_ENABLE, Allocation.NONE.name()) @@ -79,13 +78,12 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); } - @Test public void testClusterEnableOnlyPrimaries() { AllocationService strategy = createAllocationService(settingsBuilder() .put(CLUSTER_ROUTING_ALLOCATION_ENABLE, Allocation.PRIMARIES.name()) @@ -108,7 +106,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); @@ -119,7 +117,6 @@ public class EnableAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); } - @Test public void testIndexEnableNone() { AllocationService strategy = createAllocationService(settingsBuilder() .build()); @@ -143,7 +140,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); logger.info("--> start the shards (primaries)"); @@ -158,10 +155,6 @@ public class EnableAllocationTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().shardsWithState("disabled", STARTED).size(), equalTo(0)); } - - - - @Test public void testEnableClusterBalance() { final boolean useClusterSetting = randomBoolean(); final Rebalance allowedOnes = RandomPicks.randomFrom(getRandom(), EnumSet.of(Rebalance.PRIMARIES, Rebalance.REPLICAS, Rebalance.ALL)); @@ -191,7 +184,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4)); logger.info("--> start the shards (primaries)"); @@ -212,7 +205,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(newNode("node3")) ).build(); ClusterState prevState = clusterState; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8)); assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(0)); @@ -232,7 +225,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { } nodeSettingsService.clusterChanged(new ClusterChangedEvent("foo", clusterState, prevState)); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat("expected 6 shards to be started 2 to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(6)); assertThat("expected 2 shards to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(2)); @@ -265,7 +258,6 @@ public class EnableAllocationTests extends ESAllocationTestCase { } - @Test public void testEnableClusterBalanceNoReplicas() { final boolean useClusterSetting = randomBoolean(); Settings build = settingsBuilder() @@ -292,7 +284,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(newNode("node1")) .put(newNode("node2")) ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(6)); logger.info("--> start the shards (primaries)"); @@ -308,7 +300,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(newNode("node3")) ).build(); ClusterState prevState = clusterState; - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(6)); assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(0)); @@ -324,7 +316,7 @@ public class EnableAllocationTests extends ESAllocationTestCase { .put(IndexMetaData.builder(meta).settings(settingsBuilder().put(meta.getSettings()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE, randomBoolean() ? Rebalance.PRIMARIES : Rebalance.ALL).build()))).build(); } nodeSettingsService.clusterChanged(new ClusterChangedEvent("foo", clusterState, prevState)); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertThat("expected 4 primaries to be started and 2 to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4)); assertThat("expected 2 primaries to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(2)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java index 8396f61b7f7..126799f5937 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -62,7 +61,6 @@ public class MockDiskUsagesIT extends ESIntegTestCase { return pluginList(MockInternalClusterInfoService.TestPlugin.class); } - @Test //@TestLogging("org.elasticsearch.cluster:TRACE,org.elasticsearch.cluster.routing.allocation.decider:TRACE") public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception { List nodes = internalCluster().startNodesAsync(3).get(); diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 308b00031b8..e7ca4f8382b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; @@ -38,8 +37,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class ClusterSerializationTests extends ESAllocationTestCase { - - @Test public void testClusterStateSerialization() throws Exception { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1)) @@ -54,17 +51,15 @@ public class ClusterSerializationTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(new ClusterName("clusterName1")).nodes(nodes).metaData(metaData).routingTable(routingTable).build(); AllocationService strategy = createAllocationService(); - clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState).routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build(); ClusterState serializedClusterState = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), newNode("node1")); assertThat(serializedClusterState.getClusterName().value(), equalTo(clusterState.getClusterName().value())); - + assertThat(serializedClusterState.routingTable().prettyPrint(), equalTo(clusterState.routingTable().prettyPrint())); } - - @Test public void testRoutingTableSerialization() throws Exception { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1)) @@ -79,7 +74,7 @@ public class ClusterSerializationTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build(); AllocationService strategy = createAllocationService(); - RoutingTable source = strategy.reroute(clusterState).routingTable(); + RoutingTable source = strategy.reroute(clusterState, "reroute").routingTable(); BytesStreamOutput outStream = new BytesStreamOutput(); source.writeTo(outStream); diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java index 573cb131050..13bf980fc11 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.containsString; @@ -37,7 +36,6 @@ import static org.hamcrest.Matchers.containsString; * */ public class ClusterStateToStringTests extends ESAllocationTestCase { - @Test public void testClusterStateSerialization() throws Exception { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test_idx").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1)) @@ -53,7 +51,7 @@ public class ClusterStateToStringTests extends ESAllocationTestCase { ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build(); AllocationService strategy = createAllocationService(); - clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState).routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build(); String clusterStateString = clusterState.toString(); assertNotNull(clusterStateString); @@ -61,7 +59,5 @@ public class ClusterStateToStringTests extends ESAllocationTestCase { assertThat(clusterStateString, containsString("test_idx")); assertThat(clusterStateString, containsString("test_template")); assertThat(clusterStateString, containsString("node_foo")); - } - } diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java index 87280f6acb3..b3050392c98 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java @@ -22,72 +22,383 @@ package org.elasticsearch.cluster.serialization; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.DiffableUtils; -import org.elasticsearch.cluster.DiffableUtils.KeyedReader; +import org.elasticsearch.cluster.DiffableUtils.MapDiff; +import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamableReader; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; +import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; public class DiffableTests extends ESTestCase { - @Test - public void testJdkMapDiff() throws IOException { - Map before = new HashMap<>(); - before.put("foo", new TestDiffable("1")); - before.put("bar", new TestDiffable("2")); - before.put("baz", new TestDiffable("3")); - before = unmodifiableMap(before); - Map map = new HashMap<>(); - map.putAll(before); - map.remove("bar"); - map.put("baz", new TestDiffable("4")); - map.put("new", new TestDiffable("5")); - Map after = unmodifiableMap(new HashMap<>(map)); - Diff diff = DiffableUtils.diff(before, after); - BytesStreamOutput out = new BytesStreamOutput(); - diff.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); - Map serialized = DiffableUtils.readJdkMapDiff(in, TestDiffable.PROTO).apply(before); - assertThat(serialized.size(), equalTo(3)); - assertThat(serialized.get("foo").value(), equalTo("1")); - assertThat(serialized.get("baz").value(), equalTo("4")); - assertThat(serialized.get("new").value(), equalTo("5")); + public void testJKDMapDiff() throws IOException { + new JdkMapDriver() { + @Override + protected boolean diffableValues() { + return true; + } + + @Override + protected TestDiffable createValue(Integer key, boolean before) { + return new TestDiffable(String.valueOf(before ? key : key + 1)); + } + + @Override + protected MapDiff diff(Map before, Map after) { + return DiffableUtils.diff(before, after, keySerializer); + } + + @Override + protected MapDiff readDiff(StreamInput in) throws IOException { + return useProtoForDiffableSerialization + ? DiffableUtils.readJdkMapDiff(in, keySerializer, TestDiffable.PROTO) + : DiffableUtils.readJdkMapDiff(in, keySerializer, diffableValueSerializer()); + } + }.execute(); + + new JdkMapDriver() { + @Override + protected boolean diffableValues() { + return false; + } + + @Override + protected String createValue(Integer key, boolean before) { + return String.valueOf(before ? key : key + 1); + } + + @Override + protected MapDiff diff(Map before, Map after) { + return DiffableUtils.diff(before, after, keySerializer, nonDiffableValueSerializer()); + } + + @Override + protected MapDiff readDiff(StreamInput in) throws IOException { + return DiffableUtils.readJdkMapDiff(in, keySerializer, nonDiffableValueSerializer()); + } + }.execute(); } - @Test public void testImmutableOpenMapDiff() throws IOException { - ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); - builder.put("foo", new TestDiffable("1")); - builder.put("bar", new TestDiffable("2")); - builder.put("baz", new TestDiffable("3")); - ImmutableOpenMap before = builder.build(); - builder = ImmutableOpenMap.builder(before); - builder.remove("bar"); - builder.put("baz", new TestDiffable("4")); - builder.put("new", new TestDiffable("5")); - ImmutableOpenMap after = builder.build(); - Diff diff = DiffableUtils.diff(before, after); - BytesStreamOutput out = new BytesStreamOutput(); - diff.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); - ImmutableOpenMap serialized = DiffableUtils.readImmutableOpenMapDiff(in, new KeyedReader() { + new ImmutableOpenMapDriver() { @Override - public TestDiffable readFrom(StreamInput in, String key) throws IOException { + protected boolean diffableValues() { + return true; + } + + @Override + protected TestDiffable createValue(Integer key, boolean before) { + return new TestDiffable(String.valueOf(before ? key : key + 1)); + } + + @Override + protected MapDiff diff(ImmutableOpenMap before, ImmutableOpenMap after) { + return DiffableUtils.diff(before, after, keySerializer); + } + + @Override + protected MapDiff readDiff(StreamInput in) throws IOException { + return useProtoForDiffableSerialization + ? DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, TestDiffable.PROTO) + : DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, diffableValueSerializer()); + } + }.execute(); + + new ImmutableOpenMapDriver() { + @Override + protected boolean diffableValues() { + return false; + } + + @Override + protected String createValue(Integer key, boolean before) { + return String.valueOf(before ? key : key + 1); + } + + @Override + protected MapDiff diff(ImmutableOpenMap before, ImmutableOpenMap after) { + return DiffableUtils.diff(before, after, keySerializer, nonDiffableValueSerializer()); + } + + @Override + protected MapDiff readDiff(StreamInput in) throws IOException { + return DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, nonDiffableValueSerializer()); + } + }.execute(); + } + + public void testImmutableOpenIntMapDiff() throws IOException { + new ImmutableOpenIntMapDriver() { + @Override + protected boolean diffableValues() { + return true; + } + + @Override + protected TestDiffable createValue(Integer key, boolean before) { + return new TestDiffable(String.valueOf(before ? key : key + 1)); + } + + @Override + protected MapDiff diff(ImmutableOpenIntMap before, ImmutableOpenIntMap after) { + return DiffableUtils.diff(before, after, keySerializer); + } + + @Override + protected MapDiff readDiff(StreamInput in) throws IOException { + return useProtoForDiffableSerialization + ? DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, TestDiffable.PROTO) + : DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, diffableValueSerializer()); + } + }.execute(); + + new ImmutableOpenIntMapDriver() { + @Override + protected boolean diffableValues() { + return false; + } + + @Override + protected String createValue(Integer key, boolean before) { + return String.valueOf(before ? key : key + 1); + } + + @Override + protected MapDiff diff(ImmutableOpenIntMap before, ImmutableOpenIntMap after) { + return DiffableUtils.diff(before, after, keySerializer, nonDiffableValueSerializer()); + } + + @Override + protected MapDiff readDiff(StreamInput in) throws IOException { + return DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, nonDiffableValueSerializer()); + } + }.execute(); + } + + /** + * Class that abstracts over specific map implementation type and value kind (Diffable or not) + * @param map type + * @param value type + */ + public abstract class MapDriver { + protected final Set keys = randomPositiveIntSet(); + protected final Set keysToRemove = new HashSet<>(randomSubsetOf(randomInt(keys.size()), keys.toArray(new Integer[keys.size()]))); + protected final Set keysThatAreNotRemoved = Sets.difference(keys, keysToRemove); + protected final Set keysToOverride = new HashSet<>(randomSubsetOf(randomInt(keysThatAreNotRemoved.size()), + keysThatAreNotRemoved.toArray(new Integer[keysThatAreNotRemoved.size()]))); + protected final Set keysToAdd = Sets.difference(randomPositiveIntSet(), keys); // make sure keysToAdd does not contain elements in keys + protected final Set keysUnchanged = Sets.difference(keysThatAreNotRemoved, keysToOverride); + + protected final DiffableUtils.KeySerializer keySerializer = randomBoolean() + ? DiffableUtils.getIntKeySerializer() + : DiffableUtils.getVIntKeySerializer(); + + protected final boolean useProtoForDiffableSerialization = randomBoolean(); + + private Set randomPositiveIntSet() { + int maxSetSize = randomInt(6); + Set result = new HashSet<>(); + for (int i = 0; i < maxSetSize; i++) { + // due to duplicates, set size can be smaller than maxSetSize + result.add(randomIntBetween(0, 100)); + } + return result; + } + + /** + * whether we operate on {@link org.elasticsearch.cluster.Diffable} values + */ + protected abstract boolean diffableValues(); + + /** + * functions that determines value in "before" or "after" map based on key + */ + protected abstract V createValue(Integer key, boolean before); + + /** + * creates map based on JDK-based map + */ + protected abstract T createMap(Map values); + + /** + * calculates diff between two maps + */ + protected abstract MapDiff diff(T before, T after); + + /** + * reads diff of maps from stream + */ + protected abstract MapDiff readDiff(StreamInput in) throws IOException; + + /** + * gets element at key "key" in map "map" + */ + protected abstract V get(T map, Integer key); + + /** + * returns size of given map + */ + protected abstract int size(T map); + + /** + * executes the actual test + */ + public void execute() throws IOException { + logger.debug("Keys in 'before' map: {}", keys); + logger.debug("Keys to remove: {}", keysToRemove); + logger.debug("Keys to override: {}", keysToOverride); + logger.debug("Keys to add: {}", keysToAdd); + + logger.debug("--> creating 'before' map"); + Map before = new HashMap<>(); + for (Integer key : keys) { + before.put(key, createValue(key, true)); + } + T beforeMap = createMap(before); + + logger.debug("--> creating 'after' map"); + Map after = new HashMap<>(); + after.putAll(before); + for (Integer key : keysToRemove) { + after.remove(key); + } + for (Integer key : keysToOverride) { + after.put(key, createValue(key, false)); + } + for (Integer key : keysToAdd) { + after.put(key, createValue(key, false)); + } + T afterMap = createMap(unmodifiableMap(after)); + + MapDiff diffMap = diff(beforeMap, afterMap); + + // check properties of diffMap + assertThat(new HashSet(diffMap.getDeletes()), equalTo(keysToRemove)); + if (diffableValues()) { + assertThat(diffMap.getDiffs().keySet(), equalTo(keysToOverride)); + for (Integer key : keysToOverride) { + assertThat(diffMap.getDiffs().get(key).apply(get(beforeMap, key)), equalTo(get(afterMap, key))); + } + assertThat(diffMap.getUpserts().keySet(), equalTo(keysToAdd)); + for (Integer key : keysToAdd) { + assertThat(diffMap.getUpserts().get(key), equalTo(get(afterMap, key))); + } + } else { + assertThat(diffMap.getDiffs(), equalTo(emptyMap())); + Set keysToAddAndOverride = Sets.union(keysToAdd, keysToOverride); + assertThat(diffMap.getUpserts().keySet(), equalTo(keysToAddAndOverride)); + for (Integer key : keysToAddAndOverride) { + assertThat(diffMap.getUpserts().get(key), equalTo(get(afterMap, key))); + } + } + + if (randomBoolean()) { + logger.debug("--> serializing diff"); + BytesStreamOutput out = new BytesStreamOutput(); + diffMap.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes()); + logger.debug("--> reading diff back"); + diffMap = readDiff(in); + } + T appliedDiffMap = diffMap.apply(beforeMap); + + // check properties of appliedDiffMap + assertThat(size(appliedDiffMap), equalTo(keys.size() - keysToRemove.size() + keysToAdd.size())); + for (Integer key : keysToRemove) { + assertThat(get(appliedDiffMap, key), nullValue()); + } + for (Integer key : keysUnchanged) { + assertThat(get(appliedDiffMap, key), equalTo(get(beforeMap, key))); + } + for (Integer key : keysToOverride) { + assertThat(get(appliedDiffMap, key), not(equalTo(get(beforeMap, key)))); + assertThat(get(appliedDiffMap, key), equalTo(get(afterMap, key))); + } + for (Integer key : keysToAdd) { + assertThat(get(appliedDiffMap, key), equalTo(get(afterMap, key))); + } + } + } + + abstract class JdkMapDriver extends MapDriver, V> { + + @Override + protected Map createMap(Map values) { + return values; + } + + @Override + protected V get(Map map, Integer key) { + return map.get(key); + } + + @Override + protected int size(Map map) { + return map.size(); + } + } + + abstract class ImmutableOpenMapDriver extends MapDriver, V> { + + @Override + protected ImmutableOpenMap createMap(Map values) { + return ImmutableOpenMap.builder().putAll(values).build(); + } + + @Override + protected V get(ImmutableOpenMap map, Integer key) { + return map.get(key); + } + + @Override + protected int size(ImmutableOpenMap map) { + return map.size(); + } + } + + + abstract class ImmutableOpenIntMapDriver extends MapDriver, V> { + + @Override + protected ImmutableOpenIntMap createMap(Map values) { + return ImmutableOpenIntMap.builder().putAll(values).build(); + } + + @Override + protected V get(ImmutableOpenIntMap map, Integer key) { + return map.get(key); + } + + @Override + protected int size(ImmutableOpenIntMap map) { + return map.size(); + } + } + + private static DiffableUtils.DiffableValueSerializer diffableValueSerializer() { + return new DiffableUtils.DiffableValueSerializer() { + @Override + public TestDiffable read(StreamInput in, K key) throws IOException { return new TestDiffable(in.readString()); } @Override - public Diff readDiffFrom(StreamInput in, String key) throws IOException { + public Diff readDiff(StreamInput in, K key) throws IOException { return AbstractDiffable.readDiffFrom(new StreamableReader() { @Override public TestDiffable readFrom(StreamInput in) throws IOException { @@ -95,13 +406,23 @@ public class DiffableTests extends ESTestCase { } }, in); } - }).apply(before); - assertThat(serialized.size(), equalTo(3)); - assertThat(serialized.get("foo").value(), equalTo("1")); - assertThat(serialized.get("baz").value(), equalTo("4")); - assertThat(serialized.get("new").value(), equalTo("5")); - + }; } + + private static DiffableUtils.NonDiffableValueSerializer nonDiffableValueSerializer() { + return new DiffableUtils.NonDiffableValueSerializer() { + @Override + public void write(String value, StreamOutput out) throws IOException { + out.writeString(value); + } + + @Override + public String read(StreamInput in, K key) throws IOException { + return in.readString(); + } + }; + } + public static class TestDiffable extends AbstractDiffable { public static final TestDiffable PROTO = new TestDiffable(""); @@ -125,6 +446,22 @@ public class DiffableTests extends ESTestCase { public void writeTo(StreamOutput out) throws IOException { out.writeString(value); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TestDiffable that = (TestDiffable) o; + + return !(value != null ? !value.equals(that.value) : that.value != null); + + } + + @Override + public int hashCode() { + return value != null ? value.hashCode() : 0; + } } } diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java index 6b998523f13..65d5b0b9fcd 100644 --- a/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java @@ -29,23 +29,23 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.indices.store.IndicesStore; +import org.elasticsearch.index.store.IndexStoreConfig; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.hamcrest.Matchers; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope = TEST) public class ClusterSettingsIT extends ESIntegTestCase { - - @Test - public void clusterNonExistingSettingsUpdate() { + public void testClusterNonExistingSettingsUpdate() { String key1 = "no_idea_what_you_are_talking_about"; int value1 = 10; @@ -58,9 +58,8 @@ public class ClusterSettingsIT extends ESIntegTestCase { assertThat(response.getTransientSettings().getAsMap().entrySet(), Matchers.emptyIterable()); } - @Test - public void clusterSettingsUpdateResponse() { - String key1 = IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC; + public void testClusterSettingsUpdateResponse() { + String key1 = IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC; int value1 = 10; String key2 = EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE; @@ -115,7 +114,6 @@ public class ClusterSettingsIT extends ESIntegTestCase { assertThat(response3.getPersistentSettings().get(key2), notNullValue()); } - @Test public void testUpdateDiscoveryPublishTimeout() { DiscoverySettings discoverySettings = internalCluster().getInstance(DiscoverySettings.class); @@ -150,7 +148,6 @@ public class ClusterSettingsIT extends ESIntegTestCase { assertThat(discoverySettings.getPublishTimeout().seconds(), equalTo(1l)); } - @Test public void testClusterUpdateSettingsWithBlocks() { String key1 = "cluster.routing.allocation.enable"; Settings transientSettings = Settings.builder().put(key1, false).build(); @@ -185,15 +182,18 @@ public class ClusterSettingsIT extends ESIntegTestCase { assertThat(response.getPersistentSettings().get(key2), notNullValue()); } - @Test(expected = IllegalArgumentException.class) public void testMissingUnits() { assertAcked(prepareCreate("test")); - // Should fail (missing units for refresh_interval): - client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.refresh_interval", "10")).execute().actionGet(); + try { + client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.refresh_interval", "10")).execute().actionGet(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("[index.refresh_interval] with value [10]")); + assertThat(e.getMessage(), containsString("unit is missing or unrecognized")); + } } - @Test public void testMissingUnitsLenient() { try { createNode(Settings.builder().put(Settings.SETTINGS_REQUIRE_UNITS, "false").build()); diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java index b7670eaafe8..6a16f906886 100644 --- a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -64,7 +63,7 @@ public class SettingsFilteringIT extends ESIntegTestCase { } @Override - public Collection indexModules(Settings indexSettings) { + public Collection nodeModules() { return Collections.singletonList(new SettingsFilteringModule()); } } @@ -85,10 +84,7 @@ public class SettingsFilteringIT extends ESIntegTestCase { } } - - @Test public void testSettingsFiltering() { - assertAcked(client().admin().indices().prepareCreate("test-idx").setSettings(Settings.builder() .put("filter_test.foo", "test") .put("filter_test.bar1", "test") @@ -105,5 +101,4 @@ public class SettingsFilteringIT extends ESIntegTestCase { assertThat(settings.get("index.filter_test.notbar"), equalTo("test")); assertThat(settings.get("index.filter_test.notfoo"), equalTo("test")); } - } diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsValidatorTests.java b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsValidatorTests.java index 1e041aae1b7..498acef2eb9 100644 --- a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsValidatorTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsValidatorTests.java @@ -20,13 +20,12 @@ package org.elasticsearch.cluster.settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class SettingsValidatorTests extends ESTestCase { - - @Test public void testValidators() throws Exception { assertThat(Validator.EMPTY.validate("", "anything goes", null), nullValue()); @@ -98,7 +97,6 @@ public class SettingsValidatorTests extends ESTestCase { assertThat(Validator.BYTES_SIZE_OR_PERCENTAGE.validate("", "0%", null), nullValue()); } - @Test public void testDynamicValidators() throws Exception { DynamicSettings.Builder ds = new DynamicSettings.Builder(); ds.addSetting("my.test.*", Validator.POSITIVE_INTEGER); diff --git a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java index 422846fc48a..9c5d80d1283 100644 --- a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java @@ -25,13 +25,15 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; @@ -39,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo; */ @ClusterScope(scope= Scope.SUITE, numDataNodes = 2) public class ClusterSearchShardsIT extends ESIntegTestCase { - + @Override protected Settings nodeSettings(int nodeOrdinal) { switch(nodeOrdinal) { @@ -51,7 +53,6 @@ public class ClusterSearchShardsIT extends ESIntegTestCase { return super.nodeSettings(nodeOrdinal); } - @Test public void testSingleShardAllocation() throws Exception { client().admin().indices().prepareCreate("test").setSettings(settingsBuilder() .put("index.number_of_shards", "1").put("index.number_of_replicas", 0).put("index.routing.allocation.include.tag", "A")).execute().actionGet(); @@ -74,7 +75,6 @@ public class ClusterSearchShardsIT extends ESIntegTestCase { } - @Test public void testMultipleShardsSingleNodeAllocation() throws Exception { client().admin().indices().prepareCreate("test").setSettings(settingsBuilder() .put("index.number_of_shards", "4").put("index.number_of_replicas", 0).put("index.routing.allocation.include.tag", "A")).execute().actionGet(); @@ -94,7 +94,6 @@ public class ClusterSearchShardsIT extends ESIntegTestCase { assertThat(response.getGroups()[0].getShardId(), equalTo(2)); } - @Test public void testMultipleIndicesAllocation() throws Exception { client().admin().indices().prepareCreate("test1").setSettings(settingsBuilder() .put("index.number_of_shards", "4").put("index.number_of_replicas", 1)).execute().actionGet(); @@ -128,7 +127,6 @@ public class ClusterSearchShardsIT extends ESIntegTestCase { assertThat(response.getNodes().length, equalTo(2)); } - @Test public void testClusterSearchShardsWithBlocks() { createIndex("test-blocks"); diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java index 236378e87e5..c5a695d16e5 100644 --- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java @@ -40,7 +40,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.util.Collections; import java.util.HashMap; @@ -58,8 +57,6 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; public class RoutingIteratorTests extends ESAllocationTestCase { - - @Test public void testEmptyIterator() { ShardShuffler shuffler = new RotationShardShuffler(0); ShardIterator shardIterator = new PlainShardIterator(new ShardId("test1", 0), shuffler.shuffle(Collections.emptyList())); @@ -91,7 +88,6 @@ public class RoutingIteratorTests extends ESAllocationTestCase { assertThat(shardIterator.remaining(), equalTo(0)); } - @Test public void testIterator1() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2)) @@ -119,7 +115,6 @@ public class RoutingIteratorTests extends ESAllocationTestCase { assertThat(shardIterator.remaining(), equalTo(0)); } - @Test public void testIterator2() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) @@ -200,7 +195,6 @@ public class RoutingIteratorTests extends ESAllocationTestCase { assertThat(shardRouting10, sameInstance(shardRouting6)); } - @Test public void testRandomRouting() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) @@ -228,7 +222,6 @@ public class RoutingIteratorTests extends ESAllocationTestCase { assertThat(shardRouting1, sameInstance(shardRouting3)); } - @Test public void testAttributePreferenceRouting() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -257,7 +250,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase { .put(newNode("node2", unmodifiableMap(node2Attributes))) .localNodeId("node1") ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); @@ -284,7 +277,6 @@ public class RoutingIteratorTests extends ESAllocationTestCase { assertThat(shardRouting.currentNodeId(), equalTo("node2")); } - @Test public void testNodeSelectorRouting(){ AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -307,7 +299,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase { .localNodeId("node1") ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); @@ -342,7 +334,6 @@ public class RoutingIteratorTests extends ESAllocationTestCase { } - @Test public void testShardsAndPreferNodeRouting() { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -363,7 +354,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase { .put(newNode("node2")) .localNodeId("node1") ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); @@ -404,7 +395,6 @@ public class RoutingIteratorTests extends ESAllocationTestCase { assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1")); } - @Test public void testReplicaShardPreferenceIters() throws Exception { AllocationService strategy = createAllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -428,7 +418,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase { .put(newNode("node3")) .localNodeId("node1") ).build(); - routingTable = strategy.reroute(clusterState).routingTable(); + routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); diff --git a/core/src/test/java/org/elasticsearch/common/Base64Tests.java b/core/src/test/java/org/elasticsearch/common/Base64Tests.java index a2bf2da460d..74691c0b739 100644 --- a/core/src/test/java/org/elasticsearch/common/Base64Tests.java +++ b/core/src/test/java/org/elasticsearch/common/Base64Tests.java @@ -18,11 +18,10 @@ */ package org.elasticsearch.common; -import java.nio.charset.StandardCharsets; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Locale; import static org.hamcrest.Matchers.is; @@ -31,8 +30,7 @@ import static org.hamcrest.Matchers.is; * */ public class Base64Tests extends ESTestCase { - - @Test // issue #6334 + // issue #6334 public void testBase64DecodeWithExtraCharactersAfterPadding() throws Exception { String plain = randomAsciiOfLengthBetween(1, 20) + ":" + randomAsciiOfLengthBetween(1, 20); String encoded = Base64.encodeBytes(plain.getBytes(StandardCharsets.UTF_8)); diff --git a/core/src/test/java/org/elasticsearch/common/BooleansTests.java b/core/src/test/java/org/elasticsearch/common/BooleansTests.java index 0baf4335cd3..6e5446cebf9 100644 --- a/core/src/test/java/org/elasticsearch/common/BooleansTests.java +++ b/core/src/test/java/org/elasticsearch/common/BooleansTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.common; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.Locale; @@ -29,8 +28,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; public class BooleansTests extends ESTestCase { - - @Test public void testIsBoolean() { String[] booleans = new String[]{"true", "false", "on", "off", "yes", "no", "0", "1"}; String[] notBooleans = new String[]{"11", "00", "sdfsdfsf", "F", "T"}; @@ -46,8 +43,8 @@ public class BooleansTests extends ESTestCase { assertThat("recognized [" + nb + "] as boolean", Booleans.isBoolean(t.toCharArray(), "prefix".length(), nb.length()), Matchers.equalTo(false)); } } - @Test - public void parseBoolean() { + + public void testParseBoolean() { assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes", "1"), randomBoolean()), is(true)); assertThat(Booleans.parseBoolean(randomFrom("false", "off", "no", "0"), randomBoolean()), is(false)); assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT), randomBoolean()), is(true)); @@ -69,8 +66,7 @@ public class BooleansTests extends ESTestCase { assertThat(Booleans.parseBoolean(chars,0, chars.length, randomBoolean()), is(true)); } - @Test - public void parseBooleanExact() { + public void testParseBooleanExact() { assertThat(Booleans.parseBooleanExact(randomFrom("true", "on", "yes", "1")), is(true)); assertThat(Booleans.parseBooleanExact(randomFrom("false", "off", "no", "0")), is(false)); try { diff --git a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java index 46e65ad1e75..5bb9c614b84 100644 --- a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java +++ b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java @@ -29,7 +29,6 @@ import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer; import org.jboss.netty.buffer.ChannelBuffer; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.EOFException; import java.io.IOException; @@ -42,6 +41,8 @@ import java.nio.channels.WritableByteChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import static org.hamcrest.Matchers.containsString; + public class ChannelsTests extends ESTestCase { byte[] randomBytes; @@ -64,7 +65,6 @@ public class ChannelsTests extends ESTestCase { super.tearDown(); } - @Test public void testReadWriteThoughArrays() throws Exception { Channels.writeToChannel(randomBytes, fileChannel); byte[] readBytes = Channels.readFromFileChannel(fileChannel, 0, randomBytes.length); @@ -72,7 +72,6 @@ public class ChannelsTests extends ESTestCase { } - @Test public void testPartialReadWriteThroughArrays() throws Exception { int length = randomIntBetween(1, randomBytes.length / 2); int offset = randomIntBetween(0, randomBytes.length - length); @@ -89,14 +88,17 @@ public class ChannelsTests extends ESTestCase { assertThat("read bytes didn't match written bytes", source.toBytes(), Matchers.equalTo(read.toBytes())); } - @Test(expected = EOFException.class) public void testBufferReadPastEOFWithException() throws Exception { int bytesToWrite = randomIntBetween(0, randomBytes.length - 1); Channels.writeToChannel(randomBytes, 0, bytesToWrite, fileChannel); - Channels.readFromFileChannel(fileChannel, 0, bytesToWrite + 1 + randomInt(1000)); + try { + Channels.readFromFileChannel(fileChannel, 0, bytesToWrite + 1 + randomInt(1000)); + fail("Expected an EOFException"); + } catch (EOFException e) { + assertThat(e.getMessage(), containsString("read past EOF")); + } } - @Test public void testBufferReadPastEOFWithoutException() throws Exception { int bytesToWrite = randomIntBetween(0, randomBytes.length - 1); Channels.writeToChannel(randomBytes, 0, bytesToWrite, fileChannel); @@ -105,7 +107,6 @@ public class ChannelsTests extends ESTestCase { assertThat(read, Matchers.lessThan(0)); } - @Test public void testReadWriteThroughBuffers() throws IOException { ByteBuffer source; if (randomBoolean()) { @@ -130,7 +131,6 @@ public class ChannelsTests extends ESTestCase { assertThat("read bytes didn't match written bytes", randomBytes, Matchers.equalTo(copyBytes)); } - @Test public void testPartialReadWriteThroughBuffers() throws IOException { int length = randomIntBetween(1, randomBytes.length / 2); int offset = randomIntBetween(0, randomBytes.length - length); @@ -163,7 +163,6 @@ public class ChannelsTests extends ESTestCase { } - @Test public void testWriteFromChannel() throws IOException { int length = randomIntBetween(1, randomBytes.length / 2); int offset = randomIntBetween(0, randomBytes.length - length); diff --git a/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java index b3397a55b32..f4b8747ccdc 100644 --- a/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java +++ b/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java @@ -19,15 +19,14 @@ package org.elasticsearch.common; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.EnumSet; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.sameInstance; public class ParseFieldTests extends ESTestCase { - - @Test public void testParse() { String[] values = new String[]{"foo_bar", "fooBar"}; ParseField field = new ParseField(randomFrom(values)); @@ -68,7 +67,6 @@ public class ParseFieldTests extends ESTestCase { } } - @Test public void testAllDeprecated() { String[] values = new String[]{"like_text", "likeText"}; diff --git a/core/src/test/java/org/elasticsearch/common/PidFileTests.java b/core/src/test/java/org/elasticsearch/common/PidFileTests.java index a98ac1d027f..1ea4f302b7b 100644 --- a/core/src/test/java/org/elasticsearch/common/PidFileTests.java +++ b/core/src/test/java/org/elasticsearch/common/PidFileTests.java @@ -19,22 +19,21 @@ package org.elasticsearch.common; -import java.nio.charset.StandardCharsets; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.BufferedWriter; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import static org.hamcrest.Matchers.containsString; + /** * UnitTest for {@link org.elasticsearch.common.PidFile} */ public class PidFileTests extends ESTestCase { - - @Test(expected = IllegalArgumentException.class) public void testParentIsFile() throws IOException { Path dir = createTempDir(); Path parent = dir.resolve("foo"); @@ -42,10 +41,14 @@ public class PidFileTests extends ESTestCase { stream.write("foo"); } - PidFile.create(parent.resolve("bar.pid"), false); + try { + PidFile.create(parent.resolve("bar.pid"), false); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("exists but is not a directory")); + } } - @Test public void testPidFile() throws IOException { Path dir = createTempDir(); Path parent = dir.resolve("foo"); diff --git a/core/src/test/java/org/elasticsearch/common/TableTests.java b/core/src/test/java/org/elasticsearch/common/TableTests.java index 1afdf592960..46da20190dc 100644 --- a/core/src/test/java/org/elasticsearch/common/TableTests.java +++ b/core/src/test/java/org/elasticsearch/common/TableTests.java @@ -20,52 +20,80 @@ package org.elasticsearch.common; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.List; import java.util.Map; -public class TableTests extends ESTestCase { +import static org.hamcrest.Matchers.is; - @Test(expected = IllegalStateException.class) +public class TableTests extends ESTestCase { public void testFailOnStartRowWithoutHeader() { Table table = new Table(); - table.startRow(); + try { + table.startRow(); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("no headers added...")); + } } - @Test(expected = IllegalStateException.class) public void testFailOnEndHeadersWithoutStart() { Table table = new Table(); - table.endHeaders(); + try { + table.endHeaders(); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("no headers added...")); + } + } - @Test(expected = IllegalStateException.class) public void testFailOnAddCellWithoutHeader() { Table table = new Table(); - table.addCell("error"); + try { + table.addCell("error"); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("no block started...")); + } + } - @Test(expected = IllegalStateException.class) public void testFailOnAddCellWithoutRow() { Table table = this.getTableWithHeaders(); - table.addCell("error"); + try { + table.addCell("error"); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("no block started...")); + } + } - @Test(expected = IllegalStateException.class) public void testFailOnEndRowWithoutStart() { Table table = this.getTableWithHeaders(); - table.endRow(); + try { + table.endRow(); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("no row started...")); + } + } - @Test(expected = IllegalStateException.class) public void testFailOnLessCellsThanDeclared() { Table table = this.getTableWithHeaders(); table.startRow(); table.addCell("foo"); - table.endRow(true); + try { + table.endRow(true); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("mismatch on number of cells 1 in a row compared to header 2")); + } + } - @Test public void testOnLessCellsThanDeclaredUnchecked() { Table table = this.getTableWithHeaders(); table.startRow(); @@ -73,16 +101,20 @@ public class TableTests extends ESTestCase { table.endRow(false); } - @Test(expected = IllegalStateException.class) public void testFailOnMoreCellsThanDeclared() { Table table = this.getTableWithHeaders(); table.startRow(); table.addCell("foo"); table.addCell("bar"); - table.addCell("foobar"); + try { + table.addCell("foobar"); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("can't add more cells to a row than the header")); + } + } - @Test public void testSimple() { Table table = this.getTableWithHeaders(); table.startRow(); diff --git a/core/src/test/java/org/elasticsearch/common/UUIDTests.java b/core/src/test/java/org/elasticsearch/common/UUIDTests.java index af5f382ec31..f82e1a464d9 100644 --- a/core/src/test/java/org/elasticsearch/common/UUIDTests.java +++ b/core/src/test/java/org/elasticsearch/common/UUIDTests.java @@ -19,36 +19,32 @@ package org.elasticsearch.common; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.HashSet; +import java.util.Set; public class UUIDTests extends ESTestCase { static UUIDGenerator timeUUIDGen = new TimeBasedUUIDGenerator(); static UUIDGenerator randomUUIDGen = new RandomBasedUUIDGenerator(); - @Test public void testRandomUUID() { verifyUUIDSet(100000, randomUUIDGen); } - @Test public void testTimeUUID() { verifyUUIDSet(100000, timeUUIDGen); } - @Test public void testThreadedTimeUUID() { testUUIDThreaded(timeUUIDGen); } - @Test public void testThreadedRandomUUID() { testUUIDThreaded(randomUUIDGen); } - HashSet verifyUUIDSet(int count, UUIDGenerator uuidSource) { + Set verifyUUIDSet(int count, UUIDGenerator uuidSource) { HashSet uuidSet = new HashSet<>(); for (int i = 0; i < count; ++i) { uuidSet.add(uuidSource.getBase64UUID()); @@ -59,7 +55,7 @@ public class UUIDTests extends ESTestCase { class UUIDGenRunner implements Runnable { int count; - public HashSet uuidSet = null; + public Set uuidSet = null; UUIDGenerator uuidSource; public UUIDGenRunner(int count, UUIDGenerator uuidSource) { diff --git a/core/src/test/java/org/elasticsearch/common/blobstore/BlobStoreTests.java b/core/src/test/java/org/elasticsearch/common/blobstore/BlobStoreTests.java index fdf07cb03e0..80afa5d51f9 100644 --- a/core/src/test/java/org/elasticsearch/common/blobstore/BlobStoreTests.java +++ b/core/src/test/java/org/elasticsearch/common/blobstore/BlobStoreTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.io.InputStream; @@ -41,8 +40,6 @@ import static org.hamcrest.CoreMatchers.notNullValue; @LuceneTestCase.SuppressFileSystems("ExtrasFS") public class BlobStoreTests extends ESTestCase { - - @Test public void testWriteRead() throws IOException { final BlobStore store = newBlobStore(); final BlobContainer container = store.blobContainer(new BlobPath()); @@ -62,7 +59,6 @@ public class BlobStoreTests extends ESTestCase { store.close(); } - @Test public void testMoveAndList() throws IOException { final BlobStore store = newBlobStore(); final BlobContainer container = store.blobContainer(new BlobPath()); diff --git a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java index 2a9f87a8663..fa4ce357a52 100644 --- a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java +++ b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -39,8 +38,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; * Tests for the Memory Aggregating Circuit Breaker */ public class MemoryCircuitBreakerTests extends ESTestCase { - - @Test public void testThreadedUpdatesToBreaker() throws Exception { final int NUM_THREADS = scaledRandomIntBetween(3, 15); final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500); @@ -82,7 +79,6 @@ public class MemoryCircuitBreakerTests extends ESTestCase { assertThat("breaker was tripped at least once", breaker.getTrippedCount(), greaterThanOrEqualTo(1L)); } - @Test public void testThreadedUpdatesToChildBreaker() throws Exception { final int NUM_THREADS = scaledRandomIntBetween(3, 15); final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500); @@ -140,7 +136,6 @@ public class MemoryCircuitBreakerTests extends ESTestCase { assertThat("breaker was tripped at least once", breaker.getTrippedCount(), greaterThanOrEqualTo(1L)); } - @Test public void testThreadedUpdatesToChildBreakerWithParentLimit() throws Exception { final int NUM_THREADS = scaledRandomIntBetween(3, 15); final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500); @@ -212,7 +207,6 @@ public class MemoryCircuitBreakerTests extends ESTestCase { assertThat("total breaker was tripped at least once", tripped.get(), greaterThanOrEqualTo(1)); } - @Test public void testConstantFactor() throws Exception { final MemoryCircuitBreaker breaker = new MemoryCircuitBreaker(new ByteSizeValue(15), 1.6, logger); String field = "myfield"; diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java index 802ea7cc628..95a65f82924 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java @@ -32,7 +32,6 @@ import org.hamcrest.Matchers; import org.jboss.netty.buffer.ChannelBuffer; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.EOFException; import java.io.IOException; @@ -61,7 +60,6 @@ public class PagedBytesReferenceTests extends ESTestCase { super.tearDown(); } - @Test public void testGet() { int length = randomIntBetween(1, PAGE_SIZE * 3); BytesReference pbr = getRandomizedPagedBytesReference(length); diff --git a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java index 4f64f0baca7..38090403668 100644 --- a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java +++ b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java @@ -22,11 +22,14 @@ package org.elasticsearch.common.cache; import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.instanceOf; @@ -410,8 +413,7 @@ public class CacheTests extends ESTestCase { Value that = (Value) o; - return value == that.value; - + return value.equals(that.value); } @Override @@ -460,6 +462,25 @@ public class CacheTests extends ESTestCase { assertEquals(replacements, notifications); } + public void testComputeIfAbsentLoadsSuccessfully() { + Map map = new HashMap<>(); + Cache cache = CacheBuilder.builder().build(); + for (int i = 0; i < numberOfEntries; i++) { + try { + cache.computeIfAbsent(i, k -> { + int value = randomInt(); + map.put(k, value); + return value; + }); + } catch (ExecutionException e) { + fail(e.getMessage()); + } + } + for (int i = 0; i < numberOfEntries; i++) { + assertEquals(map.get(i), cache.get(i)); + } + } + public void testComputeIfAbsentCallsOnce() throws InterruptedException { int numberOfThreads = randomIntBetween(2, 200); final Cache cache = CacheBuilder.builder().build(); @@ -502,6 +523,146 @@ public class CacheTests extends ESTestCase { } } + public void testDependentKeyDeadlock() throws InterruptedException { + class Key { + private final int key; + + public Key(int key) { + this.key = key; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Key key1 = (Key) o; + + return key == key1.key; + + } + + @Override + public int hashCode() { + return key % 2; + } + } + + int numberOfThreads = randomIntBetween(2, 256); + final Cache cache = CacheBuilder.builder().build(); + CountDownLatch latch = new CountDownLatch(1 + numberOfThreads); + CountDownLatch deadlockLatch = new CountDownLatch(numberOfThreads); + List threads = new ArrayList<>(); + for (int i = 0; i < numberOfThreads; i++) { + Thread thread = new Thread(() -> { + Random random = new Random(random().nextLong()); + latch.countDown(); + for (int j = 0; j < numberOfEntries; j++) { + Key key = new Key(random.nextInt(numberOfEntries)); + try { + cache.computeIfAbsent(key, k -> { + if (k.key == 0) { + return 0; + } else { + Integer value = cache.get(new Key(k.key / 2)); + return value != null ? value : 0; + } + }); + } catch (ExecutionException e) { + fail(e.getMessage()); + } + } + // successfully avoided deadlock, release the main thread + deadlockLatch.countDown(); + }); + threads.add(thread); + thread.start(); + } + + AtomicBoolean deadlock = new AtomicBoolean(); + assert !deadlock.get(); + + // start a watchdog service + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(() -> { + Set ids = threads.stream().map(t -> t.getId()).collect(Collectors.toSet()); + ThreadMXBean mxBean = ManagementFactory.getThreadMXBean(); + long[] deadlockedThreads = mxBean.findDeadlockedThreads(); + if (!deadlock.get() && deadlockedThreads != null) { + for (long deadlockedThread : deadlockedThreads) { + // ensure that we detected deadlock on our threads + if (ids.contains(deadlockedThread)) { + deadlock.set(true); + // release the main test thread to fail the test + for (int i = 0; i < numberOfThreads; i++) { + deadlockLatch.countDown(); + } + break; + } + } + } + }, 1, 1, TimeUnit.SECONDS); + + // everything is setup, release the hounds + latch.countDown(); + + // wait for either deadlock to be detected or the threads to terminate + deadlockLatch.await(); + + // shutdown the watchdog service + scheduler.shutdown(); + + assertFalse("deadlock", deadlock.get()); + } + + public void testCachePollution() throws InterruptedException { + int numberOfThreads = randomIntBetween(2, 200); + final Cache cache = CacheBuilder.builder().build(); + CountDownLatch latch = new CountDownLatch(1 + numberOfThreads); + List threads = new ArrayList<>(); + for (int i = 0; i < numberOfThreads; i++) { + Thread thread = new Thread(() -> { + latch.countDown(); + Random random = new Random(random().nextLong()); + for (int j = 0; j < numberOfEntries; j++) { + Integer key = random.nextInt(numberOfEntries); + boolean first; + boolean second; + do { + first = random.nextBoolean(); + second = random.nextBoolean(); + } while (first && second); + if (first) { + try { + cache.computeIfAbsent(key, k -> { + if (random.nextBoolean()) { + return Integer.toString(k); + } else { + throw new Exception("testCachePollution"); + } + }); + } catch (ExecutionException e) { + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(Exception.class)); + assertEquals(e.getCause().getMessage(), "testCachePollution"); + } + } else if (second) { + cache.invalidate(key); + } else { + cache.get(key); + } + } + }); + threads.add(thread); + thread.start(); + } + + latch.countDown(); + for (Thread thread : threads) { + thread.join(); + } + } + // test that the cache is not corrupted under lots of concurrent modifications, even hitting the same key // here be dragons: this test did catch one subtle bug during development; do not remove lightly public void testTorture() throws InterruptedException { diff --git a/core/src/test/java/org/elasticsearch/common/cli/TerminalTests.java b/core/src/test/java/org/elasticsearch/common/cli/TerminalTests.java index da0347790b4..dcbbc1ed337 100644 --- a/core/src/test/java/org/elasticsearch/common/cli/TerminalTests.java +++ b/core/src/test/java/org/elasticsearch/common/cli/TerminalTests.java @@ -19,16 +19,17 @@ package org.elasticsearch.common.cli; -import org.junit.Test; +import java.nio.file.NoSuchFileException; +import java.util.List; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; /** * */ public class TerminalTests extends CliToolTestCase { - - @Test public void testVerbosity() throws Exception { CaptureOutputTerminal terminal = new CaptureOutputTerminal(Terminal.Verbosity.SILENT); assertPrinted(terminal, Terminal.Verbosity.SILENT, "text"); @@ -46,10 +47,28 @@ public class TerminalTests extends CliToolTestCase { assertPrinted(terminal, Terminal.Verbosity.VERBOSE, "text"); } + public void testError() throws Exception { + try { + // actually throw so we have a stacktrace + throw new NoSuchFileException("/path/to/some/file"); + } catch (NoSuchFileException e) { + CaptureOutputTerminal terminal = new CaptureOutputTerminal(Terminal.Verbosity.NORMAL); + terminal.printError(e); + List output = terminal.getTerminalOutput(); + assertFalse(output.isEmpty()); + assertTrue(output.get(0), output.get(0).contains("NoSuchFileException")); // exception class + assertTrue(output.get(0), output.get(0).contains("/path/to/some/file")); // message + assertEquals(1, output.size()); + + // TODO: we should test stack trace is printed in debug mode...except debug is a sysprop instead of + // a command line param...maybe it should be VERBOSE instead of a separate debug prop? + } + } + private void assertPrinted(CaptureOutputTerminal logTerminal, Terminal.Verbosity verbosity, String text) { logTerminal.print(verbosity, text); assertThat(logTerminal.getTerminalOutput(), hasSize(1)); - assertThat(logTerminal.getTerminalOutput(), hasItem(is("text"))); + assertThat(logTerminal.getTerminalOutput(), hasItem(text)); logTerminal.terminalOutput.clear(); } diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/CompressedStreamOutput.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/CompressedStreamOutput.java deleted file mode 100644 index 3cf0bcd5cfd..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/CompressedStreamOutput.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import org.elasticsearch.Version; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; - -/** - */ -public abstract class CompressedStreamOutput extends StreamOutput { - - private final StreamOutput out; - - protected byte[] uncompressed; - protected int uncompressedLength; - private int position = 0; - - private boolean closed; - - public CompressedStreamOutput(StreamOutput out) throws IOException { - this.out = out; - super.setVersion(out.getVersion()); - writeHeader(out); - } - - @Override - public StreamOutput setVersion(Version version) { - out.setVersion(version); - return super.setVersion(version); - } - - @Override - public void write(int b) throws IOException { - if (position >= uncompressedLength) { - flushBuffer(); - } - uncompressed[position++] = (byte) b; - } - - @Override - public void writeByte(byte b) throws IOException { - if (position >= uncompressedLength) { - flushBuffer(); - } - uncompressed[position++] = b; - } - - @Override - public void writeBytes(byte[] input, int offset, int length) throws IOException { - // ES, check if length is 0, and don't write in this case - if (length == 0) { - return; - } - final int BUFFER_LEN = uncompressedLength; - - // simple case first: buffering only (for trivially short writes) - int free = BUFFER_LEN - position; - if (free >= length) { - System.arraycopy(input, offset, uncompressed, position, length); - position += length; - return; - } - // fill partial input as much as possible and flush - if (position > 0) { - System.arraycopy(input, offset, uncompressed, position, free); - position += free; - flushBuffer(); - offset += free; - length -= free; - } - - // then write intermediate full block, if any, without copying: - while (length >= BUFFER_LEN) { - compress(input, offset, BUFFER_LEN, out); - offset += BUFFER_LEN; - length -= BUFFER_LEN; - } - - // and finally, copy leftovers in input, if any - if (length > 0) { - System.arraycopy(input, offset, uncompressed, 0, length); - } - position = length; - } - - @Override - public void flush() throws IOException { - flushBuffer(); - out.flush(); - } - - @Override - public void close() throws IOException { - if (!closed) { - flushBuffer(); - closed = true; - doClose(); - out.close(); - } - } - - protected abstract void doClose() throws IOException; - - @Override - public void reset() throws IOException { - position = 0; - out.reset(); - } - - private void flushBuffer() throws IOException { - if (position > 0) { - compress(uncompressed, 0, position, out); - position = 0; - } - } - - protected abstract void writeHeader(StreamOutput out) throws IOException; - - /** - * Compresses the data into the output - */ - protected abstract void compress(byte[] data, int offset, int len, StreamOutput out) throws IOException; - -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/CorruptedCompressorTests.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/CorruptedCompressorTests.java deleted file mode 100644 index a18a9e3fb65..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/CorruptedCompressorTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.ChunkEncoder; -import com.ning.compress.lzf.LZFChunk; -import com.ning.compress.lzf.util.ChunkDecoderFactory; -import com.ning.compress.lzf.util.ChunkEncoderFactory; -import org.elasticsearch.test.ESTestCase; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; - -/** - * Test an extremely rare corruption produced by the pure java impl of ChunkEncoder. - */ -public class CorruptedCompressorTests extends ESTestCase { - - public void testCorruption() throws IOException { - // this test generates a hash collision: [0,1,153,64] hashes the same as [1,153,64,64] - // and then leverages the bug s/inPos/0/ to corrupt the array - // the first array is used to insert a reference from this hash to offset 6 - // and then the hash table is reused and still thinks that there is such a hash at position 6 - // and at position 7, it finds a sequence with the same hash - // so it inserts a buggy reference - byte[] b1 = new byte[] {0,1,2,3,4,(byte)153,64,64,64,9,9,9,9,9,9,9,9,9,9}; - byte[] b2 = new byte[] {1,(byte)153,0,0,0,0,(byte)153,64,64,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; - ChunkEncoder encoder = ChunkEncoderFactory.safeInstance(); - ChunkDecoder decoder = ChunkDecoderFactory.safeInstance(); - check(encoder, decoder, b1, 0, b1.length); - final int off = 6; - check(encoder, decoder, b2, off, b2.length - off); - } - - private void check(ChunkEncoder encoder, ChunkDecoder decoder, byte[] bytes, int offset, int length) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - byte[] expected = new byte[length]; - byte[] buffer = new byte[LZFChunk.MAX_CHUNK_LEN]; - byte[] output = new byte[length]; - System.arraycopy(bytes, offset, expected, 0, length); - encoder.encodeAndWriteChunk(bytes, offset, length, outputStream); - System.out.println(Arrays.toString(Arrays.copyOf(outputStream.toByteArray(), 20))); - InputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); - assertEquals(decoder.decodeChunk(inputStream, buffer, output), length); - - System.out.println(Arrays.toString(Arrays.copyOf(output, 20))); - assertArrayEquals(expected, output); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamOutput.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamOutput.java deleted file mode 100644 index 3aa2a5de806..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamOutput.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.BufferRecycler; -import com.ning.compress.lzf.ChunkEncoder; -import com.ning.compress.lzf.LZFChunk; -import com.ning.compress.lzf.util.ChunkEncoderFactory; - -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; - -public class LZFCompressedStreamOutput extends CompressedStreamOutput { - - private final BufferRecycler recycler; - private final ChunkEncoder encoder; - - public LZFCompressedStreamOutput(StreamOutput out) throws IOException { - super(out); - this.recycler = BufferRecycler.instance(); - this.uncompressed = this.recycler.allocOutputBuffer(LZFChunk.MAX_CHUNK_LEN); - this.uncompressedLength = LZFChunk.MAX_CHUNK_LEN; - this.encoder = ChunkEncoderFactory.safeInstance(recycler); - } - - @Override - public void writeHeader(StreamOutput out) throws IOException { - // nothing to do here, each chunk has a header of its own - } - - @Override - protected void compress(byte[] data, int offset, int len, StreamOutput out) throws IOException { - encoder.encodeAndWriteChunk(data, offset, len, out); - } - - @Override - protected void doClose() throws IOException { - byte[] buf = uncompressed; - if (buf != null) { - uncompressed = null; - recycler.releaseOutputBuffer(buf); - } - encoder.close(); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFXContentTests.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFXContentTests.java deleted file mode 100644 index 05135f0ed68..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFXContentTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import org.elasticsearch.common.compress.AbstractCompressedXContentTestCase; - -public class LZFXContentTests extends AbstractCompressedXContentTestCase { - - public LZFXContentTests() { - super(new LZFTestCompressor()); - } - -} diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java index 924926b9b30..8f8e8b4ea1c 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java @@ -22,20 +22,18 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.equalTo; /** * Basic Tests for {@link GeoDistance} */ public class GeoDistanceTests extends ESTestCase { - - @Test public void testGeoDistanceSerialization() throws IOException { // make sure that ordinals don't change, because we rely on then in serialization assertThat(GeoDistance.PLANE.ordinal(), equalTo(0)); @@ -54,7 +52,6 @@ public class GeoDistanceTests extends ESTestCase { } } - @Test(expected = IOException.class) public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { if (randomBoolean()) { @@ -64,11 +61,12 @@ public class GeoDistanceTests extends ESTestCase { } try (StreamInput in = StreamInput.wrap(out.bytes())) { GeoDistance.readGeoDistanceFrom(in); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown GeoDistance ordinal [")); } } } - @Test public void testDistanceCheck() { // Note, is within is an approximation, so, even though 0.52 is outside 50mi, we still get "true" GeoDistance.DistanceBoundingCheck check = GeoDistance.distanceBoundingCheck(0, 0, 50, DistanceUnit.MILES); @@ -81,7 +79,6 @@ public class GeoDistanceTests extends ESTestCase { assertThat(check.isWithin(0, -178), equalTo(false)); } - @Test public void testArcDistanceVsPlaneInEllipsis() { GeoPoint centre = new GeoPoint(48.8534100, 2.3488000); GeoPoint northernPoint = new GeoPoint(48.8801108681, 2.35152032666); diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoHashTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoHashTests.java index 063fd76f846..d9d1245fb42 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/GeoHashTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/GeoHashTests.java @@ -18,22 +18,20 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.test.ESTestCase; -import org.apache.lucene.util.XGeoHashUtils; -import org.junit.Test; /** - * Tests for {@link org.apache.lucene.util.XGeoHashUtils} + * Tests for {@link org.apache.lucene.util.GeoHashUtils} */ public class GeoHashTests extends ESTestCase { - @Test public void testGeohashAsLongRoutines() { final GeoPoint expected = new GeoPoint(); final GeoPoint actual = new GeoPoint(); //Ensure that for all points at all supported levels of precision - // that the long encoding of a geohash is compatible with its + // that the long encoding of a geohash is compatible with its // String based counterpart for (double lat=-90;lat<90;lat++) { @@ -41,13 +39,13 @@ public class GeoHashTests extends ESTestCase { { for(int p=1;p<=12;p++) { - long geoAsLong = XGeoHashUtils.longEncode(lng, lat, p); + long geoAsLong = GeoHashUtils.longEncode(lng, lat, p); // string encode from geohashlong encoded location - String geohashFromLong = XGeoHashUtils.stringEncode(geoAsLong); + String geohashFromLong = GeoHashUtils.stringEncode(geoAsLong); // string encode from full res lat lon - String geohash = XGeoHashUtils.stringEncode(lng, lat, p); + String geohash = GeoHashUtils.stringEncode(lng, lat, p); // ensure both strings are the same assertEquals(geohash, geohashFromLong); diff --git a/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java index 703ab2ffaaf..6a73717fa2f 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java @@ -28,25 +28,30 @@ import com.spatial4j.core.shape.impl.PointImpl; import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.LineString; import com.vividsolutions.jts.geom.Polygon; + +import org.elasticsearch.common.geo.builders.LineStringBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertMultiLineString; +import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertMultiPolygon; +import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertPolygon; +import static org.hamcrest.Matchers.containsString; /** * Tests for {@link ShapeBuilder} */ public class ShapeBuilderTests extends ESTestCase { public void testNewPoint() { - Point point = ShapeBuilder.newPoint(-100, 45).build(); + Point point = ShapeBuilders.newPoint(-100, 45).build(); assertEquals(-100D, point.getX(), 0.0d); assertEquals(45D, point.getY(), 0.0d); } public void testNewRectangle() { - Rectangle rectangle = ShapeBuilder.newEnvelope().topLeft(-45, 30).bottomRight(45, -30).build(); + Rectangle rectangle = ShapeBuilders.newEnvelope().topLeft(-45, 30).bottomRight(45, -30).build(); assertEquals(-45D, rectangle.getMinX(), 0.0d); assertEquals(-30D, rectangle.getMinY(), 0.0d); assertEquals(45D, rectangle.getMaxX(), 0.0d); @@ -54,7 +59,7 @@ public class ShapeBuilderTests extends ESTestCase { } public void testNewPolygon() { - Polygon polygon = ShapeBuilder.newPolygon() + Polygon polygon = ShapeBuilders.newPolygon() .point(-45, 30) .point(45, 30) .point(45, -30) @@ -69,7 +74,7 @@ public class ShapeBuilderTests extends ESTestCase { } public void testNewPolygon_coordinate() { - Polygon polygon = ShapeBuilder.newPolygon() + Polygon polygon = ShapeBuilders.newPolygon() .point(new Coordinate(-45, 30)) .point(new Coordinate(45, 30)) .point(new Coordinate(45, -30)) @@ -84,7 +89,7 @@ public class ShapeBuilderTests extends ESTestCase { } public void testNewPolygon_coordinates() { - Polygon polygon = ShapeBuilder.newPolygon() + Polygon polygon = ShapeBuilders.newPolygon() .points(new Coordinate(-45, 30), new Coordinate(45, 30), new Coordinate(45, -30), new Coordinate(-45, -30), new Coordinate(-45, 30)).toPolygon(); LineString exterior = polygon.getExteriorRing(); @@ -96,7 +101,7 @@ public class ShapeBuilderTests extends ESTestCase { public void testLineStringBuilder() { // Building a simple LineString - ShapeBuilder.newLineString() + ShapeBuilders.newLineString() .point(-130.0, 55.0) .point(-130.0, -40.0) .point(-15.0, -40.0) @@ -105,9 +110,9 @@ public class ShapeBuilderTests extends ESTestCase { .point(-45.0, -15.0) .point(-110.0, -15.0) .point(-110.0, 55.0).build(); - + // Building a linestring that needs to be wrapped - ShapeBuilder.newLineString() + ShapeBuilders.newLineString() .point(100.0, 50.0) .point(110.0, -40.0) .point(240.0, -40.0) @@ -117,17 +122,17 @@ public class ShapeBuilderTests extends ESTestCase { .point(130.0, -30.0) .point(130.0, 60.0) .build(); - + // Building a lineString on the dateline - ShapeBuilder.newLineString() + ShapeBuilders.newLineString() .point(-180.0, 80.0) .point(-180.0, 40.0) .point(-180.0, -40.0) .point(-180.0, -80.0) .build(); - + // Building a lineString on the dateline - ShapeBuilder.newLineString() + ShapeBuilders.newLineString() .point(180.0, 80.0) .point(180.0, 40.0) .point(180.0, -40.0) @@ -136,87 +141,90 @@ public class ShapeBuilderTests extends ESTestCase { } public void testMultiLineString() { - ShapeBuilder.newMultiLinestring() - .linestring() + ShapeBuilders.newMultiLinestring() + .linestring(new LineStringBuilder() .point(-100.0, 50.0) .point(50.0, 50.0) .point(50.0, 20.0) .point(-100.0, 20.0) - .end() - .linestring() + ) + .linestring(new LineStringBuilder() .point(-100.0, 20.0) .point(50.0, 20.0) .point(50.0, 0.0) .point(-100.0, 0.0) - .end() + ) .build(); - // LineString that needs to be wrappped - ShapeBuilder.newMultiLinestring() - .linestring() + ShapeBuilders.newMultiLinestring() + .linestring(new LineStringBuilder() .point(150.0, 60.0) .point(200.0, 60.0) .point(200.0, 40.0) .point(150.0, 40.0) - .end() - .linestring() + ) + .linestring(new LineStringBuilder() .point(150.0, 20.0) .point(200.0, 20.0) .point(200.0, 0.0) .point(150.0, 0.0) - .end() + ) .build(); } - @Test(expected = InvalidShapeException.class) public void testPolygonSelfIntersection() { - ShapeBuilder.newPolygon() - .point(-40.0, 50.0) - .point(40.0, 50.0) - .point(-40.0, -50.0) - .point(40.0, -50.0) - .close().build(); + try { + ShapeBuilders.newPolygon() + .point(-40.0, 50.0) + .point(40.0, 50.0) + .point(-40.0, -50.0) + .point(40.0, -50.0) + .close().build(); + fail("Expected InvalidShapeException"); + } catch (InvalidShapeException e) { + assertThat(e.getMessage(), containsString("Self-intersection at or near point (0.0")); + } } public void testGeoCircle() { double earthCircumference = 40075016.69; - Circle circle = ShapeBuilder.newCircleBuilder().center(0, 0).radius("100m").build(); + Circle circle = ShapeBuilders.newCircleBuilder().center(0, 0).radius("100m").build(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(0, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = ShapeBuilder.newCircleBuilder().center(+180, 0).radius("100m").build(); + circle = ShapeBuilders.newCircleBuilder().center(+180, 0).radius("100m").build(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(180, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = ShapeBuilder.newCircleBuilder().center(-180, 0).radius("100m").build(); + circle = ShapeBuilders.newCircleBuilder().center(-180, 0).radius("100m").build(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(-180, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = ShapeBuilder.newCircleBuilder().center(0, 90).radius("100m").build(); + circle = ShapeBuilders.newCircleBuilder().center(0, 90).radius("100m").build(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(0, 90, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = ShapeBuilder.newCircleBuilder().center(0, -90).radius("100m").build(); + circle = ShapeBuilders.newCircleBuilder().center(0, -90).radius("100m").build(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(0, -90, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); double randomLat = (randomDouble() * 180) - 90; double randomLon = (randomDouble() * 360) - 180; double randomRadius = randomIntBetween(1, (int) earthCircumference / 4); - circle = ShapeBuilder.newCircleBuilder().center(randomLon, randomLat).radius(randomRadius + "m").build(); + circle = ShapeBuilders.newCircleBuilder().center(randomLon, randomLat).radius(randomRadius + "m").build(); assertEquals((360 * randomRadius) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(randomLon, randomLat, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); } public void testPolygonWrapping() { - Shape shape = ShapeBuilder.newPolygon() + Shape shape = ShapeBuilders.newPolygon() .point(-150.0, 65.0) .point(-250.0, 65.0) .point(-250.0, -65.0) .point(-150.0, -65.0) .close().build(); - + assertMultiPolygon(shape); } public void testLineStringWrapping() { - Shape shape = ShapeBuilder.newLineString() + Shape shape = ShapeBuilders.newLineString() .point(-150.0, 65.0) .point(-250.0, 65.0) .point(-250.0, -65.0) @@ -231,7 +239,7 @@ public class ShapeBuilderTests extends ESTestCase { // expected results: 3 polygons, 1 with a hole // a giant c shape - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(174,0) .point(-176,0) .point(-176,3) @@ -243,7 +251,7 @@ public class ShapeBuilderTests extends ESTestCase { .point(174,0); // 3/4 of an embedded 'c', crossing dateline once - builder.hole() + builder.hole(new LineStringBuilder() .point(175, 1) .point(175, 7) .point(-178, 7) @@ -252,15 +260,15 @@ public class ShapeBuilderTests extends ESTestCase { .point(176, 2) .point(179, 2) .point(179,1) - .point(175, 1); + .point(175, 1)); // embedded hole right of the dateline - builder.hole() + builder.hole(new LineStringBuilder() .point(-179, 1) .point(-179, 2) .point(-177, 2) .point(-177,1) - .point(-179,1); + .point(-179,1)); Shape shape = builder.close().build(); assertMultiPolygon(shape); @@ -272,7 +280,7 @@ public class ShapeBuilderTests extends ESTestCase { // expected results: 3 polygons, 1 with a hole // a giant c shape - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-186,0) .point(-176,0) .point(-176,3) @@ -284,7 +292,7 @@ public class ShapeBuilderTests extends ESTestCase { .point(-186,0); // 3/4 of an embedded 'c', crossing dateline once - builder.hole() + builder.hole(new LineStringBuilder() .point(-185,1) .point(-181,1) .point(-181,2) @@ -293,22 +301,22 @@ public class ShapeBuilderTests extends ESTestCase { .point(-178,6) .point(-178,7) .point(-185,7) - .point(-185,1); + .point(-185,1)); // embedded hole right of the dateline - builder.hole() + builder.hole(new LineStringBuilder() .point(-179,1) .point(-177,1) .point(-177,2) .point(-179,2) - .point(-179,1); + .point(-179,1)); Shape shape = builder.close().build(); assertMultiPolygon(shape); } public void testComplexShapeWithHole() { - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-85.0018514,37.1311314) .point(-85.0016645,37.1315293) .point(-85.0016246,37.1317069) @@ -348,7 +356,7 @@ public class ShapeBuilderTests extends ESTestCase { .point(-85.0016455,37.1310491) .point(-85.0018514,37.1311314); - builder.hole() + builder.hole(new LineStringBuilder() .point(-85.0000002,37.1317672) .point(-85.0001983,37.1317538) .point(-85.0003378,37.1317582) @@ -374,14 +382,14 @@ public class ShapeBuilderTests extends ESTestCase { .point(-84.9993527,37.1317788) .point(-84.9994931,37.1318061) .point(-84.9996815,37.1317979) - .point(-85.0000002,37.1317672); + .point(-85.0000002,37.1317672)); Shape shape = builder.close().build(); assertPolygon(shape); } public void testShapeWithHoleAtEdgeEndPoints() { - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-4, 2) .point(4, 2) .point(6, 0) @@ -390,19 +398,19 @@ public class ShapeBuilderTests extends ESTestCase { .point(-6, 0) .point(-4, 2); - builder.hole() + builder.hole(new LineStringBuilder() .point(4, 1) .point(4, -1) .point(-4, -1) .point(-4, 1) - .point(4, 1); + .point(4, 1)); Shape shape = builder.close().build(); assertPolygon(shape); } public void testShapeWithPointOnDateline() { - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(180, 0) .point(176, 4) .point(176, -4) @@ -414,7 +422,7 @@ public class ShapeBuilderTests extends ESTestCase { public void testShapeWithEdgeAlongDateline() { // test case 1: test the positive side of the dateline - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(180, 0) .point(176, 4) .point(180, -4) @@ -424,7 +432,7 @@ public class ShapeBuilderTests extends ESTestCase { assertPolygon(shape); // test case 2: test the negative side of the dateline - builder = ShapeBuilder.newPolygon() + builder = ShapeBuilders.newPolygon() .point(-176, 4) .point(-180, 0) .point(-180, -4) @@ -436,128 +444,133 @@ public class ShapeBuilderTests extends ESTestCase { public void testShapeWithBoundaryHoles() { // test case 1: test the positive side of the dateline - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-177, 10) .point(176, 15) .point(172, 0) .point(176, -15) .point(-177, -10) .point(-177, 10); - builder.hole() + builder.hole(new LineStringBuilder() .point(176, 10) .point(180, 5) .point(180, -5) .point(176, -10) - .point(176, 10); + .point(176, 10)); Shape shape = builder.close().build(); assertMultiPolygon(shape); // test case 2: test the negative side of the dateline - builder = ShapeBuilder.newPolygon() + builder = ShapeBuilders.newPolygon() .point(-176, 15) .point(179, 10) .point(179, -10) .point(-176, -15) .point(-172, 0); - builder.hole() + builder.hole(new LineStringBuilder() .point(-176, 10) .point(-176, -10) .point(-180, -5) .point(-180, 5) - .point(-176, 10); + .point(-176, 10)); shape = builder.close().build(); assertMultiPolygon(shape); } public void testShapeWithTangentialHole() { // test a shape with one tangential (shared) vertex (should pass) - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(179, 10) .point(168, 15) .point(164, 0) .point(166, -15) .point(179, -10) .point(179, 10); - builder.hole() + builder.hole(new LineStringBuilder() .point(-177, 10) .point(-178, -10) .point(-180, -5) .point(-180, 5) - .point(-177, 10); + .point(-177, 10)); Shape shape = builder.close().build(); assertMultiPolygon(shape); } - @Test(expected = InvalidShapeException.class) public void testShapeWithInvalidTangentialHole() { // test a shape with one invalid tangential (shared) vertex (should throw exception) - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(179, 10) .point(168, 15) .point(164, 0) .point(166, -15) .point(179, -10) .point(179, 10); - builder.hole() + builder.hole(new LineStringBuilder() .point(164, 0) .point(175, 10) .point(175, 5) .point(179, -10) - .point(164, 0); - Shape shape = builder.close().build(); - assertMultiPolygon(shape); + .point(164, 0)); + try { + builder.close().build(); + fail("Expected InvalidShapeException"); + } catch (InvalidShapeException e) { + assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior")); + } } public void testBoundaryShapeWithTangentialHole() { // test a shape with one tangential (shared) vertex for each hole (should pass) - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-177, 10) .point(176, 15) .point(172, 0) .point(176, -15) .point(-177, -10) .point(-177, 10); - builder.hole() + builder.hole(new LineStringBuilder() .point(-177, 10) .point(-178, -10) .point(-180, -5) .point(-180, 5) - .point(-177, 10); - builder.hole() + .point(-177, 10)); + builder.hole(new LineStringBuilder() .point(172, 0) .point(176, 10) .point(176, -5) - .point(172, 0); + .point(172, 0)); Shape shape = builder.close().build(); assertMultiPolygon(shape); } - @Test(expected = InvalidShapeException.class) public void testBoundaryShapeWithInvalidTangentialHole() { // test shape with two tangential (shared) vertices (should throw exception) - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-177, 10) .point(176, 15) .point(172, 0) .point(176, -15) .point(-177, -10) .point(-177, 10); - builder.hole() + builder.hole(new LineStringBuilder() .point(-177, 10) .point(172, 0) .point(180, -5) .point(176, -10) - .point(-177, 10); - Shape shape = builder.close().build(); - assertMultiPolygon(shape); + .point(-177, 10)); + try { + builder.close().build(); + fail("Expected InvalidShapeException"); + } catch (InvalidShapeException e) { + assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior")); + } } /** * Test an enveloping polygon around the max mercator bounds */ - @Test public void testBoundaryShape() { - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-180, 90) .point(180, 90) .point(180, -90) @@ -568,10 +581,9 @@ public class ShapeBuilderTests extends ESTestCase { assertPolygon(shape); } - @Test public void testShapeWithAlternateOrientation() { // cw: should produce a multi polygon spanning hemispheres - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(180, 0) .point(176, 4) .point(-176, 4) @@ -581,7 +593,7 @@ public class ShapeBuilderTests extends ESTestCase { assertPolygon(shape); // cw: geo core will convert to ccw across the dateline - builder = ShapeBuilder.newPolygon() + builder = ShapeBuilders.newPolygon() .point(180, 0) .point(-176, 4) .point(176, 4) @@ -592,15 +604,18 @@ public class ShapeBuilderTests extends ESTestCase { assertMultiPolygon(shape); } - @Test(expected = InvalidShapeException.class) public void testInvalidShapeWithConsecutiveDuplicatePoints() { - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(180, 0) .point(176, 4) .point(176, 4) .point(-176, 4) .point(180, 0); - Shape shape = builder.close().build(); - assertPolygon(shape); + try { + builder.close().build(); + fail("Expected InvalidShapeException"); + } catch (InvalidShapeException e) { + assertThat(e.getMessage(), containsString("duplicate consecutive coordinates at: (")); + } } } diff --git a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java index 83b66719985..bba56e38ec2 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java @@ -22,10 +22,10 @@ package org.elasticsearch.common.geo; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class ShapeRelationTests extends ESTestCase { @@ -80,13 +80,16 @@ public class ShapeRelationTests extends ESTestCase { } } - @Test(expected = IOException.class) public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(3, Integer.MAX_VALUE)); try (StreamInput in = StreamInput.wrap(out.bytes())) { ShapeRelation.DISJOINT.readFrom(in); + fail("Expected IOException"); + } catch(IOException e) { + assertThat(e.getMessage(), containsString("Unknown ShapeRelation ordinal [")); } + } } } diff --git a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java index c53a3fb18cb..e3e99d0e2f0 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java @@ -22,10 +22,10 @@ package org.elasticsearch.common.geo; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class SpatialStrategyTests extends ESTestCase { @@ -66,12 +66,14 @@ public class SpatialStrategyTests extends ESTestCase { } } - @Test(expected = IOException.class) public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); try (StreamInput in = StreamInput.wrap(out.bytes())) { SpatialStrategy.TERM.readFrom(in); + fail("Expected IOException"); + } catch(IOException e) { + assertThat(e.getMessage(), containsString("Unknown SpatialStrategy ordinal [")); } } } diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java new file mode 100644 index 00000000000..f15a731e86e --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java @@ -0,0 +1,143 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo.builders; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +import static org.hamcrest.Matchers.*; + +public abstract class AbstractShapeBuilderTestCase extends ESTestCase { + + private static final int NUMBER_OF_TESTBUILDERS = 20; + private static NamedWriteableRegistry namedWriteableRegistry; + + /** + * setup for the whole base test class + */ + @BeforeClass + public static void init() { + if (namedWriteableRegistry == null) { + namedWriteableRegistry = new NamedWriteableRegistry(); + namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PointBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(ShapeBuilder.class, CircleBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(ShapeBuilder.class, EnvelopeBuilder.PROTOTYPE); + } + } + + @AfterClass + public static void afterClass() throws Exception { + namedWriteableRegistry = null; + } + + /** + * create random shape that is put under test + */ + protected abstract SB createTestShapeBuilder(); + + /** + * mutate the given shape so the returned shape is different + */ + protected abstract SB mutate(SB original) throws IOException; + + /** + * Test that creates new shape from a random test shape and checks both for equality + */ + public void testFromXContent() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + SB testShape = createTestShapeBuilder(); + XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + contentBuilder.prettyPrint(); + } + XContentBuilder builder = testShape.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + XContentParser shapeParser = XContentHelper.createParser(builder.bytes()); + XContentHelper.createParser(builder.bytes()); + shapeParser.nextToken(); + ShapeBuilder parsedShape = ShapeBuilder.parse(shapeParser); + assertNotSame(testShape, parsedShape); + assertEquals(testShape, parsedShape); + assertEquals(testShape.hashCode(), parsedShape.hashCode()); + } + } + + /** + * Test serialization and deserialization of the test shape. + */ + public void testSerialization() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + SB testShape = createTestShapeBuilder(); + SB deserializedShape = copyShape(testShape); + assertEquals(deserializedShape, testShape); + assertEquals(deserializedShape.hashCode(), testShape.hashCode()); + assertNotSame(deserializedShape, testShape); + } + } + + /** + * Test equality and hashCode properties + */ + public void testEqualsAndHashcode() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + SB firstShape = createTestShapeBuilder(); + assertFalse("shape is equal to null", firstShape.equals(null)); + assertFalse("shape is equal to incompatible type", firstShape.equals("")); + assertTrue("shape is not equal to self", firstShape.equals(firstShape)); + assertThat("same shape's hashcode returns different values if called multiple times", firstShape.hashCode(), + equalTo(firstShape.hashCode())); + assertThat("different shapes should not be equal", mutate(firstShape), not(equalTo(firstShape))); + + SB secondShape = copyShape(firstShape); + assertTrue("shape is not equal to self", secondShape.equals(secondShape)); + assertTrue("shape is not equal to its copy", firstShape.equals(secondShape)); + assertTrue("equals is not symmetric", secondShape.equals(firstShape)); + assertThat("shape copy's hashcode is different from original hashcode", secondShape.hashCode(), equalTo(firstShape.hashCode())); + + SB thirdShape = copyShape(secondShape); + assertTrue("shape is not equal to self", thirdShape.equals(thirdShape)); + assertTrue("shape is not equal to its copy", secondShape.equals(thirdShape)); + assertThat("shape copy's hashcode is different from original hashcode", secondShape.hashCode(), equalTo(thirdShape.hashCode())); + assertTrue("equals is not transitive", firstShape.equals(thirdShape)); + assertThat("shape copy's hashcode is different from original hashcode", firstShape.hashCode(), equalTo(thirdShape.hashCode())); + assertTrue("equals is not symmetric", thirdShape.equals(secondShape)); + assertTrue("equals is not symmetric", thirdShape.equals(firstShape)); + } + } + + protected SB copyShape(SB original) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + original.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + ShapeBuilder prototype = (ShapeBuilder) namedWriteableRegistry.getPrototype(ShapeBuilder.class, original.getWriteableName()); + @SuppressWarnings("unchecked") + SB copy = (SB) prototype.readFrom(in); + return copy; + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/CirlceBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/CirlceBuilderTests.java new file mode 100644 index 00000000000..6b102b87b2c --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/CirlceBuilderTests.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo.builders; + +import com.vividsolutions.jts.geom.Coordinate; + +import org.elasticsearch.common.unit.DistanceUnit; + +import java.io.IOException; + +public class CirlceBuilderTests extends AbstractShapeBuilderTestCase { + + @Override + protected CircleBuilder createTestShapeBuilder() { + double centerX = randomDoubleBetween(-180, 180, false); + double centerY = randomDoubleBetween(-90, 90, false); + return new CircleBuilder() + .center(new Coordinate(centerX, centerY)) + .radius(randomDoubleBetween(0.1, 10.0, false), randomFrom(DistanceUnit.values())); + } + + @Override + protected CircleBuilder mutate(CircleBuilder original) throws IOException { + CircleBuilder mutation = copyShape(original); + double radius = original.radius(); + DistanceUnit unit = original.unit(); + + if (randomBoolean()) { + mutation.center(new Coordinate(original.center().x/2, original.center().y/2)); + } else if (randomBoolean()) { + radius = radius/2; + } else { + DistanceUnit newRandom = unit; + while (newRandom == unit) { + newRandom = randomFrom(DistanceUnit.values()); + }; + unit = newRandom; + } + return mutation.radius(radius, unit); + } +} diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java new file mode 100644 index 00000000000..e6f3db2f8af --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo.builders; + +import com.spatial4j.core.shape.Rectangle; +import com.vividsolutions.jts.geom.Coordinate; + +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; +import org.elasticsearch.test.geo.RandomShapeGenerator; + +import java.io.IOException; + +public class EnvelopeBuilderTests extends AbstractShapeBuilderTestCase { + + @Override + protected EnvelopeBuilder createTestShapeBuilder() { + EnvelopeBuilder envelope = new EnvelopeBuilder(randomFrom(Orientation.values())); + Rectangle box = RandomShapeGenerator.xRandomRectangle(getRandom(), RandomShapeGenerator.xRandomPoint(getRandom())); + envelope.topLeft(box.getMinX(), box.getMaxY()) + .bottomRight(box.getMaxX(), box.getMinY()); + return envelope; + } + + @Override + protected EnvelopeBuilder mutate(EnvelopeBuilder original) throws IOException { + EnvelopeBuilder mutation = copyShape(original); + if (randomBoolean()) { + // toggle orientation + mutation.orientation = (original.orientation == Orientation.LEFT ? Orientation.RIGHT : Orientation.LEFT); + } else { + // move one corner to the middle of original + switch (randomIntBetween(0, 3)) { + case 0: + mutation.topLeft(new Coordinate(randomDoubleBetween(-180.0, original.bottomRight.x, true), original.topLeft.y)); + break; + case 1: + mutation.topLeft(new Coordinate(original.topLeft.x, randomDoubleBetween(original.bottomRight.y, 90.0, true))); + break; + case 2: + mutation.bottomRight(new Coordinate(randomDoubleBetween(original.topLeft.x, 180.0, true), original.bottomRight.y)); + break; + case 3: + mutation.bottomRight(new Coordinate(original.bottomRight.x, randomDoubleBetween(-90.0, original.topLeft.y, true))); + break; + } + } + return mutation; + } +} diff --git a/core/src/main/java/org/elasticsearch/action/exists/ExistsAction.java b/core/src/test/java/org/elasticsearch/common/geo/builders/PointBuilderTests.java similarity index 56% rename from core/src/main/java/org/elasticsearch/action/exists/ExistsAction.java rename to core/src/test/java/org/elasticsearch/common/geo/builders/PointBuilderTests.java index d4463aea0d6..1e94a1bab3a 100644 --- a/core/src/main/java/org/elasticsearch/action/exists/ExistsAction.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/PointBuilderTests.java @@ -16,28 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.action.exists; +package org.elasticsearch.common.geo.builders; -import org.elasticsearch.action.Action; -import org.elasticsearch.client.ElasticsearchClient; +import com.vividsolutions.jts.geom.Coordinate; -public class ExistsAction extends Action { +import org.elasticsearch.test.geo.RandomShapeGenerator; +import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; - public static final ExistsAction INSTANCE = new ExistsAction(); - public static final String NAME = "indices:data/read/exists"; +public class PointBuilderTests extends AbstractShapeBuilderTestCase { - private ExistsAction() { - super(NAME); + @Override + protected PointBuilder createTestShapeBuilder() { + return (PointBuilder) RandomShapeGenerator.createShape(getRandom(), ShapeType.POINT); } @Override - public ExistsResponse newResponse() { - return new ExistsResponse(); - } - - @Override - public ExistsRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new ExistsRequestBuilder(client, this); + protected PointBuilder mutate(PointBuilder original) { + return new PointBuilder().coordinate(new Coordinate(original.longitude()/2, original.latitude()/2)); } } diff --git a/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java b/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java index dbc174ba2d6..4063c1b7be4 100644 --- a/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java +++ b/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java @@ -20,21 +20,17 @@ package org.elasticsearch.common.hash; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; -import static org.junit.Assert.*; - public class MessageDigestsTests extends ESTestCase { private void assertHash(String expected, String test, MessageDigest messageDigest) { String actual = MessageDigests.toHexString(messageDigest.digest(test.getBytes(StandardCharsets.UTF_8))); assertEquals(expected, actual); } - @Test public void testMd5() throws Exception { assertHash("d41d8cd98f00b204e9800998ecf8427e", "", MessageDigests.md5()); assertHash("900150983cd24fb0d6963f7d28e17f72", "abc", MessageDigests.md5()); @@ -44,7 +40,6 @@ public class MessageDigestsTests extends ESTestCase { assertHash("1055d3e698d289f2af8663725127bd4b", "The quick brown fox jumps over the lazy cog", MessageDigests.md5()); } - @Test public void testSha1() throws Exception { assertHash("da39a3ee5e6b4b0d3255bfef95601890afd80709", "", MessageDigests.sha1()); assertHash("a9993e364706816aba3e25717850c26c9cd0d89d", "abc", MessageDigests.sha1()); @@ -54,7 +49,6 @@ public class MessageDigestsTests extends ESTestCase { assertHash("de9f2c7fd25e1b3afad3e85a0bd17d9b100db4b3", "The quick brown fox jumps over the lazy cog", MessageDigests.sha1()); } - @Test public void testSha256() throws Exception { assertHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "", MessageDigests.sha256()); assertHash("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc", MessageDigests.sha256()); @@ -64,7 +58,6 @@ public class MessageDigestsTests extends ESTestCase { assertHash("e4c4d8f3bf76b692de791a173e05321150f7a345b46484fe427f6acc7ecc81be", "The quick brown fox jumps over the lazy cog", MessageDigests.sha256()); } - @Test public void testToHexString() throws Exception { for (int i = 0; i < 1024; i++) { BigInteger expected = BigInteger.probablePrime(256, random()); diff --git a/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java b/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java index 8998d1bccf5..a4f35389bd3 100644 --- a/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java +++ b/core/src/test/java/org/elasticsearch/common/hppc/HppcMapsTests.java @@ -19,9 +19,9 @@ package org.elasticsearch.common.hppc; import com.carrotsearch.hppc.ObjectHashSet; + import org.elasticsearch.common.collect.HppcMaps; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -29,8 +29,6 @@ import java.util.List; import static org.hamcrest.Matchers.equalTo; public class HppcMapsTests extends ESTestCase { - - @Test public void testIntersection() throws Exception { boolean enabled = false; assert enabled = true; diff --git a/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java b/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java index 08b49e8dddc..39d24a0e792 100644 --- a/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java @@ -19,24 +19,20 @@ package org.elasticsearch.common.io; -import java.nio.charset.StandardCharsets; - -import org.elasticsearch.test.ESTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; +import org.elasticsearch.test.ESTestCase; import org.junit.Assert; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; +import static org.elasticsearch.common.io.FileTestUtils.assertFileContent; /** * Unit tests for {@link org.elasticsearch.common.io.FileSystemUtils}. @@ -60,7 +56,6 @@ public class FileSystemUtilsTests extends ESTestCase { FileSystemUtils.copyDirectoryRecursively(path, src); } - @Test public void testMoveOverExistingFileAndAppend() throws IOException { FileSystemUtils.moveFilesWithoutOverwriting(src.resolve("v1"), dst, ".new"); @@ -87,7 +82,6 @@ public class FileSystemUtilsTests extends ESTestCase { assertFileContent(dst, "dir/subdir/file5.txt", "version1"); } - @Test public void testMoveOverExistingFileAndIgnore() throws IOException { Path dest = createTempDir(); @@ -115,7 +109,6 @@ public class FileSystemUtilsTests extends ESTestCase { assertFileContent(dest, "dir/subdir/file5.txt", "version1"); } - @Test public void testMoveFilesDoesNotCreateSameFileWithSuffix() throws Exception { Path[] dirs = new Path[] { createTempDir(), createTempDir(), createTempDir()}; for (Path dir : dirs) { @@ -143,26 +136,6 @@ public class FileSystemUtilsTests extends ESTestCase { assertFileContent(dst, "dir/file2.txt.new", "UPDATED"); } - /** - * Check that a file contains a given String - * @param dir root dir for file - * @param filename relative path from root dir to file - * @param expected expected content (if null, we don't expect any file) - */ - public static void assertFileContent(Path dir, String filename, String expected) throws IOException { - Assert.assertThat(Files.exists(dir), is(true)); - Path file = dir.resolve(filename); - if (expected == null) { - Assert.assertThat("file [" + file + "] should not exist.", Files.exists(file), is(false)); - } else { - assertFileExists(file); - String fileContent = new String(Files.readAllBytes(file), java.nio.charset.StandardCharsets.UTF_8); - // trim the string content to prevent different handling on windows vs. unix and CR chars... - Assert.assertThat(fileContent.trim(), equalTo(expected.trim())); - } - } - - @Test public void testAppend() { assertEquals(FileSystemUtils.append(PathUtils.get("/foo/bar"), PathUtils.get("/hello/world/this_is/awesome"), 0), PathUtils.get("/foo/bar/hello/world/this_is/awesome")); diff --git a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java index 1dd809da62f..5c6c1e1789b 100644 --- a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java @@ -19,25 +19,27 @@ package org.elasticsearch.common.io; -import java.nio.charset.StandardCharsets; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; import java.util.Arrays; -import static org.elasticsearch.common.io.Streams.*; +import static org.elasticsearch.common.io.Streams.copy; +import static org.elasticsearch.common.io.Streams.copyToString; import static org.hamcrest.Matchers.equalTo; /** * Unit tests for {@link org.elasticsearch.common.io.Streams}. */ public class StreamsTests extends ESTestCase { - - @Test public void testCopyFromInputStream() throws IOException { byte[] content = "content".getBytes(StandardCharsets.UTF_8); ByteArrayInputStream in = new ByteArrayInputStream(content); @@ -48,7 +50,6 @@ public class StreamsTests extends ESTestCase { assertThat(Arrays.equals(content, out.toByteArray()), equalTo(true)); } - @Test public void testCopyFromByteArray() throws IOException { byte[] content = "content".getBytes(StandardCharsets.UTF_8); ByteArrayOutputStream out = new ByteArrayOutputStream(content.length); @@ -56,7 +57,6 @@ public class StreamsTests extends ESTestCase { assertThat(Arrays.equals(content, out.toByteArray()), equalTo(true)); } - @Test public void testCopyFromReader() throws IOException { String content = "content"; StringReader in = new StringReader(content); @@ -66,7 +66,6 @@ public class StreamsTests extends ESTestCase { assertThat(out.toString(), equalTo(content)); } - @Test public void testCopyFromString() throws IOException { String content = "content"; StringWriter out = new StringWriter(); @@ -74,15 +73,13 @@ public class StreamsTests extends ESTestCase { assertThat(out.toString(), equalTo(content)); } - @Test public void testCopyToString() throws IOException { String content = "content"; StringReader in = new StringReader(content); String result = copyToString(in); assertThat(result, equalTo(content)); } - - @Test + public void testBytesStreamInput() throws IOException { byte stuff[] = new byte[] { 0, 1, 2, 3 }; BytesRef stuffRef = new BytesRef(stuff, 2, 2); @@ -93,5 +90,4 @@ public class StreamsTests extends ESTestCase { assertEquals(-1, input.read()); input.close(); } - } diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index 50e51bab22e..7f232363f73 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -24,21 +24,18 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; - import java.util.Objects; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; /** * Tests for {@link BytesStreamOutput} paging behaviour. */ public class BytesStreamsTests extends ESTestCase { - - @Test public void testEmpty() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -49,7 +46,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSingleByte() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); assertEquals(0, out.size()); @@ -65,7 +61,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSingleShortPage() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -83,7 +78,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testIllegalBulkWrite() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -99,7 +93,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSingleShortPageBulkWrite() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -120,7 +113,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSingleFullPageBulkWrite() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -136,7 +128,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSingleFullPageBulkWriteWithOffset() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -156,7 +147,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSingleFullPageBulkWriteWithOffsetCrossover() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -176,7 +166,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSingleFullPage() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -194,7 +183,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testOneFullOneShortPage() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -212,7 +200,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testTwoFullOneShortPage() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -230,7 +217,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSeek() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -247,7 +233,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSkip() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); @@ -261,7 +246,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testSimpleStreams() throws Exception { assumeTrue("requires a 64-bit JRE ... ?!", Constants.JRE_IS_64BIT); BytesStreamOutput out = new BytesStreamOutput(); @@ -312,7 +296,6 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - @Test public void testNamedWriteable() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); @@ -324,7 +307,6 @@ public class BytesStreamsTests extends ESTestCase { assertEquals(namedWriteableOut, namedWriteableIn); } - @Test public void testNamedWriteableDuplicates() throws IOException { NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); @@ -337,7 +319,6 @@ public class BytesStreamsTests extends ESTestCase { } } - @Test public void testNamedWriteableUnknownCategory() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable(new TestNamedWriteable("test1", "test2")); @@ -351,7 +332,6 @@ public class BytesStreamsTests extends ESTestCase { } } - @Test public void testNamedWriteableUnknownNamedWriteable() throws IOException { NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); @@ -381,13 +361,17 @@ public class BytesStreamsTests extends ESTestCase { } } - @Test(expected = UnsupportedOperationException.class) public void testNamedWriteableNotSupportedWithoutWrapping() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); TestNamedWriteable testNamedWriteable = new TestNamedWriteable("test1", "test2"); out.writeNamedWriteable(testNamedWriteable); StreamInput in = StreamInput.wrap(out.bytes().toBytes()); - in.readNamedWriteable(BaseNamedWriteable.class); + try { + in.readNamedWriteable(BaseNamedWriteable.class); + fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException e) { + assertThat(e.getMessage(), is("can't read named writeable from StreamInput")); + } } private static abstract class BaseNamedWriteable implements NamedWriteable { @@ -440,7 +424,6 @@ public class BytesStreamsTests extends ESTestCase { // we ignore this test for now since all existing callers of BytesStreamOutput happily // call bytes() after close(). @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12620") - @Test public void testAccessAfterClose() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java new file mode 100644 index 00000000000..cec70fb61f5 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.io.stream; + +import org.elasticsearch.common.bytes.ByteBufferBytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +public class StreamTests extends ESTestCase { + public void testRandomVLongSerialization() throws IOException { + for (int i = 0; i < 1024; i++) { + long write = randomLong(); + BytesStreamOutput out = new BytesStreamOutput(); + out.writeZLong(write); + long read = out.bytes().streamInput().readZLong(); + assertEquals(write, read); + } + } + + public void testSpecificVLongSerialization() throws IOException { + List> values = + Arrays.asList( + new Tuple<>(0L, new byte[]{0}), + new Tuple<>(-1L, new byte[]{1}), + new Tuple<>(1L, new byte[]{2}), + new Tuple<>(-2L, new byte[]{3}), + new Tuple<>(2L, new byte[]{4}), + new Tuple<>(Long.MIN_VALUE, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1, 1}), + new Tuple<>(Long.MAX_VALUE, new byte[]{-2, -1, -1, -1, -1, -1, -1, -1, -1, 1}) + + ); + for (Tuple value : values) { + BytesStreamOutput out = new BytesStreamOutput(); + out.writeZLong(value.v1()); + assertArrayEquals(Long.toString(value.v1()), value.v2(), out.bytes().toBytes()); + ByteBufferBytesReference bytes = new ByteBufferBytesReference(ByteBuffer.wrap(value.v2())); + assertEquals(Arrays.toString(value.v2()), (long)value.v1(), bytes.streamInput().readZLong()); + } + } + + public void testLinkedHashMap() throws IOException { + int size = randomIntBetween(1, 1024); + boolean accessOrder = randomBoolean(); + List> list = new ArrayList<>(size); + LinkedHashMap write = new LinkedHashMap<>(size, 0.75f, accessOrder); + for (int i = 0; i < size; i++) { + int value = randomInt(); + list.add(new Tuple<>(Integer.toString(i), value)); + write.put(Integer.toString(i), value); + } + if (accessOrder) { + // randomize access order + Collections.shuffle(list, random()); + for (Tuple entry : list) { + // touch the entries to set the access order + write.get(entry.v1()); + } + } + BytesStreamOutput out = new BytesStreamOutput(); + out.writeGenericValue(write); + LinkedHashMap read = (LinkedHashMap)out.bytes().streamInput().readGenericValue(); + assertEquals(size, read.size()); + int index = 0; + for (Map.Entry entry : read.entrySet()) { + assertEquals(list.get(index).v1(), entry.getKey()); + assertEquals(list.get(index).v2(), entry.getValue()); + index++; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java b/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java index 6a1f146046d..96a4a3fdf35 100644 --- a/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java @@ -23,12 +23,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTimeZone; -import org.junit.Test; import java.util.TimeZone; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class DateMathParserTests extends ESTestCase { @@ -48,12 +48,12 @@ public class DateMathParserTests extends ESTestCase { void assertDateMathEquals(String toTest, String expected) { assertDateMathEquals(toTest, expected, 0, false, null); } - + void assertDateMathEquals(String toTest, String expected, final long now, boolean roundUp, DateTimeZone timeZone) { long gotMillis = parser.parse(toTest, callable(now), roundUp, timeZone); assertDateEquals(gotMillis, toTest, expected); } - + void assertDateEquals(long gotMillis, String original, String expected) { long expectedMillis = parser.parse(expected, callable(0)); if (gotMillis != expectedMillis) { @@ -65,7 +65,7 @@ public class DateMathParserTests extends ESTestCase { "Actual milliseconds : " + gotMillis + "\n"); } } - + public void testBasicDates() { assertDateMathEquals("2014", "2014-01-01T00:00:00.000"); assertDateMathEquals("2014-05", "2014-05-01T00:00:00.000"); @@ -92,7 +92,7 @@ public class DateMathParserTests extends ESTestCase { assertDateMathEquals("2014-05-30T20:21+03:00", "2014-05-30T17:21:00.000", 0, false, DateTimeZone.forID("-08:00")); assertDateMathEquals("2014-05-30T20:21Z", "2014-05-30T20:21:00.000", 0, false, DateTimeZone.forID("-08:00")); } - + public void testBasicMath() { assertDateMathEquals("2014-11-18||+y", "2015-11-18"); assertDateMathEquals("2014-11-18||-2y", "2012-11-18"); @@ -117,7 +117,7 @@ public class DateMathParserTests extends ESTestCase { assertDateMathEquals("2014-11-18T14:27:32||+60s", "2014-11-18T14:28:32"); assertDateMathEquals("2014-11-18T14:27:32||-3600s", "2014-11-18T13:27:32"); } - + public void testLenientEmptyMath() { assertDateMathEquals("2014-05-30T20:21||", "2014-05-30T20:21:00.000"); } @@ -133,12 +133,12 @@ public class DateMathParserTests extends ESTestCase { public void testNow() { final long now = parser.parse("2014-11-18T14:27:32", callable(0), false, null); - + assertDateMathEquals("now", "2014-11-18T14:27:32", now, false, null); assertDateMathEquals("now+M", "2014-12-18T14:27:32", now, false, null); assertDateMathEquals("now-2d", "2014-11-16T14:27:32", now, false, null); assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, null); - + // timezone does not affect now assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, DateTimeZone.forID("+02:00")); } @@ -151,14 +151,14 @@ public class DateMathParserTests extends ESTestCase { // rounding should also take into account time zone assertDateMathEquals("2014-11-18||/y", "2013-12-31T23:00:00.000Z", 0, false, DateTimeZone.forID("CET")); assertDateMathEquals("2014-11-18||/y", "2014-12-31T22:59:59.999Z", 0, true, DateTimeZone.forID("CET")); - + assertDateMathEquals("2014-11-18||/M", "2014-11-01", 0, false, null); assertDateMathEquals("2014-11-18||/M", "2014-11-30T23:59:59.999", 0, true, null); assertDateMathEquals("2014-11||/M", "2014-11-01", 0, false, null); assertDateMathEquals("2014-11||/M", "2014-11-30T23:59:59.999", 0, true, null); assertDateMathEquals("2014-11-18||/M", "2014-10-31T23:00:00.000Z", 0, false, DateTimeZone.forID("CET")); assertDateMathEquals("2014-11-18||/M", "2014-11-30T22:59:59.999Z", 0, true, DateTimeZone.forID("CET")); - + assertDateMathEquals("2014-11-18T14||/w", "2014-11-17", 0, false, null); assertDateMathEquals("2014-11-18T14||/w", "2014-11-23T23:59:59.999", 0, true, null); assertDateMathEquals("2014-11-18||/w", "2014-11-17", 0, false, null); @@ -168,12 +168,12 @@ public class DateMathParserTests extends ESTestCase { assertDateMathEquals("2014-11-18||/w", "2014-11-16T23:00:00.000Z", 0, false, DateTimeZone.forID("CET")); assertDateMathEquals("2014-11-18||/w", "2014-11-23T22:59:59.999Z", 0, true, DateTimeZone.forID("CET")); assertDateMathEquals("2014-07-22||/w", "2014-07-20T22:00:00.000Z", 0, false, DateTimeZone.forID("CET")); // with DST - + assertDateMathEquals("2014-11-18T14||/d", "2014-11-18", 0, false, null); assertDateMathEquals("2014-11-18T14||/d", "2014-11-18T23:59:59.999", 0, true, null); assertDateMathEquals("2014-11-18||/d", "2014-11-18", 0, false, null); assertDateMathEquals("2014-11-18||/d", "2014-11-18T23:59:59.999", 0, true, null); - + assertDateMathEquals("2014-11-18T14:27||/h", "2014-11-18T14", 0, false, null); assertDateMathEquals("2014-11-18T14:27||/h", "2014-11-18T14:59:59.999", 0, true, null); assertDateMathEquals("2014-11-18T14||/H", "2014-11-18T14", 0, false, null); @@ -182,12 +182,12 @@ public class DateMathParserTests extends ESTestCase { assertDateMathEquals("2014-11-18T14:27||/h", "2014-11-18T14:59:59.999", 0, true, null); assertDateMathEquals("2014-11-18T14||/H", "2014-11-18T14", 0, false, null); assertDateMathEquals("2014-11-18T14||/H", "2014-11-18T14:59:59.999", 0, true, null); - + assertDateMathEquals("2014-11-18T14:27:32||/m", "2014-11-18T14:27", 0, false, null); assertDateMathEquals("2014-11-18T14:27:32||/m", "2014-11-18T14:27:59.999", 0, true, null); assertDateMathEquals("2014-11-18T14:27||/m", "2014-11-18T14:27", 0, false, null); assertDateMathEquals("2014-11-18T14:27||/m", "2014-11-18T14:27:59.999", 0, true, null); - + assertDateMathEquals("2014-11-18T14:27:32.123||/s", "2014-11-18T14:27:32", 0, false, null); assertDateMathEquals("2014-11-18T14:27:32.123||/s", "2014-11-18T14:27:32.999", 0, true, null); assertDateMathEquals("2014-11-18T14:27:32||/s", "2014-11-18T14:27:32", 0, false, null); @@ -199,12 +199,12 @@ public class DateMathParserTests extends ESTestCase { // datemath still works on timestamps assertDateMathEquals("1418248078000||/m", "2014-12-10T21:47:00.000"); - + // also check other time units DateMathParser parser = new DateMathParser(Joda.forPattern("epoch_second||dateOptionalTime")); long datetime = parser.parse("1418248078", callable(0)); assertDateEquals(datetime, "1418248078", "2014-12-10T21:47:58.000"); - + // a timestamp before 10000 is a year assertDateMathEquals("9999", "9999-01-01T00:00:00.000"); // 10000 is also a year, breaking bwc, used to be a timestamp @@ -221,7 +221,7 @@ public class DateMathParserTests extends ESTestCase { assertThat(ExceptionsHelper.detailedMessage(e).contains(exc), equalTo(true)); } } - + public void testIllegalMathFormat() { assertParseException("Expected date math unsupported operator exception", "2014-11-18||*5", "operator not supported"); assertParseException("Expected date math incompatible rounding exception", "2014-11-18||/2m", "rounding"); @@ -229,7 +229,7 @@ public class DateMathParserTests extends ESTestCase { assertParseException("Expected date math truncation exception", "2014-11-18||+12", "truncated"); assertParseException("Expected date math truncation exception", "2014-11-18||-", "truncated"); } - + public void testIllegalDateFormat() { assertParseException("Expected bad timestamp exception", Long.toString(Long.MAX_VALUE) + "0", "failed to parse date field"); assertParseException("Expected bad date format exception", "123bogus", "with format"); @@ -250,9 +250,14 @@ public class DateMathParserTests extends ESTestCase { assertTrue(called.get()); } - @Test(expected = ElasticsearchParseException.class) public void testThatUnixTimestampMayNotHaveTimeZone() { DateMathParser parser = new DateMathParser(Joda.forPattern("epoch_millis")); - parser.parse("1234567890123", callable(42), false, DateTimeZone.forTimeZone(TimeZone.getTimeZone("CET"))); + try { + parser.parse("1234567890123", callable(42), false, DateTimeZone.forTimeZone(TimeZone.getTimeZone("CET"))); + fail("Expected ElasticsearchParseException"); + } catch(ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("failed to parse date field")); + assertThat(e.getMessage(), containsString("with format [epoch_millis]")); + } } } diff --git a/core/src/test/java/org/elasticsearch/common/logging/jdk/JDKESLoggerTests.java b/core/src/test/java/org/elasticsearch/common/logging/jdk/JDKESLoggerTests.java index 748994da9f4..92dd9ffc012 100644 --- a/core/src/test/java/org/elasticsearch/common/logging/jdk/JDKESLoggerTests.java +++ b/core/src/test/java/org/elasticsearch/common/logging/jdk/JDKESLoggerTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.logging.jdk; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -51,8 +50,7 @@ public class JDKESLoggerTests extends ESTestCase { testLogger.addHandler(testHandler); } - @Test - public void locationInfoTest() { + public void testLocationInfoTest() { esTestLogger.error("This is an error"); esTestLogger.warn("This is a warning"); esTestLogger.info("This is an info"); @@ -66,31 +64,45 @@ public class JDKESLoggerTests extends ESTestCase { assertThat(record.getLevel(), equalTo(Level.SEVERE)); assertThat(record.getMessage(), equalTo("This is an error")); assertThat(record.getSourceClassName(), equalTo(JDKESLoggerTests.class.getCanonicalName())); - assertThat(record.getSourceMethodName(), equalTo("locationInfoTest")); + assertThat(record.getSourceMethodName(), equalTo("testLocationInfoTest")); record = records.get(1); assertThat(record, notNullValue()); assertThat(record.getLevel(), equalTo(Level.WARNING)); assertThat(record.getMessage(), equalTo("This is a warning")); assertThat(record.getSourceClassName(), equalTo(JDKESLoggerTests.class.getCanonicalName())); - assertThat(record.getSourceMethodName(), equalTo("locationInfoTest")); + assertThat(record.getSourceMethodName(), equalTo("testLocationInfoTest")); record = records.get(2); assertThat(record, notNullValue()); assertThat(record.getLevel(), equalTo(Level.INFO)); assertThat(record.getMessage(), equalTo("This is an info")); assertThat(record.getSourceClassName(), equalTo(JDKESLoggerTests.class.getCanonicalName())); - assertThat(record.getSourceMethodName(), equalTo("locationInfoTest")); + assertThat(record.getSourceMethodName(), equalTo("testLocationInfoTest")); record = records.get(3); assertThat(record, notNullValue()); assertThat(record.getLevel(), equalTo(Level.FINE)); assertThat(record.getMessage(), equalTo("This is a debug")); assertThat(record.getSourceClassName(), equalTo(JDKESLoggerTests.class.getCanonicalName())); - assertThat(record.getSourceMethodName(), equalTo("locationInfoTest")); + assertThat(record.getSourceMethodName(), equalTo("testLocationInfoTest")); record = records.get(4); assertThat(record, notNullValue()); assertThat(record.getLevel(), equalTo(Level.FINEST)); assertThat(record.getMessage(), equalTo("This is a trace")); assertThat(record.getSourceClassName(), equalTo(JDKESLoggerTests.class.getCanonicalName())); - assertThat(record.getSourceMethodName(), equalTo("locationInfoTest")); + assertThat(record.getSourceMethodName(), equalTo("testLocationInfoTest")); + } + + public void testSetLogLevelString() { + // verify the string based level-setters + esTestLogger.setLevel("error"); + assertThat(esTestLogger.getLevel(), equalTo("SEVERE")); + esTestLogger.setLevel("warn"); + assertThat(esTestLogger.getLevel(), equalTo("WARNING")); + esTestLogger.setLevel("info"); + assertThat(esTestLogger.getLevel(), equalTo("INFO")); + esTestLogger.setLevel("debug"); + assertThat(esTestLogger.getLevel(), equalTo("FINE")); + esTestLogger.setLevel("trace"); + assertThat(esTestLogger.getLevel(), equalTo("FINEST")); } private static class TestHandler extends Handler { diff --git a/core/src/test/java/org/elasticsearch/common/logging/log4j/Log4jESLoggerTests.java b/core/src/test/java/org/elasticsearch/common/logging/log4j/Log4jESLoggerTests.java index d0cd3879dbc..8f9c9009071 100644 --- a/core/src/test/java/org/elasticsearch/common/logging/log4j/Log4jESLoggerTests.java +++ b/core/src/test/java/org/elasticsearch/common/logging/log4j/Log4jESLoggerTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.junit.After; -import org.junit.Test; import java.nio.file.Path; import java.util.ArrayList; @@ -84,8 +83,7 @@ public class Log4jESLoggerTests extends ESTestCase { deprecationLogger.removeAppender(deprecationAppender); } - @Test - public void locationInfoTest() { + public void testLocationInfoTest() { esTestLogger.error("This is an error"); esTestLogger.warn("This is a warning"); esTestLogger.info("This is an info"); @@ -101,7 +99,7 @@ public class Log4jESLoggerTests extends ESTestCase { LocationInfo locationInfo = event.getLocationInformation(); assertThat(locationInfo, notNullValue()); assertThat(locationInfo.getClassName(), equalTo(Log4jESLoggerTests.class.getCanonicalName())); - assertThat(locationInfo.getMethodName(), equalTo("locationInfoTest")); + assertThat(locationInfo.getMethodName(), equalTo("testLocationInfoTest")); event = events.get(1); assertThat(event, notNullValue()); assertThat(event.getLevel(), equalTo(Level.WARN)); @@ -109,7 +107,7 @@ public class Log4jESLoggerTests extends ESTestCase { locationInfo = event.getLocationInformation(); assertThat(locationInfo, notNullValue()); assertThat(locationInfo.getClassName(), equalTo(Log4jESLoggerTests.class.getCanonicalName())); - assertThat(locationInfo.getMethodName(), equalTo("locationInfoTest")); + assertThat(locationInfo.getMethodName(), equalTo("testLocationInfoTest")); event = events.get(2); assertThat(event, notNullValue()); assertThat(event.getLevel(), equalTo(Level.INFO)); @@ -117,7 +115,7 @@ public class Log4jESLoggerTests extends ESTestCase { locationInfo = event.getLocationInformation(); assertThat(locationInfo, notNullValue()); assertThat(locationInfo.getClassName(), equalTo(Log4jESLoggerTests.class.getCanonicalName())); - assertThat(locationInfo.getMethodName(), equalTo("locationInfoTest")); + assertThat(locationInfo.getMethodName(), equalTo("testLocationInfoTest")); event = events.get(3); assertThat(event, notNullValue()); assertThat(event.getLevel(), equalTo(Level.DEBUG)); @@ -125,7 +123,7 @@ public class Log4jESLoggerTests extends ESTestCase { locationInfo = event.getLocationInformation(); assertThat(locationInfo, notNullValue()); assertThat(locationInfo.getClassName(), equalTo(Log4jESLoggerTests.class.getCanonicalName())); - assertThat(locationInfo.getMethodName(), equalTo("locationInfoTest")); + assertThat(locationInfo.getMethodName(), equalTo("testLocationInfoTest")); event = events.get(4); assertThat(event, notNullValue()); assertThat(event.getLevel(), equalTo(Level.TRACE)); @@ -133,10 +131,9 @@ public class Log4jESLoggerTests extends ESTestCase { locationInfo = event.getLocationInformation(); assertThat(locationInfo, notNullValue()); assertThat(locationInfo.getClassName(), equalTo(Log4jESLoggerTests.class.getCanonicalName())); - assertThat(locationInfo.getMethodName(), equalTo("locationInfoTest")); + assertThat(locationInfo.getMethodName(), equalTo("testLocationInfoTest")); } - @Test public void testDeprecationLogger() { deprecationLogger.deprecated("This is a deprecation message"); List deprecationEvents = deprecationAppender.getEvents(); diff --git a/core/src/test/java/org/elasticsearch/common/logging/log4j/LoggingConfigurationTests.java b/core/src/test/java/org/elasticsearch/common/logging/log4j/LoggingConfigurationTests.java index 2b84bec93e9..2a08dd1e55c 100644 --- a/core/src/test/java/org/elasticsearch/common/logging/log4j/LoggingConfigurationTests.java +++ b/core/src/test/java/org/elasticsearch/common/logging/log4j/LoggingConfigurationTests.java @@ -28,16 +28,16 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import org.junit.Test; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Arrays; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * @@ -49,7 +49,6 @@ public class LoggingConfigurationTests extends ESTestCase { LogConfigurator.reset(); } - @Test public void testResolveMultipleConfigs() throws Exception { String level = Log4jESLoggerFactory.getLogger("test").getLevel(); try { @@ -79,7 +78,6 @@ public class LoggingConfigurationTests extends ESTestCase { } } - @Test public void testResolveJsonLoggingConfig() throws Exception { Path tmpDir = createTempDir(); Path loggingConf = tmpDir.resolve(loggingConfiguration("json")); @@ -97,7 +95,6 @@ public class LoggingConfigurationTests extends ESTestCase { assertThat(logSettings.get("json"), is("foo")); } - @Test public void testResolvePropertiesLoggingConfig() throws Exception { Path tmpDir = createTempDir(); Path loggingConf = tmpDir.resolve(loggingConfiguration("properties")); @@ -115,7 +112,6 @@ public class LoggingConfigurationTests extends ESTestCase { assertThat(logSettings.get("key"), is("value")); } - @Test public void testResolveYamlLoggingConfig() throws Exception { Path tmpDir = createTempDir(); Path loggingConf1 = tmpDir.resolve(loggingConfiguration("yml")); @@ -136,7 +132,6 @@ public class LoggingConfigurationTests extends ESTestCase { assertThat(logSettings.get("yaml"), is("bar")); } - @Test public void testResolveConfigInvalidFilename() throws Exception { Path tmpDir = createTempDir(); Path invalidSuffix = tmpDir.resolve(loggingConfiguration(randomFrom(LogConfigurator.ALLOWED_SUFFIXES)) + randomInvalidSuffix()); @@ -155,7 +150,6 @@ public class LoggingConfigurationTests extends ESTestCase { } // tests that custom settings are not overwritten by settings in the config file - @Test public void testResolveOrder() throws Exception { Path tmpDir = createTempDir(); Path loggingConf = tmpDir.resolve(loggingConfiguration("yaml")); @@ -182,7 +176,6 @@ public class LoggingConfigurationTests extends ESTestCase { } // tests that config file is not read when we call LogConfigurator.configure(Settings, false) - @Test public void testConfigNotRead() throws Exception { Path tmpDir = createTempDir(); Path loggingConf = tmpDir.resolve(loggingConfiguration("yaml")); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index 13ac6fd0a6f..17345fd714f 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -17,6 +17,7 @@ * under the License. */ package org.elasticsearch.common.lucene; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -28,10 +29,10 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.Version; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.*; @@ -39,15 +40,12 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; /** - * + * */ public class LuceneTests extends ESTestCase { - - - /* + /** * simple test that ensures that we bump the version on Upgrade */ - @Test public void testVersion() { // note this is just a silly sanity check, we test it in lucene, and we point to it this way assertEquals(Lucene.VERSION, Version.LATEST); @@ -359,4 +357,16 @@ public class LuceneTests extends ESTestCase { w.close(); dir.close(); } + + /** + * Test that the "unmap hack" is detected as supported by lucene. + * This works around the following bug: https://bugs.openjdk.java.net/browse/JDK-4724038 + *

    + * While not guaranteed, current status is "Critical Internal API": http://openjdk.java.net/jeps/260 + * Additionally this checks we did not screw up the security logic around the hack. + */ + public void testMMapHackSupported() throws Exception { + // add assume's here if needed for certain platforms, but we should know if it does not work. + assertTrue(MMapDirectory.UNMAP_SUPPORTED); + } } diff --git a/core/src/test/java/org/elasticsearch/common/lucene/all/SimpleAllTests.java b/core/src/test/java/org/elasticsearch/common/lucene/all/SimpleAllTests.java index ccf657cf762..f4f3034528f 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/all/SimpleAllTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/all/SimpleAllTests.java @@ -28,14 +28,22 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -45,8 +53,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class SimpleAllTests extends ESTestCase { - - @Test public void testBoostOnEagerTokenizer() throws Exception { AllEntries allEntries = new AllEntries(); allEntries.addText("field1", "all", 2.0f); @@ -91,7 +97,6 @@ public class SimpleAllTests extends ESTestCase { assertFalse(ts.incrementToken()); } - @Test public void testAllEntriesRead() throws Exception { AllEntries allEntries = new AllEntries(); allEntries.addText("field1", "something", 1.0f); @@ -122,7 +127,6 @@ public class SimpleAllTests extends ESTestCase { assertEquals(scoreDoc.score, expl.getValue(), 0.00001f); } - @Test public void testSimpleAllNoBoost() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -169,7 +173,6 @@ public class SimpleAllTests extends ESTestCase { indexWriter.close(); } - @Test public void testSimpleAllWithBoost() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -217,7 +220,6 @@ public class SimpleAllTests extends ESTestCase { indexWriter.close(); } - @Test public void testMultipleTokensAllNoBoost() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -268,7 +270,6 @@ public class SimpleAllTests extends ESTestCase { indexWriter.close(); } - @Test public void testMultipleTokensAllWithBoost() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -319,7 +320,6 @@ public class SimpleAllTests extends ESTestCase { indexWriter.close(); } - @Test public void testNoTokensWithKeywordAnalyzer() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.KEYWORD_ANALYZER)); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java b/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java index 1c7a6abeaca..ad811a38aed 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java @@ -24,17 +24,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.Term; +import org.apache.lucene.index.*; import org.apache.lucene.queries.TermsQuery; -import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -43,16 +35,8 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -67,7 +51,7 @@ public class FreqTermsEnumTests extends ESTestCase { private Map referenceAll; private Map referenceNotDeleted; private Map referenceFilter; - private Filter filter; + private Query filter; static class FreqHolder { int docFreq; @@ -153,7 +137,7 @@ public class FreqTermsEnumTests extends ESTestCase { } } } - filter = new QueryWrapperFilter(new TermsQuery(filterTerms)); + filter = new TermsQuery(filterTerms); } private void addFreqs(Document doc, Map reference) { @@ -176,21 +160,18 @@ public class FreqTermsEnumTests extends ESTestCase { super.tearDown(); } - @Test public void testAllFreqs() throws Exception { assertAgainstReference(true, true, null, referenceAll); assertAgainstReference(true, false, null, referenceAll); assertAgainstReference(false, true, null, referenceAll); } - @Test public void testNonDeletedFreqs() throws Exception { assertAgainstReference(true, true, Queries.newMatchAllQuery(), referenceNotDeleted); assertAgainstReference(true, false, Queries.newMatchAllQuery(), referenceNotDeleted); assertAgainstReference(false, true, Queries.newMatchAllQuery(), referenceNotDeleted); } - @Test public void testFilterFreqs() throws Exception { assertAgainstReference(true, true, filter, referenceFilter); assertAgainstReference(true, false, filter, referenceFilter); @@ -207,7 +188,7 @@ public class FreqTermsEnumTests extends ESTestCase { for (int i = 0; i < cycles; i++) { List terms = new ArrayList<>(Arrays.asList(this.terms)); - Collections.shuffle(terms, getRandom()); + Collections.shuffle(terms, random()); for (String term : terms) { if (!termsEnum.seekExact(new BytesRef(term))) { assertThat("term : " + term, reference.get(term).docFreq, is(0)); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java index 43e151e4867..9098289847e 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java @@ -22,19 +22,23 @@ package org.elasticsearch.common.lucene.search; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.store.RAMDirectory; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class MultiPhrasePrefixQueryTests extends ESTestCase { - - @Test - public void simpleTests() throws Exception { + public void testSimple() throws Exception { IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); Document doc = new Document(); doc.add(new Field("field", "aaa bbb ccc ddd", TextField.TYPE_NOT_STORED)); @@ -63,7 +67,6 @@ public class MultiPhrasePrefixQueryTests extends ESTestCase { assertThat(searcher.count(query), equalTo(0)); } - @Test public void testBoost() throws Exception { IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); Document doc = new Document(); @@ -77,6 +80,8 @@ public class MultiPhrasePrefixQueryTests extends ESTestCase { multiPhrasePrefixQuery.add(new Term[]{new Term("field", "aaa"), new Term("field", "bb")}); multiPhrasePrefixQuery.setBoost(randomFloat()); Query query = multiPhrasePrefixQuery.rewrite(reader); - assertThat(query.getBoost(), equalTo(multiPhrasePrefixQuery.getBoost())); + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(multiPhrasePrefixQuery.getBoost())); } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java index b6c63845979..199ffaf8c03 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java @@ -26,18 +26,15 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.SearchScript; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import static org.hamcrest.Matchers.equalTo; public class ScriptScoreFunctionTests extends ESTestCase { - /** * Tests https://github.com/elasticsearch/elasticsearch/issues/2426 */ - @Test public void testScriptScoresReturnsNaN() throws IOException { ScoreFunction scoreFunction = new ScriptScoreFunction(new Script("Float.NaN"), new FloatValueScript(Float.NaN)); LeafScoreFunction leafScoreFunction = scoreFunction.getLeafScoreFunction(null); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/MoreLikeThisQueryTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/MoreLikeThisQueryTests.java index 119c595ea9b..573138c50f7 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/MoreLikeThisQueryTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/MoreLikeThisQueryTests.java @@ -32,7 +32,6 @@ import org.apache.lucene.store.RAMDirectory; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.MoreLikeThisQuery; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; @@ -40,14 +39,11 @@ import static org.hamcrest.Matchers.equalTo; * */ public class MoreLikeThisQueryTests extends ESTestCase { - - @Test public void testSimple() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); indexWriter.commit(); - Document document = new Document(); document.add(new TextField("_id", "1", Field.Store.YES)); document.add(new TextField("text", "lucene", Field.Store.YES)); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java index 29d794ad599..e1c71b0f628 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java @@ -32,7 +32,6 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -40,7 +39,6 @@ import java.util.Arrays; import java.util.List; public class XMoreLikeThisTests extends ESTestCase { - private void addDoc(RandomIndexWriter writer, String[] texts) throws IOException { Document doc = new Document(); for (String text : texts) { @@ -49,7 +47,6 @@ public class XMoreLikeThisTests extends ESTestCase { writer.addDocument(doc); } - @Test public void testTopN() throws Exception { int numDocs = 100; int topN = 25; diff --git a/core/src/test/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInputTests.java b/core/src/test/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInputTests.java index 8f4f0136299..7113a301e7f 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInputTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInputTests.java @@ -19,18 +19,15 @@ package org.elasticsearch.common.lucene.store; -import java.nio.charset.StandardCharsets; import org.apache.lucene.store.IndexInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import static org.hamcrest.Matchers.containsString; public class ByteArrayIndexInputTests extends ESTestCase { - - @Test public void testRandomReads() throws IOException { for (int i = 0; i < 100; i++) { byte[] input = randomUnicodeOfLength(randomIntBetween(1, 1000)).getBytes(StandardCharsets.UTF_8); @@ -42,7 +39,6 @@ public class ByteArrayIndexInputTests extends ESTestCase { } } - @Test public void testRandomOverflow() throws IOException { for (int i = 0; i < 100; i++) { byte[] input = randomUnicodeOfLength(randomIntBetween(1, 1000)).getBytes(StandardCharsets.UTF_8); @@ -61,7 +57,6 @@ public class ByteArrayIndexInputTests extends ESTestCase { } } - @Test public void testSeekOverflow() throws IOException { for (int i = 0; i < 100; i++) { byte[] input = randomUnicodeOfLength(randomIntBetween(1, 1000)).getBytes(StandardCharsets.UTF_8); @@ -130,7 +125,7 @@ public class ByteArrayIndexInputTests extends ESTestCase { default: fail(); } - assertEquals((long) readPos, indexInput.getFilePointer()); + assertEquals(readPos, indexInput.getFilePointer()); } return output; } diff --git a/core/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java b/core/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java index e8540d3cbaa..99acdde7af4 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java @@ -24,11 +24,9 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMDirectory; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; @@ -36,8 +34,6 @@ import static org.hamcrest.Matchers.lessThan; * */ public class InputStreamIndexInputTests extends ESTestCase { - - @Test public void testSingleReadSingleByteLimit() throws IOException { RAMDirectory dir = new RAMDirectory(); IndexOutput output = dir.createOutput("test", IOContext.DEFAULT); @@ -74,7 +70,6 @@ public class InputStreamIndexInputTests extends ESTestCase { assertThat(is.read(), equalTo(-1)); } - @Test public void testReadMultiSingleByteLimit1() throws IOException { RAMDirectory dir = new RAMDirectory(); IndexOutput output = dir.createOutput("test", IOContext.DEFAULT); @@ -113,7 +108,6 @@ public class InputStreamIndexInputTests extends ESTestCase { assertThat(is.read(read), equalTo(-1)); } - @Test public void testSingleReadTwoBytesLimit() throws IOException { RAMDirectory dir = new RAMDirectory(); IndexOutput output = dir.createOutput("test", IOContext.DEFAULT); @@ -155,7 +149,6 @@ public class InputStreamIndexInputTests extends ESTestCase { assertThat(is.read(), equalTo(-1)); } - @Test public void testReadMultiTwoBytesLimit1() throws IOException { RAMDirectory dir = new RAMDirectory(); IndexOutput output = dir.createOutput("test", IOContext.DEFAULT); @@ -199,7 +192,6 @@ public class InputStreamIndexInputTests extends ESTestCase { assertThat(is.read(read), equalTo(-1)); } - @Test public void testReadMultiFourBytesLimit() throws IOException { RAMDirectory dir = new RAMDirectory(); IndexOutput output = dir.createOutput("test", IOContext.DEFAULT); @@ -238,7 +230,6 @@ public class InputStreamIndexInputTests extends ESTestCase { assertThat(is.read(read), equalTo(-1)); } - @Test public void testMarkRest() throws Exception { RAMDirectory dir = new RAMDirectory(); IndexOutput output = dir.createOutput("test", IOContext.DEFAULT); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java index 6c2397e092e..fb3c021fd5d 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java @@ -48,7 +48,6 @@ import org.elasticsearch.index.shard.ElasticsearchMergePolicy; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.hamcrest.MatcherAssert; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -76,7 +75,7 @@ public class VersionsTests extends ESTestCase { } return newReader; } - @Test + public void testVersions() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -128,7 +127,6 @@ public class VersionsTests extends ESTestCase { dir.close(); } - @Test public void testNestedDocuments() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -169,7 +167,6 @@ public class VersionsTests extends ESTestCase { dir.close(); } - @Test public void testBackwardCompatibility() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -236,7 +233,6 @@ public class VersionsTests extends ESTestCase { } } - @Test public void testMergingOldIndices() throws Exception { final IndexWriterConfig iwConf = new IndexWriterConfig(new KeywordAnalyzer()); iwConf.setMergePolicy(new ElasticsearchMergePolicy(iwConf.getMergePolicy())); diff --git a/core/src/test/java/org/elasticsearch/common/math/MathUtilsTests.java b/core/src/test/java/org/elasticsearch/common/math/MathUtilsTests.java index 85fd28da68e..a25b7a3a780 100644 --- a/core/src/test/java/org/elasticsearch/common/math/MathUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/math/MathUtilsTests.java @@ -20,12 +20,9 @@ package org.elasticsearch.common.math; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; public class MathUtilsTests extends ESTestCase { - - @Test - public void mod() { + public void testMod() { final int iters = scaledRandomIntBetween(1000, 10000); for (int i = 0; i < iters; ++i) { final int v = rarely() ? Integer.MIN_VALUE : rarely() ? Integer.MAX_VALUE : randomInt(); @@ -35,5 +32,4 @@ public class MathUtilsTests extends ESTestCase { assertTrue(mod < m); } } - } diff --git a/core/src/test/java/org/elasticsearch/common/network/CidrsTests.java b/core/src/test/java/org/elasticsearch/common/network/CidrsTests.java new file mode 100644 index 00000000000..ef8c55ddf90 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/network/CidrsTests.java @@ -0,0 +1,192 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.network; + +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.network.Cidrs; +import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.util.*; + +import static org.hamcrest.Matchers.*; + +public class CidrsTests extends ESTestCase { + public void testNullCidr() { + try { + Cidrs.cidrMaskToMinMax(null); + fail("expected NullPointerException"); + } catch (NullPointerException e) { + assertThat(e, hasToString(containsString("cidr"))); + } + } + + public void testSplittingSlash() { + List cases = new ArrayList<>(); + cases.add("1.2.3.4"); + cases.add("1.2.3.4/32/32"); + cases.add("1.2.3.4/"); + cases.add("/"); + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected IllegalArgumentException after splitting"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("expected [a.b.c.d, e]"))); + assertThat(e, hasToString(containsString("splitting on \"/\""))); + } + } + } + + public void testSplittingDot() { + List cases = new ArrayList<>(); + cases.add("1.2.3/32"); + cases.add("1/32"); + cases.add("1./32"); + cases.add("1../32"); + cases.add("1.../32"); + cases.add("1.2.3.4.5/32"); + cases.add("/32"); + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected IllegalArgumentException after splitting"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("unable to parse"))); + assertThat(e, hasToString(containsString("as an IP address literal"))); + } + } + } + + public void testValidSpecificCases() { + List> cases = new ArrayList<>(); + cases.add(new Tuple<>("192.168.0.0/24", new long[]{(192L << 24) + (168 << 16), (192L << 24) + (168 << 16) + (1 << 8)})); + cases.add(new Tuple<>("192.168.128.0/17", new long[]{(192L << 24) + (168 << 16) + (128 << 8), (192L << 24) + (168 << 16) + (128 << 8) + (1 << 15)})); + cases.add(new Tuple<>("128.0.0.0/1", new long[]{128L << 24, (128L << 24) + (1L << 31)})); // edge case + cases.add(new Tuple<>("0.0.0.0/0", new long[]{0, 1L << 32})); // edge case + cases.add(new Tuple<>("0.0.0.0/1", new long[]{0, 1L << 31})); // edge case + cases.add(new Tuple<>( + "192.168.1.1/32", + new long[]{(192L << 24) + (168L << 16) + (1L << 8) + 1L, (192L << 24) + (168L << 16) + (1L << 8) + 1L + 1}) + ); // edge case + for (Tuple test : cases) { + long[] actual = Cidrs.cidrMaskToMinMax(test.v1()); + assertArrayEquals(test.v1(), test.v2(), actual); + } + } + + public void testInvalidSpecificOctetCases() { + List cases = new ArrayList<>(); + cases.add("256.0.0.0/8"); // first octet out of range + cases.add("255.256.0.0/16"); // second octet out of range + cases.add("255.255.256.0/24"); // third octet out of range + cases.add("255.255.255.256/32"); // fourth octet out of range + cases.add("abc.0.0.0/8"); // octet that can not be parsed + cases.add("-1.0.0.0/8"); // first octet out of range + cases.add("128.-1.0.0/16"); // second octet out of range + cases.add("128.128.-1.0/24"); // third octet out of range + cases.add("128.128.128.-1/32"); // fourth octet out of range + + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected invalid address"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("unable to parse"))); + assertThat(e, hasToString(containsString("as an IP address literal"))); + } + } + } + + public void testInvalidSpecificNetworkMaskCases() { + List cases = new ArrayList<>(); + cases.add("128.128.128.128/-1"); // network mask out of range + cases.add("128.128.128.128/33"); // network mask out of range + cases.add("128.128.128.128/abc"); // network mask that can not be parsed + + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected invalid network mask"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("network mask"))); + } + } + } + + public void testValidCombinations() { + for (long i = 0; i < (1 << 16); i++) { + for (int mask = 16; mask <= 32; mask++) { + String test = Cidrs.octetsToCIDR(Cidrs.longToOctets(i << 16), mask); + long[] actual = Cidrs.cidrMaskToMinMax(test); + assertNotNull(test, actual); + assertEquals(test, 2, actual.length); + assertEquals(test, i << 16, actual[0]); + assertEquals(test, (i << 16) + (1L << (32 - mask)), actual[1]); + } + } + } + + public void testInvalidCombinations() { + List cases = new ArrayList<>(); + cases.add("192.168.0.1/24"); // invalid because fourth octet is not zero + cases.add("192.168.1.0/16"); // invalid because third octet is not zero + cases.add("192.1.0.0/8"); // invalid because second octet is not zero + cases.add("128.0.0.0/0"); // invalid because first octet is not zero + // create cases that have a bit set outside of the network mask + int value = 1; + for (int i = 0; i < 31; i++) { + cases.add(Cidrs.octetsToCIDR(Cidrs.longToOctets(value), 32 - i - 1)); + value <<= 1; + } + + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected invalid combination"); + } catch (IllegalArgumentException e) { + assertThat(test, e, hasToString(containsString("invalid address/network mask combination"))); + } + } + } + + public void testRandomValidCombinations() { + List> cases = new ArrayList<>(); + // random number of strings with valid octets and valid network masks + for (int i = 0; i < randomIntBetween(1, 1024); i++) { + int networkMask = randomIntBetween(0, 32); + long mask = (1L << (32 - networkMask)) - 1; + long address = randomLongInIPv4Range() & ~mask; + cases.add(new Tuple<>(Cidrs.octetsToCIDR(Cidrs.longToOctets(address), networkMask), networkMask)); + } + + for (Tuple test : cases) { + long[] actual = Cidrs.cidrMaskToMinMax(test.v1()); + assertNotNull(test.v1(), actual); + assertEquals(test.v1(), 2, actual.length); + // assert the resulting block has the right size + assertEquals(test.v1(), 1L << (32 - test.v2()), actual[1] - actual[0]); + } + } + + private long randomLongInIPv4Range() { + return randomLong() & 0x00000000FFFFFFFFL; + } +} diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkServiceTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkServiceTests.java index 0a772907a8c..7ec4756d784 100644 --- a/core/src/test/java/org/elasticsearch/common/network/NetworkServiceTests.java +++ b/core/src/test/java/org/elasticsearch/common/network/NetworkServiceTests.java @@ -24,107 +24,118 @@ import org.elasticsearch.test.ESTestCase; import java.net.InetAddress; +import static org.hamcrest.Matchers.is; + /** * Tests for network service... try to keep them safe depending upon configuration * please don't actually bind to anything, just test the addresses. */ public class NetworkServiceTests extends ESTestCase { - /** - * ensure exception if we bind to multicast ipv4 address + /** + * ensure exception if we bind to multicast ipv4 address */ public void testBindMulticastV4() throws Exception { NetworkService service = new NetworkService(Settings.EMPTY); try { - service.resolveBindHostAddress("239.1.1.1"); + service.resolveBindHostAddresses(new String[] { "239.1.1.1" }); fail("should have hit exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("invalid: multicast")); } } - - /** - * ensure exception if we bind to multicast ipv6 address + /** + * ensure exception if we bind to multicast ipv6 address */ public void testBindMulticastV6() throws Exception { NetworkService service = new NetworkService(Settings.EMPTY); try { - service.resolveBindHostAddress("FF08::108"); + service.resolveBindHostAddresses(new String[] { "FF08::108" }); fail("should have hit exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("invalid: multicast")); } } - - /** - * ensure exception if we publish to multicast ipv4 address + + /** + * ensure exception if we publish to multicast ipv4 address */ public void testPublishMulticastV4() throws Exception { NetworkService service = new NetworkService(Settings.EMPTY); try { - service.resolvePublishHostAddress("239.1.1.1"); + service.resolvePublishHostAddresses(new String[] { "239.1.1.1" }); fail("should have hit exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("invalid: multicast")); } } - - /** - * ensure exception if we publish to multicast ipv6 address + + /** + * ensure exception if we publish to multicast ipv6 address */ public void testPublishMulticastV6() throws Exception { NetworkService service = new NetworkService(Settings.EMPTY); try { - service.resolvePublishHostAddress("FF08::108"); + service.resolvePublishHostAddresses(new String[] { "FF08::108" }); fail("should have hit exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("invalid: multicast")); } } - /** - * ensure specifying wildcard ipv4 address will bind to all interfaces + /** + * ensure specifying wildcard ipv4 address will bind to all interfaces */ public void testBindAnyLocalV4() throws Exception { NetworkService service = new NetworkService(Settings.EMPTY); - assertEquals(InetAddress.getByName("0.0.0.0"), service.resolveBindHostAddress("0.0.0.0")[0]); + assertEquals(InetAddress.getByName("0.0.0.0"), service.resolveBindHostAddresses(new String[] { "0.0.0.0" })[0]); } - - /** - * ensure specifying wildcard ipv6 address will bind to all interfaces + + /** + * ensure specifying wildcard ipv6 address will bind to all interfaces */ public void testBindAnyLocalV6() throws Exception { NetworkService service = new NetworkService(Settings.EMPTY); - assertEquals(InetAddress.getByName("::"), service.resolveBindHostAddress("::")[0]); + assertEquals(InetAddress.getByName("::"), service.resolveBindHostAddresses(new String[] { "::" })[0]); } - /** - * ensure specifying wildcard ipv4 address selects reasonable publish address + /** + * ensure specifying wildcard ipv4 address selects reasonable publish address */ public void testPublishAnyLocalV4() throws Exception { - InetAddress expected = null; - try { - expected = NetworkUtils.getFirstNonLoopbackAddresses()[0]; - } catch (Exception e) { - assumeNoException("test requires up-and-running non-loopback address", e); - } - NetworkService service = new NetworkService(Settings.EMPTY); - assertEquals(expected, service.resolvePublishHostAddress("0.0.0.0")); + InetAddress address = service.resolvePublishHostAddresses(new String[] { "0.0.0.0" }); + assertFalse(address.isAnyLocalAddress()); } - /** - * ensure specifying wildcard ipv6 address selects reasonable publish address + /** + * ensure specifying wildcard ipv6 address selects reasonable publish address */ public void testPublishAnyLocalV6() throws Exception { - InetAddress expected = null; - try { - expected = NetworkUtils.getFirstNonLoopbackAddresses()[0]; - } catch (Exception e) { - assumeNoException("test requires up-and-running non-loopback address", e); - } - NetworkService service = new NetworkService(Settings.EMPTY); - assertEquals(expected, service.resolvePublishHostAddress("::")); + InetAddress address = service.resolvePublishHostAddresses(new String[] { "::" }); + assertFalse(address.isAnyLocalAddress()); + } + + /** + * ensure we can bind to multiple addresses + */ + public void testBindMultipleAddresses() throws Exception { + NetworkService service = new NetworkService(Settings.EMPTY); + InetAddress[] addresses = service.resolveBindHostAddresses(new String[]{"127.0.0.1", "127.0.0.2"}); + assertThat(addresses.length, is(2)); + } + + /** + * ensure we can't bind to multiple addresses when using wildcard + */ + public void testBindMultipleAddressesWithWildcard() throws Exception { + NetworkService service = new NetworkService(Settings.EMPTY); + try { + service.resolveBindHostAddresses(new String[]{"0.0.0.0", "127.0.0.1"}); + fail("should have hit exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("is wildcard, but multiple addresses specified")); + } } } diff --git a/core/src/test/java/org/elasticsearch/common/path/PathTrieTests.java b/core/src/test/java/org/elasticsearch/common/path/PathTrieTests.java index 1309b585300..e2a6f0fa482 100644 --- a/core/src/test/java/org/elasticsearch/common/path/PathTrieTests.java +++ b/core/src/test/java/org/elasticsearch/common/path/PathTrieTests.java @@ -19,12 +19,12 @@ package org.elasticsearch.common.path; +import org.elasticsearch.rest.support.RestUtils; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -33,8 +33,15 @@ import static org.hamcrest.Matchers.nullValue; */ public class PathTrieTests extends ESTestCase { + public static final PathTrie.Decoder NO_DECODER = new PathTrie.Decoder() { + @Override + public String decode(String value) { + return value; + } + }; + public void testPath() { - PathTrie trie = new PathTrie<>(); + PathTrie trie = new PathTrie<>(NO_DECODER); trie.insert("/a/b/c", "walla"); trie.insert("a/d/g", "kuku"); trie.insert("x/b/c", "lala"); @@ -61,13 +68,13 @@ public class PathTrieTests extends ESTestCase { } public void testEmptyPath() { - PathTrie trie = new PathTrie<>(); + PathTrie trie = new PathTrie<>(NO_DECODER); trie.insert("/", "walla"); assertThat(trie.retrieve(""), equalTo("walla")); } public void testDifferentNamesOnDifferentPath() { - PathTrie trie = new PathTrie<>(); + PathTrie trie = new PathTrie<>(NO_DECODER); trie.insert("/a/{type}", "test1"); trie.insert("/b/{name}", "test2"); @@ -81,7 +88,7 @@ public class PathTrieTests extends ESTestCase { } public void testSameNameOnDifferentPath() { - PathTrie trie = new PathTrie<>(); + PathTrie trie = new PathTrie<>(NO_DECODER); trie.insert("/a/c/{name}", "test1"); trie.insert("/b/{name}", "test2"); @@ -95,7 +102,7 @@ public class PathTrieTests extends ESTestCase { } public void testPreferNonWildcardExecution() { - PathTrie trie = new PathTrie<>(); + PathTrie trie = new PathTrie<>(NO_DECODER); trie.insert("{test}", "test1"); trie.insert("b", "test2"); trie.insert("{test}/a", "test3"); @@ -111,7 +118,7 @@ public class PathTrieTests extends ESTestCase { } public void testSamePathConcreteResolution() { - PathTrie trie = new PathTrie<>(); + PathTrie trie = new PathTrie<>(NO_DECODER); trie.insert("{x}/{y}/{z}", "test1"); trie.insert("{x}/_y/{k}", "test2"); @@ -127,7 +134,7 @@ public class PathTrieTests extends ESTestCase { } public void testNamedWildcardAndLookupWithWildcard() { - PathTrie trie = new PathTrie<>(); + PathTrie trie = new PathTrie<>(NO_DECODER); trie.insert("x/{test}", "test1"); trie.insert("{test}/a", "test2"); trie.insert("/{test}", "test3"); @@ -155,24 +162,20 @@ public class PathTrieTests extends ESTestCase { assertThat(params.get("test"), equalTo("*")); } - public void testSplitPath() { - PathTrie trie = new PathTrie<>(); - assertThat(trie.splitPath("/a/"), arrayContaining("a")); - assertThat(trie.splitPath("/a/b"),arrayContaining("a", "b")); - assertThat(trie.splitPath("/a/b/c"), arrayContaining("a", "b", "c")); - assertThat(trie.splitPath("/a/b/"), arrayContaining("a", "b", "")); - assertThat(trie.splitPath("/a/b//d"), arrayContaining("a", "b", "", "d")); - - assertThat(trie.splitPath("//_search"), arrayContaining("", "_search")); - assertThat(trie.splitPath("//_search"), arrayContaining("", "_search")); - assertThat(trie.splitPath("//_search"), arrayContaining("", "_search")); - assertThat(trie.splitPath("//_search"), arrayContaining("", "_search")); - assertThat(trie.splitPath("//log/_search"), arrayContaining("", "log", "_search")); - - assertThat(trie.splitPath("/,/_search"), arrayContaining(",", "_search")); - assertThat(trie.splitPath("/,/_search"), arrayContaining(",", "_search")); - assertThat(trie.splitPath("/,/_search"), arrayContaining(",", "_search")); - assertThat(trie.splitPath("/,/_search"), arrayContaining(",", "_search")); + //https://github.com/elastic/elasticsearch/issues/14177 + //https://github.com/elastic/elasticsearch/issues/13665 + public void testEscapedSlashWithinUrl() { + PathTrie pathTrie = new PathTrie<>(RestUtils.REST_DECODER); + pathTrie.insert("/{index}/{type}/{id}", "test"); + HashMap params = new HashMap<>(); + assertThat(pathTrie.retrieve("/index/type/a%2Fe", params), equalTo("test")); + assertThat(params.get("index"), equalTo("index")); + assertThat(params.get("type"), equalTo("type")); + assertThat(params.get("id"), equalTo("a/e")); + params.clear(); + assertThat(pathTrie.retrieve("//type/id", params), equalTo("test")); + assertThat(params.get("index"), equalTo("")); + assertThat(params.get("type"), equalTo("type")); + assertThat(params.get("id"), equalTo("id")); } - } diff --git a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java index bfa08ddfdab..4d8fbc3add5 100644 --- a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java +++ b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java @@ -20,14 +20,13 @@ package org.elasticsearch.common.property; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.LinkedHashMap; import java.util.Map; -public class PropertyPlaceholderTests extends ESTestCase { +import static org.hamcrest.Matchers.is; - @Test +public class PropertyPlaceholderTests extends ESTestCase { public void testSimple() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("{", "}", false); Map map = new LinkedHashMap<>(); @@ -40,7 +39,6 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("a {foo1} b {foo2} c", placeholderResolver)); } - @Test public void testVariousPrefixSuffix() { // Test various prefix/suffix lengths PropertyPlaceholder ppEqualsPrefix = new PropertyPlaceholder("{", "}", false); @@ -54,7 +52,6 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("bar", ppShorterPrefix.replacePlaceholders("{foo}}", placeholderResolver)); } - @Test public void testDefaultValue() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); @@ -63,7 +60,6 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("", propertyPlaceholder.replacePlaceholders("${foo:}", placeholderResolver)); } - @Test public void testIgnoredUnresolvedPlaceholder() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", true); Map map = new LinkedHashMap<>(); @@ -71,15 +67,18 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver)); } - @Test(expected = IllegalArgumentException.class) public void testNotIgnoredUnresolvedPlaceholder() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + try { + propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Could not resolve placeholder 'foo'")); + } } - @Test public void testShouldIgnoreMissing() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); @@ -87,7 +86,6 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("bar", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver)); } - @Test public void testRecursive() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); @@ -99,7 +97,6 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("abarb", propertyPlaceholder.replacePlaceholders("a${foo}b", placeholderResolver)); } - @Test public void testNestedLongerPrefix() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); @@ -111,7 +108,6 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("baz", propertyPlaceholder.replacePlaceholders("${bar${foo}}", placeholderResolver)); } - @Test public void testNestedSameLengthPrefixSuffix() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("{", "}", false); Map map = new LinkedHashMap<>(); @@ -123,7 +119,6 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}", placeholderResolver)); } - @Test public void testNestedShorterPrefix() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("{", "}}", false); Map map = new LinkedHashMap<>(); @@ -135,17 +130,20 @@ public class PropertyPlaceholderTests extends ESTestCase { assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}}}", placeholderResolver)); } - @Test(expected = IllegalArgumentException.class) public void testCircularReference() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); map.put("foo", "${bar}"); map.put("bar", "${foo}"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + try { + propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Circular placeholder reference 'foo' in property definitions")); + } } - @Test public void testShouldRemoveMissing() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); diff --git a/core/src/test/java/org/elasticsearch/common/regex/RegexTests.java b/core/src/test/java/org/elasticsearch/common/regex/RegexTests.java index 18ff2c1d002..85c2be77af5 100644 --- a/core/src/test/java/org/elasticsearch/common/regex/RegexTests.java +++ b/core/src/test/java/org/elasticsearch/common/regex/RegexTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.common.regex; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Random; import java.util.regex.Pattern; @@ -27,11 +26,9 @@ import java.util.regex.Pattern; import static org.hamcrest.Matchers.equalTo; public class RegexTests extends ESTestCase { - - @Test public void testFlags() { String[] supportedFlags = new String[]{"CASE_INSENSITIVE", "MULTILINE", "DOTALL", "UNICODE_CASE", "CANON_EQ", "UNIX_LINES", - "LITERAL", "COMMENTS", "UNICODE_CHAR_CLASS"}; + "LITERAL", "COMMENTS", "UNICODE_CHAR_CLASS", "UNICODE_CHARACTER_CLASS"}; int[] flags = new int[]{Pattern.CASE_INSENSITIVE, Pattern.MULTILINE, Pattern.DOTALL, Pattern.UNICODE_CASE, Pattern.CANON_EQ, Pattern.UNIX_LINES, Pattern.LITERAL, Pattern.COMMENTS, Regex.UNICODE_CHARACTER_CLASS}; Random random = getRandom(); @@ -55,7 +52,6 @@ public class RegexTests extends ESTestCase { } } - @Test(timeout = 1000) public void testDoubleWildcardMatch() { assertTrue(Regex.simpleMatch("ddd", "ddd")); assertTrue(Regex.simpleMatch("d*d*d", "dadd")); @@ -67,5 +63,4 @@ public class RegexTests extends ESTestCase { assertTrue(Regex.simpleMatch("fff*******ddd", "fffabcddd")); assertFalse(Regex.simpleMatch("fff******ddd", "fffabcdd")); } - } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/common/rounding/RoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/RoundingTests.java index 53998c5cadf..7819e4b60ab 100644 --- a/core/src/test/java/org/elasticsearch/common/rounding/RoundingTests.java +++ b/core/src/test/java/org/elasticsearch/common/rounding/RoundingTests.java @@ -20,18 +20,15 @@ package org.elasticsearch.common.rounding; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; public class RoundingTests extends ESTestCase { - /** * simple test case to illustrate how Rounding.Interval works on readable input */ - @Test public void testInterval() { int interval = 10; Rounding.Interval rounding = new Rounding.Interval(interval); @@ -44,7 +41,6 @@ public class RoundingTests extends ESTestCase { assertEquals(message, 0, r % interval); } - @Test public void testIntervalRandom() { final long interval = randomIntBetween(1, 100); Rounding.Interval rounding = new Rounding.Interval(interval); @@ -65,7 +61,6 @@ public class RoundingTests extends ESTestCase { * offset shifts input value back before rounding (so here 6 - 7 -> -1) * then shifts rounded Value back (here -10 -> -3) */ - @Test public void testOffsetRounding() { final long interval = 10; final long offset = 7; @@ -86,7 +81,6 @@ public class RoundingTests extends ESTestCase { /** * test OffsetRounding with an internal interval rounding on random inputs */ - @Test public void testOffsetRoundingRandom() { for (int i = 0; i < 1000; ++i) { final long interval = randomIntBetween(1, 100); diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java index cc6f9cb1c11..e90691ee403 100644 --- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java +++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.ISODateTimeFormat; -import org.junit.Test; import java.util.concurrent.TimeUnit; @@ -35,10 +34,8 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; /** */ public class TimeZoneRoundingTests extends ESTestCase { - final static DateTimeZone JERUSALEM_TIMEZONE = DateTimeZone.forID("Asia/Jerusalem"); - @Test public void testUTCTimeUnitRounding() { Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).build(); assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-01T00:00:00.000Z"))); @@ -53,7 +50,6 @@ public class TimeZoneRoundingTests extends ESTestCase { assertThat(tzRounding.nextRoundingValue(utc("2012-01-08T00:00:00.000Z")), equalTo(utc("2012-01-15T00:00:00.000Z"))); } - @Test public void testUTCIntervalRounding() { Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(12)).build(); assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-03T00:00:00.000Z"))); @@ -74,7 +70,6 @@ public class TimeZoneRoundingTests extends ESTestCase { /** * test TimeIntervalTimeZoneRounding, (interval < 12h) with time zone shift */ - @Test public void testTimeIntervalTimeZoneRounding() { Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(6)).timeZone(DateTimeZone.forOffsetHours(-1)).build(); assertThat(tzRounding.round(utc("2009-02-03T00:01:01")), equalTo(utc("2009-02-02T19:00:00.000Z"))); @@ -90,7 +85,6 @@ public class TimeZoneRoundingTests extends ESTestCase { /** * test DayIntervalTimeZoneRounding, (interval >= 12h) with time zone shift */ - @Test public void testDayIntervalTimeZoneRounding() { Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(12)).timeZone(DateTimeZone.forOffsetHours(-8)).build(); assertThat(tzRounding.round(utc("2009-02-03T00:01:01")), equalTo(utc("2009-02-02T20:00:00.000Z"))); @@ -103,7 +97,6 @@ public class TimeZoneRoundingTests extends ESTestCase { assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T08:00:00.000Z")), equalTo(utc("2009-02-03T20:00:00.000Z"))); } - @Test public void testDayTimeZoneRounding() { int timezoneOffset = -2; Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(DateTimeZone.forOffsetHours(timezoneOffset)) @@ -139,7 +132,6 @@ public class TimeZoneRoundingTests extends ESTestCase { assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T02:00:00")), equalTo(utc("2009-02-04T02:00:00"))); } - @Test public void testTimeTimeZoneRounding() { // hour unit Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forOffsetHours(-2)).build(); @@ -151,7 +143,6 @@ public class TimeZoneRoundingTests extends ESTestCase { assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T01:00:00")), equalTo(utc("2009-02-03T02:00:00"))); } - @Test public void testTimeUnitRoundingDST() { Rounding tzRounding; // testing savings to non savings switch @@ -203,7 +194,6 @@ public class TimeZoneRoundingTests extends ESTestCase { /** * randomized test on TimeUnitRounding with random time units and time zone offsets */ - @Test public void testTimeZoneRoundingRandom() { for (int i = 0; i < 1000; ++i) { DateTimeUnit timeUnit = randomTimeUnit(); @@ -223,7 +213,6 @@ public class TimeZoneRoundingTests extends ESTestCase { /** * randomized test on TimeIntervalRounding with random interval and time zone offsets */ - @Test public void testIntervalRoundingRandom() { for (int i = 0; i < 1000; ++i) { // max random interval is a year, can be negative @@ -245,7 +234,6 @@ public class TimeZoneRoundingTests extends ESTestCase { /** * special test for DST switch from #9491 */ - @Test public void testAmbiguousHoursAfterDSTSwitch() { Rounding tzRounding; tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(JERUSALEM_TIMEZONE).build(); @@ -284,7 +272,6 @@ public class TimeZoneRoundingTests extends ESTestCase { * test for #10025, strict local to UTC conversion can cause joda exceptions * on DST start */ - @Test public void testLenientConversionDST() { DateTimeZone tz = DateTimeZone.forID("America/Sao_Paulo"); long start = time("2014-10-18T20:50:00.000", tz); diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java index a00c9af037d..5eb8ef8ca04 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java @@ -20,18 +20,15 @@ package org.elasticsearch.common.settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.rest.RestRequest; -import org.junit.Test; import java.io.IOException; import static org.hamcrest.CoreMatchers.equalTo; public class SettingsFilterTests extends ESTestCase { - - @Test public void testAddingAndRemovingFilters() { SettingsFilter settingsFilter = new SettingsFilter(Settings.EMPTY); settingsFilter.addFilter("foo"); @@ -49,7 +46,6 @@ public class SettingsFilterTests extends ESTestCase { assertThat(settingsFilter.getPatterns(), equalTo("")); } - @Test public void testSettingsFiltering() throws IOException { testFiltering(Settings.builder() diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java index a17bb4fd9d9..e3f2bc1bb2b 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java @@ -19,12 +19,9 @@ package org.elasticsearch.common.settings; -import org.elasticsearch.common.settings.bar.BarTestClass; -import org.elasticsearch.common.settings.foo.FooTestClass; import org.elasticsearch.common.settings.loader.YamlSettingsLoader; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.List; @@ -32,13 +29,17 @@ import java.util.Map; import java.util.Set; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** */ public class SettingsTests extends ESTestCase { - - @Test public void testCamelCaseSupport() { Settings settings = settingsBuilder() .put("test.camelCase", "bar") @@ -47,7 +48,6 @@ public class SettingsTests extends ESTestCase { assertThat(settings.get("test.camel_case"), equalTo("bar")); } - @Test public void testLoadFromDelimitedString() { Settings settings = settingsBuilder() .loadFromDelimitedString("key1=value1;key2=value2", ';') @@ -66,33 +66,29 @@ public class SettingsTests extends ESTestCase { assertThat(settings.toDelimitedString(';'), equalTo("key1=value1;key2=value2;")); } - @Test public void testReplacePropertiesPlaceholderSystemProperty() { - System.setProperty("sysProp1", "sysVal1"); - try { - Settings settings = settingsBuilder() - .put("setting1", "${sysProp1}") - .replacePropertyPlaceholders() - .build(); - assertThat(settings.get("setting1"), equalTo("sysVal1")); - } finally { - System.clearProperty("sysProp1"); - } - + String value = System.getProperty("java.home"); + assertFalse(value.isEmpty()); Settings settings = settingsBuilder() - .put("setting1", "${sysProp1:defaultVal1}") + .put("setting1", "${java.home}") + .replacePropertyPlaceholders() + .build(); + assertThat(settings.get("setting1"), equalTo(value)); + + assertNull(System.getProperty("_test_property_should_not_exist")); + settings = settingsBuilder() + .put("setting1", "${_test_property_should_not_exist:defaultVal1}") .replacePropertyPlaceholders() .build(); assertThat(settings.get("setting1"), equalTo("defaultVal1")); settings = settingsBuilder() - .put("setting1", "${sysProp1:}") + .put("setting1", "${_test_property_should_not_exist:}") .replacePropertyPlaceholders() .build(); assertThat(settings.get("setting1"), is(nullValue())); } - @Test public void testReplacePropertiesPlaceholderIgnoreEnvUnset() { Settings settings = settingsBuilder() .put("setting1", "${env.UNSET_ENV_VAR}") @@ -101,7 +97,6 @@ public class SettingsTests extends ESTestCase { assertThat(settings.get("setting1"), is(nullValue())); } - @Test public void testReplacePropertiesPlaceholderIgnoresPrompt() { Settings settings = settingsBuilder() .put("setting1", "${prompt.text}") @@ -112,7 +107,6 @@ public class SettingsTests extends ESTestCase { assertThat(settings.get("setting2"), is("${prompt.secret}")); } - @Test public void testUnFlattenedSettings() { Settings settings = settingsBuilder() .put("foo", "abc") @@ -137,7 +131,6 @@ public class SettingsTests extends ESTestCase { } - @Test public void testFallbackToFlattenedSettings() { Settings settings = settingsBuilder() .put("foo", "abc") @@ -163,7 +156,6 @@ public class SettingsTests extends ESTestCase { Matchers.hasEntry("foo.baz", "ghi"))); } - @Test public void testGetAsSettings() { Settings settings = settingsBuilder() .put("foo", "abc") @@ -175,7 +167,6 @@ public class SettingsTests extends ESTestCase { assertThat(fooSettings.get("baz"), equalTo("ghi")); } - @Test public void testNames() { Settings settings = settingsBuilder() .put("bar", "baz") @@ -195,7 +186,6 @@ public class SettingsTests extends ESTestCase { assertTrue(names.contains("baz")); } - @Test public void testThatArraysAreOverriddenCorrectly() throws IOException { // overriding a single value with an array Settings settings = settingsBuilder() @@ -300,9 +290,7 @@ public class SettingsTests extends ESTestCase { assertThat(settings.get("value"), is(nullValue())); } - @Test public void testPrefixNormalization() { - Settings settings = settingsBuilder().normalizePrefix("foo.").build(); assertThat(settings.names().size(), equalTo(0)); @@ -337,6 +325,4 @@ public class SettingsTests extends ESTestCase { assertThat(settings.getAsMap().size(), equalTo(1)); assertThat(settings.get("foo.test"), equalTo("test")); } - - } diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java index 0f90b8c3728..18591d9a592 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; @@ -32,7 +31,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class JsonSettingsLoaderTests extends ESTestCase { - @Test public void testSimpleJsonSettings() throws Exception { String json = "/org/elasticsearch/common/settings/loader/test-settings.json"; Settings settings = settingsBuilder() diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java index 60bf80a6e9d..48703044ecd 100644 --- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java @@ -23,16 +23,15 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; /** * */ public class YamlSettingsLoaderTests extends ESTestCase { - @Test public void testSimpleYamlSettings() throws Exception { String yaml = "/org/elasticsearch/common/settings/loader/test-settings.yml"; Settings settings = settingsBuilder() @@ -51,20 +50,28 @@ public class YamlSettingsLoaderTests extends ESTestCase { assertThat(settings.getAsArray("test1.test3")[1], equalTo("test3-2")); } - @Test(expected = SettingsException.class) public void testIndentation() { String yaml = "/org/elasticsearch/common/settings/loader/indentation-settings.yml"; - settingsBuilder() - .loadFromStream(yaml, getClass().getResourceAsStream(yaml)) - .build(); - } - - @Test(expected = SettingsException.class) - public void testIndentationWithExplicitDocumentStart() { - String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml"; - settingsBuilder() + try { + settingsBuilder() .loadFromStream(yaml, getClass().getResourceAsStream(yaml)) .build(); + fail("Expected SettingsException"); + } catch(SettingsException e ) { + assertThat(e.getMessage(), containsString("Failed to load settings")); + } + } + + public void testIndentationWithExplicitDocumentStart() { + String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml"; + try { + settingsBuilder() + .loadFromStream(yaml, getClass().getResourceAsStream(yaml)) + .build(); + fail("Expected SettingsException"); + } catch (SettingsException e) { + assertThat(e.getMessage(), containsString("Failed to load settings")); + } } public void testDuplicateKeysThrowsException() { diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java index 5e8a55a3471..70ea1d19cbb 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeUnitTests.java @@ -20,17 +20,19 @@ package org.elasticsearch.common.unit; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.elasticsearch.common.unit.ByteSizeUnit.*; +import static org.elasticsearch.common.unit.ByteSizeUnit.BYTES; +import static org.elasticsearch.common.unit.ByteSizeUnit.GB; +import static org.elasticsearch.common.unit.ByteSizeUnit.KB; +import static org.elasticsearch.common.unit.ByteSizeUnit.MB; +import static org.elasticsearch.common.unit.ByteSizeUnit.PB; +import static org.elasticsearch.common.unit.ByteSizeUnit.TB; import static org.hamcrest.Matchers.equalTo; /** * */ public class ByteSizeUnitTests extends ESTestCase { - - @Test public void testBytes() { assertThat(BYTES.toBytes(1), equalTo(1l)); assertThat(BYTES.toKB(1024), equalTo(1l)); @@ -38,7 +40,6 @@ public class ByteSizeUnitTests extends ESTestCase { assertThat(BYTES.toGB(1024 * 1024 * 1024), equalTo(1l)); } - @Test public void testKB() { assertThat(KB.toBytes(1), equalTo(1024l)); assertThat(KB.toKB(1), equalTo(1l)); @@ -46,7 +47,6 @@ public class ByteSizeUnitTests extends ESTestCase { assertThat(KB.toGB(1024 * 1024), equalTo(1l)); } - @Test public void testMB() { assertThat(MB.toBytes(1), equalTo(1024l * 1024)); assertThat(MB.toKB(1), equalTo(1024l)); @@ -54,7 +54,6 @@ public class ByteSizeUnitTests extends ESTestCase { assertThat(MB.toGB(1024), equalTo(1l)); } - @Test public void testGB() { assertThat(GB.toBytes(1), equalTo(1024l * 1024 * 1024)); assertThat(GB.toKB(1), equalTo(1024l * 1024)); @@ -62,7 +61,6 @@ public class ByteSizeUnitTests extends ESTestCase { assertThat(GB.toGB(1), equalTo(1l)); } - @Test public void testTB() { assertThat(TB.toBytes(1), equalTo(1024l * 1024 * 1024 * 1024)); assertThat(TB.toKB(1), equalTo(1024l * 1024 * 1024)); @@ -71,7 +69,6 @@ public class ByteSizeUnitTests extends ESTestCase { assertThat(TB.toTB(1), equalTo(1l)); } - @Test public void testPB() { assertThat(PB.toBytes(1), equalTo(1024l * 1024 * 1024 * 1024 * 1024)); assertThat(PB.toKB(1), equalTo(1024l * 1024 * 1024 * 1024)); diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java index 200d04aa8bd..56e61798325 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java @@ -22,8 +22,8 @@ package org.elasticsearch.common.unit; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.test.ESTestCase; import org.hamcrest.MatcherAssert; -import org.junit.Test; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -31,23 +31,18 @@ import static org.hamcrest.Matchers.is; * */ public class ByteSizeValueTests extends ESTestCase { - - @Test public void testActualPeta() { MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.PB).bytes(), equalTo(4503599627370496l)); } - @Test public void testActualTera() { MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.TB).bytes(), equalTo(4398046511104l)); } - @Test public void testActual() { MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.GB).bytes(), equalTo(4294967296l)); } - @Test public void testSimple() { assertThat(ByteSizeUnit.BYTES.toBytes(10), is(new ByteSizeValue(10, ByteSizeUnit.BYTES).bytes())); assertThat(ByteSizeUnit.KB.toKB(10), is(new ByteSizeValue(10, ByteSizeUnit.KB).kb())); @@ -64,7 +59,6 @@ public class ByteSizeValueTests extends ESTestCase { assertThat(value1, equalTo(value2)); } - @Test public void testToString() { assertThat("10b", is(new ByteSizeValue(10, ByteSizeUnit.BYTES).toString())); assertThat("1.5kb", is(new ByteSizeValue((long) (1024 * 1.5), ByteSizeUnit.BYTES).toString())); @@ -75,7 +69,6 @@ public class ByteSizeValueTests extends ESTestCase { assertThat("1536pb", is(new ByteSizeValue((long) (1024 * 1.5), ByteSizeUnit.PB).toString())); } - @Test public void testParsing() { assertThat(ByteSizeValue.parseBytesSizeValue("42PB", "testParsing").toString(), is("42pb")); assertThat(ByteSizeValue.parseBytesSizeValue("42 PB", "testParsing").toString(), is("42pb")); @@ -128,28 +121,48 @@ public class ByteSizeValueTests extends ESTestCase { assertThat(ByteSizeValue.parseBytesSizeValue("1 b", "testParsing").toString(), is("1b")); } - @Test(expected = ElasticsearchParseException.class) public void testFailOnMissingUnits() { - ByteSizeValue.parseBytesSizeValue("23", "test"); + try { + ByteSizeValue.parseBytesSizeValue("23", "test"); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("failed to parse setting [test]")); + } } - @Test(expected = ElasticsearchParseException.class) public void testFailOnUnknownUnits() { - ByteSizeValue.parseBytesSizeValue("23jw", "test"); + try { + ByteSizeValue.parseBytesSizeValue("23jw", "test"); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("failed to parse setting [test]")); + } } - @Test(expected = ElasticsearchParseException.class) public void testFailOnEmptyParsing() { - assertThat(ByteSizeValue.parseBytesSizeValue("", "emptyParsing").toString(), is("23kb")); + try { + assertThat(ByteSizeValue.parseBytesSizeValue("", "emptyParsing").toString(), is("23kb")); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("failed to parse setting [emptyParsing]")); + } } - @Test(expected = ElasticsearchParseException.class) public void testFailOnEmptyNumberParsing() { - assertThat(ByteSizeValue.parseBytesSizeValue("g", "emptyNumberParsing").toString(), is("23b")); + try { + assertThat(ByteSizeValue.parseBytesSizeValue("g", "emptyNumberParsing").toString(), is("23b")); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("failed to parse [g]")); + } } - @Test(expected = ElasticsearchParseException.class) public void testNoDotsAllowed() { - ByteSizeValue.parseBytesSizeValue("42b.", null, "test"); + try { + ByteSizeValue.parseBytesSizeValue("42b.", null, "test"); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("failed to parse setting [test]")); + } } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java index e29cd1e6862..25c3a136271 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.common.unit; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; @@ -29,8 +28,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class DistanceUnitTests extends ESTestCase { - - @Test public void testSimpleDistanceUnit() { assertThat(DistanceUnit.KILOMETERS.convert(10, DistanceUnit.MILES), closeTo(16.09344, 0.001)); assertThat(DistanceUnit.MILES.convert(10, DistanceUnit.MILES), closeTo(10, 0.001)); @@ -41,8 +38,7 @@ public class DistanceUnitTests extends ESTestCase { assertThat(DistanceUnit.KILOMETERS.convert(1000,DistanceUnit.METERS), closeTo(1, 0.001)); assertThat(DistanceUnit.METERS.convert(1, DistanceUnit.KILOMETERS), closeTo(1000, 0.001)); } - - @Test + public void testDistanceUnitParsing() { assertThat(DistanceUnit.Distance.parseDistance("50km").unit, equalTo(DistanceUnit.KILOMETERS)); assertThat(DistanceUnit.Distance.parseDistance("500m").unit, equalTo(DistanceUnit.METERS)); @@ -53,7 +49,7 @@ public class DistanceUnitTests extends ESTestCase { assertThat(DistanceUnit.Distance.parseDistance("12in").unit, equalTo(DistanceUnit.INCH)); assertThat(DistanceUnit.Distance.parseDistance("23mm").unit, equalTo(DistanceUnit.MILLIMETERS)); assertThat(DistanceUnit.Distance.parseDistance("23cm").unit, equalTo(DistanceUnit.CENTIMETERS)); - + double testValue = 12345.678; for (DistanceUnit unit : DistanceUnit.values()) { assertThat("Unit can be parsed from '" + unit.toString() + "'", DistanceUnit.fromString(unit.toString()), equalTo(unit)); @@ -62,4 +58,19 @@ public class DistanceUnitTests extends ESTestCase { } } + /** + * This test ensures that we are aware of accidental reordering in the distance unit ordinals, + * since equality in e.g. CircleShapeBuilder, hashCode and serialization rely on them + */ + public void testDistanceUnitNames() { + assertEquals(0, DistanceUnit.INCH.ordinal()); + assertEquals(1, DistanceUnit.YARD.ordinal()); + assertEquals(2, DistanceUnit.FEET.ordinal()); + assertEquals(3, DistanceUnit.KILOMETERS.ordinal()); + assertEquals(4, DistanceUnit.NAUTICALMILES.ordinal()); + assertEquals(5, DistanceUnit.MILLIMETERS.ordinal()); + assertEquals(6, DistanceUnit.CENTIMETERS.ordinal()); + assertEquals(7, DistanceUnit.MILES.ordinal()); + assertEquals(8, DistanceUnit.METERS.ordinal()); + } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java index 807b4a72bf0..4c64e04ec34 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java @@ -24,17 +24,16 @@ import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.number.IsCloseTo.closeTo; public class FuzzinessTests extends ESTestCase { - - @Test public void testNumerics() { String[] options = new String[]{"1.0", "1", "1.000000"}; assertThat(Fuzziness.build(randomFrom(options)).asByte(), equalTo((byte) 1)); @@ -45,7 +44,6 @@ public class FuzzinessTests extends ESTestCase { assertThat(Fuzziness.build(randomFrom(options)).asShort(), equalTo((short) 1)); } - @Test public void testParseFromXContent() throws IOException { final int iters = randomIntBetween(10, 50); for (int i = 0; i < iters; i++) { @@ -61,7 +59,7 @@ public class FuzzinessTests extends ESTestCase { assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER)); Fuzziness parse = Fuzziness.parse(parser); assertThat(parse.asFloat(), equalTo(floatValue)); - assertThat(parse.asDouble(), closeTo((double) floatValue, 0.000001)); + assertThat(parse.asDouble(), closeTo(floatValue, 0.000001)); assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); } { @@ -140,10 +138,7 @@ public class FuzzinessTests extends ESTestCase { } - @Test public void testAuto() { - final int codePoints = randomIntBetween(0, 10); - String string = randomRealisticUnicodeOfCodepointLength(codePoints); assertThat(Fuzziness.AUTO.asByte(), equalTo((byte) 1)); assertThat(Fuzziness.AUTO.asInt(), equalTo(1)); assertThat(Fuzziness.AUTO.asFloat(), equalTo(1f)); @@ -154,7 +149,6 @@ public class FuzzinessTests extends ESTestCase { } - @Test public void testAsDistance() { final int iters = randomIntBetween(10, 50); for (int i = 0; i < iters; i++) { @@ -164,7 +158,6 @@ public class FuzzinessTests extends ESTestCase { } } - @Test public void testSerialization() throws IOException { Fuzziness fuzziness = Fuzziness.AUTO; Fuzziness deserializedFuzziness = doSerializeRoundtrip(fuzziness); @@ -175,7 +168,6 @@ public class FuzzinessTests extends ESTestCase { assertEquals(fuzziness, deserializedFuzziness); } - @Test public void testSerializationAuto() throws IOException { Fuzziness fuzziness = Fuzziness.AUTO; Fuzziness deserializedFuzziness = doSerializeRoundtrip(fuzziness); diff --git a/core/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java index b9ac8e52875..e918a579df0 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.unit; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.is; @@ -29,8 +28,6 @@ import static org.hamcrest.Matchers.is; * Tests for the {@link RatioValue} class */ public class RatioValueTests extends ESTestCase { - - @Test public void testParsing() { assertThat(RatioValue.parseRatioValue("100%").toString(), is("100.0%")); assertThat(RatioValue.parseRatioValue("0%").toString(), is("0.0%")); @@ -46,7 +43,6 @@ public class RatioValueTests extends ESTestCase { assertThat(RatioValue.parseRatioValue("0.001").toString(), is("0.1%")); } - @Test public void testNegativeCase() { testInvalidRatio("100.0001%"); testInvalidRatio("-0.1%"); diff --git a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java index c1595acf581..f2f85e0c7f5 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java @@ -19,16 +19,14 @@ package org.elasticsearch.common.unit; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; /** * */ public class SizeValueTests extends ESTestCase { - - @Test public void testThatConversionWorks() { SizeValue sizeValue = new SizeValue(1000); assertThat(sizeValue.kilo(), is(1l)); @@ -55,15 +53,18 @@ public class SizeValueTests extends ESTestCase { assertThat(sizeValue.toString(), is("1000p")); } - @Test public void testThatParsingWorks() { assertThat(SizeValue.parseSizeValue("1k").toString(), is(new SizeValue(1000).toString())); assertThat(SizeValue.parseSizeValue("1p").toString(), is(new SizeValue(1, SizeUnit.PETA).toString())); assertThat(SizeValue.parseSizeValue("1G").toString(), is(new SizeValue(1, SizeUnit.GIGA).toString())); } - @Test(expected = IllegalArgumentException.class) public void testThatNegativeValuesThrowException() { - new SizeValue(-1); + try { + new SizeValue(-1); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("may not be negative")); + } } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java index 19738bc28dd..ec0e26091df 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java @@ -24,11 +24,11 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; import org.joda.time.PeriodType; -import org.junit.Test; import java.io.IOException; import java.util.concurrent.TimeUnit; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; @@ -136,18 +136,30 @@ public class TimeValueTests extends ESTestCase { assertEqualityAfterSerialize(new TimeValue(1, TimeUnit.NANOSECONDS)); } - @Test(expected = ElasticsearchParseException.class) public void testFailOnUnknownUnits() { - TimeValue.parseTimeValue("23tw", null, "test"); + try { + TimeValue.parseTimeValue("23tw", null, "test"); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("Failed to parse")); + } } - @Test(expected = ElasticsearchParseException.class) public void testFailOnMissingUnits() { - TimeValue.parseTimeValue("42", null, "test"); + try { + TimeValue.parseTimeValue("42", null, "test"); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("Failed to parse")); + } } - @Test(expected = ElasticsearchParseException.class) public void testNoDotsAllowed() { - TimeValue.parseTimeValue("42ms.", null, "test"); + try { + TimeValue.parseTimeValue("42ms.", null, "test"); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("Failed to parse")); + } } } diff --git a/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java b/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java index 735fda16797..172a064b698 100644 --- a/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java @@ -19,28 +19,20 @@ package org.elasticsearch.common.util; -import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; /** * */ public class ArrayUtilsTests extends ESTestCase { - - @Test - public void binarySearch() throws Exception { - + public void testBinarySearch() throws Exception { for (int j = 0; j < 100; j++) { - int index = Math.min(randomInt(0, 10), 9); double tolerance = Math.random() * 0.01; double lookForValue = randomFreq(0.9) ? -1 : Double.NaN; // sometimes we'll look for NaN @@ -110,5 +102,4 @@ public class ArrayUtilsTests extends ESTestCase { } assertArrayEquals(sourceOfTruth.toArray(new String[0]), ArrayUtils.concat(first, second)); } - } diff --git a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java index 91c74762dff..df26f2d55b8 100644 --- a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java @@ -19,17 +19,21 @@ package org.elasticsearch.common.util; -import com.carrotsearch.hppc.ObjectLongMap; import com.carrotsearch.hppc.ObjectLongHashMap; +import com.carrotsearch.hppc.ObjectLongMap; import com.carrotsearch.hppc.cursors.ObjectLongCursor; + import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.TestUtil; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; import java.util.Map.Entry; +import java.util.Set; public class BytesRefHashTests extends ESSingleNodeTestCase { @@ -92,7 +96,6 @@ public class BytesRefHashTests extends ESSingleNodeTestCase { /** * Test method for {@link org.apache.lucene.util.BytesRefHash#size()}. */ - @Test public void testSize() { BytesRefBuilder ref = new BytesRefBuilder(); int num = scaledRandomIntBetween(2, 20); @@ -123,7 +126,6 @@ public class BytesRefHashTests extends ESSingleNodeTestCase { * {@link org.apache.lucene.util.BytesRefHash#get(int, BytesRef)} * . */ - @Test public void testGet() { BytesRefBuilder ref = new BytesRefBuilder(); BytesRef scratch = new BytesRef(); @@ -163,7 +165,6 @@ public class BytesRefHashTests extends ESSingleNodeTestCase { * {@link org.apache.lucene.util.BytesRefHash#add(org.apache.lucene.util.BytesRef)} * . */ - @Test public void testAdd() { BytesRefBuilder ref = new BytesRefBuilder(); BytesRef scratch = new BytesRef(); @@ -199,7 +200,6 @@ public class BytesRefHashTests extends ESSingleNodeTestCase { hash.close(); } - @Test public void testFind() throws Exception { BytesRefBuilder ref = new BytesRefBuilder(); BytesRef scratch = new BytesRef(); diff --git a/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java b/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java index 2bdaea34d1e..5c6a93254aa 100644 --- a/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java @@ -21,14 +21,11 @@ package org.elasticsearch.common.util; import org.elasticsearch.common.util.CancellableThreads.Interruptable; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.concurrent.CountDownLatch; public class CancellableThreadsTests extends ESTestCase { - public static class CustomException extends RuntimeException { - public CustomException(String msg) { super(msg); } @@ -53,7 +50,6 @@ public class CancellableThreadsTests extends ESTestCase { } - @Test public void testCancellableThreads() throws InterruptedException { Thread[] threads = new Thread[randomIntBetween(3, 10)]; final TestPlan[] plans = new TestPlan[threads.length]; diff --git a/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java b/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java index 73611c80be6..4c3612da8e0 100644 --- a/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java @@ -24,7 +24,6 @@ import org.apache.lucene.util.BytesRefArray; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.Counter; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.*; @@ -33,14 +32,11 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class CollectionUtilsTests extends ESTestCase { - - @Test - public void rotateEmpty() { + public void testRotateEmpty() { assertTrue(CollectionUtils.rotate(Collections.emptyList(), randomInt()).isEmpty()); } - @Test - public void rotate() { + public void testRotate() { final int iters = scaledRandomIntBetween(10, 100); for (int k = 0; k < iters; ++k) { final int size = randomIntBetween(1, 100); @@ -65,7 +61,6 @@ public class CollectionUtilsTests extends ESTestCase { } } - @Test public void testSortAndDedupByteRefArray() { SortedSet set = new TreeSet<>(); final int numValues = scaledRandomIntBetween(0, 10000); @@ -78,7 +73,7 @@ public class CollectionUtilsTests extends ESTestCase { array.append(new BytesRef(s)); } if (randomBoolean()) { - Collections.shuffle(tmpList, getRandom()); + Collections.shuffle(tmpList, random()); for (BytesRef ref : tmpList) { array.append(ref); } @@ -99,7 +94,6 @@ public class CollectionUtilsTests extends ESTestCase { } - @Test public void testSortByteRefArray() { List values = new ArrayList<>(); final int numValues = scaledRandomIntBetween(0, 10000); @@ -110,7 +104,7 @@ public class CollectionUtilsTests extends ESTestCase { array.append(new BytesRef(s)); } if (randomBoolean()) { - Collections.shuffle(values, getRandom()); + Collections.shuffle(values, random()); } int[] indices = new int[array.size()]; for (int i = 0; i < indices.length; i++) { diff --git a/core/src/test/java/org/elasticsearch/common/util/ExtensionPointTests.java b/core/src/test/java/org/elasticsearch/common/util/ExtensionPointTests.java new file mode 100644 index 00000000000..25564deb07e --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/util/ExtensionPointTests.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.util; + +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.common.inject.*; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +public class ExtensionPointTests extends ESTestCase { + + public void testClassSet() { + final ExtensionPoint.ClassSet allocationDeciders = new ExtensionPoint.ClassSet<>("allocation_decider", AllocationDecider.class, AllocationDeciders.class); + allocationDeciders.registerExtension(TestAllocationDecider.class); + Injector injector = new ModulesBuilder().add(new Module() { + @Override + public void configure(Binder binder) { + binder.bind(Settings.class).toInstance(Settings.EMPTY); + binder.bind(Consumer.class).asEagerSingleton(); + allocationDeciders.bind(binder); + } + }).createInjector(); + assertEquals(1, TestAllocationDecider.instances.get()); + + } + + public static class Consumer { + @Inject + public Consumer(Set deciders, TestAllocationDecider other) { + // we require the TestAllocationDecider more than once to ensure it's bound as a singleton + } + } + + public static class TestAllocationDecider extends AllocationDecider { + static final AtomicInteger instances = new AtomicInteger(0); + + @Inject + public TestAllocationDecider(Settings settings) { + super(settings); + instances.incrementAndGet(); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java b/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java index 09221b5151f..aa21f323185 100644 --- a/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java @@ -22,13 +22,16 @@ package org.elasticsearch.common.util; import com.carrotsearch.hppc.LongLongHashMap; import com.carrotsearch.hppc.LongLongMap; import com.carrotsearch.hppc.cursors.LongLongCursor; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; -import java.util.*; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; public class LongHashTests extends ESSingleNodeTestCase { - LongHash hash; private void newHash() { @@ -86,7 +89,6 @@ public class LongHashTests extends ESSingleNodeTestCase { hash.close(); } - @Test public void testSize() { int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { @@ -106,7 +108,6 @@ public class LongHashTests extends ESSingleNodeTestCase { hash.close(); } - @Test public void testKey() { int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { @@ -138,7 +139,6 @@ public class LongHashTests extends ESSingleNodeTestCase { hash.close(); } - @Test public void testAdd() { int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { @@ -167,7 +167,6 @@ public class LongHashTests extends ESSingleNodeTestCase { hash.close(); } - @Test public void testFind() throws Exception { int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { @@ -206,5 +205,4 @@ public class LongHashTests extends ESSingleNodeTestCase { assertTrue("key: " + key + " count: " + count + " long: " + l, key < count); } } - } diff --git a/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java b/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java index 2036535a610..35fa7bec058 100644 --- a/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java @@ -20,13 +20,11 @@ package org.elasticsearch.common.util; import com.carrotsearch.hppc.LongObjectHashMap; + import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; public class LongObjectHashMapTests extends ESSingleNodeTestCase { - - @Test - public void duel() { + public void testDuel() { final LongObjectHashMap map1 = new LongObjectHashMap<>(); final LongObjectPagedHashMap map2 = new LongObjectPagedHashMap<>(randomInt(42), 0.6f + randomFloat() * 0.39f, BigArraysTests.randombigArrays()); final int maxKey = randomIntBetween(1, 10000); diff --git a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java index 1a4deb42556..94f154d4e5d 100644 --- a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.elasticsearch.bwcompat.OldIndexBackwardsCompatibilityIT; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.FileSystemUtils; @@ -88,7 +89,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { } } ++metaStateVersion; - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid), metaStateVersion, shardDataPaths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid, AllocationId.newInitializing()), metaStateVersion, shardDataPaths); } final Path path = randomFrom(shardDataPaths); ShardPath targetPath = new ShardPath(false, path, path, uuid, new ShardId("foo", 0)); @@ -199,7 +200,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) { String uuid = Strings.randomBase64UUID(); final ShardId shardId = new ShardId("foo", 0); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid), 1, nodeEnvironment.availableShardPaths(shardId)); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid, AllocationId.newInitializing()), 1, nodeEnvironment.availableShardPaths(shardId)); MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment); boolean multiDataPaths = nodeEnvironment.nodeDataPaths().length > 1; boolean needsUpgrading = helper.needsUpgrading(shardId); @@ -267,7 +268,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { } }; String uuid = Strings.randomBase64UUID(); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid), 1, paths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid, AllocationId.newInitializing()), 1, paths); final ShardPath shardPath = helper.pickShardPath(new ShardId("foo", 0)); assertEquals(expectedPath, shardPath.getDataPath()); assertEquals(expectedPath, shardPath.getShardStatePath()); diff --git a/core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java b/core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java index 4923a3ea979..80e2524f298 100644 --- a/core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java @@ -19,13 +19,10 @@ package org.elasticsearch.common.util; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.net.URI; public class URIPatternTests extends ESTestCase { - - @Test public void testURIPattern() throws Exception { assertTrue(new URIPattern("http://test.local/").match(new URI("http://test.local/"))); assertFalse(new URIPattern("http://test.local/somepath").match(new URI("http://test.local/"))); diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/CountDownTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/CountDownTests.java index 6a45bfbd3d2..1a32064fe7d 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/CountDownTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/CountDownTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.util.concurrent; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -31,8 +30,6 @@ import static org.hamcrest.Matchers.greaterThan; public class CountDownTests extends ESTestCase { - - @Test public void testConcurrent() throws InterruptedException { final AtomicInteger count = new AtomicInteger(0); final CountDown countDown = new CountDown(scaledRandomIntBetween(10, 1000)); @@ -46,7 +43,7 @@ public class CountDownTests extends ESTestCase { try { latch.await(); } catch (InterruptedException e) { - throw new RuntimeException(); + throw new RuntimeException(e); } while (true) { if(frequently()) { @@ -80,8 +77,7 @@ public class CountDownTests extends ESTestCase { assertThat(countDown.isCountedDown(), equalTo(true)); assertThat(count.get(), Matchers.equalTo(1)); } - - @Test + public void testSingleThreaded() { int atLeast = scaledRandomIntBetween(10, 1000); final CountDown countDown = new CountDown(atLeast); @@ -100,6 +96,5 @@ public class CountDownTests extends ESTestCase { } assertThat(atLeast, greaterThan(0)); } - } } diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java index 520b619caaa..deac15b50d3 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java @@ -22,19 +22,12 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.PriorityBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.Matchers.equalTo; @@ -44,12 +37,10 @@ import static org.hamcrest.Matchers.is; * */ public class PrioritizedExecutorsTests extends ESTestCase { - - @Test public void testPriorityQueue() throws Exception { PriorityBlockingQueue queue = new PriorityBlockingQueue<>(); List priorities = Arrays.asList(Priority.values()); - Collections.shuffle(priorities); + Collections.shuffle(priorities, random()); for (Priority priority : priorities) { queue.add(priority); @@ -65,7 +56,6 @@ public class PrioritizedExecutorsTests extends ESTestCase { } } - @Test public void testSubmitPrioritizedExecutorWithRunnables() throws Exception { ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName())); List results = new ArrayList<>(8); @@ -95,7 +85,6 @@ public class PrioritizedExecutorsTests extends ESTestCase { terminate(executor); } - @Test public void testExecutePrioritizedExecutorWithRunnables() throws Exception { ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName())); List results = new ArrayList<>(8); @@ -125,7 +114,6 @@ public class PrioritizedExecutorsTests extends ESTestCase { terminate(executor); } - @Test public void testSubmitPrioritizedExecutorWithCallables() throws Exception { ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName())); List results = new ArrayList<>(8); @@ -155,7 +143,6 @@ public class PrioritizedExecutorsTests extends ESTestCase { terminate(executor); } - @Test public void testSubmitPrioritizedExecutorWithMixed() throws Exception { ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName())); List results = new ArrayList<>(8); @@ -185,7 +172,6 @@ public class PrioritizedExecutorsTests extends ESTestCase { terminate(executor); } - @Test public void testTimeout() throws Exception { ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(EsExecutors.daemonThreadFactory(getTestName())); PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName())); @@ -247,7 +233,6 @@ public class PrioritizedExecutorsTests extends ESTestCase { assertTrue(terminate(timer, executor)); } - @Test public void testTimeoutCleanup() throws Exception { ThreadPool threadPool = new ThreadPool("test"); final ScheduledThreadPoolExecutor timer = (ScheduledThreadPoolExecutor) threadPool.scheduler(); diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java index 9b01b785f2b..9338beccb9a 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.util.concurrent; import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.concurrent.CopyOnWriteArrayList; @@ -34,8 +33,6 @@ import static org.hamcrest.Matchers.is; /** */ public class RefCountedTests extends ESTestCase { - - @Test public void testRefCount() throws IOException { MyRefCounted counted = new MyRefCounted(); @@ -87,7 +84,6 @@ public class RefCountedTests extends ESTestCase { } } - @Test public void testMultiThreaded() throws InterruptedException { final MyRefCounted counted = new MyRefCounted(); Thread[] threads = new Thread[randomIntBetween(2, 5)]; diff --git a/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java index 76fc51d8b67..5ce816927ca 100644 --- a/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java @@ -20,17 +20,12 @@ package org.elasticsearch.common.util.iterable; import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matchers; import java.util.Arrays; import java.util.Iterator; -import java.util.List; import java.util.NoSuchElementException; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.object.HasToString.hasToString; -import static org.junit.Assert.*; public class IterablesTests extends ESTestCase { public void testGetOverList() { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java new file mode 100644 index 00000000000..1b917263b79 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public abstract class BaseXContentTestCase extends ESTestCase { + + public abstract XContentType xcontentType(); + + public void testBasics() throws IOException { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { + generator.writeStartObject(); + generator.writeEndObject(); + } + byte[] data = os.toByteArray(); + assertEquals(xcontentType(), XContentFactory.xContentType(data)); + } + + public void testRawField() throws Exception { + for (boolean useStream : new boolean[] {false, true}) { + for (XContentType xcontentType : XContentType.values()) { + doTestRawField(xcontentType.xContent(), useStream); + } + } + } + + void doTestRawField(XContent source, boolean useStream) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (XContentGenerator generator = source.createGenerator(os)) { + generator.writeStartObject(); + generator.writeFieldName("foo"); + generator.writeNull(); + generator.writeEndObject(); + } + final byte[] rawData = os.toByteArray(); + + os = new ByteArrayOutputStream(); + try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { + generator.writeStartObject(); + if (useStream) { + generator.writeRawField("bar", new ByteArrayInputStream(rawData)); + } else { + generator.writeRawField("bar", new BytesArray(rawData)); + } + generator.writeEndObject(); + } + + XContentParser parser = xcontentType().xContent().createParser(os.toByteArray()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("bar", parser.currentName()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("foo", parser.currentName()); + assertEquals(Token.VALUE_NULL, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + } + + public void testRawValue() throws Exception { + for (XContentType xcontentType : XContentType.values()) { + doTestRawValue(xcontentType.xContent()); + } + } + + void doTestRawValue(XContent source) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (XContentGenerator generator = source.createGenerator(os)) { + generator.writeStartObject(); + generator.writeFieldName("foo"); + generator.writeNull(); + generator.writeEndObject(); + } + final byte[] rawData = os.toByteArray(); + + os = new ByteArrayOutputStream(); + try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { + generator.writeRawValue(new BytesArray(rawData)); + } + + XContentParser parser = xcontentType().xContent().createParser(os.toByteArray()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("foo", parser.currentName()); + assertEquals(Token.VALUE_NULL, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + } +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java index 0437292406e..4b8d356eb72 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java @@ -55,7 +55,25 @@ public class ObjectParserTests extends ESTestCase { assertEquals(s.test, "foo"); assertEquals(s.testNumber, 2); assertEquals(s.ints, Arrays.asList(1, 2, 3, 4)); - assertEquals(objectParser.toString(), "ObjectParser{name='foo', fields=[FieldParser{preferred_name=test, supportedTokens=[VALUE_STRING], type=STRING}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY], type=INT_ARRAY}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY], type=INT_ARRAY}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}]}"); + assertEquals(objectParser.toString(), "ObjectParser{name='foo', fields=[FieldParser{preferred_name=test, supportedTokens=[VALUE_STRING], type=STRING}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY, VALUE_STRING, VALUE_NUMBER], type=INT_ARRAY}, FieldParser{preferred_name=test_array, supportedTokens=[START_ARRAY, VALUE_STRING, VALUE_NUMBER], type=INT_ARRAY}, FieldParser{preferred_name=test_number, supportedTokens=[VALUE_STRING, VALUE_NUMBER], type=INT}]}"); + } + + public void testObjectOrDefault() throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser("{\"object\" : { \"test\": 2}}"); + ObjectParser objectParser = new ObjectParser("foo", StaticTestStruct::new); + objectParser.declareInt(StaticTestStruct::setTest, new ParseField("test")); + objectParser.declareObjectOrDefault(StaticTestStruct::setObject, objectParser, StaticTestStruct::new, new ParseField("object")); + StaticTestStruct s = objectParser.parse(parser); + assertEquals(s.object.test, 2); + parser = XContentType.JSON.xContent().createParser("{\"object\" : false }"); + s = objectParser.parse(parser); + assertNull(s.object); + + parser = XContentType.JSON.xContent().createParser("{\"object\" : true }"); + s = objectParser.parse(parser); + assertNotNull(s.object); + assertEquals(s.object.test, 0); + } public void testExceptions() throws IOException { @@ -174,15 +192,35 @@ public class ObjectParserTests extends ESTestCase { XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); builder.startObject(); builder.field("int_field", randomBoolean() ? "1" : 1); - builder.array("int_array_field", randomBoolean() ? "1" : 1); + if (randomBoolean()) { + builder.array("int_array_field", randomBoolean() ? "1" : 1); + } else { + builder.field("int_array_field", randomBoolean() ? "1" : 1); + } builder.field("double_field", randomBoolean() ? "2.1" : 2.1d); - builder.array("double_array_field", randomBoolean() ? "2.1" : 2.1d); + if (randomBoolean()) { + builder.array("double_array_field", randomBoolean() ? "2.1" : 2.1d); + } else { + builder.field("double_array_field", randomBoolean() ? "2.1" : 2.1d); + } builder.field("float_field", randomBoolean() ? "3.1" : 3.1f); - builder.array("float_array_field", randomBoolean() ? "3.1" : 3.1); + if (randomBoolean()) { + builder.array("float_array_field", randomBoolean() ? "3.1" : 3.1); + } else { + builder.field("float_array_field", randomBoolean() ? "3.1" : 3.1); + } builder.field("long_field", randomBoolean() ? "4" : 4); - builder.array("long_array_field", randomBoolean() ? "4" : 4); + if (randomBoolean()) { + builder.array("long_array_field", randomBoolean() ? "4" : 4); + } else { + builder.field("long_array_field", randomBoolean() ? "4" : 4); + } builder.field("string_field", "5"); - builder.array("string_array_field", "5"); + if (randomBoolean()) { + builder.array("string_array_field", "5"); + } else { + builder.field("string_array_field", "5"); + } boolean nullValue = randomBoolean(); builder.field("boolean_field", nullValue); builder.field("string_or_null", nullValue ? null : "5"); diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java index 0f803509bc5..7489ea8f9dc 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java @@ -21,10 +21,10 @@ package org.elasticsearch.common.xcontent; import com.fasterxml.jackson.dataformat.cbor.CBORConstants; import com.fasterxml.jackson.dataformat.smile.SmileConstants; + import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -35,24 +35,18 @@ import static org.hamcrest.Matchers.equalTo; * */ public class XContentFactoryTests extends ESTestCase { - - - @Test public void testGuessJson() throws IOException { testGuessType(XContentType.JSON); } - @Test public void testGuessSmile() throws IOException { testGuessType(XContentType.SMILE); } - @Test public void testGuessYaml() throws IOException { testGuessType(XContentType.YAML); } - @Test public void testGuessCbor() throws IOException { testGuessType(XContentType.CBOR); } @@ -103,4 +97,18 @@ public class XContentFactoryTests extends ESTestCase { is = new ByteArrayInputStream(new byte[] {(byte) 1}); assertNull(XContentFactory.xContentType(is)); } + + public void testJsonFromBytesOptionallyPrecededByUtf8Bom() throws Exception { + byte[] bytes = new byte[] {(byte) '{', (byte) '}'}; + assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + + bytes = new byte[] {(byte) 0x20, (byte) '{', (byte) '}'}; + assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + + bytes = new byte[] {(byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) '{', (byte) '}'}; + assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + + bytes = new byte[] {(byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) 0x20, (byte) '{', (byte) '}'}; + assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + } } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java deleted file mode 100644 index 9cfedcc1cf5..00000000000 --- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.builder; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.ESTestCase; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -/** - * - */ -public class BuilderRawFieldTests extends ESTestCase { - - @Test - public void testJsonRawField() throws IOException { - testRawField(XContentType.JSON); - } - - @Test - public void testSmileRawField() throws IOException { - testRawField(XContentType.SMILE); - } - - @Test - public void testYamlRawField() throws IOException { - testRawField(XContentType.YAML); - } - - @Test - public void testCborRawField() throws IOException { - testRawField(XContentType.CBOR); - } - - private void testRawField(XContentType type) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(type); - builder.startObject(); - builder.field("field1", "value1"); - builder.rawField("_source", XContentFactory.contentBuilder(type).startObject().field("s_field", "s_value").endObject().bytes()); - builder.field("field2", "value2"); - builder.rawField("payload_i", new BytesArray(Long.toString(1))); - builder.field("field3", "value3"); - builder.rawField("payload_d", new BytesArray(Double.toString(1.1))); - builder.field("field4", "value4"); - builder.rawField("payload_s", new BytesArray("test")); - builder.field("field5", "value5"); - builder.endObject(); - - XContentParser parser = XContentFactory.xContent(type).createParser(builder.bytes()); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field1")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value1")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("_source")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("s_field")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("s_value")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field2")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value2")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("payload_i")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER)); - assertThat(parser.numberType(), equalTo(XContentParser.NumberType.INT)); - assertThat(parser.longValue(), equalTo(1l)); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field3")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value3")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("payload_d")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER)); - assertThat(parser.numberType(), equalTo(XContentParser.NumberType.DOUBLE)); - assertThat(parser.doubleValue(), equalTo(1.1d)); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field4")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value4")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("payload_s")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("test")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo("field5")); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING)); - assertThat(parser.text(), equalTo("value5")); - - assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java index 4ce1a7c4630..7ffafc004ab 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.xcontent.builder; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.FastCharArrayWriter; @@ -31,8 +32,8 @@ import org.elasticsearch.common.xcontent.XContentGenerator; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; @@ -54,11 +55,9 @@ import static org.hamcrest.Matchers.equalTo; * */ public class XContentBuilderTests extends ESTestCase { - - @Test public void testPrettyWithLfAtEnd() throws Exception { - FastCharArrayWriter writer = new FastCharArrayWriter(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(writer); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); generator.usePrettyPrint(); generator.usePrintLineFeedAtEnd(); @@ -71,31 +70,30 @@ public class XContentBuilderTests extends ESTestCase { // double close, and check there is no error... generator.close(); - assertThat(writer.unsafeCharArray()[writer.size() - 1], equalTo('\n')); + byte[] bytes = os.toByteArray(); + assertThat((char) bytes[bytes.length - 1], equalTo('\n')); } - @Test - public void verifyReuseJsonGenerator() throws Exception { - FastCharArrayWriter writer = new FastCharArrayWriter(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(writer); + public void testReuseJsonGenerator() throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); generator.writeStartObject(); generator.writeStringField("test", "value"); generator.writeEndObject(); generator.flush(); - assertThat(writer.toStringTrim(), equalTo("{\"test\":\"value\"}")); + assertThat(new BytesRef(os.toByteArray()), equalTo(new BytesRef("{\"test\":\"value\"}"))); // try again... - writer.reset(); + os.reset(); generator.writeStartObject(); generator.writeStringField("test", "value"); generator.writeEndObject(); generator.flush(); // we get a space at the start here since it thinks we are not in the root object (fine, we will ignore it in the real code we use) - assertThat(writer.toStringTrim(), equalTo("{\"test\":\"value\"}")); + assertThat(new BytesRef(os.toByteArray()), equalTo(new BytesRef(" {\"test\":\"value\"}"))); } - @Test public void testRaw() throws IOException { { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -141,7 +139,6 @@ public class XContentBuilderTests extends ESTestCase { } } - @Test public void testSimpleGenerator() throws Exception { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject().field("test", "value").endObject(); @@ -152,14 +149,12 @@ public class XContentBuilderTests extends ESTestCase { assertThat(builder.string(), equalTo("{\"test\":\"value\"}")); } - @Test public void testOverloadedList() throws Exception { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject().field("test", Arrays.asList("1", "2")).endObject(); assertThat(builder.string(), equalTo("{\"test\":[\"1\",\"2\"]}")); } - @Test public void testWritingBinaryToStream() throws Exception { BytesStreamOutput bos = new BytesStreamOutput(); @@ -177,7 +172,6 @@ public class XContentBuilderTests extends ESTestCase { System.out.println("DATA: " + sData); } - @Test public void testFieldCaseConversion() throws Exception { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).fieldCaseConversion(CAMELCASE); builder.startObject().field("test_name", "value").endObject(); @@ -188,14 +182,12 @@ public class XContentBuilderTests extends ESTestCase { assertThat(builder.string(), equalTo("{\"test_name\":\"value\"}")); } - @Test public void testByteConversion() throws Exception { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject().field("test_name", (Byte)(byte)120).endObject(); assertThat(builder.bytes().toUtf8(), equalTo("{\"test_name\":120}")); } - @Test public void testDateTypesConversion() throws Exception { Date date = new Date(); String expectedDate = XContentBuilder.defaultDatePrinter.print(date.getTime()); @@ -222,7 +214,6 @@ public class XContentBuilderTests extends ESTestCase { assertThat(builder.string(), equalTo("{\"calendar\":\"" + expectedCalendar + "\"}")); } - @Test public void testCopyCurrentStructure() throws Exception { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject() @@ -277,19 +268,16 @@ public class XContentBuilderTests extends ESTestCase { assertThat(i, equalTo(terms.size())); } - @Test public void testHandlingOfPath() throws IOException { Path path = PathUtils.get("path"); checkPathSerialization(path); } - @Test public void testHandlingOfPath_relative() throws IOException { Path path = PathUtils.get("..", "..", "path"); checkPathSerialization(path); } - @Test public void testHandlingOfPath_absolute() throws IOException { Path path = createTempDir().toAbsolutePath(); checkPathSerialization(path); @@ -305,7 +293,6 @@ public class XContentBuilderTests extends ESTestCase { assertThat(pathBuilder.string(), equalTo(stringBuilder.string())); } - @Test public void testHandlingOfPath_XContentBuilderStringName() throws IOException { Path path = PathUtils.get("path"); XContentBuilderString name = new XContentBuilderString("file"); @@ -319,7 +306,6 @@ public class XContentBuilderTests extends ESTestCase { assertThat(pathBuilder.string(), equalTo(stringBuilder.string())); } - @Test public void testHandlingOfCollectionOfPaths() throws IOException { Path path = PathUtils.get("path"); diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java index 20e4d4163bf..173a58cff89 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentParserTests.java @@ -24,13 +24,10 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; public class CborXContentParserTests extends ESTestCase { - - @Test public void testEmptyValue() throws IOException { BytesReference ref = XContentFactory.cborBuilder().startObject().field("field", "").endObject().bytes(); diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/DummyIndicesAnalysisModule.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java similarity index 72% rename from core/src/test/java/org/elasticsearch/indices/analysis/DummyIndicesAnalysisModule.java rename to core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java index 9d14f67ec60..928b8a6a5a9 100644 --- a/core/src/test/java/org/elasticsearch/indices/analysis/DummyIndicesAnalysisModule.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/CborXContentTests.java @@ -17,14 +17,16 @@ * under the License. */ -package org.elasticsearch.indices.analysis; +package org.elasticsearch.common.xcontent.cbor; -import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; -public class DummyIndicesAnalysisModule extends AbstractModule { +public class CborXContentTests extends BaseXContentTestCase { @Override - protected void configure() { - bind(DummyIndicesAnalysis.class).asEagerSingleton(); + public XContentType xcontentType() { + return XContentType.CBOR; } + } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java index 903914bc5fe..bf2dd442b64 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentGenerator; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -36,9 +35,7 @@ import static org.hamcrest.Matchers.nullValue; * */ public class JsonVsCborTests extends ESTestCase { - - @Test - public void compareParsingTokens() throws IOException { + public void testCompareParsingTokens() throws IOException { BytesStreamOutput xsonOs = new BytesStreamOutput(); XContentGenerator xsonGen = XContentFactory.xContent(XContentType.CBOR).createGenerator(xsonOs); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/indices/analysis/KuromojiIndicesAnalysisModule.java b/core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java similarity index 72% rename from plugins/analysis-kuromoji/src/main/java/org/elasticsearch/indices/analysis/KuromojiIndicesAnalysisModule.java rename to core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java index 8046aece373..8a739eef4b8 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/indices/analysis/KuromojiIndicesAnalysisModule.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java @@ -17,16 +17,16 @@ * under the License. */ -package org.elasticsearch.indices.analysis; +package org.elasticsearch.common.xcontent.json; -import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; -/** - */ -public class KuromojiIndicesAnalysisModule extends AbstractModule { +public class JsonXContentTests extends BaseXContentTestCase { @Override - protected void configure() { - bind(KuromojiIndicesAnalysis.class).asEagerSingleton(); + public XContentType xcontentType() { + return XContentType.JSON; } + } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java index e1a89ff242f..9e686fe78f1 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java @@ -25,11 +25,9 @@ import org.elasticsearch.common.xcontent.XContentGenerator; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -37,18 +35,7 @@ import static org.hamcrest.Matchers.nullValue; * */ public class JsonVsSmileTests extends ESTestCase { - -// @Test public void testBinarySmileField() throws Exception { -// JsonGenerator gen = new SmileFactory().createJsonGenerator(new ByteArrayOutputStream()); -//// JsonGenerator gen = new JsonFactory().createJsonGenerator(new ByteArrayOutputStream(), JsonEncoding.UTF8); -// gen.writeStartObject(); -// gen.writeFieldName("field1"); -// gen.writeBinary(new byte[]{1, 2, 3}); -// gen.writeEndObject(); -// } - - @Test - public void compareParsingTokens() throws IOException { + public void testCompareParsingTokens() throws IOException { BytesStreamOutput xsonOs = new BytesStreamOutput(); XContentGenerator xsonGen = XContentFactory.xContent(XContentType.SMILE).createGenerator(xsonOs); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalIndexModule.java b/core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java old mode 100755 new mode 100644 similarity index 72% rename from core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalIndexModule.java rename to core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java index bcc6fc055eb..6961e84416d --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalIndexModule.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/smile/SmileXContentTests.java @@ -17,17 +17,16 @@ * under the License. */ -package org.elasticsearch.index.mapper.externalvalues; +package org.elasticsearch.common.xcontent.smile; -import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; -/** - * - */ -public class ExternalIndexModule extends AbstractModule { +public class SmileXContentTests extends BaseXContentTestCase { @Override - protected void configure() { - bind(RegisterExternalTypes.class).asEagerSingleton(); + public XContentType xcontentType() { + return XContentType.SMILE; } + } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java index 067c5538281..a8091fc1122 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.common.xcontent.support; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.Arrays; import java.util.HashMap; @@ -51,9 +50,7 @@ public class XContentHelperTests extends ESTestCase { return Arrays.asList(values); } - @Test public void testMergingListValuesAreMapsOfOne() { - Map defaults = getMap("test", getList(getNamedMap("name1", "t1", "1"), getNamedMap("name2", "t2", "2"))); Map content = getMap("test", getList(getNamedMap("name2", "t3", "3"), getNamedMap("name4", "t4", "4"))); Map expected = getMap("test", diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index abce42b6232..1c4ff9874a5 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; @@ -48,8 +47,6 @@ import static org.hamcrest.core.IsEqual.equalTo; /** */ public class XContentMapValuesTests extends ESTestCase { - - @Test public void testFilter() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field("test1", "value1") @@ -106,7 +103,6 @@ public class XContentMapValuesTests extends ESTestCase { } @SuppressWarnings({"unchecked"}) - @Test public void testExtractValue() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field("test", "value") @@ -196,8 +192,6 @@ public class XContentMapValuesTests extends ESTestCase { assertThat(XContentMapValues.extractValue("path1.xxx.path2.yyy.test", map).toString(), equalTo("value")); } - @SuppressWarnings({"unchecked"}) - @Test public void testExtractRawValue() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field("test", "value") @@ -237,8 +231,7 @@ public class XContentMapValuesTests extends ESTestCase { assertThat(XContentMapValues.extractRawValues("path1.xxx.path2.yyy.test", map).get(0).toString(), equalTo("value")); } - @Test - public void prefixedNamesFilteringTest() { + public void testPrefixedNamesFilteringTest() { Map map = new HashMap<>(); map.put("obj", "value"); map.put("obj_name", "value_name"); @@ -248,9 +241,8 @@ public class XContentMapValuesTests extends ESTestCase { } - @Test @SuppressWarnings("unchecked") - public void nestedFilteringTest() { + public void testNestedFiltering() { Map map = new HashMap<>(); map.put("field", "value"); map.put("array", @@ -295,8 +287,7 @@ public class XContentMapValuesTests extends ESTestCase { } @SuppressWarnings("unchecked") - @Test - public void completeObjectFilteringTest() { + public void testCompleteObjectFiltering() { Map map = new HashMap<>(); map.put("field", "value"); map.put("obj", @@ -340,8 +331,7 @@ public class XContentMapValuesTests extends ESTestCase { } @SuppressWarnings("unchecked") - @Test - public void filterIncludesUsingStarPrefix() { + public void testFilterIncludesUsingStarPrefix() { Map map = new HashMap<>(); map.put("field", "value"); map.put("obj", @@ -382,8 +372,7 @@ public class XContentMapValuesTests extends ESTestCase { } - @Test - public void filterWithEmptyIncludesExcludes() { + public void testFilterWithEmptyIncludesExcludes() { Map map = new HashMap<>(); map.put("field", "value"); Map filteredMap = XContentMapValues.filter(map, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY); @@ -392,8 +381,6 @@ public class XContentMapValuesTests extends ESTestCase { } - @SuppressWarnings({"unchecked"}) - @Test public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .startObject("obj") @@ -406,7 +393,6 @@ public class XContentMapValuesTests extends ESTestCase { assertThat(mapTuple.v2(), equalTo(filteredSource)); } - @Test public void testThatFilterIncludesEmptyObjectWhenUsingExcludes() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .startObject("obj") @@ -419,7 +405,6 @@ public class XContentMapValuesTests extends ESTestCase { assertThat(mapTuple.v2(), equalTo(filteredSource)); } - @Test public void testNotOmittingObjectsWithExcludedProperties() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .startObject("obj") @@ -436,7 +421,6 @@ public class XContentMapValuesTests extends ESTestCase { } @SuppressWarnings({"unchecked"}) - @Test public void testNotOmittingObjectWithNestedExcludedObject() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .startObject("obj1") @@ -470,7 +454,6 @@ public class XContentMapValuesTests extends ESTestCase { } @SuppressWarnings({"unchecked"}) - @Test public void testIncludingObjectWithNestedIncludedObject() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .startObject("obj1") @@ -568,5 +551,4 @@ public class XContentMapValuesTests extends ESTestCase { parser.list()); } } - } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java index 9669b0992de..ed7aee33eba 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java @@ -20,9 +20,12 @@ package org.elasticsearch.common.xcontent.support.filtering; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -170,7 +173,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase return sample(newXContentBuilder(filters)); } - @Test public void testNoFiltering() throws Exception { XContentBuilder expected = sample(); @@ -179,23 +181,18 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("**")); } - @Test public void testNoMatch() throws Exception { XContentBuilder expected = newXContentBuilder().startObject().endObject(); assertXContentBuilder(expected, sample("xyz")); } - @Test public void testSimpleField() throws Exception { - XContentBuilder expected = newXContentBuilder().startObject() - .field("title", "My awesome book") - .endObject(); + XContentBuilder expected = newXContentBuilder().startObject().field("title", "My awesome book").endObject(); assertXContentBuilder(expected, sample("title")); } - @Test public void testSimpleFieldWithWildcard() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .field("price", 27.99) @@ -252,7 +249,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("pr*")); } - @Test public void testMultipleFields() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .field("title", "My awesome book") @@ -262,7 +258,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("title", "pages")); } - @Test public void testSimpleArray() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startArray("tags") @@ -274,7 +269,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("tags")); } - @Test public void testSimpleArrayOfObjects() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startArray("authors") @@ -296,7 +290,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("authors.*name")); } - @Test public void testSimpleArrayOfObjectsProperty() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startArray("authors") @@ -313,7 +306,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("authors.l*")); } - @Test public void testRecurseField1() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startArray("authors") @@ -366,7 +358,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("**.name")); } - @Test public void testRecurseField2() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startObject("properties") @@ -411,7 +402,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("properties.**.name")); } - @Test public void testRecurseField3() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startObject("properties") @@ -441,7 +431,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("properties.*.en.**.name")); } - @Test public void testRecurseField4() throws Exception { XContentBuilder expected = newXContentBuilder().startObject() .startObject("properties") @@ -473,7 +462,6 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expected, sample("properties.**.distributors.name")); } - @Test public void testRawField() throws Exception { XContentBuilder expectedRawField = newXContentBuilder().startObject().field("foo", 0).startObject("raw").field("content", "hello world!").endObject().endObject(); @@ -487,18 +475,12 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", raw).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", raw).endObject()); - // Test method: rawField(String fieldName, byte[] content) - assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject()); - assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject()); - assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", raw.toBytes()).endObject()); - // Test method: rawField(String fieldName, InputStream content) assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*").startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); } - @Test public void testArrays() throws Exception { // Test: Array of values (no filtering) XContentBuilder expected = newXContentBuilder().startObject().startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject(); @@ -519,6 +501,5 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase // Test: Array of objects (with partial filtering) expected = newXContentBuilder().startObject().startArray("tags").startObject().field("firstname", "ipsum").endObject().endArray().endObject(); assertXContentBuilder(expected, newXContentBuilder("t*.firstname").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject()); - } } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java new file mode 100644 index 00000000000..0ca2eafa417 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support.filtering; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class FilterPathGeneratorFilteringTests extends ESTestCase { + + private final JsonFactory JSON_FACTORY = new JsonFactory(); + + public void testFilters() throws Exception { + final String SAMPLE = "{'a':0,'b':true,'c':'c_value','d':[0,1,2],'e':[{'f1':'f1_value','f2':'f2_value'},{'g1':'g1_value','g2':'g2_value'}],'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"; + + assertResult(SAMPLE, "a", "{'a':0}"); + assertResult(SAMPLE, "b", "{'b':true}"); + assertResult(SAMPLE, "c", "{'c':'c_value'}"); + assertResult(SAMPLE, "d", "{'d':[0,1,2]}"); + assertResult(SAMPLE, "e", "{'e':[{'f1':'f1_value','f2':'f2_value'},{'g1':'g1_value','g2':'g2_value'}]}"); + assertResult(SAMPLE, "h", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "z", ""); + + assertResult(SAMPLE, "e.f1", "{'e':[{'f1':'f1_value'}]}"); + assertResult(SAMPLE, "e.f2", "{'e':[{'f2':'f2_value'}]}"); + assertResult(SAMPLE, "e.f*", "{'e':[{'f1':'f1_value','f2':'f2_value'}]}"); + assertResult(SAMPLE, "e.*2", "{'e':[{'f2':'f2_value'},{'g2':'g2_value'}]}"); + + assertResult(SAMPLE, "h.i", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "h.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "*.i", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "*.i.j", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.*.j", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "*.i.j.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.*.j.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.*.k", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "*.i.j.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.*.j.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.*.k.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.*.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "h.i.j.k.*", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "h.*.j.*.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + assertResult(SAMPLE, "**.l", "{'h':{'i':{'j':{'k':{'l':'l_value'}}}}}"); + + assertResult(SAMPLE, "**.*2", "{'e':[{'f2':'f2_value'},{'g2':'g2_value'}]}"); + } + + public void testFiltersWithDots() throws Exception { + assertResult("{'a':0,'b.c':'value','b':{'c':'c_value'}}", "b.c", "{'b':{'c':'c_value'}}"); + assertResult("{'a':0,'b.c':'value','b':{'c':'c_value'}}", "b\\.c", "{'b.c':'value'}"); + } + + private void assertResult(String input, String filter, String expected) throws Exception { + try (BytesStreamOutput os = new BytesStreamOutput()) { + try (FilteringGeneratorDelegate generator = new FilteringGeneratorDelegate(JSON_FACTORY.createGenerator(os), new FilterPathBasedFilter(new String[]{filter}), true, true)) { + try (JsonParser parser = JSON_FACTORY.createParser(replaceQuotes(input))) { + while (parser.nextToken() != null) { + generator.copyCurrentStructure(parser); + } + } + } + assertThat(os.bytes().toUtf8(), equalTo(replaceQuotes(expected))); + } + } + + private String replaceQuotes(String s) { + return s.replace('\'', '"'); + } +} diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java new file mode 100644 index 00000000000..50683007717 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java @@ -0,0 +1,351 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support.filtering; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.*; + +public class FilterPathTests extends ESTestCase { + + public void testSimpleFilterPath() { + final String input = "test"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("test")); + + FilterPath next = filterPath.getNext(); + assertNotNull(next); + assertThat(next.matches(), is(true)); + assertThat(next.getSegment(), isEmptyString()); + assertSame(next, FilterPath.EMPTY); + } + + public void testFilterPathWithSubField() { + final String input = "foo.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testFilterPathWithSubFields() { + final String input = "foo.bar.quz"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("quz")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testEmptyFilterPath() { + FilterPath[] filterPaths = FilterPath.compile(""); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(0)); + } + + public void testNullFilterPath() { + FilterPath[] filterPaths = FilterPath.compile((String) null); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(0)); + } + + public void testFilterPathWithEscapedDots() { + String input = "w.0.0.t"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("w")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("0")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("0")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("t")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + + input = "w\\.0\\.0\\.t"; + + filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("w.0.0.t")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + + + input = "w\\.0.0\\.t"; + + filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + filterPath = filterPaths[0]; + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("w.0")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("0.t")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testSimpleWildcardFilterPath() { + FilterPath[] filterPaths = FilterPath.compile("*"); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isSimpleWildcard(), is(true)); + assertThat(filterPath.getSegment(), equalTo("*")); + + FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2)); + assertNotNull(next); + assertSame(next, FilterPath.EMPTY); + } + + public void testWildcardInNameFilterPath() { + String input = "f*o.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("f*o")); + assertThat(filterPath.matchProperty("foo"), notNullValue()); + assertThat(filterPath.matchProperty("flo"), notNullValue()); + assertThat(filterPath.matchProperty("foooo"), notNullValue()); + assertThat(filterPath.matchProperty("boo"), nullValue()); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testDoubleWildcardFilterPath() { + FilterPath[] filterPaths = FilterPath.compile("**"); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isDoubleWildcard(), is(true)); + assertThat(filterPath.getSegment(), equalTo("**")); + + FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2)); + assertNotNull(next); + assertSame(next, FilterPath.EMPTY); + } + + public void testStartsWithDoubleWildcardFilterPath() { + String input = "**.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("**")); + + FilterPath next = filterPath.matchProperty(randomAsciiOfLength(2)); + assertNotNull(next); + assertThat(next.matches(), is(false)); + assertThat(next.getSegment(), equalTo("bar")); + + next = next.getNext(); + assertNotNull(next); + assertThat(next.matches(), is(true)); + assertThat(next.getSegment(), isEmptyString()); + assertSame(next, FilterPath.EMPTY); + } + + public void testContainsDoubleWildcardFilterPath() { + String input = "foo.**.bar"; + + FilterPath[] filterPaths = FilterPath.compile(input); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(1)); + + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isDoubleWildcard(), equalTo(true)); + assertThat(filterPath.getSegment(), equalTo("**")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } + + public void testMultipleFilterPaths() { + String[] inputs = {"foo.**.bar.*", "test.dot\\.ted"}; + + FilterPath[] filterPaths = FilterPath.compile(inputs); + assertNotNull(filterPaths); + assertThat(filterPaths, arrayWithSize(2)); + + // foo.**.bar.* + FilterPath filterPath = filterPaths[0]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("foo")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.isDoubleWildcard(), equalTo(true)); + assertThat(filterPath.getSegment(), equalTo("**")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("bar")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.isSimpleWildcard(), equalTo(true)); + assertThat(filterPath.getSegment(), equalTo("*")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + + // test.dot\.ted + filterPath = filterPaths[1]; + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("test")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(false)); + assertThat(filterPath.getSegment(), equalTo("dot.ted")); + + filterPath = filterPath.getNext(); + assertNotNull(filterPath); + assertThat(filterPath.matches(), is(true)); + assertThat(filterPath.getSegment(), isEmptyString()); + assertSame(filterPath, FilterPath.EMPTY); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGeneratorBenchmark.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGeneratorBenchmark.java deleted file mode 100644 index 97ce4fcb838..00000000000 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilteringJsonGeneratorBenchmark.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.xcontent.support.filtering; - -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Locale; - -/** - * Benchmark class to compare filtered and unfiltered XContent generators. - */ -public class FilteringJsonGeneratorBenchmark { - - public static void main(String[] args) throws IOException { - final XContent XCONTENT = JsonXContent.jsonXContent; - - System.out.println("Executing " + FilteringJsonGeneratorBenchmark.class + "..."); - - System.out.println("Warming up..."); - run(XCONTENT, 500_000, 100, 0.5); - System.out.println("Warmed up."); - - System.out.println("nb documents | nb fields | nb fields written | % fields written | time (millis) | rate (docs/sec) | avg size"); - - for (int nbFields : Arrays.asList(10, 25, 50, 100, 250)) { - for (int nbDocs : Arrays.asList(100, 1000, 10_000, 100_000, 500_000)) { - for (double ratio : Arrays.asList(0.0, 1.0, 0.99, 0.95, 0.9, 0.75, 0.5, 0.25, 0.1, 0.05, 0.01)) { - run(XCONTENT, nbDocs, nbFields, ratio); - } - } - } - System.out.println("Done."); - } - - private static void run(XContent xContent, long nbIterations, int nbFields, double ratio) throws IOException { - String[] fields = fields(nbFields); - String[] filters = fields((int) (nbFields * ratio)); - - long size = 0; - BytesStreamOutput os = new BytesStreamOutput(); - - long start = System.nanoTime(); - for (int i = 0; i < nbIterations; i++) { - XContentBuilder builder = new XContentBuilder(xContent, os, filters); - builder.startObject(); - - for (String field : fields) { - builder.field(field, System.nanoTime()); - } - builder.endObject(); - - size += builder.bytes().length(); - os.reset(); - } - double milliseconds = (System.nanoTime() - start) / 1_000_000d; - - System.out.printf(Locale.ROOT, "%12d | %9d | %17d | %14.2f %% | %10.3f ms | %15.2f | %8.0f %n", - nbIterations, nbFields, - (int) (nbFields * ratio), - (ratio * 100d), - milliseconds, - ((double) nbIterations) / (milliseconds / 1000d), - size / ((double) nbIterations)); - } - - /** - * Returns a String array of field names starting from "field_0" with a length of n. - * If n=3, the array is ["field_0","field_1","field_2"] - */ - private static String[] fields(int n) { - String[] fields = new String[n]; - for (int i = 0; i < n; i++) { - fields[i] = "field_" + i; - } - return fields; - } -} diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/indices/analysis/IcuIndicesAnalysisModule.java b/core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java similarity index 72% rename from plugins/analysis-icu/src/main/java/org/elasticsearch/indices/analysis/IcuIndicesAnalysisModule.java rename to core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java index e7587205a04..17c2a590ec1 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/indices/analysis/IcuIndicesAnalysisModule.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/yaml/YamlXContentTests.java @@ -17,16 +17,16 @@ * under the License. */ -package org.elasticsearch.indices.analysis; +package org.elasticsearch.common.xcontent.yaml; -import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.xcontent.BaseXContentTestCase; +import org.elasticsearch.common.xcontent.XContentType; -/** - */ -public class IcuIndicesAnalysisModule extends AbstractModule { +public class YamlXContentTests extends BaseXContentTestCase { @Override - protected void configure() { - bind(IcuIndicesAnalysis.class).asEagerSingleton(); + public XContentType xcontentType() { + return XContentType.YAML; } + } diff --git a/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java b/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java index 50637cb801a..b696c445f30 100644 --- a/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java +++ b/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java @@ -22,12 +22,11 @@ package org.elasticsearch.consistencylevel; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; @@ -37,9 +36,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class WriteConsistencyLevelIT extends ESIntegTestCase { - - - @Test public void testWriteConsistencyLevelReplication2() throws Exception { prepareCreate("test", 1, Settings.settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 2)).execute().actionGet(); @@ -57,7 +53,7 @@ public class WriteConsistencyLevelIT extends ESIntegTestCase { fail("can't index, does not match consistency"); } catch (UnavailableShardsException e) { assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); - assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet write consistency of [QUORUM] (have 1, needed 2). Timeout: [100ms], request: index {[test][type1][1], source[{ type1 : { \"id\" : \"1\", \"name\" : \"test\" } }]}")); + assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet write consistency of [QUORUM] (have 1, needed 2). Timeout: [100ms], request: [index {[test][type1][1], source[{ type1 : { \"id\" : \"1\", \"name\" : \"test\" } }]}]")); // but really, all is well } @@ -68,7 +64,7 @@ public class WriteConsistencyLevelIT extends ESIntegTestCase { assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); - // this should work, since we now have + // this should work, since we now have client().prepareIndex("test", "type1", "1").setSource(source("1", "test")) .setConsistencyLevel(WriteConsistencyLevel.QUORUM) .setTimeout(timeValueSeconds(1)).execute().actionGet(); @@ -80,7 +76,7 @@ public class WriteConsistencyLevelIT extends ESIntegTestCase { fail("can't index, does not match consistency"); } catch (UnavailableShardsException e) { assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE)); - assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet write consistency of [ALL] (have 2, needed 3). Timeout: [100ms], request: index {[test][type1][1], source[{ type1 : { \"id\" : \"1\", \"name\" : \"test\" } }]}")); + assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet write consistency of [ALL] (have 2, needed 3). Timeout: [100ms], request: [index {[test][type1][1], source[{ type1 : { \"id\" : \"1\", \"name\" : \"test\" } }]}]")); // but really, all is well } diff --git a/core/src/test/java/org/elasticsearch/count/simple/SimpleCountIT.java b/core/src/test/java/org/elasticsearch/count/simple/SimpleCountIT.java deleted file mode 100644 index 344abee3da7..00000000000 --- a/core/src/test/java/org/elasticsearch/count/simple/SimpleCountIT.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.count.simple; - -import org.apache.lucene.util.Constants; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; - -import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.is; - -public class SimpleCountIT extends ESIntegTestCase { - - @Test - public void testCountRandomPreference() throws InterruptedException, ExecutionException { - createIndex("test"); - indexRandom(true, client().prepareIndex("test", "type", "1").setSource("field", "value"), - client().prepareIndex("test", "type", "2").setSource("field", "value"), - client().prepareIndex("test", "type", "3").setSource("field", "value"), - client().prepareIndex("test", "type", "4").setSource("field", "value"), - client().prepareIndex("test", "type", "5").setSource("field", "value"), - client().prepareIndex("test", "type", "6").setSource("field", "value")); - - int iters = scaledRandomIntBetween(10, 100); - for (int i = 0; i < iters; i++) { - - String randomPreference = randomUnicodeOfLengthBetween(0, 4); - // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) - while (randomPreference.startsWith("_")) { - randomPreference = randomUnicodeOfLengthBetween(0, 4); - } - // id is not indexed, but lets see that we automatically convert to - CountResponse countResponse = client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).setPreference(randomPreference).get(); - assertHitCount(countResponse, 6l); - } - } - - @Test - public void simpleIpTests() throws Exception { - createIndex("test"); - - client().admin().indices().preparePutMapping("test").setType("type1") - .setSource(XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("from").field("type", "ip").endObject() - .startObject("to").field("type", "ip").endObject() - .endObject().endObject().endObject()) - .execute().actionGet(); - - client().prepareIndex("test", "type1", "1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefresh(true).execute().actionGet(); - - CountResponse countResponse = client().prepareCount() - .setQuery(boolQuery().must(rangeQuery("from").lt("192.168.0.7")).must(rangeQuery("to").gt("192.168.0.7"))) - .execute().actionGet(); - - assertHitCount(countResponse, 1l); - } - - @Test - public void simpleIdTests() { - createIndex("test"); - - client().prepareIndex("test", "type", "XXX1").setSource("field", "value").setRefresh(true).execute().actionGet(); - // id is not indexed, but lets see that we automatically convert to - CountResponse countResponse = client().prepareCount().setQuery(QueryBuilders.termQuery("_id", "XXX1")).execute().actionGet(); - assertHitCount(countResponse, 1l); - - countResponse = client().prepareCount().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")).execute().actionGet(); - assertHitCount(countResponse, 1l); - - // id is not index, but we can automatically support prefix as well - countResponse = client().prepareCount().setQuery(QueryBuilders.prefixQuery("_id", "XXX")).execute().actionGet(); - assertHitCount(countResponse, 1l); - - countResponse = client().prepareCount().setQuery(QueryBuilders.queryStringQuery("_id:XXX*").lowercaseExpandedTerms(false)).execute().actionGet(); - assertHitCount(countResponse, 1l); - } - - @Test - public void simpleCountEarlyTerminationTests() throws Exception { - // set up one shard only to test early termination - prepareCreate("test").setSettings( - SETTING_NUMBER_OF_SHARDS, 1, - SETTING_NUMBER_OF_REPLICAS, 0).get(); - ensureGreen(); - int max = randomIntBetween(3, 29); - List docbuilders = new ArrayList<>(max); - - for (int i = 1; i <= max; i++) { - String id = String.valueOf(i); - docbuilders.add(client().prepareIndex("test", "type1", id).setSource("field", i)); - } - - indexRandom(true, docbuilders); - ensureGreen(); - refresh(); - - // sanity check - CountResponse countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max)).execute().actionGet(); - assertHitCount(countResponse, max); - - // threshold <= actual count - for (int i = 1; i <= max; i++) { - countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max)).setTerminateAfter(i).execute().actionGet(); - assertHitCount(countResponse, i); - assertTrue(countResponse.terminatedEarly()); - } - - // threshold > actual count - countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max)).setTerminateAfter(max + randomIntBetween(1, max)).execute().actionGet(); - assertHitCount(countResponse, max); - assertFalse(countResponse.terminatedEarly()); - } - - @Test - public void localDependentDateTests() throws Exception { - assumeFalse("Locals are buggy on JDK9EA", Constants.JRE_IS_MINIMUM_JAVA9 && systemPropertyAsBoolean("tests.security.manager", false)); - assertAcked(prepareCreate("test") - .addMapping("type1", - jsonBuilder().startObject() - .startObject("type1") - .startObject("properties") - .startObject("date_field") - .field("type", "date") - .field("format", "E, d MMM yyyy HH:mm:ss Z") - .field("locale", "de") - .endObject() - .endObject() - .endObject() - .endObject())); - ensureGreen(); - for (int i = 0; i < 10; i++) { - client().prepareIndex("test", "type1", "" + i).setSource("date_field", "Mi, 06 Dez 2000 02:55:00 -0800").execute().actionGet(); - client().prepareIndex("test", "type1", "" + (10 + i)).setSource("date_field", "Do, 07 Dez 2000 02:55:00 -0800").execute().actionGet(); - } - - refresh(); - for (int i = 0; i < 10; i++) { - CountResponse countResponse = client().prepareCount("test") - .setQuery(QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Do, 07 Dez 2000 00:00:00 -0800")) - .execute().actionGet(); - assertHitCount(countResponse, 10l); - - countResponse = client().prepareCount("test") - .setQuery(QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Fr, 08 Dez 2000 00:00:00 -0800")) - .execute().actionGet(); - assertHitCount(countResponse, 20l); - } - } - - @Test - public void testThatNonEpochDatesCanBeSearched() throws Exception { - assertAcked(prepareCreate("test") - .addMapping("type1", - jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("date_field").field("type", "date").field("format", "yyyyMMddHH").endObject().endObject() - .endObject().endObject())); - ensureGreen("test"); - - XContentBuilder document = jsonBuilder() - .startObject() - .field("date_field", "2015060210") - .endObject(); - assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true)); - - document = jsonBuilder() - .startObject() - .field("date_field", "2014060210") - .endObject(); - assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true)); - - refresh(); - - assertHitCount(client().prepareCount("test").get(), 2); - - CountResponse countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("date_field").from("2015010100").to("2015123123")).get(); - assertHitCount(countResponse, 1); - - countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("date_field").from(2015010100).to(2015123123)).get(); - assertHitCount(countResponse, 1); - - countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("date_field").from(2015010100).to(2015123123).timeZone("UTC")).get(); - assertHitCount(countResponse, 1); - } -} diff --git a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java index 92e68cec436..8e44c7a5442 100644 --- a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java +++ b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java @@ -23,21 +23,18 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; + import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; /** * */ public class JacksonLocationTests extends ESTestCase { - - @Test public void testLocationExtraction() throws IOException { // { // "index" : "test", diff --git a/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java b/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java index b1bddc1c549..bb531c41da8 100644 --- a/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java +++ b/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java @@ -28,20 +28,24 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.LocalDateTime; import org.joda.time.MutableDateTime; -import org.joda.time.format.*; -import org.junit.Test; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.DateTimeFormatterBuilder; +import org.joda.time.format.DateTimeParser; +import org.joda.time.format.ISODateTimeFormat; import java.util.Date; import java.util.Locale; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; /** * */ public class SimpleJodaTests extends ESTestCase { - - @Test public void testMultiParsers() { DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); DateTimeParser[] parsers = new DateTimeParser[3]; @@ -55,7 +59,6 @@ public class SimpleJodaTests extends ESTestCase { formatter.parseMillis("2009-11-15 14:12:12"); } - @Test public void testIsoDateFormatDateTimeNoMillisUTC() { DateTimeFormatter formatter = ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.UTC); long millis = formatter.parseMillis("1970-01-01T00:00:00Z"); @@ -63,7 +66,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(millis, equalTo(0l)); } - @Test public void testUpperBound() { MutableDateTime dateTime = new MutableDateTime(3000, 12, 31, 23, 59, 59, 999, DateTimeZone.UTC); DateTimeFormatter formatter = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC); @@ -74,7 +76,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(dateTime.toString(), equalTo("2000-01-01T23:59:59.999Z")); } - @Test public void testIsoDateFormatDateOptionalTimeUTC() { DateTimeFormatter formatter = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC); long millis = formatter.parseMillis("1970-01-01T00:00:00Z"); @@ -105,7 +106,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(millis, equalTo(TimeValue.timeValueHours(2).millis())); } - @Test public void testIsoVsCustom() { DateTimeFormatter formatter = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC); long millis = formatter.parseMillis("1970-01-01T00:00:00"); @@ -120,7 +120,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(millis, equalTo(0l)); } - @Test public void testWriteAndParse() { DateTimeFormatter dateTimeWriter = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC); DateTimeFormatter formatter = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC); @@ -128,7 +127,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(formatter.parseMillis(dateTimeWriter.print(date.getTime())), equalTo(date.getTime())); } - @Test public void testSlashInFormat() { FormatDateTimeFormatter formatter = Joda.forPattern("MM/yyyy"); formatter.parser().parseMillis("01/2001"); @@ -145,14 +143,12 @@ public class SimpleJodaTests extends ESTestCase { } } - @Test public void testMultipleFormats() { FormatDateTimeFormatter formatter = Joda.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"); long millis = formatter.parser().parseMillis("1970/01/01 00:00:00"); assertThat("1970/01/01 00:00:00", is(formatter.printer().print(millis))); } - @Test public void testMultipleDifferentFormats() { FormatDateTimeFormatter formatter = Joda.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"); String input = "1970/01/01 00:00:00"; @@ -166,7 +162,6 @@ public class SimpleJodaTests extends ESTestCase { Joda.forPattern(" date_time || date_time_no_millis"); } - @Test public void testInvalidPatterns() { expectInvalidPattern("does_not_exist_pattern", "Invalid format: [does_not_exist_pattern]: Illegal pattern component: o"); expectInvalidPattern("OOOOO", "Invalid format: [OOOOO]: Illegal pattern component: OOOOO"); @@ -186,7 +181,6 @@ public class SimpleJodaTests extends ESTestCase { } } - @Test public void testRounding() { long TIME = utcTimeInMillis("2009-02-03T01:01:01"); MutableDateTime time = new MutableDateTime(DateTimeZone.UTC); @@ -198,7 +192,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(time.dayOfMonth().roundFloor().toString(), equalTo("2009-02-03T00:00:00.000Z")); } - @Test public void testRoundingSetOnTime() { MutableDateTime time = new MutableDateTime(DateTimeZone.UTC); time.setRounding(time.getChronology().monthOfYear(), MutableDateTime.ROUND_FLOOR); @@ -227,7 +220,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(time.getMillis(), equalTo(utcTimeInMillis("2011-05-02T00:00:00.000Z"))); } - @Test public void testRoundingWithTimeZone() { MutableDateTime time = new MutableDateTime(DateTimeZone.UTC); time.setZone(DateTimeZone.forOffsetHours(-2)); @@ -252,7 +244,6 @@ public class SimpleJodaTests extends ESTestCase { assertThat(time.getMillis(), equalTo(utcTime.getMillis() - TimeValue.timeValueHours(22).millis())); } - @Test public void testThatEpochsCanBeParsed() { boolean parseMilliSeconds = randomBoolean(); @@ -274,7 +265,6 @@ public class SimpleJodaTests extends ESTestCase { } } - @Test public void testThatNegativeEpochsCanBeParsed() { // problem: negative epochs can be arbitrary in size... boolean parseMilliSeconds = randomBoolean(); @@ -305,16 +295,24 @@ public class SimpleJodaTests extends ESTestCase { } } - @Test(expected = IllegalArgumentException.class) public void testForInvalidDatesInEpochSecond() { FormatDateTimeFormatter formatter = Joda.forPattern("epoch_second"); - formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901", "12345678901234")); + try { + formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901", "12345678901234")); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Invalid format")); + } } - @Test(expected = IllegalArgumentException.class) public void testForInvalidDatesInEpochMillis() { FormatDateTimeFormatter formatter = Joda.forPattern("epoch_millis"); - formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901234")); + try { + formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901234")); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Invalid format")); + } } public void testThatEpochParserIsPrinter() { @@ -670,7 +668,6 @@ public class SimpleJodaTests extends ESTestCase { assertDateFormatParsingThrowingException("strictYearMonthDay", "2014-05-5"); } - @Test public void testThatRootObjectParsingIsStrict() throws Exception { String[] datesThatWork = new String[] { "2014/10/10", "2014/10/10 12:12:12", "2014-05-05", "2014-05-05T12:12:12.123Z" }; String[] datesThatShouldNotWork = new String[]{ "5-05-05", "2014-5-05", "2014-05-5", diff --git a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java index 77fd17eec80..9c702acb2c4 100644 --- a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java +++ b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java @@ -19,9 +19,34 @@ package org.elasticsearch.deps.lucene; -import org.apache.lucene.document.*; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; @@ -29,7 +54,6 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -40,8 +64,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class SimpleLuceneTests extends ESTestCase { - - @Test public void testSortValues() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -61,7 +83,6 @@ public class SimpleLuceneTests extends ESTestCase { } } - @Test public void testSimpleNumericOps() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -93,7 +114,6 @@ public class SimpleLuceneTests extends ESTestCase { * of the field. This means that heavily accessed fields that use field selector should be added * first (with load and break). */ - @Test public void testOrdering() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -122,7 +142,6 @@ public class SimpleLuceneTests extends ESTestCase { indexWriter.close(); } - @Test public void testBoost() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -157,7 +176,6 @@ public class SimpleLuceneTests extends ESTestCase { indexWriter.close(); } - @Test public void testNRTSearchOnClosedWriter() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -182,7 +200,6 @@ public class SimpleLuceneTests extends ESTestCase { * A test just to verify that term freqs are not stored for numeric fields. int1 is not storing termFreq * and int2 does. */ - @Test public void testNumericTermDocsFreqs() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); diff --git a/core/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java b/core/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java index 0e0ff0f84e5..66dc0542678 100644 --- a/core/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java +++ b/core/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java @@ -22,25 +22,31 @@ package org.elasticsearch.deps.lucene; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.vectorhighlight.CustomFieldQuery; import org.apache.lucene.search.vectorhighlight.FastVectorHighlighter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * */ public class VectorHighlighterTests extends ESTestCase { - - @Test public void testVectorHighlighter() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -63,7 +69,6 @@ public class VectorHighlighterTests extends ESTestCase { assertThat(fragment, equalTo("the big bad dog")); } - @Test public void testVectorHighlighterPrefixQuery() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -101,7 +106,6 @@ public class VectorHighlighterTests extends ESTestCase { assertThat(fragment, notNullValue()); } - @Test public void testVectorHighlighterNoStore() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -123,7 +127,6 @@ public class VectorHighlighterTests extends ESTestCase { assertThat(fragment, nullValue()); } - @Test public void testVectorHighlighterNoTermVector() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java index f717102bce9..2a1b146da92 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java @@ -116,6 +116,11 @@ public class DiscoveryModuleTests extends ModuleTestCase { } + @Override + public DiscoveryStats stats() { + return null; + } + @Override public Lifecycle.State lifecycleState() { return null; diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java index ca95e50685f..f9778f6438f 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java @@ -51,6 +51,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.store.IndicesStoreIntegrationIT; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration; import org.elasticsearch.test.disruption.*; @@ -61,7 +63,6 @@ import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.*; @@ -71,8 +72,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.*; @@ -135,9 +134,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { .put("discovery.zen.join_timeout", "10s") // still long to induce failures but to long so test won't time out .put(DiscoverySettings.PUBLISH_TIMEOUT, "1s") // <-- for hitting simulated network failures quickly .put("http.enabled", false) // just to make test quicker - .put("transport.host", "127.0.0.1") // only bind on one IF we use v4 here by default - .put("transport.bind_host", "127.0.0.1") - .put("transport.publish_host", "127.0.0.1") .put("gateway.local.list_timeout", "10s") // still long to induce failures but to long so test won't time out .build(); @@ -170,9 +166,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { /** * Test that no split brain occurs under partial network partition. See https://github.com/elasticsearch/elasticsearch/issues/2488 */ - @Test - public void failWithMinimumMasterNodesConfigured() throws Exception { - + public void testFailWithMinimumMasterNodesConfigured() throws Exception { List nodes = startCluster(3); // Figure out what is the elected master node @@ -213,7 +207,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { /** * Verify that nodes fault detection works after master (re) election */ - @Test public void testNodesFDAfterMasterReelection() throws Exception { startCluster(4); @@ -244,7 +237,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { /** * Verify that the proper block is applied when nodes loose their master */ - @Test public void testVerifyApiBlocksDuringPartition() throws Exception { startCluster(3); @@ -326,7 +318,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { * This test isolates the master from rest of the cluster, waits for a new master to be elected, restores the partition * and verifies that all node agree on the new cluster state */ - @Test public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception { final List nodes = startCluster(3); @@ -394,7 +385,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { *

    * This test is a superset of tests run in the Jepsen test suite, with the exception of versioned updates */ - @Test // NOTE: if you remove the awaitFix, make sure to port the test to the 1.x branch @LuceneTestCase.AwaitsFix(bugUrl = "needs some more work to stabilize") @TestLogging("action.index:TRACE,action.get:TRACE,discovery:TRACE,cluster.service:TRACE,indices.recovery:TRACE,indices.cluster:TRACE") @@ -483,7 +473,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { docsPerIndexer = 1 + randomInt(5); logger.info("indexing " + docsPerIndexer + " docs per indexer during partition"); countDownLatchRef.set(new CountDownLatch(docsPerIndexer * indexers.size())); - Collections.shuffle(semaphores); + Collections.shuffle(semaphores, random()); for (Semaphore semaphore : semaphores) { assertThat(semaphore.availablePermits(), equalTo(0)); semaphore.release(docsPerIndexer); @@ -530,7 +520,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { /** * Test that cluster recovers from a long GC on master that causes other nodes to elect a new one */ - @Test public void testMasterNodeGCs() throws Exception { List nodes = startCluster(3, -1); @@ -572,7 +561,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { * that already are following another elected master node. These nodes should reject this cluster state and prevent * them from following the stale master. */ - @Test public void testStaleMasterNotHijackingMajority() throws Exception { // 3 node cluster with unicast discovery and minimum_master_nodes set to 2: final List nodes = startCluster(3, 2); @@ -633,7 +621,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { // but will be queued and once the old master node un-freezes it gets executed. // The old master node will send this update + the cluster state where he is flagged as master to the other // nodes that follow the new master. These nodes should ignore this update. - internalCluster().getInstance(ClusterService.class, oldMasterNode).submitStateUpdateTask("sneaky-update", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + internalCluster().getInstance(ClusterService.class, oldMasterNode).submitStateUpdateTask("sneaky-update", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) throws Exception { return ClusterState.builder(currentState).build(); @@ -682,7 +670,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { * Test that a document which is indexed on the majority side of a partition, is available from the minority side, * once the partition is healed */ - @Test @TestLogging(value = "cluster.service:TRACE") public void testRejoinDocumentExistsInAllShardCopies() throws Exception { List nodes = startCluster(3); @@ -696,7 +683,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { ensureGreen("test"); nodes = new ArrayList<>(nodes); - Collections.shuffle(nodes, getRandom()); + Collections.shuffle(nodes, random()); String isolatedNode = nodes.get(0); String notIsolatedNode = nodes.get(1); @@ -738,8 +725,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { * The temporal unicast responses is empty. When partition is solved the one ping response contains a master node. * The rejoining node should take this master node and connect. */ - @Test - public void unicastSinglePingResponseContainsMaster() throws Exception { + public void testUnicastSinglePingResponseContainsMaster() throws Exception { List nodes = startCluster(4, -1, new int[]{0}); // Figure out what is the elected master node final String masterNode = internalCluster().getMasterName(); @@ -774,9 +760,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { assertMaster(masterNode, nodes); } - @Test @TestLogging("discovery.zen:TRACE,cluster.service:TRACE") - public void isolatedUnicastNodes() throws Exception { + public void testIsolatedUnicastNodes() throws Exception { List nodes = startCluster(4, -1, new int[]{0}); // Figure out what is the elected master node final String unicastTarget = nodes.get(0); @@ -814,7 +799,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { /** * Test cluster join with issues in cluster state publishing * */ - @Test public void testClusterJoinDespiteOfPublishingIssues() throws Exception { List nodes = startCluster(2, 1); @@ -828,23 +812,26 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { DiscoveryNodes discoveryNodes = internalCluster().getInstance(ClusterService.class, nonMasterNode).state().nodes(); + TransportService masterTranspotService = internalCluster().getInstance(TransportService.class, discoveryNodes.masterNode().getName()); + logger.info("blocking requests from non master [{}] to master [{}]", nonMasterNode, masterNode); MockTransportService nonMasterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, nonMasterNode); - nonMasterTransportService.addFailToSendNoConnectRule(discoveryNodes.masterNode()); + nonMasterTransportService.addFailToSendNoConnectRule(masterTranspotService); assertNoMaster(nonMasterNode); logger.info("blocking cluster state publishing from master [{}] to non master [{}]", masterNode, nonMasterNode); MockTransportService masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterNode); + TransportService localTransportService = internalCluster().getInstance(TransportService.class, discoveryNodes.localNode().getName()); if (randomBoolean()) { - masterTransportService.addFailToSendNoConnectRule(discoveryNodes.localNode(), PublishClusterStateAction.SEND_ACTION_NAME); + masterTransportService.addFailToSendNoConnectRule(localTransportService, PublishClusterStateAction.SEND_ACTION_NAME); } else { - masterTransportService.addFailToSendNoConnectRule(discoveryNodes.localNode(), PublishClusterStateAction.COMMIT_ACTION_NAME); + masterTransportService.addFailToSendNoConnectRule(localTransportService, PublishClusterStateAction.COMMIT_ACTION_NAME); } logger.info("allowing requests from non master [{}] to master [{}], waiting for two join request", nonMasterNode, masterNode); final CountDownLatch countDownLatch = new CountDownLatch(2); - nonMasterTransportService.addDelegate(discoveryNodes.masterNode(), new MockTransportService.DelegateTransport(nonMasterTransportService.original()) { + nonMasterTransportService.addDelegate(masterTranspotService, new MockTransportService.DelegateTransport(nonMasterTransportService.original()) { @Override public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { if (action.equals(MembershipAction.DISCOVERY_JOIN_ACTION_NAME)) { @@ -857,8 +844,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { countDownLatch.await(); logger.info("waiting for cluster to reform"); - masterTransportService.clearRule(discoveryNodes.localNode()); - nonMasterTransportService.clearRule(discoveryNodes.masterNode()); + masterTransportService.clearRule(localTransportService); + nonMasterTransportService.clearRule(localTransportService); ensureStableCluster(2); @@ -867,8 +854,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { internalCluster().stopRandomNonMasterNode(); } - - @Test public void testClusterFormingWithASlowNode() throws Exception { configureUnicastCluster(3, null, 2); @@ -894,7 +879,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { * sure that the node is removed form the cluster, that the node start pinging and that * the cluster reforms when healed. */ - @Test @TestLogging("discovery.zen:TRACE,action:TRACE") public void testNodeNotReachableFromMaster() throws Exception { startCluster(3); @@ -911,9 +895,9 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { logger.info("blocking request from master [{}] to [{}]", masterNode, nonMasterNode); MockTransportService masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterNode); if (randomBoolean()) { - masterTransportService.addUnresponsiveRule(internalCluster().getInstance(ClusterService.class, nonMasterNode).localNode()); + masterTransportService.addUnresponsiveRule(internalCluster().getInstance(TransportService.class, nonMasterNode)); } else { - masterTransportService.addFailToSendNoConnectRule(internalCluster().getInstance(ClusterService.class, nonMasterNode).localNode()); + masterTransportService.addFailToSendNoConnectRule(internalCluster().getInstance(TransportService.class, nonMasterNode)); } logger.info("waiting for [{}] to be removed from cluster", nonMasterNode); @@ -932,8 +916,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { * This test creates a scenario where a primary shard (0 replicas) relocates and is in POST_RECOVERY on the target * node but already deleted on the source node. Search request should still work. */ - @Test - public void searchWithRelocationAndSlowClusterStateProcessing() throws Exception { + public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Exception { configureUnicastCluster(3, null, 1); InternalTestCluster.Async masterNodeFuture = internalCluster().startMasterOnlyNodeAsync(); InternalTestCluster.Async node_1Future = internalCluster().startDataOnlyNodeAsync(); @@ -969,10 +952,9 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { // wait for relocation to finish endRelocationLatch.await(); // now search for the documents and see if we get a reply - assertThat(client().prepareCount().get().getCount(), equalTo(100l)); + assertThat(client().prepareSearch().setSize(0).get().getHits().totalHits(), equalTo(100l)); } - @Test public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception { // test for https://github.com/elastic/elasticsearch/issues/8823 configureUnicastCluster(2, null, 1); @@ -997,7 +979,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { // tests if indices are really deleted even if a master transition inbetween @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11665") - @Test public void testIndicesDeleted() throws Exception { configureUnicastCluster(3, null, 2); InternalTestCluster.Async> masterNodes= internalCluster().startMasterOnlyNodesAsync(2); @@ -1057,7 +1038,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { new NetworkDisconnectPartition(getRandom()), new SlowClusterStateProcessing(getRandom()) ); - Collections.shuffle(list); + Collections.shuffle(list, random()); setDisruptionScheme(list.get(0)); return list.get(0); } diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java index 9d1ce5c2af4..e7a10b0f62b 100644 --- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java @@ -39,7 +39,6 @@ import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -48,7 +47,6 @@ import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; public class ZenFaultDetectionTests extends ESTestCase { - protected ThreadPool threadPool; protected static final Version version0 = Version.fromId(/*0*/99); @@ -129,7 +127,6 @@ public class ZenFaultDetectionTests extends ESTestCase { return builder.build(); } - @Test public void testNodesFaultDetectionConnectOnDisconnect() throws InterruptedException { Settings.Builder settings = Settings.builder(); boolean shouldRetry = randomBoolean(); @@ -178,9 +175,7 @@ public class ZenFaultDetectionTests extends ESTestCase { assertThat(failureReason[0], matcher); } - @Test public void testMasterFaultDetectionConnectOnDisconnect() throws InterruptedException { - Settings.Builder settings = Settings.builder(); boolean shouldRetry = randomBoolean(); // make sure we don't ping @@ -198,7 +193,7 @@ public class ZenFaultDetectionTests extends ESTestCase { masterFD.addListener(new MasterFaultDetection.Listener() { @Override - public void onMasterFailure(DiscoveryNode masterNode, String reason) { + public void onMasterFailure(DiscoveryNode masterNode, Throwable cause, String reason) { failureNode[0] = masterNode; failureReason[0] = reason; notified.countDown(); @@ -216,4 +211,4 @@ public class ZenFaultDetectionTests extends ESTestCase { assertThat(failureReason[0], matcher); } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java index ec7d81b0409..efbfa800437 100644 --- a/core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration; import org.junit.Before; -import org.junit.Test; import java.util.List; import java.util.concurrent.ExecutionException; @@ -51,7 +50,6 @@ public class ZenUnicastDiscoveryIT extends ESIntegTestCase { discoveryConfig = null; } - @Test public void testNormalClusterForming() throws ExecutionException, InterruptedException { int currentNumNodes = randomIntBetween(3, 5); @@ -74,7 +72,6 @@ public class ZenUnicastDiscoveryIT extends ESIntegTestCase { } } - @Test // Without the 'include temporalResponses responses to nodesToConnect' improvement in UnicastZenPing#sendPings this // test fails, because 2 nodes elect themselves as master and the health request times out b/c waiting_for_nodes=N // can't be satisfied. diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java index eddc4d9bae7..c4955561905 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.*; @@ -50,12 +49,11 @@ public class ElectMasterServiceTests extends ESTestCase { nodes.add(node); } - Collections.shuffle(nodes, getRandom()); + Collections.shuffle(nodes, random()); return nodes; } - @Test - public void sortByMasterLikelihood() { + public void testSortByMasterLikelihood() { List nodes = generateRandomNodes(); List sortedNodes = electMasterService().sortByMasterLikelihood(nodes); assertEquals(nodes.size(), sortedNodes.size()); @@ -72,8 +70,7 @@ public class ElectMasterServiceTests extends ESTestCase { } - @Test - public void electMaster() { + public void testElectMaster() { List nodes = generateRandomNodes(); ElectMasterService service = electMasterService(); int min_master_nodes = randomIntBetween(0, nodes.size()); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java index 70c9430b531..42cb7cf43f4 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingService; @@ -86,7 +87,7 @@ public class NodeJoinControllerTests extends ESTestCase { nodes.add(node); pendingJoins.add(joinNodeAsync(node)); } - nodeJoinController.stopAccumulatingJoins(); + nodeJoinController.stopAccumulatingJoins("test"); for (int i = randomInt(5); i > 0; i--) { DiscoveryNode node = newNode(nodeId++); nodes.add(node); @@ -119,7 +120,7 @@ public class NodeJoinControllerTests extends ESTestCase { pendingJoins.add(future); assertThat(future.isDone(), equalTo(false)); } - nodeJoinController.stopAccumulatingJoins(); + nodeJoinController.stopAccumulatingJoins("test"); for (Future future : pendingJoins) { try { future.get(); @@ -243,7 +244,7 @@ public class NodeJoinControllerTests extends ESTestCase { // add - Collections.shuffle(nodesToJoin); + Collections.shuffle(nodesToJoin, random()); logger.debug("--> joining [{}] unique master nodes. Total of [{}] join requests", initialJoins, nodesToJoin.size()); for (DiscoveryNode node : nodesToJoin) { pendingJoins.add(joinNodeAsync(node)); @@ -268,7 +269,7 @@ public class NodeJoinControllerTests extends ESTestCase { } } - Collections.shuffle(nodesToJoin); + Collections.shuffle(nodesToJoin, random()); logger.debug("--> joining [{}] nodes, with repetition a total of [{}]", finalJoins, nodesToJoin.size()); for (DiscoveryNode node : nodesToJoin) { pendingJoins.add(joinNodeAsync(node)); @@ -284,7 +285,7 @@ public class NodeJoinControllerTests extends ESTestCase { logger.debug("--> testing accumulation stopped"); nodeJoinController.startAccumulatingJoins(); - nodeJoinController.stopAccumulatingJoins(); + nodeJoinController.stopAccumulatingJoins("test"); } @@ -315,7 +316,7 @@ public class NodeJoinControllerTests extends ESTestCase { nodesToJoin.add(node); } } - Collections.shuffle(nodesToJoin); + Collections.shuffle(nodesToJoin, random()); logger.debug("--> joining [{}] nodes, with repetition a total of [{}]", initialJoins, nodesToJoin.size()); for (DiscoveryNode node : nodesToJoin) { pendingJoins.add(joinNodeAsync(node)); @@ -487,17 +488,17 @@ public class NodeJoinControllerTests extends ESTestCase { @Override public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List startedShards, boolean withReroute) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } @Override public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, List failedShards) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } @Override - public RoutingAllocation.Result reroute(ClusterState clusterState, boolean debug) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + protected RoutingAllocation.Result reroute(ClusterState clusterState, String reason, boolean debug) { + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } } diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java index fbe6baab7c8..0b5f9997dba 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java @@ -22,6 +22,7 @@ package org.elasticsearch.discovery.zen; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; @@ -34,7 +35,11 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.LocalTransportAddress; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.discovery.Discovery; +import org.elasticsearch.discovery.DiscoveryStats; import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.discovery.zen.fd.FaultDetection; import org.elasticsearch.discovery.zen.membership.MembershipAction; @@ -48,7 +53,6 @@ import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportService; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.net.InetAddress; @@ -71,8 +75,6 @@ import static org.hamcrest.Matchers.sameInstance; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) @ESIntegTestCase.SuppressLocalMode public class ZenDiscoveryIT extends ESIntegTestCase { - - @Test public void testChangeRejoinOnMasterOptionIsDynamic() throws Exception { Settings nodeSettings = Settings.settingsBuilder() .put("discovery.type", "zen") // <-- To override the local setting if set externally @@ -88,7 +90,6 @@ public class ZenDiscoveryIT extends ESIntegTestCase { assertThat(zenDiscovery.isRejoinOnMasterGone(), is(false)); } - @Test public void testNoShardRelocationsOccurWhenElectedMasterNodeFails() throws Exception { Settings defaultSettings = Settings.builder() .put(FaultDetection.SETTING_PING_TIMEOUT, "1s") @@ -135,7 +136,6 @@ public class ZenDiscoveryIT extends ESIntegTestCase { assertThat(numRecoveriesAfterNewMaster, equalTo(numRecoveriesBeforeNewMaster)); } - @Test @TestLogging(value = "action.admin.cluster.health:TRACE") public void testNodeFailuresAreProcessedOnce() throws ExecutionException, InterruptedException, IOException { Settings defaultSettings = Settings.builder() @@ -180,7 +180,6 @@ public class ZenDiscoveryIT extends ESIntegTestCase { assertThat(statesFound, Matchers.hasSize(2)); } - @Test public void testNodeRejectsClusterStateWithWrongMasterNode() throws Exception { Settings settings = Settings.builder() .put("discovery.type", "zen") @@ -229,7 +228,6 @@ public class ZenDiscoveryIT extends ESIntegTestCase { assertThat(ExceptionsHelper.detailedMessage(reference.get()), containsString("cluster state from a different master than the current one, rejecting")); } - @Test public void testHandleNodeJoin_incompatibleMinVersion() throws UnknownHostException { Settings nodeSettings = Settings.settingsBuilder() .put("discovery.type", "zen") // <-- To override the local setting if set externally @@ -254,7 +252,6 @@ public class ZenDiscoveryIT extends ESIntegTestCase { assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [1.6.0] that is lower than the minimum compatible version [" + Version.V_2_0_0_beta1.minimumCompatibilityVersion() + "]")); } - @Test public void testJoinElectedMaster_incompatibleMinVersion() { ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.V_2_0_0_beta1); @@ -264,4 +261,37 @@ public class ZenDiscoveryIT extends ESIntegTestCase { assertThat("Can't join master because version 1.6.0 is lower than the minimum compatable version 2.0.0 can support", electMasterService.electMaster(Collections.singletonList(node)), nullValue()); } + public void testDiscoveryStats() throws IOException { + String expectedStatsJsonResponse = "{\n" + + " \"discovery\" : {\n" + + " \"cluster_state_queue\" : {\n" + + " \"total\" : 0,\n" + + " \"pending\" : 0,\n" + + " \"committed\" : 0\n" + + " }\n" + + " }\n" + + "}"; + + Settings nodeSettings = Settings.settingsBuilder() + .put("discovery.type", "zen") // <-- To override the local setting if set externally + .build(); + internalCluster().startNode(nodeSettings); + + logger.info("--> request node discovery stats"); + NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setDiscovery(true).get(); + assertThat(statsResponse.getNodes().length, equalTo(1)); + + DiscoveryStats stats = statsResponse.getNodes()[0].getDiscoveryStats(); + assertThat(stats.getQueueStats(), notNullValue()); + assertThat(stats.getQueueStats().getTotal(), equalTo(0)); + assertThat(stats.getQueueStats().getCommitted(), equalTo(0)); + assertThat(stats.getQueueStats().getPending(), equalTo(0)); + + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + builder.startObject(); + stats.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + assertThat(builder.string(), equalTo(expectedStatsJsonResponse)); + } } diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java index e35f8d55c8f..c54489bceba 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Collections; @@ -33,8 +32,6 @@ import java.util.Collections; import static org.hamcrest.Matchers.equalTo; public class ZenPingTests extends ESTestCase { - - @Test public void testPingCollection() { DiscoveryNode[] nodes = new DiscoveryNode[randomIntBetween(1, 30)]; long maxIdPerNode[] = new long[nodes.length]; @@ -65,7 +62,7 @@ public class ZenPingTests extends ESTestCase { } // shuffle - Collections.shuffle(pings); + Collections.shuffle(pings, random()); ZenPing.PingCollection collection = new ZenPing.PingCollection(); collection.addPings(pings.toArray(new ZenPing.PingResponse[pings.size()])); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java index 509740ee575..a3b2caca311 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java @@ -38,15 +38,12 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.netty.NettyTransport; -import org.junit.Test; import java.net.InetSocketAddress; import static org.hamcrest.Matchers.equalTo; public class UnicastZenPingIT extends ESTestCase { - - @Test public void testSimplePings() throws InterruptedException { Settings settings = Settings.EMPTY; int startPort = 11000 + randomIntBetween(0, 1000); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java index a8e9f00eb7f..bc5e97ce08e 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java @@ -162,6 +162,31 @@ public class PendingClusterStatesQueueTests extends ESTestCase { } } + public void testQueueStats() { + List states = randomStates(scaledRandomIntBetween(10, 100), "master"); + PendingClusterStatesQueue queue = createQueueWithStates(states); + assertThat(queue.stats().getTotal(), equalTo(states.size())); + assertThat(queue.stats().getPending(), equalTo(states.size())); + assertThat(queue.stats().getCommitted(), equalTo(0)); + + List committedContexts = randomCommitStates(queue); + assertThat(queue.stats().getTotal(), equalTo(states.size())); + assertThat(queue.stats().getPending(), equalTo(states.size() - committedContexts.size())); + assertThat(queue.stats().getCommitted(), equalTo(committedContexts.size())); + + ClusterState highestCommitted = null; + for (ClusterStateContext context : committedContexts) { + if (highestCommitted == null || context.state.supersedes(highestCommitted)) { + highestCommitted = context.state; + } + } + + queue.markAsProcessed(highestCommitted); + assertThat(queue.stats().getTotal(), equalTo(states.size() - committedContexts.size())); + assertThat(queue.stats().getPending(), equalTo(states.size() - committedContexts.size())); + assertThat(queue.stats().getCommitted(), equalTo(0)); + } + protected List randomCommitStates(PendingClusterStatesQueue queue) { List committedContexts = new ArrayList<>(); for (int iter = randomInt(queue.pendingStates.size() - 1); iter >= 0; iter--) { diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java index b787195dbcb..8dea09ba093 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -47,7 +48,6 @@ import org.elasticsearch.transport.*; import org.elasticsearch.transport.local.LocalTransport; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.*; @@ -61,7 +61,6 @@ import static org.hamcrest.Matchers.*; @TestLogging("discovery.zen.publish:TRACE") public class PublishClusterStateActionTests extends ESTestCase { - protected ThreadPool threadPool; protected Map nodes = new HashMap<>(); @@ -224,7 +223,6 @@ public class PublishClusterStateActionTests extends ESTestCase { return new MockPublishAction(settings, transportService, nodesProvider, listener, discoverySettings, ClusterName.DEFAULT); } - @Test public void testSimpleClusterStatePublishing() throws Exception { MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, Version.CURRENT).setAsMaster(); MockNode nodeB = createMockNode("nodeB", Settings.EMPTY, Version.CURRENT); @@ -304,9 +302,7 @@ public class PublishClusterStateActionTests extends ESTestCase { assertSameStateFromFull(nodeC.clusterState, clusterState); } - @Test public void testUnexpectedDiffPublishing() throws Exception { - MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, Version.CURRENT, new ClusterStateListener() { @Override public void clusterChanged(ClusterChangedEvent event) { @@ -330,7 +326,6 @@ public class PublishClusterStateActionTests extends ESTestCase { assertSameStateFromDiff(nodeB.clusterState, clusterState); } - @Test public void testDisablingDiffPublishing() throws Exception { Settings noDiffPublishingSettings = Settings.builder().put(DiscoverySettings.PUBLISH_DIFF_ENABLE, false).build(); @@ -368,7 +363,6 @@ public class PublishClusterStateActionTests extends ESTestCase { /** * Test not waiting on publishing works correctly (i.e., publishing times out) */ - @Test public void testSimultaneousClusterStatePublishing() throws Exception { int numberOfNodes = randomIntBetween(2, 10); int numberOfIterations = scaledRandomIntBetween(5, 50); @@ -416,9 +410,7 @@ public class PublishClusterStateActionTests extends ESTestCase { } } - @Test public void testSerializationFailureDuringDiffPublishing() throws Exception { - MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, Version.CURRENT, new ClusterStateListener() { @Override public void clusterChanged(ClusterChangedEvent event) { @@ -665,7 +657,7 @@ public class PublishClusterStateActionTests extends ESTestCase { logger.info("--> committing states"); - Collections.shuffle(states, random()); + Randomness.shuffle(states); for (ClusterState state : states) { node.action.handleCommitRequest(new PublishClusterStateAction.CommitClusterStateRequest(state.stateUUID()), channel); assertThat(channel.response.get(), equalTo((TransportResponse) TransportResponse.Empty.INSTANCE)); @@ -726,7 +718,7 @@ public class PublishClusterStateActionTests extends ESTestCase { private void assertProperMetaDataForVersion(MetaData metaData, long version) { for (long i = 1; i <= version; i++) { assertThat(metaData.index("test" + i), notNullValue()); - assertThat(metaData.index("test" + i).numberOfShards(), equalTo((int) i)); + assertThat(metaData.index("test" + i).getNumberOfShards(), equalTo((int) i)); } assertThat(metaData.index("test" + (version + 1)), nullValue()); assertThat(metaData.transientSettings().get("test"), equalTo(Long.toString(version))); diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java index 6cf8ba7ef36..47ab7deb995 100644 --- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java +++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java @@ -19,25 +19,26 @@ package org.elasticsearch.document; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.flush.FlushResponse; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; -import static org.elasticsearch.client.Requests.*; +import static org.elasticsearch.client.Requests.clearIndicesCacheRequest; +import static org.elasticsearch.client.Requests.getRequest; +import static org.elasticsearch.client.Requests.indexRequest; +import static org.elasticsearch.client.Requests.refreshRequest; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @@ -47,17 +48,14 @@ import static org.hamcrest.Matchers.nullValue; * */ public class DocumentActionsIT extends ESIntegTestCase { - protected void createIndex() { createIndex(getConcreteIndexName()); } - protected String getConcreteIndexName() { return "test"; } - @Test public void testIndexActions() throws Exception { createIndex(); NumShards numShards = getNumShards(getConcreteIndexName()); @@ -82,10 +80,10 @@ public class DocumentActionsIT extends ESIntegTestCase { assertNoFailures(clearIndicesCacheResponse); assertThat(clearIndicesCacheResponse.getSuccessfulShards(), equalTo(numShards.totalNumShards)); - logger.info("Optimizing"); + logger.info("Force Merging"); waitForRelocation(ClusterHealthStatus.GREEN); - OptimizeResponse optimizeResponse = optimize(); - assertThat(optimizeResponse.getSuccessfulShards(), equalTo(numShards.totalNumShards)); + ForceMergeResponse mergeResponse = forceMerge(); + assertThat(mergeResponse.getSuccessfulShards(), equalTo(numShards.totalNumShards)); GetResponse getResult; @@ -157,29 +155,21 @@ public class DocumentActionsIT extends ESIntegTestCase { // check count for (int i = 0; i < 5; i++) { // test successful - CountResponse countResponse = client().prepareCount("test").setQuery(termQuery("_type", "type1")).execute().actionGet(); + SearchResponse countResponse = client().prepareSearch("test").setSize(0).setQuery(termQuery("_type", "type1")).execute().actionGet(); assertNoFailures(countResponse); - assertThat(countResponse.getCount(), equalTo(2l)); + assertThat(countResponse.getHits().totalHits(), equalTo(2l)); assertThat(countResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(countResponse.getFailedShards(), equalTo(0)); - // test failed (simply query that can't be parsed) - try { - client().count(countRequest("test").source("{ term : { _type : \"type1 } }")).actionGet(); - } catch(SearchPhaseExecutionException e) { - assertThat(e.shardFailures().length, equalTo(numShards.numPrimaries)); - } - // count with no query is a match all one - countResponse = client().prepareCount("test").execute().actionGet(); + countResponse = client().prepareSearch("test").setSize(0).execute().actionGet(); assertThat("Failures " + countResponse.getShardFailures(), countResponse.getShardFailures() == null ? 0 : countResponse.getShardFailures().length, equalTo(0)); - assertThat(countResponse.getCount(), equalTo(2l)); + assertThat(countResponse.getHits().totalHits(), equalTo(2l)); assertThat(countResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries)); assertThat(countResponse.getFailedShards(), equalTo(0)); } } - @Test public void testBulk() throws Exception { createIndex(); NumShards numShards = getNumShards(getConcreteIndexName()); diff --git a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java index 529b60562bf..4f28cf19d7b 100644 --- a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java +++ b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java @@ -19,7 +19,7 @@ package org.elasticsearch.document; -import org.elasticsearch.action.ActionWriteResponse; +import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -32,7 +32,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -43,11 +42,9 @@ import static org.hamcrest.Matchers.nullValue; /** */ public class ShardInfoIT extends ESIntegTestCase { - private int numCopies; private int numNodes; - @Test public void testIndexAndDelete() throws Exception { prepareIndex(1); IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get(); @@ -56,15 +53,13 @@ public class ShardInfoIT extends ESIntegTestCase { assertShardInfo(deleteResponse); } - @Test public void testUpdate() throws Exception { prepareIndex(1); UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}").setDocAsUpsert(true).get(); assertShardInfo(updateResponse); } - @Test - public void testBulk_withIndexAndDeleteItems() throws Exception { + public void testBulkWithIndexAndDeleteItems() throws Exception { prepareIndex(1); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); for (int i = 0; i < 10; i++) { @@ -86,8 +81,7 @@ public class ShardInfoIT extends ESIntegTestCase { } } - @Test - public void testBulk_withUpdateItems() throws Exception { + public void testBulkWithUpdateItems() throws Exception { prepareIndex(1); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); for (int i = 0; i < 10; i++) { @@ -123,11 +117,11 @@ public class ShardInfoIT extends ESIntegTestCase { } } - private void assertShardInfo(ActionWriteResponse response) { + private void assertShardInfo(ReplicationResponse response) { assertShardInfo(response, numCopies, numNodes); } - private void assertShardInfo(ActionWriteResponse response, int expectedTotal, int expectedSuccessful) { + private void assertShardInfo(ReplicationResponse response, int expectedTotal, int expectedSuccessful) { assertThat(response.getShardInfo().getTotal(), greaterThanOrEqualTo(expectedTotal)); assertThat(response.getShardInfo().getSuccessful(), greaterThanOrEqualTo(expectedSuccessful)); } diff --git a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java index 06cf1e2cbfd..79f9efbb814 100644 --- a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java +++ b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.env; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.net.URL; @@ -34,7 +33,6 @@ import static org.hamcrest.CoreMatchers.nullValue; * Simple unit-tests for Environment.java */ public class EnvironmentTests extends ESTestCase { - public Environment newEnvironment() throws IOException { return newEnvironment(Settings.EMPTY); } @@ -47,7 +45,6 @@ public class EnvironmentTests extends ESTestCase { return new Environment(build); } - @Test public void testRepositoryResolution() throws IOException { Environment environment = newEnvironment(); assertThat(environment.resolveRepoFile("/test/repos/repo1"), nullValue()); diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 6b766e56277..acee455bb6c 100644 --- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -26,9 +26,10 @@ import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Files; @@ -39,16 +40,12 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.CoreMatchers.equalTo; @LuceneTestCase.SuppressFileSystems("ExtrasFS") // TODO: fix test to allow extras public class NodeEnvironmentTests extends ESTestCase { + private final IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("foo", Settings.EMPTY); - private final Settings idxSettings = Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).build(); - - @Test public void testNodeLockSingleEnvironment() throws IOException { NodeEnvironment env = newNodeEnvironment(Settings.builder() .put("node.max_local_storage_nodes", 1).build()); @@ -75,7 +72,6 @@ public class NodeEnvironmentTests extends ESTestCase { } - @Test public void testNodeLockMultipleEnvironment() throws IOException { final NodeEnvironment first = newNodeEnvironment(); String[] dataPaths = first.getSettings().getAsArray("path.data"); @@ -88,7 +84,6 @@ public class NodeEnvironmentTests extends ESTestCase { IOUtils.close(first, second); } - @Test public void testShardLock() throws IOException { final NodeEnvironment env = newNodeEnvironment(); @@ -105,9 +100,8 @@ public class NodeEnvironmentTests extends ESTestCase { Files.createDirectories(path.resolve("0")); Files.createDirectories(path.resolve("1")); } - Settings settings = settingsBuilder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10)).build(); try { - env.lockAllForIndex(new Index("foo"), settings, randomIntBetween(0, 10)); + env.lockAllForIndex(new Index("foo"), idxSettings, randomIntBetween(0, 10)); fail("shard 0 is locked"); } catch (LockObtainFailedException ex) { // expected @@ -117,7 +111,7 @@ public class NodeEnvironmentTests extends ESTestCase { // can lock again? env.shardLock(new ShardId("foo", 0)).close(); - List locks = env.lockAllForIndex(new Index("foo"), settings, randomIntBetween(0, 10)); + List locks = env.lockAllForIndex(new Index("foo"), idxSettings, randomIntBetween(0, 10)); try { env.shardLock(new ShardId("foo", 0)); fail("shard is locked"); @@ -129,7 +123,6 @@ public class NodeEnvironmentTests extends ESTestCase { env.close(); } - @Test public void testGetAllIndices() throws Exception { final NodeEnvironment env = newNodeEnvironment(); final int numIndices = randomIntBetween(1, 10); @@ -147,7 +140,6 @@ public class NodeEnvironmentTests extends ESTestCase { env.close(); } - @Test public void testDeleteSafe() throws IOException, InterruptedException { final NodeEnvironment env = newNodeEnvironment(); ShardLock fooLock = env.shardLock(new ShardId("foo", 0)); @@ -235,7 +227,6 @@ public class NodeEnvironmentTests extends ESTestCase { env.close(); } - @Test public void testStressShardLock() throws IOException, InterruptedException { class Int { int value = 0; @@ -252,7 +243,7 @@ public class NodeEnvironmentTests extends ESTestCase { flipFlop[i] = new AtomicInteger(); } - Thread[] threads = new Thread[randomIntBetween(2,5)]; + Thread[] threads = new Thread[randomIntBetween(2, 5)]; final CountDownLatch latch = new CountDownLatch(1); final int iters = scaledRandomIntBetween(10000, 100000); for (int i = 0; i < threads.length; i++) { @@ -265,7 +256,7 @@ public class NodeEnvironmentTests extends ESTestCase { fail(e.getMessage()); } for (int i = 0; i < iters; i++) { - int shard = randomIntBetween(0, counts.length-1); + int shard = randomIntBetween(0, counts.length - 1); try { try (ShardLock autoCloses = env.shardLock(new ShardId("foo", shard), scaledRandomIntBetween(0, 10))) { counts[shard].value++; @@ -297,23 +288,20 @@ public class NodeEnvironmentTests extends ESTestCase { env.close(); } - @Test public void testCustomDataPaths() throws Exception { String[] dataPaths = tmpPaths(); NodeEnvironment env = newNodeEnvironment(dataPaths, "/tmp", Settings.EMPTY); - Settings s1 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build(); - Settings s2 = Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build(); + IndexSettings s1 = IndexSettingsModule.newIndexSettings("myindex", Settings.EMPTY); + IndexSettings s2 = IndexSettingsModule.newIndexSettings("myindex", Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build()); ShardId sid = new ShardId("myindex", 0); Index i = new Index("myindex"); - assertFalse("no settings should mean no custom data path", NodeEnvironment.hasCustomDataPath(s1)); - assertTrue("settings with path_data should have a custom data path", NodeEnvironment.hasCustomDataPath(s2)); + assertFalse("no settings should mean no custom data path", s1.hasCustomDataPath()); + assertTrue("settings with path_data should have a custom data path", s2.hasCustomDataPath()); assertThat(env.availableShardPaths(sid), equalTo(env.availableShardPaths(sid))); - assertFalse(NodeEnvironment.hasCustomDataPath(s1)); assertThat(env.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/0/myindex/0"))); - assertTrue(NodeEnvironment.hasCustomDataPath(s2)); assertThat("shard paths with a custom data_path should contain only regular paths", env.availableShardPaths(sid), diff --git a/core/src/test/java/org/elasticsearch/exists/SimpleExistsIT.java b/core/src/test/java/org/elasticsearch/exists/SimpleExistsIT.java deleted file mode 100644 index 3046a85be0e..00000000000 --- a/core/src/test/java/org/elasticsearch/exists/SimpleExistsIT.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.exists; - -import org.elasticsearch.action.exists.ExistsResponse; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; - -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists; - -public class SimpleExistsIT extends ESIntegTestCase { - - - @Test - public void testExistsRandomPreference() throws Exception { - createIndex("test"); - indexRandom(true, client().prepareIndex("test", "type", "1").setSource("field", "value"), - client().prepareIndex("test", "type", "2").setSource("field", "value"), - client().prepareIndex("test", "type", "3").setSource("field", "value"), - client().prepareIndex("test", "type", "4").setSource("field", "value"), - client().prepareIndex("test", "type", "5").setSource("field", "value"), - client().prepareIndex("test", "type", "6").setSource("field", "value")); - - int iters = scaledRandomIntBetween(10, 100); - for (int i = 0; i < iters; i++) { - - String randomPreference = randomUnicodeOfLengthBetween(0, 4); - // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) - while (randomPreference.startsWith("_")) { - randomPreference = randomUnicodeOfLengthBetween(0, 4); - } - // id is not indexed, but lets see that we automatically convert to - ExistsResponse existsResponse = client().prepareExists().setQuery(QueryBuilders.matchAllQuery()).setPreference(randomPreference).get(); - assertExists(existsResponse, true); - } - } - - - @Test - public void simpleIpTests() throws Exception { - createIndex("test"); - - client().admin().indices().preparePutMapping("test").setType("type1") - .setSource(XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("from").field("type", "ip").endObject() - .startObject("to").field("type", "ip").endObject() - .endObject().endObject().endObject()) - .execute().actionGet(); - - client().prepareIndex("test", "type1", "1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefresh(true).execute().actionGet(); - - ExistsResponse existsResponse = client().prepareExists() - .setQuery(boolQuery().must(rangeQuery("from").lt("192.168.0.7")).must(rangeQuery("to").gt("192.168.0.7"))).get(); - - assertExists(existsResponse, true); - - existsResponse = client().prepareExists().setQuery(boolQuery().must(rangeQuery("from").lt("192.168.0.4")).must(rangeQuery("to").gt("192.168.0.11"))).get(); - - assertExists(existsResponse, false); - } - - @Test - public void simpleIdTests() { - createIndex("test"); - - client().prepareIndex("test", "type", "XXX1").setSource("field", "value").setRefresh(true).execute().actionGet(); - // id is not indexed, but lets see that we automatically convert to - ExistsResponse existsResponse = client().prepareExists().setQuery(QueryBuilders.termQuery("_id", "XXX1")).execute().actionGet(); - assertExists(existsResponse, true); - - existsResponse = client().prepareExists().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")).execute().actionGet(); - assertExists(existsResponse, true); - - existsResponse = client().prepareExists().setQuery(QueryBuilders.prefixQuery("_id", "XXX")).execute().actionGet(); - assertExists(existsResponse, true); - - existsResponse = client().prepareExists().setQuery(QueryBuilders.queryStringQuery("_id:XXX*").lowercaseExpandedTerms(false)).execute().actionGet(); - assertExists(existsResponse, true); - } - - @Test - public void simpleNonExistenceTests() throws Exception { - createIndex("test"); - client().prepareIndex("test", "type1", "1").setSource("field", 2).execute().actionGet(); - client().prepareIndex("test", "type1", "2").setSource("field", 5).execute().actionGet(); - client().prepareIndex("test", "type", "XXX1").setSource("str_field", "value").execute().actionGet(); - ensureGreen(); - refresh(); - ExistsResponse existsResponse = client().prepareExists("test").setQuery(QueryBuilders.rangeQuery("field").gte(6).lte(8)).execute().actionGet(); - assertExists(existsResponse, false); - - existsResponse = client().prepareExists("test").setQuery(QueryBuilders.queryStringQuery("_id:XXY*").lowercaseExpandedTerms(false)).execute().actionGet(); - assertExists(existsResponse, false); - } - -} diff --git a/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java b/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java index d2b1e6d4e38..54480e02b17 100644 --- a/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java +++ b/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java @@ -32,7 +32,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.ISODateTimeFormat; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -50,8 +49,6 @@ import static org.hamcrest.Matchers.notNullValue; /** */ public class ExplainActionIT extends ESIntegTestCase { - - @Test public void testSimple() throws Exception { assertAcked(prepareCreate("test") .addAlias(new Alias("alias")) @@ -116,8 +113,6 @@ public class ExplainActionIT extends ESIntegTestCase { assertThat(response.getId(), equalTo("2")); } - @SuppressWarnings("unchecked") - @Test public void testExplainWithFields() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen("test"); @@ -177,7 +172,6 @@ public class ExplainActionIT extends ESIntegTestCase { } @SuppressWarnings("unchecked") - @Test public void testExplainWitSource() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen("test"); @@ -213,7 +207,6 @@ public class ExplainActionIT extends ESIntegTestCase { assertThat(((Map) response.getGetResult().getSource().get("obj1")).get("field1").toString(), equalTo("value1")); } - @Test public void testExplainWithFilteredAlias() throws Exception { assertAcked(prepareCreate("test") .addMapping("test", "field2", "type=string") @@ -230,7 +223,6 @@ public class ExplainActionIT extends ESIntegTestCase { assertFalse(response.isMatch()); } - @Test public void testExplainWithFilteredAliasFetchSource() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .addMapping("test", "field2", "type=string") @@ -257,8 +249,7 @@ public class ExplainActionIT extends ESIntegTestCase { assertThat((String)response.getGetResult().getSource().get("field1"), equalTo("value1")); } - @Test - public void explainDateRangeInQueryString() { + public void testExplainDateRangeInQueryString() { createIndex("test"); String aMonthAgo = ISODateTimeFormat.yearMonthDay().print(new DateTime(DateTimeZone.UTC).minusMonths(1)); @@ -277,10 +268,7 @@ public class ExplainActionIT extends ESIntegTestCase { return randomBoolean() ? "test" : "alias"; } - - @Test - public void streamExplainTest() throws Exception { - + public void testStreamExplain() throws Exception { Explanation exp = Explanation.match(2f, "some explanation"); // write @@ -308,6 +296,5 @@ public class ExplainActionIT extends ESIntegTestCase { result = Lucene.readExplanation(esBuffer); assertThat(exp.toString(),equalTo(result.toString())); - } } diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java index d3d1d8b46c1..5f3b0567e32 100644 --- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.fieldstats; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.action.fieldstats.FieldStatsResponse; @@ -47,12 +48,12 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { )); ensureGreen("test"); - byte minByte = Byte.MAX_VALUE; - byte maxByte = Byte.MIN_VALUE; - short minShort = Short.MAX_VALUE; - short maxShort = Short.MIN_VALUE; - int minInt = Integer.MAX_VALUE; - int maxInt = Integer.MIN_VALUE; + long minByte = Byte.MAX_VALUE; + long maxByte = Byte.MIN_VALUE; + long minShort = Short.MAX_VALUE; + long maxShort = Short.MIN_VALUE; + long minInt = Integer.MAX_VALUE; + long maxInt = Integer.MIN_VALUE; long minLong = Long.MAX_VALUE; long maxLong = Long.MIN_VALUE; float minFloat = Float.MAX_VALUE; @@ -66,11 +67,11 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { List request = new ArrayList<>(numDocs); for (int doc = 0; doc < numDocs; doc++) { byte b = randomByte(); - minByte = (byte) Math.min(minByte, b); - maxByte = (byte) Math.max(maxByte, b); + minByte = Math.min(minByte, b); + maxByte = Math.max(maxByte, b); short s = randomShort(); - minShort = (short) Math.min(minShort, s); - maxShort = (short) Math.max(maxShort, s); + minShort = Math.min(minShort, s); + maxShort = Math.max(maxShort, s); int i = randomInt(); minInt = Math.min(minInt, i); maxInt = Math.max(maxInt, i); @@ -106,18 +107,18 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { assertThat(stats.getDensity(), equalTo(100)); } - assertThat(response.getAllFieldStats().get("byte").getMinValue(), equalTo(Byte.toString(minByte))); - assertThat(response.getAllFieldStats().get("byte").getMaxValue(), equalTo(Byte.toString(maxByte))); - assertThat(response.getAllFieldStats().get("short").getMinValue(), equalTo(Short.toString(minShort))); - assertThat(response.getAllFieldStats().get("short").getMaxValue(), equalTo(Short.toString(maxShort))); - assertThat(response.getAllFieldStats().get("integer").getMinValue(), equalTo(Integer.toString(minInt))); - assertThat(response.getAllFieldStats().get("integer").getMaxValue(), equalTo(Integer.toString(maxInt))); - assertThat(response.getAllFieldStats().get("long").getMinValue(), equalTo(Long.toString(minLong))); - assertThat(response.getAllFieldStats().get("long").getMaxValue(), equalTo(Long.toString(maxLong))); - assertThat(response.getAllFieldStats().get("float").getMinValue(), equalTo(Float.toString(minFloat))); - assertThat(response.getAllFieldStats().get("float").getMaxValue(), equalTo(Float.toString(maxFloat))); - assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(Double.toString(minDouble))); - assertThat(response.getAllFieldStats().get("double").getMaxValue(), equalTo(Double.toString(maxDouble))); + assertThat(response.getAllFieldStats().get("byte").getMinValue(), equalTo(minByte)); + assertThat(response.getAllFieldStats().get("byte").getMaxValue(), equalTo(maxByte)); + assertThat(response.getAllFieldStats().get("short").getMinValue(), equalTo(minShort)); + assertThat(response.getAllFieldStats().get("short").getMaxValue(), equalTo(maxShort)); + assertThat(response.getAllFieldStats().get("integer").getMinValue(), equalTo(minInt)); + assertThat(response.getAllFieldStats().get("integer").getMaxValue(), equalTo(maxInt)); + assertThat(response.getAllFieldStats().get("long").getMinValue(), equalTo(minLong)); + assertThat(response.getAllFieldStats().get("long").getMaxValue(), equalTo(maxLong)); + assertThat(response.getAllFieldStats().get("float").getMinValue(), equalTo(minFloat)); + assertThat(response.getAllFieldStats().get("float").getMaxValue(), equalTo(maxFloat)); + assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(minDouble)); + assertThat(response.getAllFieldStats().get("double").getMaxValue(), equalTo(maxDouble)); } public void testFieldStatsIndexLevel() throws Exception { @@ -139,32 +140,32 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { // default: FieldStatsResponse response = client().prepareFieldStats().setFields("value").get(); assertAllSuccessful(response); - assertThat(response.getAllFieldStats().get("value").getMinValue(), equalTo(Long.toString(-10))); - assertThat(response.getAllFieldStats().get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getAllFieldStats().get("value").getMinValue(), equalTo(-10l)); + assertThat(response.getAllFieldStats().get("value").getMaxValue(), equalTo(300l)); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMinValue(), equalTo(Long.toString(-10))); - assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMinValue(), equalTo(-10l)); + assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMaxValue(), equalTo(300l)); // Level: cluster response = client().prepareFieldStats().setFields("value").setLevel("cluster").get(); assertAllSuccessful(response); - assertThat(response.getAllFieldStats().get("value").getMinValue(), equalTo(Long.toString(-10))); - assertThat(response.getAllFieldStats().get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getAllFieldStats().get("value").getMinValue(), equalTo(-10l)); + assertThat(response.getAllFieldStats().get("value").getMaxValue(), equalTo(300l)); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMinValue(), equalTo(Long.toString(-10))); - assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMinValue(), equalTo(-10l)); + assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMaxValue(), equalTo(300l)); // Level: indices response = client().prepareFieldStats().setFields("value").setLevel("indices").get(); assertAllSuccessful(response); assertThat(response.getAllFieldStats(), nullValue()); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(3)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(Long.toString(-10))); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(Long.toString(100))); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(Long.toString(101))); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(Long.toString(200))); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(Long.toString(201))); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(-10l)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(100l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(101l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(200l)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(201l)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(300l)); // Illegal level option: try { @@ -200,10 +201,10 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { FieldStatsResponse response = client().prepareFieldStats().setFields("value").setLevel("indices").get(); assertAllSuccessful(response); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(Long.toString(1))); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(Long.toString(2))); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("a")); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo("b")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1l)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(2l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(new BytesRef("a"))); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(new BytesRef("b"))); } public void testFieldStatsFiltering() throws Exception { @@ -230,8 +231,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { assertAllSuccessful(response); assertThat(response.getAllFieldStats(), nullValue()); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(Long.toString(201))); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(201l)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(300l)); response = client().prepareFieldStats() .setFields("value") @@ -241,10 +242,10 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { assertAllSuccessful(response); assertThat(response.getAllFieldStats(), nullValue()); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(Long.toString(-10))); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(Long.toString(100))); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(Long.toString(101))); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(Long.toString(200))); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(-10l)); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(100l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(101l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(200l)); response = client().prepareFieldStats() .setFields("value") @@ -254,10 +255,10 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { assertAllSuccessful(response); assertThat(response.getAllFieldStats(), nullValue()); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(Long.toString(101))); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(Long.toString(200))); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(Long.toString(201))); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(101l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(200l)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(201l)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(300l)); response = client().prepareFieldStats() .setFields("value") @@ -285,8 +286,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { assertAllSuccessful(response); assertThat(response.getAllFieldStats(), nullValue()); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(Long.toString(101))); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(Long.toString(200))); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(101l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(200l)); response = client().prepareFieldStats() .setFields("value") @@ -296,8 +297,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { assertAllSuccessful(response); assertThat(response.getAllFieldStats(), nullValue()); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(Long.toString(201))); - assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(Long.toString(300))); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(201l)); + assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(300l)); } public void testIncompatibleFilter() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java index 5db3130913c..e25b95be578 100644 --- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java +++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java @@ -19,11 +19,15 @@ package org.elasticsearch.fieldstats; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.action.fieldstats.FieldStatsResponse; import org.elasticsearch.action.fieldstats.IndexConstraint; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import java.util.ArrayList; import java.util.List; @@ -63,7 +67,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { } public void testString() { - createIndex("test", Settings.EMPTY, "field", "value", "type=string"); + createIndex("test", Settings.EMPTY, "test", "field", "type=string"); for (int value = 0; value <= 10; value++) { client().prepareIndex("test", "test").setSource("field", String.format(Locale.ENGLISH, "%03d", value)).get(); } @@ -73,13 +77,15 @@ public class FieldStatsTests extends ESSingleNodeTestCase { assertThat(result.getAllFieldStats().get("field").getMaxDoc(), equalTo(11l)); assertThat(result.getAllFieldStats().get("field").getDocCount(), equalTo(11l)); assertThat(result.getAllFieldStats().get("field").getDensity(), equalTo(100)); - assertThat(result.getAllFieldStats().get("field").getMinValue(), equalTo(String.format(Locale.ENGLISH, "%03d", 0))); - assertThat(result.getAllFieldStats().get("field").getMaxValue(), equalTo(String.format(Locale.ENGLISH, "%03d", 10))); + assertThat(result.getAllFieldStats().get("field").getMinValue(), equalTo(new BytesRef(String.format(Locale.ENGLISH, "%03d", 0)))); + assertThat(result.getAllFieldStats().get("field").getMaxValue(), equalTo(new BytesRef(String.format(Locale.ENGLISH, "%03d", 10)))); + assertThat(result.getAllFieldStats().get("field").getMinValueAsString(), equalTo(String.format(Locale.ENGLISH, "%03d", 0))); + assertThat(result.getAllFieldStats().get("field").getMaxValueAsString(), equalTo(String.format(Locale.ENGLISH, "%03d", 10))); } public void testDouble() { String fieldName = "field"; - createIndex("test", Settings.EMPTY, fieldName, "value", "type=double"); + createIndex("test", Settings.EMPTY, "test", fieldName, "type=double"); for (double value = -1; value <= 9; value++) { client().prepareIndex("test", "test").setSource(fieldName, value).get(); } @@ -89,13 +95,14 @@ public class FieldStatsTests extends ESSingleNodeTestCase { assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11l)); assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11l)); assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100)); - assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(Double.toString(-1))); - assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(Double.toString(9))); + assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1d)); + assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d)); + assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Double.toString(-1))); } public void testFloat() { String fieldName = "field"; - createIndex("test", Settings.EMPTY, fieldName, "value", "type=float"); + createIndex("test", Settings.EMPTY, "test", fieldName, "type=float"); for (float value = -1; value <= 9; value++) { client().prepareIndex("test", "test").setSource(fieldName, value).get(); } @@ -105,12 +112,14 @@ public class FieldStatsTests extends ESSingleNodeTestCase { assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11l)); assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11l)); assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100)); - assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(Float.toString(-1))); - assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(Float.toString(9))); + assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1f)); + assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9f)); + assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Float.toString(-1))); + assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(Float.toString(9))); } private void testNumberRange(String fieldName, String fieldType, long min, long max) { - createIndex("test", Settings.EMPTY, fieldName, "value", "type=" + fieldType); + createIndex("test", Settings.EMPTY, "test", fieldName, "type=" + fieldType); for (long value = min; value <= max; value++) { client().prepareIndex("test", "test").setSource(fieldName, value).get(); } @@ -121,8 +130,10 @@ public class FieldStatsTests extends ESSingleNodeTestCase { assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(numDocs)); assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(numDocs)); assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100)); - assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(java.lang.Long.toString(min))); - assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(java.lang.Long.toString(max))); + assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(min)); + assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(max)); + assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(java.lang.Long.toString(min))); + assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(java.lang.Long.toString(max))); client().admin().indices().prepareDelete("test").get(); } @@ -169,11 +180,11 @@ public class FieldStatsTests extends ESSingleNodeTestCase { } public void testInvalidField() { - createIndex("test1", Settings.EMPTY, "field1", "value", "type=string"); + createIndex("test1", Settings.EMPTY, "test", "field1", "type=string"); client().prepareIndex("test1", "test").setSource("field1", "a").get(); client().prepareIndex("test1", "test").setSource("field1", "b").get(); - createIndex("test2", Settings.EMPTY, "field2", "value", "type=string"); + createIndex("test2", Settings.EMPTY, "test", "field2", "type=string"); client().prepareIndex("test2", "test").setSource("field2", "a").get(); client().prepareIndex("test2", "test").setSource("field2", "b").get(); client().admin().indices().prepareRefresh().get(); @@ -191,15 +202,15 @@ public class FieldStatsTests extends ESSingleNodeTestCase { assertThat(result.getTotalShards(), equalTo(2)); assertThat(result.getSuccessfulShards(), equalTo(1)); assertThat(result.getShardFailures()[0].reason(), either(containsString("field [field1] doesn't exist")).or(containsString("field [field2] doesn't exist"))); - assertThat(result.getIndicesMergedFieldStats().get("_all").get("field1").getMinValue(), equalTo("a")); - assertThat(result.getIndicesMergedFieldStats().get("_all").get("field1").getMaxValue(), equalTo("b")); + assertThat(result.getIndicesMergedFieldStats().get("_all").get("field1").getMinValueAsString(), equalTo("a")); + assertThat(result.getIndicesMergedFieldStats().get("_all").get("field1").getMaxValueAsString(), equalTo("b")); } public void testNumberFiltering() { createIndex("test1", Settings.EMPTY, "type", "value", "type=long"); - client().prepareIndex("test1", "test").setSource("value", 1).get(); + client().prepareIndex("test1", "test").setSource("value", 1l).get(); createIndex("test2", Settings.EMPTY, "type", "value", "type=long"); - client().prepareIndex("test2", "test").setSource("value", 3).get(); + client().prepareIndex("test2", "test").setSource("value", 3l).get(); client().admin().indices().prepareRefresh().get(); FieldStatsResponse response = client().prepareFieldStats() @@ -207,8 +218,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3l)); response = client().prepareFieldStats() .setFields("value") @@ -230,7 +241,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1l)); response = client().prepareFieldStats() .setFields("value") @@ -238,7 +249,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1l)); response = client().prepareFieldStats() .setFields("value") @@ -253,7 +264,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3l)); response = client().prepareFieldStats() .setFields("value") @@ -261,7 +272,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3l)); response = client().prepareFieldStats() .setFields("value") @@ -276,8 +287,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("1")); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("3")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(1l)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(3l)); response = client().prepareFieldStats() .setFields("value") @@ -288,10 +299,15 @@ public class FieldStatsTests extends ESSingleNodeTestCase { } public void testDateFiltering() { + DateTime dateTime1 = new DateTime(2014, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC); + String dateTime1Str = DateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().print(dateTime1); + DateTime dateTime2 = new DateTime(2014, 1, 2, 0, 0, 0, 0, DateTimeZone.UTC); + String dateTime2Str = DateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().print(dateTime2); + createIndex("test1", Settings.EMPTY, "type", "value", "type=date"); - client().prepareIndex("test1", "test").setSource("value", "2014-01-01T00:00:00.000Z").get(); + client().prepareIndex("test1", "test").setSource("value", dateTime1Str).get(); createIndex("test2", Settings.EMPTY, "type", "value", "type=date"); - client().prepareIndex("test2", "test").setSource("value", "2014-01-02T00:00:00.000Z").get(); + client().prepareIndex("test2", "test").setSource("value", dateTime2Str).get(); client().admin().indices().prepareRefresh().get(); FieldStatsResponse response = client().prepareFieldStats() @@ -299,8 +315,10 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(dateTime1.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(dateTime2.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(), equalTo(dateTime1Str)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(), equalTo(dateTime2Str)); response = client().prepareFieldStats() .setFields("value") @@ -315,7 +333,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(dateTime1.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(), equalTo(dateTime1Str)); response = client().prepareFieldStats() .setFields("value") @@ -323,7 +342,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(dateTime2.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(), equalTo(dateTime2Str)); response = client().prepareFieldStats() .setFields("value") @@ -338,7 +358,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(dateTime2.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(), equalTo(dateTime2Str)); response = client().prepareFieldStats() .setFields("value") @@ -346,8 +367,10 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(dateTime1.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(dateTime2.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(), equalTo(dateTime1Str)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(), equalTo(dateTime2Str)); response = client().prepareFieldStats() .setFields("value") @@ -355,8 +378,56 @@ public class FieldStatsTests extends ESSingleNodeTestCase { .setLevel("indices") .get(); assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2)); - assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo("2014-01-01T00:00:00.000Z")); - assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo("2014-01-02T00:00:00.000Z")); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(dateTime1.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValue(), equalTo(dateTime2.getMillis())); + assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(), equalTo(dateTime1Str)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(), equalTo(dateTime2Str)); + } + + public void testDateFiltering_optionalFormat() { + createIndex("test1", Settings.EMPTY, "type", "value", "type=date,format=strict_date_optional_time"); + client().prepareIndex("test1", "type").setSource("value", "2014-01-01T00:00:00.000Z").get(); + createIndex("test2", Settings.EMPTY, "type", "value", "type=date,format=strict_date_optional_time"); + client().prepareIndex("test2", "type").setSource("value", "2014-01-02T00:00:00.000Z").get(); + client().admin().indices().prepareRefresh().get(); + + DateTime dateTime1 = new DateTime(2014, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC); + DateTime dateTime2 = new DateTime(2014, 1, 2, 0, 0, 0, 0, DateTimeZone.UTC); + FieldStatsResponse response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, String.valueOf(dateTime1.getMillis()), "epoch_millis"), new IndexConstraint("value", MAX, LTE, String.valueOf(dateTime2.getMillis()), "epoch_millis")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(), equalTo("2014-01-02T00:00:00.000Z")); + + try { + client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GT, String.valueOf(dateTime1.getMillis()), "xyz")) + .setLevel("indices") + .get(); + fail("IllegalArgumentException should have been thrown"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Invalid format")); + } + } + + public void testEmptyIndex() { + createIndex("test1", Settings.EMPTY, "type", "value", "type=date"); + FieldStatsResponse response = client().prepareFieldStats() + .setFields("value") + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1)); + assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(0)); + + response = client().prepareFieldStats() + .setFields("value") + .setIndexContraints(new IndexConstraint("value", MIN, GTE, "1998-01-01T00:00:00.000Z")) + .setLevel("indices") + .get(); + assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0)); } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java index 88f47e7b83a..e81db454e02 100644 --- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -45,7 +44,6 @@ import static org.hamcrest.Matchers.sameInstance; /** */ public class AsyncShardFetchTests extends ESTestCase { - private final DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, Version.CURRENT); private final Response response1 = new Response(node1); private final Throwable failure1 = new Throwable("simulated failure 1"); @@ -69,7 +67,6 @@ public class AsyncShardFetchTests extends ESTestCase { terminate(threadPool); } - @Test public void testClose() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).build(); test.addSimulation(node1.getId(), response1); @@ -92,8 +89,6 @@ public class AsyncShardFetchTests extends ESTestCase { } } - - @Test public void testFullCircleSingleNodeSuccess() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).build(); test.addSimulation(node1.getId(), response1); @@ -113,7 +108,6 @@ public class AsyncShardFetchTests extends ESTestCase { assertThat(fetchData.getData().get(node1), sameInstance(response1)); } - @Test public void testFullCircleSingleNodeFailure() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).build(); // add a failed response for node1 @@ -146,7 +140,6 @@ public class AsyncShardFetchTests extends ESTestCase { assertThat(fetchData.getData().get(node1), sameInstance(response1)); } - @Test public void testTwoNodesOnSetup() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).put(node2).build(); test.addSimulation(node1.getId(), response1); @@ -175,7 +168,6 @@ public class AsyncShardFetchTests extends ESTestCase { assertThat(fetchData.getData().get(node2), sameInstance(response2)); } - @Test public void testTwoNodesOnSetupAndFailure() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).put(node2).build(); test.addSimulation(node1.getId(), response1); @@ -202,7 +194,6 @@ public class AsyncShardFetchTests extends ESTestCase { assertThat(fetchData.getData().get(node1), sameInstance(response1)); } - @Test public void testTwoNodesAddedInBetween() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).build(); test.addSimulation(node1.getId(), response1); diff --git a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java index e9b0e4a89f1..6b28b7f7897 100644 --- a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java @@ -26,26 +26,22 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; /** */ public class DanglingIndicesStateTests extends ESTestCase { - private static Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - @Test public void testCleanupWhenEmpty() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env); @@ -58,7 +54,6 @@ public class DanglingIndicesStateTests extends ESTestCase { } } - @Test public void testDanglingProcessing() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env); @@ -102,7 +97,6 @@ public class DanglingIndicesStateTests extends ESTestCase { } } - @Test public void testRenameOfIndexState() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env); diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 8c6e7a1dbf4..c804239c694 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -34,12 +34,11 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; @@ -50,9 +49,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { private final ESLogger logger = Loggers.getLogger(GatewayIndexStateIT.class); - @Test public void testMappingMetaDataParsed() throws Exception { - logger.info("--> starting 1 nodes"); internalCluster().startNode(); @@ -79,9 +76,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { assertThat(mappingMd.routing().required(), equalTo(true)); } - @Test public void testSimpleOpenClose() throws Exception { - logger.info("--> starting 2 nodes"); internalCluster().startNodesAsync(2).get(); @@ -94,7 +89,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { ensureGreen(); ClusterStateResponse stateResponse = client().admin().cluster().prepareState().execute().actionGet(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries)); assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards)); @@ -105,7 +100,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { client().admin().indices().prepareClose("test").get(); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); logger.info("--> verifying that the state is green"); @@ -131,7 +126,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { ensureGreen(); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries)); assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards)); @@ -142,7 +137,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { logger.info("--> closing test index..."); client().admin().indices().prepareClose("test").execute().actionGet(); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); logger.info("--> restarting nodes..."); @@ -151,7 +146,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { ensureGreen(); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); logger.info("--> trying to index into a closed index ..."); @@ -169,7 +164,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { ensureGreen(); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries)); assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards)); @@ -181,7 +176,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase { client().prepareIndex("test", "type1", "2").setSource("field1", "value1").execute().actionGet(); } - @Test public void testJustMasterNode() throws Exception { logger.info("--> cleaning nodes"); @@ -206,7 +200,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase { assertThat(clusterStateResponse.getState().metaData().hasIndex("test"), equalTo(true)); } - @Test public void testJustMasterNodeAndJustDataNode() throws Exception { logger.info("--> cleaning nodes"); @@ -223,7 +216,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase { client().prepareIndex("test", "type1").setSource("field1", "value1").setTimeout("100ms").execute().actionGet(); } - @Test public void testTwoNodesSingleDoc() throws Exception { logger.info("--> cleaning nodes"); @@ -246,7 +238,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { client().admin().indices().prepareClose("test").execute().actionGet(); ClusterStateResponse stateResponse = client().admin().cluster().prepareState().execute().actionGet(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); logger.info("--> opening the index..."); @@ -263,7 +255,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase { } } - @Test public void testDanglingIndicesConflictWithAlias() throws Exception { logger.info("--> starting two nodes"); internalCluster().startNodesAsync(2).get(); @@ -323,7 +314,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); } - @Test public void testDanglingIndices() throws Exception { logger.info("--> starting two nodes"); diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index d5cb28c385c..15ddc9dd771 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.Iterator; @@ -78,7 +77,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase { .nodes(generateDiscoveryNodes(masterEligible)) .build(); // new cluster state will have initializing shards on node 1 - RoutingTable routingTableNewClusterState = strategy.reroute(init).routingTable(); + RoutingTable routingTableNewClusterState = strategy.reroute(init, "reroute").routingTable(); if (initializing == false) { // pretend all initialized, nothing happened ClusterState temp = ClusterState.builder(init).routingTable(routingTableNewClusterState).metaData(metaDataOldClusterState).build(); @@ -131,7 +130,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase { .routingTable(routingTableIndexCreated) .nodes(generateDiscoveryNodes(masterEligible)) .build(); - RoutingTable routingTableInitializing = strategy.reroute(init).routingTable(); + RoutingTable routingTableInitializing = strategy.reroute(init, "reroute").routingTable(); ClusterState temp = ClusterState.builder(init).routingTable(routingTableInitializing).build(); RoutingTable routingTableStarted = strategy.applyStartedShards(temp, temp.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable(); @@ -184,14 +183,13 @@ public class GatewayMetaStateTests extends ESAllocationTestCase { if (expectMetaData) { assertThat(indices.hasNext(), equalTo(true)); - assertThat(indices.next().getNewMetaData().index(), equalTo("test")); + assertThat(indices.next().getNewMetaData().getIndex(), equalTo("test")); assertThat(indices.hasNext(), equalTo(false)); } else { assertThat(indices.hasNext(), equalTo(false)); } } - @Test public void testVersionChangeIsAlwaysWritten() throws Exception { // test that version changes are always written boolean initializing = randomBoolean(); @@ -203,7 +201,6 @@ public class GatewayMetaStateTests extends ESAllocationTestCase { assertState(event, stateInMemory, expectMetaData); } - @Test public void testNewShardsAlwaysWritten() throws Exception { // make sure new shards on data only node always written boolean initializing = true; @@ -215,7 +212,6 @@ public class GatewayMetaStateTests extends ESAllocationTestCase { assertState(event, stateInMemory, expectMetaData); } - @Test public void testAllUpToDateNothingWritten() throws Exception { // make sure state is not written again if we wrote already boolean initializing = false; @@ -227,7 +223,6 @@ public class GatewayMetaStateTests extends ESAllocationTestCase { assertState(event, stateInMemory, expectMetaData); } - @Test public void testNoWriteIfNothingChanged() throws Exception { boolean initializing = false; boolean versionChanged = false; @@ -239,7 +234,6 @@ public class GatewayMetaStateTests extends ESAllocationTestCase { assertState(newEventWithNothingChanged, stateInMemory, expectMetaData); } - @Test public void testWriteClosedIndex() throws Exception { // test that the closing of an index is written also on data only node boolean masterEligible = randomBoolean(); diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java index ccb07461015..486092fd401 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java @@ -24,14 +24,10 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.cluster.NoopClusterService; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; - public class GatewayServiceTests extends ESTestCase { - - private GatewayService createService(Settings.Builder settings) { return new GatewayService(Settings.builder() .put("http.enabled", "false") @@ -40,9 +36,7 @@ public class GatewayServiceTests extends ESTestCase { } - @Test public void testDefaultRecoverAfterTime() throws IOException { - // check that the default is not set GatewayService service = createService(Settings.builder()); assertNull(service.recoverAfterTime()); diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index 1d51c308869..441314b1e35 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -19,31 +19,20 @@ package org.elasticsearch.gateway; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.*; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.*; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import java.io.InputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -52,25 +41,13 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.StreamSupport; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.startsWith; +import static org.hamcrest.Matchers.*; @LuceneTestCase.SuppressFileSystems("ExtrasFS") // TODO: fix test to work with ExtrasFS public class MetaDataStateFormatTests extends ESTestCase { - - /** * Ensure we can read a pre-generated cluster state. */ @@ -139,7 +116,6 @@ public class MetaDataStateFormatTests extends ESTestCase { } } - @Test public void testVersionMismatch() throws IOException { Path[] dirs = new Path[randomIntBetween(1, 5)]; for (int i = 0; i < dirs.length; i++) { @@ -209,7 +185,7 @@ public class MetaDataStateFormatTests extends ESTestCase { long filePointer = raf.position(); ByteBuffer bb = ByteBuffer.wrap(new byte[1]); raf.read(bb); - + bb.flip(); byte oldValue = bb.get(0); byte newValue = (byte) ~oldValue; @@ -316,7 +292,6 @@ public class MetaDataStateFormatTests extends ESTestCase { assertEquals(state.clusterUUID(), uuid); } - @Test public void testLoadState() throws IOException { final ToXContent.Params params = ToXContent.EMPTY_PARAMS; final Path[] dirs = new Path[randomIntBetween(1, 5)]; @@ -355,7 +330,7 @@ public class MetaDataStateFormatTests extends ESTestCase { } List dirList = Arrays.asList(dirs); - Collections.shuffle(dirList, getRandom()); + Collections.shuffle(dirList, random()); MetaData loadedMetaData = format.loadLatestState(logger, dirList.toArray(new Path[0])); MetaData latestMetaData = meta.get(numStates-1); assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_"))); @@ -365,9 +340,9 @@ public class MetaDataStateFormatTests extends ESTestCase { for (IndexMetaData original : latestMetaData) { IndexMetaData deserialized = indices.get(original.getIndex()); assertThat(deserialized, notNullValue()); - assertThat(deserialized.version(), equalTo(original.version())); - assertThat(deserialized.numberOfReplicas(), equalTo(original.numberOfReplicas())); - assertThat(deserialized.numberOfShards(), equalTo(original.numberOfShards())); + assertThat(deserialized.getVersion(), equalTo(original.getVersion())); + assertThat(deserialized.getNumberOfReplicas(), equalTo(original.getNumberOfReplicas())); + assertThat(deserialized.getNumberOfShards(), equalTo(original.getNumberOfShards())); } // now corrupt all the latest ones and make sure we fail to load the state @@ -490,9 +465,9 @@ public class MetaDataStateFormatTests extends ESTestCase { long temp; result = string.hashCode(); result = 31 * result + aInt; - result = 31 * result + (int) (aLong ^ (aLong >>> 32)); + result = 31 * result + Long.hashCode(aLong); temp = Double.doubleToLongBits(aDouble); - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + Long.hashCode(temp); result = 31 * result + (aBoolean ? 1 : 0); return result; } diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java index 90e61e3bfc7..1c3ec79dd94 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.gateway; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -27,24 +26,24 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.Index; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; -import org.junit.Test; +import org.elasticsearch.test.InternalTestCluster.RestartCallback; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.LinkedHashMap; -import java.util.concurrent.Future; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.elasticsearch.test.InternalTestCluster.RestartCallback; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class MetaDataWriteDataNodesIT extends ESIntegTestCase { - - @Test public void testMetaWrittenAlsoOnDataNode() throws Exception { // this test checks that index state is written on data only nodes if they have a shard allocated String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); @@ -56,7 +55,6 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase { assertIndexInMetaState(masterNode, "test"); } - @Test public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception { // this test checks that the index state is removed from a data only node once all shards have been allocated away from it String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); @@ -70,19 +68,18 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase { index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); ensureGreen(); assertIndexInMetaState(node1, index); - assertIndexNotInMetaState(node2, index); + assertIndexDirectoryDeleted(node2, index); assertIndexInMetaState(masterNode, index); logger.debug("relocating index..."); client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node2)).get(); client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get(); ensureGreen(); - assertIndexNotInMetaState(node1, index); + assertIndexDirectoryDeleted(node1, index); assertIndexInMetaState(node2, index); assertIndexInMetaState(masterNode, index); } - @Test public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); final String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY); @@ -116,7 +113,7 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase { // make sure it was also written on red node although index is closed ImmutableOpenMap indicesMetaData = getIndicesMetaDataOnNode(dataNode); assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("integer_field")); - assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(indicesMetaData.get(index).getState(), equalTo(IndexMetaData.State.CLOSE)); /* Try the same and see if this also works if node was just restarted. * Each node holds an array of indices it knows of and checks if it should @@ -141,56 +138,49 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase { // make sure it was also written on red node although index is closed indicesMetaData = getIndicesMetaDataOnNode(dataNode); assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("float_field")); - assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(indicesMetaData.get(index).getState(), equalTo(IndexMetaData.State.CLOSE)); // finally check that meta data is also written of index opened again assertAcked(client().admin().indices().prepareOpen(index).get()); indicesMetaData = getIndicesMetaDataOnNode(dataNode); - assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(indicesMetaData.get(index).getState(), equalTo(IndexMetaData.State.OPEN)); } - protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception { - assertMetaState(nodeName, indexName, false); + protected void assertIndexDirectoryDeleted(final String nodeName, final String indexName) throws Exception { + assertBusy(() -> { + logger.info("checking if index directory exists..."); + assertFalse("Expecting index directory of " + indexName + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, indexName)); + } + ); } - protected void assertIndexInMetaState(String nodeName, String indexName) throws Exception { - assertMetaState(nodeName, indexName, true); - } - - - private void assertMetaState(final String nodeName, final String indexName, final boolean shouldBe) throws Exception { - awaitBusy(() -> { + protected void assertIndexInMetaState(final String nodeName, final String indexName) throws Exception { + assertBusy(() -> { logger.info("checking if meta state exists..."); try { - return shouldBe == metaStateExists(nodeName, indexName); + assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, getIndicesMetaDataOnNode(nodeName).containsKey(indexName)); } catch (Throwable t) { logger.info("failed to load meta state", t); - // TODO: loading of meta state fails rarely if the state is deleted while we try to load it - // this here is a hack, would be much better to use for example a WatchService - return false; + fail("could not load meta state"); } - }); - boolean inMetaSate = metaStateExists(nodeName, indexName); - if (shouldBe) { - assertTrue("expected " + indexName + " in meta state of node " + nodeName, inMetaSate); - } else { - assertFalse("expected " + indexName + " to not be in meta state of node " + nodeName, inMetaSate); } + ); } - private boolean metaStateExists(String nodeName, String indexName) throws Exception { - ImmutableOpenMap indices = getIndicesMetaDataOnNode(nodeName); - boolean inMetaSate = false; - for (ObjectObjectCursor index : indices) { - inMetaSate = inMetaSate || index.key.equals(indexName); + + private boolean indexDirectoryExists(String nodeName, String indexName) { + NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName); + for (Path path : nodeEnv.indexPaths(new Index(indexName))) { + if (Files.exists(path)) { + return true; + } } - return inMetaSate; + return false; } private ImmutableOpenMap getIndicesMetaDataOnNode(String nodeName) throws Exception { GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName); - MetaData nodeMetaData = null; - nodeMetaData = nodeMetaState.loadMetaState(); + MetaData nodeMetaData = nodeMetaState.loadMetaState(); return nodeMetaData.getIndices(); } } diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index de66f9519c7..8bcb9c45402 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -33,14 +32,12 @@ import static org.hamcrest.Matchers.nullValue; /** */ public class MetaStateServiceTests extends ESTestCase { - private static Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - @Test public void testWriteLoadIndex() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(randomSettings(), env); @@ -51,7 +48,6 @@ public class MetaStateServiceTests extends ESTestCase { } } - @Test public void testLoadMissingIndex() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(randomSettings(), env); @@ -59,7 +55,6 @@ public class MetaStateServiceTests extends ESTestCase { } } - @Test public void testWriteLoadGlobal() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(randomSettings(), env); @@ -72,7 +67,6 @@ public class MetaStateServiceTests extends ESTestCase { } } - @Test public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(randomSettings(), env); @@ -89,8 +83,7 @@ public class MetaStateServiceTests extends ESTestCase { } } - @Test - public void tesLoadGlobal() throws Exception { + public void testLoadGlobal() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(randomSettings(), env); diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index ce6a8b0a6e6..73cbb51faed 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.Collections; @@ -64,13 +63,11 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { * Verifies that the canProcess method of primary allocation behaves correctly * and processes only the applicable shard. */ - @Test public void testNoProcessReplica() { ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); } - @Test public void testNoProcessPrimayNotAllcoatedBefore() { ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, true, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); @@ -79,7 +76,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { /** * Tests that when async fetch returns that there is no data, the shard will not be allocated. */ - @Test public void testNoAsyncFetchData() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); boolean changed = testAllocator.allocateUnassigned(allocation); @@ -91,7 +87,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { /** * Tests when the node returns that no data was found for it (-1), it will be moved to ignore unassigned. */ - @Test public void testNoAllocationFound() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); testAllocator.addData(node1, -1); @@ -104,7 +99,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { /** * Tests when the node returns that no data was found for it (-1), it will be moved to ignore unassigned. */ - @Test public void testStoreException() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); testAllocator.addData(node1, 3, new CorruptIndexException("test", "test")); @@ -117,7 +111,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { /** * Tests that when there is a node to allocate the shard to, it will be allocated to it. */ - @Test public void testFoundAllocationAndAllocating() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); testAllocator.addData(node1, 10); @@ -132,7 +125,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { * Tests that when there is a node to allocate to, but it is throttling (and it is the only one), * it will be moved to ignore unassigned until it can be allocated to. */ - @Test public void testFoundAllocationButThrottlingDecider() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(throttleAllocationDeciders()); testAllocator.addData(node1, 10); @@ -146,7 +138,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { * Tests that when there is a node to be allocated to, but it the decider said "no", we still * force the allocation to it. */ - @Test public void testFoundAllocationButNoDecider() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(noAllocationDeciders()); testAllocator.addData(node1, 10); @@ -160,7 +151,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { /** * Tests that the highest version node is chosed for allocation. */ - @Test public void testAllocateToTheHighestVersion() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); testAllocator.addData(node1, 10).addData(node2, 12); @@ -175,7 +165,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { * Tests that when restoring from snapshot, even if we didn't find any node to allocate on, the shard * will remain in the unassigned list to be allocated later. */ - @Test public void testRestoreIgnoresNoNodesToAllocate() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder(shardId.getIndex()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)) @@ -187,7 +176,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), state.getRoutingNodes(), state.nodes(), null); + RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), state.getRoutingNodes(), state.nodes(), null, System.nanoTime()); testAllocator.addData(node1, -1).addData(node2, -1); boolean changed = testAllocator.allocateUnassigned(allocation); @@ -199,7 +188,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { * Tests that only when enough copies of the shard exists we are going to allocate it. This test * verifies that with same version (1), and quorum allocation. */ - @Test public void testEnoughCopiesFoundForAllocation() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder(shardId.getIndex()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2)) @@ -212,7 +200,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -220,7 +208,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node1, 1); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -228,7 +216,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node2, 1); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0)); @@ -241,7 +229,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { * Tests that only when enough copies of the shard exists we are going to allocate it. This test * verifies that even with different version, we treat different versions as a copy, and count them. */ - @Test public void testEnoughCopiesFoundForAllocationWithDifferentVersion() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder(shardId.getIndex()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2)) @@ -254,7 +241,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -262,7 +249,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node1, 1); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -270,7 +257,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node2, 2); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0)); @@ -279,7 +266,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node2.id())); } - @Test public void testAllocationOnAnyNodeWithSharedFs() { ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, false, ShardRoutingState.UNASSIGNED, 0, @@ -304,8 +290,6 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(nAndV.nodes, contains(node2, node1, node3)); } - - @Test public void testAllocationOnAnyNodeShouldPutNodesWithExceptionsLast() { ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, false, ShardRoutingState.UNASSIGNED, 0, @@ -344,7 +328,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), null); + return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); } class TestAllocator extends PrimaryShardAllocator { diff --git a/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java index b30c5096c4b..edde1720474 100644 --- a/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java @@ -20,14 +20,14 @@ package org.elasticsearch.gateway; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; -import org.junit.Test; import java.util.concurrent.TimeUnit; @@ -35,7 +35,6 @@ import static org.elasticsearch.client.Requests.clusterHealthRequest; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @@ -47,13 +46,11 @@ import static org.hamcrest.Matchers.notNullValue; */ @ClusterScope(numDataNodes =0, scope= Scope.TEST) public class QuorumGatewayIT extends ESIntegTestCase { - @Override protected int numberOfReplicas() { return 2; } - @Test public void testChangeInitialShardsRecovery() throws Exception { logger.info("--> starting 3 nodes"); final String[] nodes = internalCluster().startNodesAsync(3).get().toArray(new String[0]); @@ -71,9 +68,9 @@ public class QuorumGatewayIT extends ESIntegTestCase { refresh(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 2l); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2l); } - + final String nodeToRemove = nodes[between(0,2)]; logger.info("--> restarting 1 nodes -- kill 2"); internalCluster().fullRestart(new RestartCallback() { @@ -81,7 +78,7 @@ public class QuorumGatewayIT extends ESIntegTestCase { public Settings onNodeStopped(String nodeName) throws Exception { return Settings.EMPTY; } - + @Override public boolean doRestart(String nodeName) { return nodeToRemove.equals(nodeName); @@ -111,13 +108,11 @@ public class QuorumGatewayIT extends ESIntegTestCase { assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 2l); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2l); } } - @Test public void testQuorumRecovery() throws Exception { - logger.info("--> starting 3 nodes"); internalCluster().startNodesAsync(3).get(); // we are shutting down nodes - make sure we don't have 2 clusters if we test network @@ -136,7 +131,7 @@ public class QuorumGatewayIT extends ESIntegTestCase { refresh(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 2l); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2l); } logger.info("--> restart all nodes"); internalCluster().fullRestart(new RestartCallback() { @@ -158,18 +153,18 @@ public class QuorumGatewayIT extends ESIntegTestCase { activeClient.prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject().field("field", "value3").endObject()).get(); assertNoFailures(activeClient.admin().indices().prepareRefresh().get()); for (int i = 0; i < 10; i++) { - assertHitCount(activeClient.prepareCount().setQuery(matchAllQuery()).get(), 3l); + assertHitCount(activeClient.prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 3l); } } } - + }); logger.info("--> all nodes are started back, verifying we got the latest version"); logger.info("--> running cluster_health (wait for the shards to startup)"); ensureGreen(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 3l); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 3l); } } } diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index 5766ef30d0f..3dd6597a6eb 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.util.Set; @@ -37,7 +36,6 @@ import static org.hamcrest.Matchers.hasItem; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class RecoverAfterNodesIT extends ESIntegTestCase { - private final static TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { @@ -56,7 +54,6 @@ public class RecoverAfterNodesIT extends ESIntegTestCase { return internalCluster().client(name); } - @Test public void testRecoverAfterNodes() throws Exception { logger.info("--> start node (1)"); Client clientNode1 = startNode(settingsBuilder().put("gateway.recover_after_nodes", 3)); @@ -82,7 +79,6 @@ public class RecoverAfterNodesIT extends ESIntegTestCase { assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode3).isEmpty(), equalTo(true)); } - @Test public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start master_node (1)"); Client master1 = startNode(settingsBuilder().put("gateway.recover_after_master_nodes", 2).put("node.data", false).put("node.master", true)); @@ -119,7 +115,6 @@ public class RecoverAfterNodesIT extends ESIntegTestCase { assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data2).isEmpty(), equalTo(true)); } - @Test public void testRecoverAfterDataNodes() throws Exception { logger.info("--> start master_node (1)"); Client master1 = startNode(settingsBuilder().put("gateway.recover_after_data_nodes", 2).put("node.data", false).put("node.master", true)); diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryBackwardsCompatibilityIT.java index fbd8b973fad..2184fda47c4 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoveryBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryBackwardsCompatibilityIT.java @@ -19,8 +19,8 @@ package org.elasticsearch.gateway; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.settings.Settings; @@ -29,7 +29,6 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESBackcompatTestCase; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.HashMap; @@ -40,8 +39,6 @@ import static org.hamcrest.Matchers.greaterThan; @ESIntegTestCase.ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST, numClientNodes = 0, transportClientRatio = 0.0) public class RecoveryBackwardsCompatibilityIT extends ESBackcompatTestCase { - - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -60,7 +57,6 @@ public class RecoveryBackwardsCompatibilityIT extends ESBackcompatTestCase { return 3; } - @Test public void testReusePeerRecovery() throws Exception { assertAcked(prepareCreate("test").setSettings(Settings.builder().put(indexSettings()) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) @@ -83,7 +79,7 @@ public class RecoveryBackwardsCompatibilityIT extends ESBackcompatTestCase { logger.info("--> upgrade cluster"); logClusterState(); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.settingsBuilder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none")).execute().actionGet(); @@ -91,7 +87,7 @@ public class RecoveryBackwardsCompatibilityIT extends ESBackcompatTestCase { client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.settingsBuilder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "all")).execute().actionGet(); ensureGreen(); - countResponse = client().prepareCount().get(); + countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setDetailed(true).get(); diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 341139ba88b..01c76b465a9 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -34,10 +34,11 @@ import org.elasticsearch.indices.flush.SyncedFlushUtil; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.store.MockFSDirectoryService; -import org.junit.Test; +import org.elasticsearch.test.store.MockFSIndexStore; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -45,7 +46,6 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; @@ -55,8 +55,6 @@ import static org.hamcrest.Matchers.notNullValue; @ClusterScope(numDataNodes = 0, scope = Scope.TEST) public class RecoveryFromGatewayIT extends ESIntegTestCase { - - @Test public void testOneNodeRecoverFromGateway() throws Exception { internalCluster().startNode(); @@ -78,7 +76,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { .startArray("appAccountIds").value(14).endArray().endObject()).execute().actionGet(); refresh(); - assertHitCount(client().prepareCount().setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout internalCluster().fullRestart(); @@ -87,7 +85,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow(); client().admin().indices().prepareRefresh().execute().actionGet(); - assertHitCount(client().prepareCount().setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); internalCluster().fullRestart(); @@ -95,12 +93,10 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow(); client().admin().indices().prepareRefresh().execute().actionGet(); - assertHitCount(client().prepareCount().setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); } - @Test public void testSingleNodeNoFlush() throws Exception { - internalCluster().startNode(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -148,10 +144,10 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { refresh(); for (int i = 0; i <= randomInt(10); i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), value1Docs + value2Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("field", "value1")).get(), value1Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("field", "value2")).get(), value2Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("num", 179)).get(), value1Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), value1Docs + value2Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("field", "value1")).get(), value1Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("field", "value2")).get(), value2Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("num", 179)).get(), value1Docs); } if (!indexToAllShards) { // we have to verify primaries are started for them to be restored @@ -164,10 +160,10 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow(); for (int i = 0; i <= randomInt(10); i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), value1Docs + value2Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("field", "value1")).get(), value1Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("field", "value2")).get(), value2Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("num", 179)).get(), value1Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), value1Docs + value2Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("field", "value1")).get(), value1Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("field", "value2")).get(), value2Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("num", 179)).get(), value1Docs); } internalCluster().fullRestart(); @@ -177,23 +173,21 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow(); for (int i = 0; i <= randomInt(10); i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), value1Docs + value2Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("field", "value1")).get(), value1Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("field", "value2")).get(), value2Docs); - assertHitCount(client().prepareCount().setQuery(termQuery("num", 179)).get(), value1Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), value1Docs + value2Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("field", "value1")).get(), value1Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("field", "value2")).get(), value2Docs); + assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("num", 179)).get(), value1Docs); } } - - @Test - public void testSingleNodeWithFlush() throws Exception { + public void testSingleNodeWithFlush() throws Exception { internalCluster().startNode(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).execute().actionGet(); flush(); client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject().field("field", "value2").endObject()).execute().actionGet(); refresh(); - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout @@ -204,7 +198,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); } internalCluster().fullRestart(); @@ -213,13 +207,11 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); } } - @Test public void testTwoNodeFirstNodeCleared() throws Exception { - final String firstNode = internalCluster().startNode(); internalCluster().startNode(); @@ -232,7 +224,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureGreen(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); } internalCluster().fullRestart(new RestartCallback() { @@ -252,11 +244,10 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureGreen(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); } } - @Test public void testLatestVersionLoaded() throws Exception { // clean two nodes internalCluster().startNodesAsync(2, settingsBuilder().put("gateway.recover_after_nodes", 2).build()).get(); @@ -270,7 +261,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureGreen(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2); } String metaDataUuid = client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(); @@ -290,7 +281,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { client.admin().indices().prepareRefresh().execute().actionGet(); for (int i = 0; i < 10; i++) { - assertHitCount(client.prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 3); + assertHitCount(client.prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 3); } logger.info("--> add some metadata, additional type and template"); @@ -319,22 +310,21 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { assertThat(client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(), equalTo(metaDataUuid)); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 3); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 3); } ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.metaData().index("test").mapping("type2"), notNullValue()); assertThat(state.metaData().templates().get("template_1").template(), equalTo("te*")); - assertThat(state.metaData().index("test").aliases().get("test_alias"), notNullValue()); - assertThat(state.metaData().index("test").aliases().get("test_alias").filter(), notNullValue()); + assertThat(state.metaData().index("test").getAliases().get("test_alias"), notNullValue()); + assertThat(state.metaData().index("test").getAliases().get("test_alias").filter(), notNullValue()); } - @Test @TestLogging("gateway:TRACE,indices.recovery:TRACE,index.engine:TRACE") public void testReusePeerRecovery() throws Exception { final Settings settings = settingsBuilder() .put("action.admin.cluster.node.shutdown.delay", "10ms") - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) .put("gateway.recover_after_nodes", 4) .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CONCURRENT_RECOVERIES, 4) @@ -360,7 +350,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { } logger.info("Running Cluster Health"); ensureGreen(); - client().admin().indices().prepareOptimize("test").setMaxNumSegments(100).get(); // just wait for merges + client().admin().indices().prepareForceMerge("test").setMaxNumSegments(100).get(); // just wait for merges client().admin().indices().prepareFlush().setWaitIfOngoing(true).setForce(true).get(); boolean useSyncIds = randomBoolean(); @@ -438,7 +428,6 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { } } - @Test public void testRecoveryDifferentNodeOrderStartup() throws Exception { // we need different data paths so we make sure we start the second node fresh @@ -461,7 +450,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase { ensureYellow(); assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true)); - assertHitCount(client().prepareCount("test").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 1); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 1); } } diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java index e692b620d2f..9a053b36527 100644 --- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.gateway; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; @@ -28,15 +27,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; @@ -50,7 +42,6 @@ import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.test.ESAllocationTestCase; import org.junit.Before; -import org.junit.Test; import java.util.Collections; import java.util.EnumSet; @@ -64,7 +55,6 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { - private final ShardId shardId = new ShardId("test", 0); private final DiscoveryNode node1 = newNode("node1"); private final DiscoveryNode node2 = newNode("node2"); @@ -80,7 +70,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { /** * Verifies that when we are still fetching data in an async manner, the replica shard moves to ignore unassigned. */ - @Test public void testNoAsyncFetchData() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders()); testAllocator.clean(); @@ -93,7 +82,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { * Verifies that on index creation, we don't go and fetch data, but keep the replica shard unassigned to let * the shard allocator to allocate it. There isn't a copy around to find anyhow. */ - @Test public void testNoAsyncFetchOnIndexCreation() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), Settings.EMPTY, UnassignedInfo.Reason.INDEX_CREATED); testAllocator.clean(); @@ -107,7 +95,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { * Verifies that for anything but index creation, fetch data ends up being called, since we need to go and try * and find a better copy for the shard. */ - @Test public void testAsyncFetchOnAnythingButIndexCreation() { UnassignedInfo.Reason reason = RandomPicks.randomFrom(getRandom(), EnumSet.complementOf(EnumSet.of(UnassignedInfo.Reason.INDEX_CREATED))); RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), Settings.EMPTY, reason); @@ -119,7 +106,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { /** * Verifies that when there is a full match (syncId and files) we allocate it to matching node. */ - @Test public void testSimpleFullMatchAllocation() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders()); DiscoveryNode nodeToMatch = randomBoolean() ? node2 : node3; @@ -133,7 +119,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { /** * Verifies that when there is a sync id match but no files match, we allocate it to matching node. */ - @Test public void testSyncIdMatch() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders()); DiscoveryNode nodeToMatch = randomBoolean() ? node2 : node3; @@ -147,7 +132,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { /** * Verifies that when there is no sync id match but files match, we allocate it to matching node. */ - @Test public void testFileChecksumMatch() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders()); DiscoveryNode nodeToMatch = randomBoolean() ? node2 : node3; @@ -164,7 +148,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { * adding a replica and having that replica actually recover and cause the corruption to be identified * See CorruptFileTest# */ - @Test public void testNoPrimaryData() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders()); testAllocator.addData(node2, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")); @@ -177,7 +160,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { * Verifies that when there is primary data, but no data at all on other nodes, the shard keeps * unassigned to be allocated later on. */ - @Test public void testNoDataForReplicaOnAnyNode() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders()); testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")); @@ -190,7 +172,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { * Verifies that when there is primary data, but no matching data at all on other nodes, the shard keeps * unassigned to be allocated later on. */ - @Test public void testNoMatchingFilesForReplicaOnAnyNode() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders()); testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")) @@ -204,7 +185,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { * When there is no decision or throttle decision across all nodes for the shard, make sure the shard * moves to the ignore unassigned list. */ - @Test public void testNoOrThrottleDecidersRemainsInUnassigned() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(randomBoolean() ? noAllocationDeciders() : throttleAllocationDeciders()); testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")) @@ -218,7 +198,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { * Tests when the node to allocate to due to matching is being throttled, we move the shard to ignored * to wait till throttling on it is done. */ - @Test public void testThrottleWhenAllocatingToMatchingNode() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{new TestAllocateDecision(Decision.YES), new AllocationDecider(Settings.EMPTY) { @@ -237,7 +216,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); } - @Test public void testDelayedAllocation() { RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT); @@ -246,6 +224,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { // we sometime return empty list of files, make sure we test this as well testAllocator.addData(node2, false, null); } + AllocationService.updateLeftDelayOfUnassignedShards(allocation, Settings.EMPTY); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -254,13 +233,13 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT); testAllocator.addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")); + AllocationService.updateLeftDelayOfUnassignedShards(allocation, Settings.EMPTY); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node2.id())); } - @Test public void testCancelRecoveryBetterSyncId() { RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders()); testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")) @@ -272,7 +251,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).get(0).shardId(), equalTo(shardId)); } - @Test public void testNotCancellingRecoveryIfSyncedOnExistingRecovery() { RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders()); testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")) @@ -283,7 +261,6 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(0)); } - @Test public void testNotCancellingRecovery() { RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders()); testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")) @@ -313,7 +290,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY); + return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY, System.nanoTime()); } private RoutingAllocation onePrimaryOnNode1And1ReplicaRecovering(AllocationDeciders deciders) { @@ -332,7 +309,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY); + return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY, System.nanoTime()); } class TestAllocator extends ReplicaShardAllocator { diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index b26e3ec220a..f41f4adc74e 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -25,7 +25,11 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.get.*; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -38,7 +42,6 @@ import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.junit.Test; import java.io.IOException; import java.util.Collections; @@ -49,12 +52,17 @@ import java.util.Set; import static java.util.Collections.singleton; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; public class GetActionIT extends ESIntegTestCase { - - @Test - public void simpleGetTests() { + public void testSimpleGet() { assertAcked(prepareCreate("test") .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1)) .addAlias(new Alias("alias"))); @@ -182,8 +190,7 @@ public class GetActionIT extends ESIntegTestCase { return randomBoolean() ? "test" : "alias"; } - @Test - public void simpleMultiGetTests() throws Exception { + public void testSimpleMultiGet() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1))); ensureGreen(); @@ -235,8 +242,7 @@ public class GetActionIT extends ESIntegTestCase { assertThat(response.getResponses()[0].getResponse().getField("field").getValues().get(0).toString(), equalTo("value1")); } - @Test - public void realtimeGetWithCompressBackcompat() throws Exception { + public void testRealtimeGetWithCompressBackcompat() throws Exception { assertAcked(prepareCreate("test") .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1).put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)) .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("_source").field("compress", true).endObject().endObject().endObject())); @@ -255,7 +261,6 @@ public class GetActionIT extends ESIntegTestCase { assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo(fieldValue)); } - @Test public void testGetDocWithMultivaluedFields() throws Exception { String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties") @@ -330,7 +335,6 @@ public class GetActionIT extends ESIntegTestCase { assertThat(response.getFields().get("field").getValues().get(1).toString(), equalTo("2")); } - @Test public void testThatGetFromTranslogShouldWorkWithExcludeBackcompat() throws Exception { String index = "test"; String type = "type1"; @@ -364,7 +368,6 @@ public class GetActionIT extends ESIntegTestCase { assertThat(responseBeforeFlush.getSourceAsString(), is(responseAfterFlush.getSourceAsString())); } - @Test public void testThatGetFromTranslogShouldWorkWithIncludeBackcompat() throws Exception { String index = "test"; String type = "type1"; @@ -399,7 +402,6 @@ public class GetActionIT extends ESIntegTestCase { } @SuppressWarnings("unchecked") - @Test public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFieldsBackcompat() throws Exception { String index = "test"; String type = "type1"; @@ -455,7 +457,6 @@ public class GetActionIT extends ESIntegTestCase { assertThat(responseBeforeFlushWithExtraFilters.getSourceAsString(), is(responseAfterFlushWithExtraFilters.getSourceAsString())); } - @Test public void testGetWithVersion() { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1))); @@ -555,7 +556,6 @@ public class GetActionIT extends ESIntegTestCase { assertThat(response.getVersion(), equalTo(2l)); } - @Test public void testMultiGetWithVersion() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1))); @@ -671,8 +671,7 @@ public class GetActionIT extends ESIntegTestCase { assertThat(response.getResponses()[2].getResponse().getSourceAsMap().get("field").toString(), equalTo("value2")); } - @Test - public void testGetFields_metaData() throws Exception { + public void testGetFieldsMetaData() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent") @@ -726,8 +725,7 @@ public class GetActionIT extends ESIntegTestCase { assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1")); } - @Test - public void testGetFields_nonLeafField() throws Exception { + public void testGetFieldsNonLeafField() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .addMapping("my-type1", jsonBuilder().startObject().startObject("my-type1").startObject("properties") .startObject("field1").startObject("properties") @@ -757,9 +755,8 @@ public class GetActionIT extends ESIntegTestCase { } } - @Test @TestLogging("index.shard.service:TRACE,cluster.service:TRACE,action.admin.indices.flush:TRACE") - public void testGetFields_complexField() throws Exception { + public void testGetFieldsComplexField() throws Exception { assertAcked(prepareCreate("my-index") .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1)) .addMapping("my-type2", jsonBuilder().startObject().startObject("my-type2").startObject("properties") @@ -850,8 +847,7 @@ public class GetActionIT extends ESIntegTestCase { assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2")); } - @Test - public void testGet_allField() throws Exception { + public void testGetAllField() throws Exception { assertAcked(prepareCreate("test") .addAlias(new Alias("alias")) .addMapping("my-type1", jsonBuilder() @@ -875,7 +871,6 @@ public class GetActionIT extends ESIntegTestCase { assertThat(getResponse.getField("_all").getValue().toString(), equalTo("some text" + " ")); } - @Test public void testUngeneratedFieldsThatAreNeverStored() throws IOException { String createIndexSource = "{\n" + " \"settings\": {\n" + @@ -899,8 +894,7 @@ public class GetActionIT extends ESIntegTestCase { " \"input\": [\n" + " \"Nevermind\",\n" + " \"Nirvana\"\n" + - " ],\n" + - " \"output\": \"Nirvana - Nevermind\"\n" + + " ]\n" + " }\n" + "}"; @@ -916,7 +910,6 @@ public class GetActionIT extends ESIntegTestCase { assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList); } - @Test public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { String createIndexSource = "{\n" + " \"settings\": {\n" + @@ -955,7 +948,6 @@ public class GetActionIT extends ESIntegTestCase { assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); } - @Test public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabledBackcompat() throws IOException { indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(false, false); String[] fieldsList = {}; @@ -969,7 +961,6 @@ public class GetActionIT extends ESIntegTestCase { assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList); } - @Test public void testUngeneratedFieldsPartOfSourceEitherStoredOrSourceEnabledBackcompat() throws IOException { boolean stored = randomBoolean(); boolean sourceEnabled = true; @@ -1014,7 +1005,6 @@ public class GetActionIT extends ESIntegTestCase { client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); } - @Test public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { String createIndexSource = "{\n" + " \"settings\": {\n" + @@ -1048,7 +1038,6 @@ public class GetActionIT extends ESIntegTestCase { assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); } - @Test public void testGeneratedStringFieldsUnstored() throws IOException { indexSingleDocumentWithStringFieldsGeneratedFromText(false, randomBoolean()); String[] fieldsList = {"_all", "_field_names"}; @@ -1062,7 +1051,6 @@ public class GetActionIT extends ESIntegTestCase { assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList); } - @Test public void testGeneratedStringFieldsStored() throws IOException { indexSingleDocumentWithStringFieldsGeneratedFromText(true, randomBoolean()); String[] fieldsList = {"_all"}; @@ -1107,8 +1095,6 @@ public class GetActionIT extends ESIntegTestCase { index("test", "doc", "1", doc); } - - @Test public void testGeneratedNumberFieldsUnstored() throws IOException { indexSingleDocumentWithNumericFieldsGeneratedFromText(false, randomBoolean()); String[] fieldsList = {"token_count", "text.token_count"}; @@ -1122,7 +1108,6 @@ public class GetActionIT extends ESIntegTestCase { assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList); } - @Test public void testGeneratedNumberFieldsStored() throws IOException { indexSingleDocumentWithNumericFieldsGeneratedFromText(true, randomBoolean()); String[] fieldsList = {"token_count", "text.token_count"}; diff --git a/core/src/test/java/org/elasticsearch/http/netty/HttpPublishPortIT.java b/core/src/test/java/org/elasticsearch/http/netty/HttpPublishPortIT.java index 687c01520d0..4d73b52576a 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/HttpPublishPortIT.java +++ b/core/src/test/java/org/elasticsearch/http/netty/HttpPublishPortIT.java @@ -25,16 +25,16 @@ import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class HttpPublishPortIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.settingsBuilder() @@ -44,7 +44,6 @@ public class HttpPublishPortIT extends ESIntegTestCase { .build(); } - @Test public void testHttpPublishPort() throws Exception { NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setHttp(true).get(); assertThat(response.getNodes(), arrayWithSize(greaterThanOrEqualTo(1))); diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java index 0a4b057dd3d..cb111a71988 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java @@ -19,33 +19,42 @@ package org.elasticsearch.http.netty; +import org.elasticsearch.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.common.util.MockBigArrays; -import org.elasticsearch.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.threadpool.ThreadPool; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.*; -import org.jboss.netty.handler.codec.http.*; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelConfig; +import org.jboss.netty.channel.ChannelFactory; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelPipeline; +import org.jboss.netty.handler.codec.http.DefaultHttpHeaders; +import org.jboss.netty.handler.codec.http.HttpHeaders; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpVersion; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.net.SocketAddress; import java.util.ArrayList; import java.util.List; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class NettyHttpChannelTests extends ESTestCase { - private NetworkService networkService; private ThreadPool threadPool; private MockBigArrays bigArrays; @@ -69,7 +78,6 @@ public class NettyHttpChannelTests extends ESTestCase { } } - @Test public void testCorsEnabledWithoutAllowOrigins() { // Set up a HTTP transport with only the CORS enabled setting Settings settings = Settings.builder() @@ -93,7 +101,6 @@ public class NettyHttpChannelTests extends ESTestCase { assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue()); } - @Test public void testCorsEnabledWithAllowOrigins() { // create a http transport with CORS enabled and allow origin configured Settings settings = Settings.builder() diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java index 74e30d5f319..95cb5b46b5f 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java @@ -18,13 +18,13 @@ */ package org.elasticsearch.http.netty; -import java.nio.charset.StandardCharsets; import org.elasticsearch.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.http.netty.NettyHttpServerTransport.HttpChannelPipelineFactory; import org.elasticsearch.http.netty.pipelining.OrderedDownstreamChannelEvent; import org.elasticsearch.http.netty.pipelining.OrderedUpstreamMessageEvent; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -44,8 +44,8 @@ import org.jboss.netty.handler.codec.http.HttpResponse; import org.jboss.netty.handler.codec.http.QueryStringDecoder; import org.junit.After; import org.junit.Before; -import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -55,8 +55,9 @@ import java.util.concurrent.Executors; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.http.netty.NettyHttpClient.returnHttpResponseBodies; -import static org.elasticsearch.http.netty.NettyHttpServerTransport.HttpChannelPipelineFactory; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH; import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK; import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; @@ -65,7 +66,6 @@ import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; * This test just tests, if he pipelining works in general with out any connection the elasticsearch handler */ public class NettyHttpServerPipeliningTests extends ESTestCase { - private NetworkService networkService; private ThreadPool threadPool; private MockPageCacheRecycler mockPageCacheRecycler; @@ -90,9 +90,11 @@ public class NettyHttpServerPipeliningTests extends ESTestCase { } } - @Test public void testThatHttpPipeliningWorksWhenEnabled() throws Exception { - Settings settings = settingsBuilder().put("http.pipelining", true).build(); + Settings settings = settingsBuilder() + .put("http.pipelining", true) + .put("http.port", "0") + .build(); httpServerTransport = new CustomNettyHttpServerTransport(settings); httpServerTransport.start(); InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses()); @@ -105,9 +107,11 @@ public class NettyHttpServerPipeliningTests extends ESTestCase { } } - @Test public void testThatHttpPipeliningCanBeDisabled() throws Exception { - Settings settings = settingsBuilder().put("http.pipelining", false).build(); + Settings settings = settingsBuilder() + .put("http.pipelining", false) + .put("http.port", "0") + .build(); httpServerTransport = new CustomNettyHttpServerTransport(settings); httpServerTransport.start(); InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses()); @@ -216,7 +220,7 @@ public class NettyHttpServerPipeliningTests extends ESTestCase { Thread.sleep(timeout); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); - throw new RuntimeException(); + throw new RuntimeException(e1); } } diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java index 964f9851b09..f4ce3756e61 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java @@ -23,8 +23,9 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.jboss.netty.handler.codec.http.HttpResponse; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -34,8 +35,6 @@ import java.util.Locale; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.http.netty.NettyHttpClient.returnOpaqueIds; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasSize; @@ -44,13 +43,11 @@ import static org.hamcrest.Matchers.hasSize; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 1) public class NettyPipeliningDisabledIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put(Node.HTTP_ENABLED, true).put("http.pipelining", false).build(); } - @Test public void testThatNettyHttpServerDoesNotSupportPipelining() throws Exception { ensureGreen(); List requests = Arrays.asList("/", "/_nodes/stats", "/", "/_cluster/state", "/", "/_nodes", "/"); diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java index eafd242ec33..9e5971c1d4f 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java @@ -23,8 +23,9 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.jboss.netty.handler.codec.http.HttpResponse; -import org.junit.Test; import java.util.Arrays; import java.util.Collection; @@ -33,21 +34,17 @@ import java.util.Locale; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.http.netty.NettyHttpClient.returnOpaqueIds; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, numDataNodes = 1) public class NettyPipeliningEnabledIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put(Node.HTTP_ENABLED, true).put("http.pipelining", true).build(); } - @Test public void testThatNettyHttpServerSupportsPipelining() throws Exception { List requests = Arrays.asList("/", "/_nodes/stats", "/", "/_cluster/state", "/"); diff --git a/core/src/test/java/org/elasticsearch/http/netty/pipelining/HttpPipeliningHandlerTests.java b/core/src/test/java/org/elasticsearch/http/netty/pipelining/HttpPipeliningHandlerTests.java index f21153e45a7..28cdd241e15 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/pipelining/HttpPipeliningHandlerTests.java +++ b/core/src/test/java/org/elasticsearch/http/netty/pipelining/HttpPipeliningHandlerTests.java @@ -22,16 +22,31 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.test.ESTestCase; import org.jboss.netty.bootstrap.ClientBootstrap; import org.jboss.netty.bootstrap.ServerBootstrap; -import org.jboss.netty.channel.*; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.channel.ChannelPipeline; +import org.jboss.netty.channel.ChannelPipelineFactory; +import org.jboss.netty.channel.Channels; +import org.jboss.netty.channel.MessageEvent; +import org.jboss.netty.channel.SimpleChannelUpstreamHandler; import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; -import org.jboss.netty.handler.codec.http.*; +import org.jboss.netty.handler.codec.http.DefaultHttpChunk; +import org.jboss.netty.handler.codec.http.DefaultHttpRequest; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpClientCodec; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpRequestDecoder; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseEncoder; import org.jboss.netty.util.HashedWheelTimer; import org.jboss.netty.util.Timeout; import org.jboss.netty.util.TimerTask; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -43,7 +58,10 @@ import java.util.concurrent.atomic.AtomicBoolean; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.jboss.netty.buffer.ChannelBuffers.EMPTY_BUFFER; import static org.jboss.netty.buffer.ChannelBuffers.copiedBuffer; -import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.*; +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST; +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.TRANSFER_ENCODING; import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.CHUNKED; import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE; import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK; @@ -58,8 +76,6 @@ public class HttpPipeliningHandlerTests extends ESTestCase { private static final long RESPONSE_TIMEOUT = 10000L; private static final long CONNECTION_TIMEOUT = 10000L; private static final String CONTENT_TYPE_TEXT = "text/plain; charset=UTF-8"; - // TODO make me random - private static final InetSocketAddress HOST_ADDR = new InetSocketAddress(InetAddress.getLoopbackAddress(), 9080); private static final String PATH1 = "/1"; private static final String PATH2 = "/2"; private static final String SOME_RESPONSE_TEXT = "some response for "; @@ -72,6 +88,8 @@ public class HttpPipeliningHandlerTests extends ESTestCase { private HashedWheelTimer timer; + private InetSocketAddress boundAddress; + @Before public void startBootstraps() { clientBootstrap = new ClientBootstrap(new NioClientSocketChannelFactory()); @@ -100,7 +118,8 @@ public class HttpPipeliningHandlerTests extends ESTestCase { } }); - serverBootstrap.bind(HOST_ADDR); + Channel channel = serverBootstrap.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); + boundAddress = (InetSocketAddress) channel.getLocalAddress(); timer = new HashedWheelTimer(); } @@ -115,12 +134,11 @@ public class HttpPipeliningHandlerTests extends ESTestCase { clientBootstrap.releaseExternalResources(); } - @Test - public void shouldReturnMessagesInOrder() throws InterruptedException { + public void testShouldReturnMessagesInOrder() throws InterruptedException { responsesIn = new CountDownLatch(1); responses.clear(); - final ChannelFuture connectionFuture = clientBootstrap.connect(HOST_ADDR); + final ChannelFuture connectionFuture = clientBootstrap.connect(boundAddress); assertTrue(connectionFuture.await(CONNECTION_TIMEOUT)); final Channel clientChannel = connectionFuture.getChannel(); @@ -128,11 +146,11 @@ public class HttpPipeliningHandlerTests extends ESTestCase { // NetworkAddress.formatAddress makes a proper HOST header. final HttpRequest request1 = new DefaultHttpRequest( HTTP_1_1, HttpMethod.GET, PATH1); - request1.headers().add(HOST, NetworkAddress.formatAddress(HOST_ADDR)); + request1.headers().add(HOST, NetworkAddress.formatAddress(boundAddress)); final HttpRequest request2 = new DefaultHttpRequest( HTTP_1_1, HttpMethod.GET, PATH2); - request2.headers().add(HOST, NetworkAddress.formatAddress(HOST_ADDR)); + request2.headers().add(HOST, NetworkAddress.formatAddress(boundAddress)); clientChannel.write(request1); clientChannel.write(request2); diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index dd32b309a0f..d228ca4c0cc 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -18,40 +18,367 @@ */ package org.elasticsearch.index; +import org.apache.lucene.index.AssertingDirectoryReader; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.*; +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.Similarity; +import org.elasticsearch.Version; +import org.elasticsearch.cache.recycler.PageCacheRecycler; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.ModuleTestCase; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.env.ShardLock; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.cache.query.QueryCache; +import org.elasticsearch.index.cache.query.index.IndexQueryCache; +import org.elasticsearch.index.cache.query.none.NoneQueryCache; import org.elasticsearch.index.engine.EngineException; -import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexSearcherWrapper; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.index.store.IndexStore; +import org.elasticsearch.index.store.IndexStoreConfig; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.IndicesWarmer; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.indices.cache.query.IndicesQueryCache; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener; +import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.indices.memory.IndexingMemoryController; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.ScriptContextRegistry; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.engine.MockEngineFactory; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; -public class IndexModuleTests extends ModuleTestCase { +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; - public void testWrapperIsBound() { - IndexModule module = new IndexModule(IndexMetaData.PROTO); - assertInstanceBinding(module, IndexSearcherWrapper.class,(x) -> x == null); - module.indexSearcherWrapper = Wrapper.class; - assertBinding(module, IndexSearcherWrapper.class, Wrapper.class); +public class IndexModuleTests extends ESTestCase { + private Index index; + private Settings settings; + private IndexSettings indexSettings; + private Environment environment; + private NodeEnvironment nodeEnvironment; + private NodeServicesProvider nodeServicesProvider; + private IndexService.ShardStoreDeleter deleter = new IndexService.ShardStoreDeleter() { + @Override + public void deleteShardStore(String reason, ShardLock lock, IndexSettings indexSettings) throws IOException { + } + @Override + public void addPendingDelete(ShardId shardId, IndexSettings indexSettings) { + } + }; + private MapperRegistry mapperRegistry; + + static NodeServicesProvider newNodeServiceProvider(Settings settings, Environment environment, Client client, ScriptEngineService... scriptEngineServices) throws IOException { + // TODO this can be used in other place too - lets first refactor the IndicesQueriesRegistry + ThreadPool threadPool = new ThreadPool("test"); + IndicesWarmer warmer = new IndicesWarmer(settings, threadPool); + IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings); + CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + PageCacheRecycler recycler = new PageCacheRecycler(settings, threadPool); + BigArrays bigArrays = new BigArrays(recycler, circuitBreakerService); + IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndicesFieldDataCacheListener(circuitBreakerService), threadPool); + Set scriptEngines = new HashSet<>(); + scriptEngines.addAll(Arrays.asList(scriptEngineServices)); + ScriptService scriptService = new ScriptService(settings, environment, scriptEngines, new ResourceWatcherService(settings, threadPool), new ScriptContextRegistry(Collections.emptyList())); + IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry(settings, Collections.emptySet(), new NamedWriteableRegistry()); + // nocommit null: + IndexingMemoryController indexingMemoryController = new IndexingMemoryController(settings, threadPool, null); + return new NodeServicesProvider(threadPool, indicesQueryCache, null, warmer, bigArrays, client, scriptService, indicesQueriesRegistry, indicesFieldDataCache, circuitBreakerService, indexingMemoryController); } - public void testEngineFactoryBound() { - IndexModule module = new IndexModule(IndexMetaData.PROTO); - assertBinding(module, EngineFactory.class, InternalEngineFactory.class); - module.engineFactoryImpl = MockEngineFactory.class; - assertBinding(module, EngineFactory.class, MockEngineFactory.class); + @Override + public void setUp() throws Exception { + super.setUp(); + index = new Index("foo"); + settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("path.home", createTempDir().toString()).build(); + indexSettings = IndexSettingsModule.newIndexSettings(index, settings); + environment = new Environment(settings); + nodeServicesProvider = newNodeServiceProvider(settings, environment, null); + nodeEnvironment = new NodeEnvironment(settings, environment); + mapperRegistry = new IndicesModule().getMapperRegistry(); } - public void testOtherServiceBound() { - final IndexMetaData meta = IndexMetaData.builder(IndexMetaData.PROTO).index("foo").build(); - IndexModule module = new IndexModule(meta); - assertBinding(module, IndexService.class, IndexService.class); - assertBinding(module, IndexServicesProvider.class, IndexServicesProvider.class); - assertInstanceBinding(module, IndexMetaData.class, (x) -> x == meta); + @Override + public void tearDown() throws Exception { + super.tearDown(); + nodeEnvironment.close(); + nodeServicesProvider.getThreadPool().shutdown(); + if (nodeServicesProvider.getThreadPool().awaitTermination(10, TimeUnit.SECONDS) == false) { + nodeServicesProvider.getThreadPool().shutdownNow(); + } + } + + public void testWrapperIsBound() throws IOException { + IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); + module.setSearcherWrapper((s) -> new Wrapper()); + module.engineFactory.set(new MockEngineFactory(AssertingDirectoryReader.class)); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + assertTrue(indexService.getSearcherWrapper() instanceof Wrapper); + assertSame(indexService.getEngineFactory(), module.engineFactory.get()); + indexService.close("simon says", false); + } + + + public void testRegisterIndexStore() throws IOException { + final Index index = new Index("foo"); + final Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("path.home", createTempDir().toString()).put(IndexModule.STORE_TYPE, "foo_store").build(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); + IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); + module.addIndexStore("foo_store", FooStore::new); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + assertTrue(indexService.getIndexStore() instanceof FooStore); + try { + module.addIndexStore("foo_store", FooStore::new); + fail("already registered"); + } catch (IllegalArgumentException ex) { + // fine + } + indexService.close("simon says", false); + } + + public void testOtherServiceBound() throws IOException { + final AtomicBoolean atomicBoolean = new AtomicBoolean(false); + final IndexEventListener eventListener = new IndexEventListener() { + @Override + public void beforeIndexDeleted(IndexService indexService) { + atomicBoolean.set(true); + } + }; + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); + IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); + Consumer listener = (s) -> {}; + module.addIndexSettingsListener(listener); + module.addIndexEventListener(eventListener); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + IndexSettings x = indexService.getIndexSettings(); + assertEquals(x.getSettings().getAsMap(), indexSettings.getSettings().getAsMap()); + assertEquals(x.getIndex(), index); + assertSame(x.getUpdateListeners().get(0), listener); + indexService.getIndexEventListener().beforeIndexDeleted(null); + assertTrue(atomicBoolean.get()); + indexService.close("simon says", false); + } + + + public void testListener() throws IOException { + IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); + Consumer listener = (s) -> { + }; + module.addIndexSettingsListener(listener); + + try { + module.addIndexSettingsListener(listener); + fail("already added"); + } catch (IllegalStateException ex) { + + } + + try { + module.addIndexSettingsListener(null); + fail("must not be null"); + } catch (IllegalArgumentException ex) { + + } + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + IndexSettings x = indexService.getIndexSettings(); + assertEquals(1, x.getUpdateListeners().size()); + assertSame(x.getUpdateListeners().get(0), listener); + indexService.close("simon says", false); + } + + public void testAddSimilarity() throws IOException { + Settings indexSettings = Settings.settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.similarity.my_similarity.type", "test_similarity") + .put("index.similarity.my_similarity.key", "there is a key") + .put("path.home", createTempDir().toString()) + .build(); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); + module.addSimilarity("test_similarity", (string, settings) -> new SimilarityProvider() { + @Override + public String name() { + return string; + } + + @Override + public Similarity get() { + return new TestSimilarity(settings.get("key")); + } + }); + + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + SimilarityService similarityService = indexService.similarityService(); + assertNotNull(similarityService.getSimilarity("my_similarity")); + assertTrue(similarityService.getSimilarity("my_similarity").get() instanceof TestSimilarity); + assertEquals("my_similarity", similarityService.getSimilarity("my_similarity").name()); + assertEquals("there is a key", ((TestSimilarity) similarityService.getSimilarity("my_similarity").get()).key); + indexService.close("simon says", false); + } + + public void testSetupUnknownSimilarity() throws IOException { + Settings indexSettings = Settings.settingsBuilder() + .put("index.similarity.my_similarity.type", "test_similarity") + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("path.home", createTempDir().toString()) + .build(); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); + try { + module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + } catch (IllegalArgumentException ex) { + assertEquals("Unknown Similarity type [test_similarity] for [my_similarity]", ex.getMessage()); + } + } + + public void testSetupWithoutType() throws IOException { + Settings indexSettings = Settings.settingsBuilder() + .put("index.similarity.my_similarity.foo", "bar") + .put("path.home", createTempDir().toString()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); + try { + module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + } catch (IllegalArgumentException ex) { + assertEquals("Similarity [my_similarity] must have an associated type", ex.getMessage()); + } + } + + public void testCannotRegisterProvidedImplementations() { + Settings indexSettings = Settings.settingsBuilder() + .put("path.home", createTempDir().toString()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); + try { + module.registerQueryCache("index", IndexQueryCache::new); + fail("only once"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Can't register the same [query_cache] more than once for [index]"); + } + + try { + module.registerQueryCache("none", (settings, x) -> new NoneQueryCache(settings)); + fail("only once"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Can't register the same [query_cache] more than once for [none]"); + } + + try { + module.registerQueryCache("index", null); + fail("must not be null"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "provider must not be null"); + } + } + + public void testRegisterCustomQueryCache() throws IOException { + Settings indexSettings = Settings.settingsBuilder() + .put(IndexModule.QUERY_CACHE_TYPE, "custom") + .put("path.home", createTempDir().toString()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); + module.registerQueryCache("custom", (a, b) -> new CustomQueryCache()); + try { + module.registerQueryCache("custom", (a, b) -> new CustomQueryCache()); + fail("only once"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Can't register the same [query_cache] more than once for [custom]"); + } + + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + assertTrue(indexService.cache().query() instanceof CustomQueryCache); + indexService.close("simon says", false); + } + + public void testDefaultQueryCacheImplIsSelected() throws IOException { + Settings indexSettings = Settings.settingsBuilder() + .put("path.home", createTempDir().toString()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); + assertTrue(indexService.cache().query() instanceof IndexQueryCache); + indexService.close("simon says", false); + } + + class CustomQueryCache implements QueryCache { + + @Override + public void clear(String reason) { + } + + @Override + public void close() throws IOException { + } + + @Override + public Index index() { + return new Index("test"); + } + + @Override + public Weight doCache(Weight weight, QueryCachingPolicy policy) { + return weight; + } + } + + private static class TestSimilarity extends Similarity { + private final Similarity delegate = new BM25Similarity(); + private final String key; + + + public TestSimilarity(String key) { + if (key == null) { + throw new AssertionError("key is null"); + } + this.key = key; + } + + @Override + public long computeNorm(FieldInvertState state) { + return delegate.computeNorm(state); + } + + @Override + public SimWeight computeWeight(CollectionStatistics collectionStats, TermStatistics... termStats) { + return delegate.computeWeight(collectionStats, termStats); + } + + @Override + public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException { + return delegate.simScorer(weight, context); + } + } + + + + public static final class FooStore extends IndexStore { + + public FooStore(IndexSettings indexSettings, IndexStoreConfig config) { + super(indexSettings, config); + } } public static final class Wrapper extends IndexSearcherWrapper { @@ -62,7 +389,7 @@ public class IndexModuleTests extends ModuleTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return null; } } diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java index 5db3e546697..c41051ec59c 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java @@ -25,16 +25,14 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.junit.Test; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutionException; +import static org.hamcrest.Matchers.containsString; + public class IndexRequestBuilderIT extends ESIntegTestCase { - - - @Test public void testSetSource() throws InterruptedException, ExecutionException { createIndex("test"); ensureYellow(); @@ -52,10 +50,13 @@ public class IndexRequestBuilderIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.termQuery("test_field", "foobar")).get(); ElasticsearchAssertions.assertHitCount(searchResponse, builders.length); } - - @Test(expected = IllegalArgumentException.class) - public void testOddNumberOfSourceObjetc() { - client().prepareIndex("test", "test").setSource((Object)"test_field", (Object)"foobar", new Object()); - } + public void testOddNumberOfSourceObjects() { + try { + client().prepareIndex("test", "test").setSource("test_field", "foobar", new Object()); + fail ("Expected IllegalArgumentException"); + } catch(IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("The number of object passed must be even but was [3]")); + } + } } diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index 412141cdc8d..3544cb1a257 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -28,20 +28,19 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; /** Unit test(s) for IndexService */ public class IndexServiceTests extends ESSingleNodeTestCase { - - @Test public void testDetermineShadowEngineShouldBeUsed() { Settings regularSettings = Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) @@ -73,10 +72,9 @@ public class IndexServiceTests extends ESSingleNodeTestCase { return new CompressedXContent(builder.string()); } - @Test public void testFilteringAliases() throws Exception { - IndexService indexService = newIndexService(); + IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); add(indexService, "all", null); @@ -85,53 +83,59 @@ public class IndexServiceTests extends ESSingleNodeTestCase { assertThat(indexService.getMetaData().getAliases().containsKey("dogs"), equalTo(true)); assertThat(indexService.getMetaData().getAliases().containsKey("turtles"), equalTo(false)); - assertThat(indexService.aliasFilter("cats").toString(), equalTo("animal:cat")); - assertThat(indexService.aliasFilter("cats", "dogs").toString(), equalTo("animal:cat animal:dog")); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "cats").toString(), equalTo("animal:cat")); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "cats", "dogs").toString(), equalTo("animal:cat animal:dog")); // Non-filtering alias should turn off all filters because filters are ORed - assertThat(indexService.aliasFilter("all"), nullValue()); - assertThat(indexService.aliasFilter("cats", "all"), nullValue()); - assertThat(indexService.aliasFilter("all", "cats"), nullValue()); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "all"), nullValue()); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "cats", "all"), nullValue()); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "all", "cats"), nullValue()); add(indexService, "cats", filter(termQuery("animal", "feline"))); add(indexService, "dogs", filter(termQuery("animal", "canine"))); - assertThat(indexService.aliasFilter("dogs", "cats").toString(), equalTo("animal:canine animal:feline")); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline")); } - @Test public void testAliasFilters() throws Exception { IndexService indexService = newIndexService(); + IndexShard shard = indexService.getShard(0); + add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); - assertThat(indexService.aliasFilter(), nullValue()); - assertThat(indexService.aliasFilter("dogs").toString(), equalTo("animal:dog")); - assertThat(indexService.aliasFilter("dogs", "cats").toString(), equalTo("animal:dog animal:cat")); + assertThat(indexService.aliasFilter(shard.getQueryShardContext()), nullValue()); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs").toString(), equalTo("animal:dog")); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:dog animal:cat")); add(indexService, "cats", filter(termQuery("animal", "feline"))); add(indexService, "dogs", filter(termQuery("animal", "canine"))); - assertThat(indexService.aliasFilter("dogs", "cats").toString(), equalTo("animal:canine animal:feline")); + assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline")); } - @Test(expected = InvalidAliasNameException.class) public void testRemovedAliasFilter() throws Exception { IndexService indexService = newIndexService(); + IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); remove(indexService, "cats"); - indexService.aliasFilter("cats"); + try { + indexService.aliasFilter(shard.getQueryShardContext(), "cats"); + fail("Expected InvalidAliasNameException"); + } catch (InvalidAliasNameException e) { + assertThat(e.getMessage(), containsString("Invalid alias name [cats]")); + } } - - @Test public void testUnknownAliasFilter() throws Exception { IndexService indexService = newIndexService(); + IndexShard shard = indexService.getShard(0); + add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); try { - indexService.aliasFilter("unknown"); + indexService.aliasFilter(shard.getQueryShardContext(), "unknown"); fail(); } catch (InvalidAliasNameException e) { // all is well diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java new file mode 100644 index 00000000000..3f97fe402fa --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -0,0 +1,161 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +public class IndexSettingsTests extends ESTestCase { + + + public void testRunListener() { + Version version = VersionUtils.getPreviousVersion(); + Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build(); + final AtomicInteger integer = new AtomicInteger(0); + Consumer settingsConsumer = (s) -> integer.set(s.getAsInt("index.test.setting.int", -1)); + IndexMetaData metaData = newIndexMeta("index", theSettings); + IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY, Collections.singleton(settingsConsumer)); + assertEquals(version, settings.getIndexVersionCreated()); + assertEquals("0xdeadbeef", settings.getUUID()); + + assertEquals(1, settings.getUpdateListeners().size()); + assertFalse(settings.updateIndexMetaData(metaData)); + assertEquals(metaData.getSettings().getAsMap(), settings.getSettings().getAsMap()); + assertEquals(0, integer.get()); + assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 42).build()))); + assertEquals(42, integer.get()); + } + + public void testMergedSettingsArePassed() { + Version version = VersionUtils.getPreviousVersion(); + Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version) + .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build(); + final AtomicInteger integer = new AtomicInteger(0); + final StringBuilder builder = new StringBuilder(); + Consumer settingsConsumer = (s) -> { + integer.set(s.getAsInt("index.test.setting.int", -1)); + builder.append(s.get("index.not.updated", "")); + }; + IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY, Collections.singleton(settingsConsumer)); + assertEquals(0, integer.get()); + assertEquals("", builder.toString()); + IndexMetaData newMetaData = newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.test.setting.int", 42).build()); + assertTrue(settings.updateIndexMetaData(newMetaData)); + assertSame(settings.getIndexMetaData(), newMetaData); + assertEquals(42, integer.get()); + assertEquals("", builder.toString()); + integer.set(0); + assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.not.updated", "boom").build()))); + assertEquals("boom", builder.toString()); + assertEquals(42, integer.get()); + + } + + public void testListenerCanThrowException() { + Version version = VersionUtils.getPreviousVersion(); + Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build(); + final AtomicInteger integer = new AtomicInteger(0); + Consumer settingsConsumer = (s) -> integer.set(s.getAsInt("index.test.setting.int", -1)); + Consumer exceptionConsumer = (s) -> {throw new RuntimeException("boom");}; + List> list = new ArrayList<>(); + list.add(settingsConsumer); + list.add(exceptionConsumer); + Collections.shuffle(list, random()); + IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY, list); + assertEquals(0, integer.get()); + assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 42).build()))); + assertEquals(42, integer.get()); + } + + public void testSettingsConsistency() { + Version version = VersionUtils.getPreviousVersion(); + IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build()); + IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList()); + assertEquals(version, settings.getIndexVersionCreated()); + assertEquals("_na_", settings.getUUID()); + try { + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("index.test.setting.int", 42).build())); + fail("version has changed"); + } catch (IllegalArgumentException ex) { + assertTrue(ex.getMessage(), ex.getMessage().startsWith("version mismatch on settings update expected: ")); + } + + metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build()); + settings = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList()); + try { + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("index.test.setting.int", 42).build())); + fail("uuid missing/change"); + } catch (IllegalArgumentException ex) { + assertEquals("uuid mismatch on settings update expected: 0xdeadbeef but was: _na_", ex.getMessage()); + } + assertEquals(metaData.getSettings().getAsMap(), settings.getSettings().getAsMap()); + } + + + public void testNodeSettingsAreContained() { + final int numShards = randomIntBetween(1, 10); + final int numReplicas = randomIntBetween(0, 10); + Settings theSettings = Settings.settingsBuilder(). + put("index.foo.bar", 42) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards).build(); + + Settings nodeSettings = Settings.settingsBuilder().put("node.foo.bar", 43).build(); + final AtomicInteger indexValue = new AtomicInteger(0); + final AtomicInteger nodeValue = new AtomicInteger(0); + Consumer settingsConsumer = (s) -> {indexValue.set(s.getAsInt("index.foo.bar", -1)); nodeValue.set(s.getAsInt("node.foo.bar", -1));}; + IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), nodeSettings, Collections.singleton(settingsConsumer)); + assertEquals(numReplicas, settings.getNumberOfReplicas()); + assertEquals(numShards, settings.getNumberOfShards()); + assertEquals(0, indexValue.get()); + assertEquals(0, nodeValue.get()); + + assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.settingsBuilder(). + put("index.foo.bar", 42) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numReplicas + 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards).build()))); + + assertEquals(42, indexValue.get()); + assertEquals(43, nodeValue.get()); + assertSame(nodeSettings, settings.getNodeSettings()); + + + } + + private IndexMetaData newIndexMeta(String name, Settings indexSettings) { + Settings build = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(indexSettings) + .build(); + IndexMetaData metaData = IndexMetaData.builder(name).settings(build).build(); + return metaData; + } + + +} diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java index dd73e41c9f0..e214cea9cc1 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; @@ -52,12 +51,11 @@ import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import org.junit.Test; import java.io.IOException; import java.nio.file.Path; -import java.util.Collection; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; @@ -66,8 +64,13 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; /** * Tests for indices that use shadow replicas and a shared filesystem @@ -167,7 +170,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { } - @Test public void testIndexWithFewDocuments() throws Exception { final Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -246,7 +248,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertAcked(client().admin().indices().prepareDelete(IDX)); } - @Test public void testReplicaToPrimaryPromotion() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -305,7 +306,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertThat(gResp2.getField("foo").getValue().toString(), equalTo("foobar")); } - @Test public void testPrimaryRelocation() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -366,7 +366,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); } - @Test public void testPrimaryRelocationWithConcurrentIndexing() throws Throwable { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -439,7 +438,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertHitCount(resp, numPhase1Docs + numPhase2Docs); } - @Test public void testPrimaryRelocationWhereRecoveryFails() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = Settings.builder() @@ -478,7 +476,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { final AtomicBoolean keepFailing = new AtomicBoolean(true); MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, node1)); - mockTransportService.addDelegate(internalCluster().getInstance(Discovery.class, node3).localNode(), + mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, node3), new MockTransportService.DelegateTransport(mockTransportService.original()) { @Override @@ -535,7 +533,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertHitCount(resp, counter.get()); } - @Test public void testIndexWithShadowReplicasCleansUp() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -576,7 +573,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { * Tests that shadow replicas can be "naturally" rebalanced and relocated * around the cluster. By "naturally" I mean without using the reroute API */ - @Test public void testShadowReplicaNaturalRelocation() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -630,7 +626,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertPathHasBeenCleared(dataPath); } - @Test public void testShadowReplicasUsingFieldData() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -699,7 +694,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { }); } - @Test public void testIndexOnSharedFSRecoversToAnyNode() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java new file mode 100644 index 00000000000..f62d44df43f --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index; + +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.settings.Validator; +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; + +import java.util.Collection; +import java.util.Collections; +import java.util.function.Consumer; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +@ClusterScope(scope = SUITE, numDataNodes = 1, numClientNodes = 0) +public class SettingsListenerIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return pluginList(SettingsListenerPlugin.class); + } + + public static class SettingsListenerPlugin extends Plugin { + private final SettingsTestingService service = new SettingsTestingService(); + + /** + * The name of the plugin. + */ + @Override + public String name() { + return "settings-listener"; + } + + /** + * The description of the plugin. + */ + @Override + public String description() { + return "Settings Listenern Plugin"; + } + + public void onModule(ClusterModule clusterModule) { + clusterModule.registerIndexDynamicSetting("index.test.new.setting", Validator.INTEGER); + } + + @Override + public void onIndexModule(IndexModule module) { + if (module.getIndex().getName().equals("test")) { // only for the test index + module.addIndexSettingsListener(service); + service.accept(module.getSettings()); + } + } + + @Override + public Collection nodeModules() { + return Collections.singletonList(new SettingsListenerModule(service)); + } + } + + public static class SettingsListenerModule extends AbstractModule { + private final SettingsTestingService service; + + public SettingsListenerModule(SettingsTestingService service) { + this.service = service; + } + + @Override + protected void configure() { + bind(SettingsTestingService.class).toInstance(service); + } + } + + public static class SettingsTestingService implements Consumer { + public volatile int value; + + @Override + public void accept(Settings settings) { + value = settings.getAsInt("index.test.new.setting", -1); + } + } + + public void testListener() { + assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() + .put("index.test.new.setting", 21) + .build()).get()); + + for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + assertEquals(21, instance.value); + } + + client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() + .put("index.test.new.setting", 42)).get(); + for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + assertEquals(42, instance.value); + } + + assertAcked(client().admin().indices().prepareCreate("other").setSettings(Settings.builder() + .put("index.test.new.setting", 21) + .build()).get()); + + for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + assertEquals(42, instance.value); + } + + client().admin().indices().prepareUpdateSettings("other").setSettings(Settings.builder() + .put("index.test.new.setting", 84)).get(); + + for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) { + assertEquals(42, instance.value); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java b/core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java index aed603ce394..21ecdf710b7 100644 --- a/core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java +++ b/core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java @@ -19,7 +19,7 @@ package org.elasticsearch.index; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.cluster.ClusterState; @@ -34,7 +34,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; -import org.junit.Test; import java.util.Collection; import java.util.List; @@ -76,7 +75,6 @@ public class TransportIndexFailuresIT extends ESIntegTestCase { return 1; } - @Test public void testNetworkPartitionDuringReplicaIndexOp() throws Exception { final String INDEX = "testidx"; @@ -117,12 +115,12 @@ public class TransportIndexFailuresIT extends ESIntegTestCase { logger.info("--> preventing index/replica operations"); TransportService mockTransportService = internalCluster().getInstance(TransportService.class, primaryNode); ((MockTransportService) mockTransportService).addFailToSendNoConnectRule( - internalCluster().getInstance(Discovery.class, replicaNode).localNode(), + internalCluster().getInstance(TransportService.class, replicaNode), singleton(IndexAction.NAME + "[r]") ); mockTransportService = internalCluster().getInstance(TransportService.class, replicaNode); ((MockTransportService) mockTransportService).addFailToSendNoConnectRule( - internalCluster().getInstance(Discovery.class, primaryNode).localNode(), + internalCluster().getInstance(TransportService.class, primaryNode), singleton(IndexAction.NAME + "[r]") ); diff --git a/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java b/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java index 3f7ea542305..d54d1a953bc 100644 --- a/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java @@ -21,14 +21,11 @@ package org.elasticsearch.index; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; public class VersionTypeTests extends ESTestCase { - @Test public void testInternalVersionConflict() throws Exception { - assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(10, Versions.MATCH_ANY, randomBoolean())); assertFalse(VersionType.INTERNAL.isVersionConflictForReads(10, Versions.MATCH_ANY)); // if we don't have a version in the index we accept everything @@ -70,7 +67,6 @@ public class VersionTypeTests extends ESTestCase { // updatedVersion = (currentVersion == Versions.NOT_SET || currentVersion == Versions.NOT_FOUND) ? 1 : currentVersion + 1; } - @Test public void testVersionValidation() { assertTrue(VersionType.EXTERNAL.validateVersionForWrites(randomIntBetween(1, Integer.MAX_VALUE))); assertFalse(VersionType.EXTERNAL.validateVersionForWrites(Versions.MATCH_ANY)); @@ -101,9 +97,7 @@ public class VersionTypeTests extends ESTestCase { assertFalse(VersionType.INTERNAL.validateVersionForReads(randomIntBetween(Integer.MIN_VALUE, -1))); } - @Test public void testExternalVersionConflict() throws Exception { - assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean())); assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_SET, 10, randomBoolean())); // MATCH_ANY must throw an exception in the case of external version, as the version must be set! it used as the new value @@ -139,9 +133,7 @@ public class VersionTypeTests extends ESTestCase { // updatedVersion = index.version(); } - @Test public void testExternalGTEVersionConflict() throws Exception { - assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean())); assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_SET, 10, randomBoolean())); // MATCH_ANY must throw an exception in the case of external version, as the version must be set! it used as the new value @@ -168,9 +160,7 @@ public class VersionTypeTests extends ESTestCase { } - @Test public void testForceVersionConflict() throws Exception { - assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean())); assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_SET, 10, randomBoolean())); @@ -201,9 +191,7 @@ public class VersionTypeTests extends ESTestCase { assertFalse(VersionType.FORCE.isVersionConflictForReads(10, Versions.MATCH_ANY)); } - @Test public void testUpdateVersion() { - assertThat(VersionType.INTERNAL.updateVersion(Versions.NOT_SET, 10), equalTo(1l)); assertThat(VersionType.INTERNAL.updateVersion(Versions.NOT_FOUND, 10), equalTo(1l)); assertThat(VersionType.INTERNAL.updateVersion(1, 1), equalTo(2l)); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java index d31cc0a5c65..17bd9d587b3 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -30,7 +29,6 @@ import java.io.StringReader; import static org.elasticsearch.common.settings.Settings.settingsBuilder; public class ASCIIFoldingTokenFilterFactoryTests extends ESTokenStreamTestCase { - @Test public void testDefault() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -44,7 +42,6 @@ public class ASCIIFoldingTokenFilterFactoryTests extends ESTokenStreamTestCase { assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testPreserveOriginal() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -58,5 +55,4 @@ public class ASCIIFoldingTokenFilterFactoryTests extends ESTokenStreamTestCase { tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java index 03986188c6c..7cd16e350a4 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java @@ -25,32 +25,31 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.ar.ArabicNormalizationFilter; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.fa.PersianNormalizationFilter; +import org.apache.lucene.analysis.hunspell.Dictionary; import org.apache.lucene.analysis.miscellaneous.KeywordRepeatFilter; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; -import org.elasticsearch.common.inject.ProvisionException; +import org.elasticsearch.common.inject.ModuleTestCase; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.indices.analysis.HunspellService; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.VersionUtils; import org.hamcrest.MatcherAssert; import java.io.BufferedWriter; import java.io.IOException; +import java.io.InputStream; import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.Set; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -59,22 +58,21 @@ import static org.hamcrest.Matchers.*; /** * */ -public class AnalysisModuleTests extends ESTestCase { +public class AnalysisModuleTests extends ModuleTestCase { - private Injector injector; + public AnalysisService getAnalysisService(Settings settings) throws IOException { + return getAnalysisService(getNewRegistry(settings), settings); + } - public AnalysisService getAnalysisService(Settings settings) { + public AnalysisService getAnalysisService(AnalysisRegistry registry, Settings settings) throws IOException { Index index = new Index("test"); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - AnalysisModule analysisModule = new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)); - analysisModule.addTokenFilter("myfilter", MyFilterTokenFilterFactory.class); - injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - analysisModule) - .createChildInjector(parentInjector); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, settings); + return registry.build(idxSettings); + } - return injector.getInstance(AnalysisService.class); + public AnalysisRegistry getNewRegistry(Settings settings) { + return new AnalysisRegistry(null, new Environment(settings), + Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new), Collections.emptyMap(), Collections.emptyMap()); } private Settings loadFromClasspath(String path) { @@ -85,12 +83,12 @@ public class AnalysisModuleTests extends ESTestCase { } - public void testSimpleConfigurationJson() { + public void testSimpleConfigurationJson() throws IOException { Settings settings = loadFromClasspath("/org/elasticsearch/index/analysis/test1.json"); testSimpleConfiguration(settings); } - public void testSimpleConfigurationYaml() { + public void testSimpleConfigurationYaml() throws IOException { Settings settings = loadFromClasspath("/org/elasticsearch/index/analysis/test1.yml"); testSimpleConfiguration(settings); } @@ -108,12 +106,12 @@ public class AnalysisModuleTests extends ESTestCase { .put("path.home", createTempDir().toString()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_0) .build(); - AnalysisService analysisService2 = getAnalysisService(settings2); + AnalysisRegistry newRegistry = getNewRegistry(settings2); + AnalysisService analysisService2 = getAnalysisService(newRegistry, settings2); - // indicesanalysisservice always has the current version - IndicesAnalysisService indicesAnalysisService2 = injector.getInstance(IndicesAnalysisService.class); - assertThat(indicesAnalysisService2.analyzer("default"), is(instanceOf(NamedAnalyzer.class))); - NamedAnalyzer defaultNamedAnalyzer = (NamedAnalyzer) indicesAnalysisService2.analyzer("default"); + // registry always has the current version + assertThat(newRegistry.getAnalyzer("default"), is(instanceOf(NamedAnalyzer.class))); + NamedAnalyzer defaultNamedAnalyzer = (NamedAnalyzer) newRegistry.getAnalyzer("default"); assertThat(defaultNamedAnalyzer.analyzer(), is(instanceOf(StandardAnalyzer.class))); assertEquals(Version.CURRENT.luceneVersion, defaultNamedAnalyzer.analyzer().getVersion()); @@ -121,6 +119,9 @@ public class AnalysisModuleTests extends ESTestCase { assertThat(analysisService2.analyzer("standard").analyzer(), is(instanceOf(StandardAnalyzer.class))); assertEquals(Version.V_0_90_0.luceneVersion, analysisService2.analyzer("standard").analyzer().getVersion()); assertEquals(Version.V_0_90_0.luceneVersion, analysisService2.analyzer("thai").analyzer().getVersion()); + + assertThat(analysisService2.analyzer("custom7").analyzer(), is(instanceOf(StandardAnalyzer.class))); + assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), analysisService2.analyzer("custom7").analyzer().getVersion()); } private void assertTokenFilter(String name, Class clazz) throws IOException { @@ -135,7 +136,7 @@ public class AnalysisModuleTests extends ESTestCase { assertThat(stream, instanceOf(clazz)); } - private void testSimpleConfiguration(Settings settings) { + private void testSimpleConfiguration(Settings settings) throws IOException { AnalysisService analysisService = getAnalysisService(settings); Analyzer analyzer = analysisService.analyzer("custom1").analyzer(); @@ -236,7 +237,7 @@ public class AnalysisModuleTests extends ESTestCase { return wordListFile; } - public void testUnderscoreInAnalyzerName() { + public void testUnderscoreInAnalyzerName() throws IOException { Settings settings = Settings.builder() .put("index.analysis.analyzer._invalid_name.tokenizer", "keyword") .put("path.home", createTempDir().toString()) @@ -245,13 +246,12 @@ public class AnalysisModuleTests extends ESTestCase { try { getAnalysisService(settings); fail("This should fail with IllegalArgumentException because the analyzers name starts with _"); - } catch (ProvisionException e) { - assertTrue(e.getCause() instanceof IllegalArgumentException); - assertThat(e.getCause().getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")).or(equalTo("analyzer name must not start with '_'. got \"_invalidName\""))); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")).or(equalTo("analyzer name must not start with '_'. got \"_invalidName\""))); } } - public void testUnderscoreInAnalyzerNameAlias() { + public void testUnderscoreInAnalyzerNameAlias() throws IOException { Settings settings = Settings.builder() .put("index.analysis.analyzer.valid_name.tokenizer", "keyword") .put("index.analysis.analyzer.valid_name.alias", "_invalid_name") @@ -261,13 +261,12 @@ public class AnalysisModuleTests extends ESTestCase { try { getAnalysisService(settings); fail("This should fail with IllegalArgumentException because the analyzers alias starts with _"); - } catch (ProvisionException e) { - assertTrue(e.getCause() instanceof IllegalArgumentException); - assertThat(e.getCause().getMessage(), equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")); } } - public void testBackwardCompatible() { + public void testBackwardCompatible() throws IOException { Settings settings = settingsBuilder() .put("index.analysis.analyzer.custom1.tokenizer", "standard") .put("index.analysis.analyzer.custom1.position_offset_gap", "128") @@ -288,7 +287,7 @@ public class AnalysisModuleTests extends ESTestCase { assertThat(custom2.getPositionIncrementGap("custom2"), equalTo(256)); } - public void testWithBothSettings() { + public void testWithBothSettings() throws IOException { Settings settings = settingsBuilder() .put("index.analysis.analyzer.custom.tokenizer", "standard") .put("index.analysis.analyzer.custom.position_offset_gap", "128") @@ -300,14 +299,13 @@ public class AnalysisModuleTests extends ESTestCase { try { getAnalysisService(settings); fail("Analyzer has both position_offset_gap and position_increment_gap should fail"); - } catch (ProvisionException e) { - assertTrue(e.getCause() instanceof IllegalArgumentException); - assertThat(e.getCause().getMessage(), equalTo("Custom Analyzer [custom] defined both [position_offset_gap] and [position_increment_gap]" + + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("Custom Analyzer [custom] defined both [position_offset_gap] and [position_increment_gap]" + ", use only [position_increment_gap]")); } } - public void testDeprecatedPositionOffsetGap() { + public void testDeprecatedPositionOffsetGap() throws IOException { Settings settings = settingsBuilder() .put("index.analysis.analyzer.custom.tokenizer", "standard") .put("index.analysis.analyzer.custom.position_offset_gap", "128") @@ -317,10 +315,22 @@ public class AnalysisModuleTests extends ESTestCase { try { getAnalysisService(settings); fail("Analyzer should fail if it has position_offset_gap"); - } catch (ProvisionException e) { - assertTrue(e.getCause() instanceof IllegalArgumentException); - assertThat(e.getCause().getMessage(), equalTo("Option [position_offset_gap] in Custom Analyzer [custom] " + + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("Option [position_offset_gap] in Custom Analyzer [custom] " + "has been renamed, please use [position_increment_gap] instead.")); } } + + public void testRegisterHunspellDictionary() throws Exception { + Settings settings = settingsBuilder() + .put("path.home", createTempDir().toString()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + AnalysisModule module = new AnalysisModule(new Environment(settings)); + InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff"); + InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic"); + Dictionary dictionary = new Dictionary(aff, dic); + module.registerHunspellDictionary("foo", dictionary); + assertInstanceBinding(module, HunspellService.class, (x) -> x.getDictionary("foo") == dictionary); + } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java index cd3a8c643ca..cd5138b4e0c 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java @@ -19,49 +19,48 @@ package org.elasticsearch.index.analysis; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.VersionUtils; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.io.IOException; +import java.util.*; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.instanceOf; public class AnalysisServiceTests extends ESTestCase { - private static AnalyzerProviderFactory analyzerProvider(final String name) { - return new AnalyzerProviderFactory() { - @Override - public AnalyzerProvider create(String name, Settings settings) { - return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer()); - } - }; + private static AnalyzerProvider analyzerProvider(final String name) { + return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer()); } - public void testDefaultAnalyzers() { + public void testDefaultAnalyzers() throws IOException { Version version = VersionUtils.randomVersion(getRandom()); - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndicesAnalysisService indicesAnalysisService = new IndicesAnalysisService(settings); - AnalysisService analysisService = new AnalysisService(new Index("index"), settings, indicesAnalysisService, - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).put("path.home", createTempDir().toString()).build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); } - public void testOverrideDefaultAnalyzer() { + public void testOverrideDefaultAnalyzer() throws IOException { Version version = VersionUtils.randomVersion(getRandom()); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndicesAnalysisService indicesAnalysisService = new IndicesAnalysisService(settings); - AnalysisService analysisService = new AnalysisService(new Index("index"), settings, indicesAnalysisService, + AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings(new Index("index"), settings), Collections.singletonMap("default", analyzerProvider("default")), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); @@ -72,10 +71,9 @@ public class AnalysisServiceTests extends ESTestCase { public void testOverrideDefaultIndexAnalyzer() { Version version = VersionUtils.randomVersionBetween(getRandom(), Version.V_3_0_0, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndicesAnalysisService indicesAnalysisService = new IndicesAnalysisService(settings); try { - AnalysisService analysisService = new AnalysisService(new Index("index"), settings, indicesAnalysisService, - Collections.singletonMap("default_index", new PreBuiltAnalyzerProviderFactory("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer())), + AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings(new Index("index"), settings), + Collections.singletonMap("default_index", new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer())), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); fail("Expected ISE"); } catch (IllegalArgumentException e) { @@ -87,8 +85,7 @@ public class AnalysisServiceTests extends ESTestCase { public void testBackCompatOverrideDefaultIndexAnalyzer() { Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_3_0_0)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndicesAnalysisService indicesAnalysisService = new IndicesAnalysisService(settings); - AnalysisService analysisService = new AnalysisService(new Index("index"), settings, indicesAnalysisService, + AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings(new Index("index"), settings), Collections.singletonMap("default_index", analyzerProvider("default_index")), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); @@ -99,8 +96,7 @@ public class AnalysisServiceTests extends ESTestCase { public void testOverrideDefaultSearchAnalyzer() { Version version = VersionUtils.randomVersion(getRandom()); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndicesAnalysisService indicesAnalysisService = new IndicesAnalysisService(settings); - AnalysisService analysisService = new AnalysisService(new Index("index"), settings, indicesAnalysisService, + AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings(new Index("index"), settings), Collections.singletonMap("default_search", analyzerProvider("default_search")), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); @@ -111,14 +107,101 @@ public class AnalysisServiceTests extends ESTestCase { public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() { Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_3_0_0)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndicesAnalysisService indicesAnalysisService = new IndicesAnalysisService(settings); - Map analyzers = new HashMap<>(); + Map analyzers = new HashMap<>(); analyzers.put("default_index", analyzerProvider("default_index")); analyzers.put("default_search", analyzerProvider("default_search")); - AnalysisService analysisService = new AnalysisService(new Index("index"), settings, indicesAnalysisService, + AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings(new Index("index"), settings), analyzers, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); } + + public void testConfigureCamelCaseTokenFilter() throws IOException { + // tests a filter that + Settings settings = Settings.builder().put("path.home", createTempDir().toString()).build(); + Settings indexSettings = settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") + .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") + .put("index.analysis.analyzer.custom_analyzer_1.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build(); + + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), indexSettings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + try (NamedAnalyzer custom_analyser = analysisService.analyzer("custom_analyzer")) { + assertNotNull(custom_analyser); + TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee"); + tokenStream.reset(); + CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); + List token = new ArrayList<>(); + while(tokenStream.incrementToken()) { + token.add(charTermAttribute.toString()); + } + assertEquals(token.toString(), 2, token.size()); + assertEquals("j2se", token.get(0)); + assertEquals("j2ee", token.get(1)); + } + + try (NamedAnalyzer custom_analyser = analysisService.analyzer("custom_analyzer_1")) { + assertNotNull(custom_analyser); + TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee"); + tokenStream.reset(); + CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); + List token = new ArrayList<>(); + while(tokenStream.incrementToken()) { + token.add(charTermAttribute.toString()); + } + assertEquals(token.toString(), 6, token.size()); + assertEquals("j", token.get(0)); + assertEquals("2", token.get(1)); + assertEquals("se", token.get(2)); + assertEquals("j", token.get(3)); + assertEquals("2", token.get(4)); + assertEquals("ee", token.get(5)); + } + } + + public void testCameCaseOverride() throws IOException { + Settings settings = Settings.builder().put("path.home", createTempDir().toString()).build(); + Settings indexSettings = settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") + .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") + .put("index.analysis.analyzer.custom_analyzer_1.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), indexSettings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + + TokenFilterFactory word_delimiter = analysisService.tokenFilter("word_delimiter"); + TokenFilterFactory override = analysisService.tokenFilter("wordDelimiter"); + assertNotEquals(word_delimiter.name(), override.name()); + assertNotSame(analysisService.tokenFilter("wordDelimiter"), analysisService.tokenFilter("word_delimiter")); + assertSame(analysisService.tokenFilter("porterStem"), analysisService.tokenFilter("porter_stem")); + + //unconfigured + IndexSettings idxSettings1 = IndexSettingsModule.newIndexSettings(new Index("index"), settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); + AnalysisService analysisService1 = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings1); + assertSame(analysisService1.tokenFilter("wordDelimiter"), analysisService1.tokenFilter("word_delimiter")); + assertSame(analysisService1.tokenFilter("porterStem"), analysisService1.tokenFilter("porter_stem")); + } + + public void testBuiltInAnalyzersAreCached() throws IOException { + Settings settings = Settings.builder().put("path.home", createTempDir().toString()).build(); + Settings indexSettings = settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), indexSettings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + AnalysisService otherAnalysisSergice = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + final int numIters = randomIntBetween(5, 20); + for (int i = 0; i < numIters; i++) { + PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values()); + assertSame(analysisService.analyzer(preBuiltAnalyzers.name()), otherAnalysisSergice.analyzer(preBuiltAnalyzers.name())); + } + } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java index abfe52097ed..061e0d9d29f 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java @@ -22,15 +22,12 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.util.CharArraySet; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.is; public class AnalysisTests extends ESTestCase { - @Test public void testParseStemExclusion() { - /* Comma separated list */ Settings settings = settingsBuilder().put("stem_exclusion", "foo,bar").build(); CharArraySet set = Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET); @@ -45,5 +42,4 @@ public class AnalysisTests extends ESTestCase { assertThat(set.contains("bar"), is(true)); assertThat(set.contains("baz"), is(false)); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java index 72eac6860ba..1404716b0c8 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java @@ -21,23 +21,20 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.analysis.HunspellService; +import org.elasticsearch.test.IndexSettingsModule; +import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; public class AnalysisTestsHelper { - public static AnalysisService createAnalysisServiceFromClassPath(Path baseDir, String resource) { + public static AnalysisService createAnalysisServiceFromClassPath(Path baseDir, String resource) throws IOException { Settings settings = Settings.settingsBuilder() .loadFromStream(resource, AnalysisTestsHelper.class.getResourceAsStream(resource)) .put("path.home", baseDir.toString()) @@ -47,27 +44,13 @@ public class AnalysisTestsHelper { } public static AnalysisService createAnalysisServiceFromSettings( - Settings settings) { + Settings settings) throws IOException { Index index = new Index("test"); if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) { settings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); } - IndicesModule indicesModule = new IndicesModule(settings) { - @Override - public void configure() { - // skip services - bindHunspellExtension(); - } - }; - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), - new EnvironmentModule(new Environment(settings)), indicesModule).createInjector(); - - AnalysisModule analysisModule = new AnalysisModule(settings, - parentInjector.getInstance(IndicesAnalysisService.class)); - - Injector injector = new ModulesBuilder().add(new IndexSettingsModule(index, settings), - new IndexNameModule(index), analysisModule).createChildInjector(parentInjector); - - return injector.getInstance(AnalysisService.class); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, settings); + Environment environment = new Environment(settings); + return new AnalysisRegistry(new HunspellService(settings, environment, Collections.emptyMap()), environment).build(idxSettings); } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java index 98ed9d28703..d2e2d4cc6e2 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/CJKFilterFactoryTests.java @@ -22,16 +22,13 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; public class CJKFilterFactoryTests extends ESTokenStreamTestCase { - private static final String RESOURCE = "/org/elasticsearch/index/analysis/cjk_analysis.json"; - @Test public void testDefault() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_bigram"); @@ -42,7 +39,6 @@ public class CJKFilterFactoryTests extends ESTokenStreamTestCase { assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testNoFlags() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_no_flags"); @@ -52,8 +48,7 @@ public class CJKFilterFactoryTests extends ESTokenStreamTestCase { tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - - @Test + public void testHanOnly() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_han_only"); @@ -63,8 +58,7 @@ public class CJKFilterFactoryTests extends ESTokenStreamTestCase { tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - - @Test + public void testHanUnigramOnly() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("cjk_han_unigram_only"); @@ -74,7 +68,4 @@ public class CJKFilterFactoryTests extends ESTokenStreamTestCase { tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - - - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java index 0171b4cc695..dd08d470136 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java @@ -20,26 +20,18 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import static org.elasticsearch.common.settings.Settings.settingsBuilder; /** */ public class CharFilterTests extends ESTokenStreamTestCase { - - @Test public void testMappingCharFilter() throws Exception { Index index = new Index("test"); Settings settings = settingsBuilder() @@ -50,24 +42,16 @@ public class CharFilterTests extends ESTokenStreamTestCase { .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "my_mapping") .put("path.home", createTempDir().toString()) .build(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class))) - .createChildInjector(parentInjector); - - AnalysisService analysisService = injector.getInstance(AnalysisService.class); - + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter"); - + assertTokenStreamContents(analyzer1.tokenStream("test", "jeff quit phish"), new String[]{"jeff", "qit", "fish"}); // Repeat one more time to make sure that char filter is reinitialized correctly assertTokenStreamContents(analyzer1.tokenStream("test", "jeff quit phish"), new String[]{"jeff", "qit", "fish"}); } - @Test public void testHtmlStripCharFilter() throws Exception { Index index = new Index("test"); Settings settings = settingsBuilder() @@ -76,14 +60,8 @@ public class CharFilterTests extends ESTokenStreamTestCase { .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "html_strip") .put("path.home", createTempDir().toString()) .build(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class))) - .createChildInjector(parentInjector); - - AnalysisService analysisService = injector.getInstance(AnalysisService.class); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter"); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java index 28b30e9ff5d..e685c21422b 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java @@ -24,57 +24,40 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.lucene.all.AllEntries; import org.elasticsearch.common.lucene.all.AllTokenStream; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.MatcherAssert; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItems; -import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.*; /** */ public class CompoundAnalysisTests extends ESTestCase { - - @Test public void testDefaultsCompoundAnalysis() throws Exception { Index index = new Index("test"); Settings settings = getJsonSettings(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - AnalysisModule analysisModule = new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)); - analysisModule.addTokenFilter("myfilter", MyFilterTokenFilterFactory.class); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - analysisModule) - .createChildInjector(parentInjector); - - AnalysisService analysisService = injector.getInstance(AnalysisService.class); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings), + Collections.emptyMap(),Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings); TokenFilterFactory filterFactory = analysisService.tokenFilter("dict_dec"); MatcherAssert.assertThat(filterFactory, instanceOf(DictionaryCompoundWordTokenFilterFactory.class)); } - @Test public void testDictionaryDecompounder() throws Exception { Settings[] settingsArr = new Settings[]{getJsonSettings(), getYamlSettings()}; for (Settings settings : settingsArr) { @@ -86,16 +69,9 @@ public class CompoundAnalysisTests extends ESTestCase { private List analyze(Settings settings, String analyzerName, String text) throws IOException { Index index = new Index("test"); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - AnalysisModule analysisModule = new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)); - analysisModule.addTokenFilter("myfilter", MyFilterTokenFilterFactory.class); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - analysisModule) - .createChildInjector(parentInjector); - - AnalysisService analysisService = injector.getInstance(AnalysisService.class); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings), + Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings); Analyzer analyzer = analysisService.analyzer(analyzerName).analyzer(); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java index f81fef1f816..02c4e1a2642 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactoryTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -29,8 +28,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; public class HunspellTokenFilterFactoryTests extends ESTestCase { - - @Test public void testDedup() throws IOException { Settings settings = settingsBuilder() .put("path.home", createTempDir().toString()) @@ -59,5 +56,4 @@ public class HunspellTokenFilterFactoryTests extends ESTestCase { hunspellTokenFilter = (HunspellTokenFilterFactory) tokenFilter; assertThat(hunspellTokenFilter.dedup(), is(false)); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java index df382b7cc7a..99c936cd346 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/KeepFilterFactoryTests.java @@ -22,10 +22,8 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.test.ESTokenStreamTestCase; import org.junit.Assert; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -33,18 +31,14 @@ import java.io.StringReader; import static org.hamcrest.Matchers.instanceOf; public class KeepFilterFactoryTests extends ESTokenStreamTestCase { - private static final String RESOURCE = "/org/elasticsearch/index/analysis/keep_analysis.json"; - - @Test - public void testLoadWithoutSettings() { + public void testLoadWithoutSettings() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("keep"); Assert.assertNull(tokenFilter); } - @Test public void testLoadOverConfiguredSettings() { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir().toString()) @@ -55,12 +49,12 @@ public class KeepFilterFactoryTests extends ESTokenStreamTestCase { try { AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); Assert.fail("path and array are configured"); - } catch (Exception e) { - assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } catch (IllegalArgumentException e) { + } catch (IOException e) { + fail("expected IAE"); } } - @Test public void testKeepWordsPathSettings() { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir().toString()) @@ -71,8 +65,9 @@ public class KeepFilterFactoryTests extends ESTokenStreamTestCase { // test our none existing setup is picked up AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); fail("expected an exception due to non existent keep_words_path"); - } catch (Throwable e) { - assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } catch (IllegalArgumentException e) { + } catch (IOException e) { + fail("expected IAE"); } settings = Settings.settingsBuilder().put(settings) @@ -82,13 +77,13 @@ public class KeepFilterFactoryTests extends ESTokenStreamTestCase { // test our none existing setup is picked up AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); fail("expected an exception indicating that you can't use [keep_words_path] with [keep_words] "); - } catch (Throwable e) { - assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } catch (IllegalArgumentException e) { + } catch (IOException e) { + fail("expected IAE"); } } - @Test public void testCaseInsensitiveMapping() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_keep_filter"); @@ -100,7 +95,6 @@ public class KeepFilterFactoryTests extends ESTokenStreamTestCase { assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[]{1, 2}); } - @Test public void testCaseSensitiveMapping() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_case_sensitive_keep_filter"); @@ -111,5 +105,4 @@ public class KeepFilterFactoryTests extends ESTokenStreamTestCase { tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[]{1}); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java index fd8f70f6f74..1e8a0ba16ed 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/KeepTypesFilterFactoryTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -31,8 +30,6 @@ import java.io.StringReader; import static org.hamcrest.Matchers.instanceOf; public class KeepTypesFilterFactoryTests extends ESTokenStreamTestCase { - - @Test public void testKeepTypes() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir().toString()) diff --git a/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java index 6f283f95550..e133ffc79ae 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactoryTests.java @@ -23,14 +23,11 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; public class LimitTokenCountFilterFactoryTests extends ESTokenStreamTestCase { - - @Test public void testDefault() throws IOException { Settings settings = Settings.settingsBuilder() .put("index.analysis.filter.limit_default.type", "limit") @@ -55,7 +52,6 @@ public class LimitTokenCountFilterFactoryTests extends ESTokenStreamTestCase { } } - @Test public void testSettings() throws IOException { { Settings settings = Settings.settingsBuilder() diff --git a/core/src/test/java/org/elasticsearch/index/analysis/NGramTokenizerFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/NGramTokenizerFactoryTests.java index 11fcf066fe6..d931b478f3e 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/NGramTokenizerFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/NGramTokenizerFactoryTests.java @@ -29,8 +29,9 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.io.StringReader; @@ -45,17 +46,15 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.scaledRandomIntB import static org.hamcrest.Matchers.instanceOf; public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { - - - @Test public void testParseTokenChars() { final Index index = new Index("test"); final String name = "ngr"; final Settings indexSettings = newAnalysisSettingsBuilder().build(); + IndexSettings indexProperties = IndexSettingsModule.newIndexSettings(index, indexSettings); for (String tokenChars : Arrays.asList("letters", "number", "DIRECTIONALITY_UNDEFINED")) { final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", tokenChars).build(); try { - new NGramTokenizerFactory(index, indexSettings, name, settings).create(); + new NGramTokenizerFactory(indexProperties, null, name, settings).create(); fail(); } catch (IllegalArgumentException expected) { // OK @@ -63,59 +62,57 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { } for (String tokenChars : Arrays.asList("letter", " digit ", "punctuation", "DIGIT", "CoNtRoL", "dash_punctuation")) { final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", tokenChars).build(); - new NGramTokenizerFactory(index, indexSettings, name, settings).create(); + indexProperties = IndexSettingsModule.newIndexSettings(index, indexSettings); + + new NGramTokenizerFactory(indexProperties, null, name, settings).create(); // no exception } } - @Test public void testNoTokenChars() throws IOException { final Index index = new Index("test"); final String name = "ngr"; final Settings indexSettings = newAnalysisSettingsBuilder().build(); final Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 4).putArray("token_chars", new String[0]).build(); - Tokenizer tokenizer = new NGramTokenizerFactory(index, indexSettings, name, settings).create(); + Tokenizer tokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); tokenizer.setReader(new StringReader("1.34")); assertTokenStreamContents(tokenizer, new String[] {"1.", "1.3", "1.34", ".3", ".34", "34"}); } - @Test public void testPreTokenization() throws IOException { // Make sure that pretokenization works well and that it can be used even with token chars which are supplementary characters final Index index = new Index("test"); final String name = "ngr"; final Settings indexSettings = newAnalysisSettingsBuilder().build(); Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "letter,digit").build(); - Tokenizer tokenizer = new NGramTokenizerFactory(index, indexSettings, name, settings).create(); + Tokenizer tokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); tokenizer.setReader(new StringReader("Åbc déf g\uD801\uDC00f ")); assertTokenStreamContents(tokenizer, new String[] {"Åb", "Åbc", "bc", "dé", "déf", "éf", "g\uD801\uDC00", "g\uD801\uDC00f", "\uD801\uDC00f"}); settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "letter,digit,punctuation,whitespace,symbol").build(); - tokenizer = new NGramTokenizerFactory(index, indexSettings, name, settings).create(); + tokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); tokenizer.setReader(new StringReader(" a!$ 9")); assertTokenStreamContents(tokenizer, new String[] {" a", " a!", "a!", "a!$", "!$", "!$ ", "$ ", "$ 9", " 9"}); } - @Test public void testPreTokenizationEdge() throws IOException { // Make sure that pretokenization works well and that it can be used even with token chars which are supplementary characters final Index index = new Index("test"); final String name = "ngr"; final Settings indexSettings = newAnalysisSettingsBuilder().build(); Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "letter,digit").build(); - Tokenizer tokenizer = new EdgeNGramTokenizerFactory(index, indexSettings, name, settings).create(); + Tokenizer tokenizer = new EdgeNGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); tokenizer.setReader(new StringReader("Åbc déf g\uD801\uDC00f ")); assertTokenStreamContents(tokenizer, new String[] {"Åb", "Åbc", "dé", "déf", "g\uD801\uDC00", "g\uD801\uDC00f"}); settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("token_chars", "letter,digit,punctuation,whitespace,symbol").build(); - tokenizer = new EdgeNGramTokenizerFactory(index, indexSettings, name, settings).create(); + tokenizer = new EdgeNGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); tokenizer.setReader(new StringReader(" a!$ 9")); assertTokenStreamContents(tokenizer, new String[] {" a", " a!"}); } - - @Test + public void testBackwardsCompatibilityEdgeNgramTokenizer() throws Exception { int iters = scaledRandomIntBetween(20, 100); final Index index = new Index("test"); @@ -131,7 +128,7 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { } Settings settings = builder.build(); Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build(); - Tokenizer edgeNGramTokenizer = new EdgeNGramTokenizerFactory(index, indexSettings, name, settings).create(); + Tokenizer edgeNGramTokenizer = new EdgeNGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); edgeNGramTokenizer.setReader(new StringReader("foo bar")); if (compatVersion) { assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class)); @@ -142,7 +139,7 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { } else { Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("side", "back").build(); Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build(); - Tokenizer edgeNGramTokenizer = new EdgeNGramTokenizerFactory(index, indexSettings, name, settings).create(); + Tokenizer edgeNGramTokenizer = new EdgeNGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); edgeNGramTokenizer.setReader(new StringReader("foo bar")); assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class)); } @@ -150,14 +147,13 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).put("side", "back").build(); Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); try { - new EdgeNGramTokenizerFactory(index, indexSettings, name, settings).create(); + new EdgeNGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); fail("should fail side:back is not supported anymore"); } catch (IllegalArgumentException ex) { } - + } - - @Test + public void testBackwardsCompatibilityNgramTokenizer() throws Exception { int iters = scaledRandomIntBetween(20, 100); for (int i = 0; i < iters; i++) { @@ -172,9 +168,9 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { } Settings settings = builder.build(); Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build(); - Tokenizer nGramTokenizer = new NGramTokenizerFactory(index, indexSettings, name, settings).create(); + Tokenizer nGramTokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); nGramTokenizer.setReader(new StringReader("foo bar")); - if (compatVersion) { + if (compatVersion) { assertThat(nGramTokenizer, instanceOf(Lucene43NGramTokenizer.class)); } else { assertThat(nGramTokenizer, instanceOf(NGramTokenizer.class)); @@ -183,14 +179,13 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { } else { Settings settings = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3).build(); Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build(); - Tokenizer nGramTokenizer = new NGramTokenizerFactory(index, indexSettings, name, settings).create(); + Tokenizer nGramTokenizer = new NGramTokenizerFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(); nGramTokenizer.setReader(new StringReader("foo bar")); assertThat(nGramTokenizer, instanceOf(Lucene43NGramTokenizer.class)); } } } - - @Test + public void testBackwardsCompatibilityEdgeNgramTokenFilter() throws Exception { int iters = scaledRandomIntBetween(20, 100); for (int i = 0; i < iters; i++) { @@ -211,10 +206,10 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build(); Tokenizer tokenizer = new MockTokenizer(); tokenizer.setReader(new StringReader("foo bar")); - TokenStream edgeNGramTokenFilter = new EdgeNGramTokenFilterFactory(index, indexSettings, name, settings).create(tokenizer); + TokenStream edgeNGramTokenFilter = new EdgeNGramTokenFilterFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(tokenizer); if (reverse) { assertThat(edgeNGramTokenFilter, instanceOf(ReverseStringFilter.class)); - } else if (compatVersion) { + } else if (compatVersion) { assertThat(edgeNGramTokenFilter, instanceOf(Lucene43EdgeNGramTokenFilter.class)); } else { assertThat(edgeNGramTokenFilter, instanceOf(EdgeNGramTokenFilter.class)); @@ -230,7 +225,7 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { Settings indexSettings = newAnalysisSettingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build(); Tokenizer tokenizer = new MockTokenizer(); tokenizer.setReader(new StringReader("foo bar")); - TokenStream edgeNGramTokenFilter = new EdgeNGramTokenFilterFactory(index, indexSettings, name, settings).create(tokenizer); + TokenStream edgeNGramTokenFilter = new EdgeNGramTokenFilterFactory(IndexSettingsModule.newIndexSettings(index, indexSettings), null, name, settings).create(tokenizer); if (reverse) { assertThat(edgeNGramTokenFilter, instanceOf(ReverseStringFilter.class)); } else { @@ -240,9 +235,9 @@ public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { } } - + private Version randomVersion(Random random) throws IllegalArgumentException, IllegalAccessException { - Field[] declaredFields = Version.class.getDeclaredFields(); + Field[] declaredFields = Version.class.getFields(); List versionFields = new ArrayList<>(); for (Field field : declaredFields) { if ((field.getModifiers() & Modifier.STATIC) != 0 && field.getName().startsWith("V_") && field.getType() == Version.class) { diff --git a/core/src/test/java/org/elasticsearch/index/analysis/NumericAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/NumericAnalyzerTests.java index a9ca96cc5f4..89940558d51 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/NumericAnalyzerTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/NumericAnalyzerTests.java @@ -24,7 +24,6 @@ import org.apache.lucene.analysis.NumericTokenStream.NumericTermAttribute; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -32,8 +31,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class NumericAnalyzerTests extends ESTestCase { - - @Test public void testAttributeEqual() throws IOException { final int precisionStep = 8; final double value = randomDouble(); @@ -59,5 +56,4 @@ public class NumericAnalyzerTests extends ESTestCase { ts1.end(); ts2.end(); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java index 9c578ef6385..6fa2e21fbd1 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.analysis; */ import java.io.IOException; -import java.lang.Thread.UncaughtExceptionHandler; import java.util.Arrays; import java.util.regex.Pattern; @@ -110,45 +109,6 @@ public class PatternAnalyzerTests extends ESTokenStreamTestCase { /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { Analyzer a = new PatternAnalyzer(Pattern.compile(","), true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); - - // dodge jre bug http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7104012 - final UncaughtExceptionHandler savedHandler = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread thread, Throwable throwable) { - assumeTrue("not failing due to jre bug ", !isJREBug7104012(throwable)); - // otherwise its some other bug, pass to default handler - savedHandler.uncaughtException(thread, throwable); - } - }); - - try { - Thread.getDefaultUncaughtExceptionHandler(); - checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER); - } catch (ArrayIndexOutOfBoundsException ex) { - assumeTrue("not failing due to jre bug ", !isJREBug7104012(ex)); - throw ex; // otherwise rethrow - } finally { - Thread.setDefaultUncaughtExceptionHandler(savedHandler); - } - } - - static boolean isJREBug7104012(Throwable t) { - if (!(t instanceof ArrayIndexOutOfBoundsException)) { - // BaseTokenStreamTestCase now wraps exc in a new RuntimeException: - t = t.getCause(); - if (!(t instanceof ArrayIndexOutOfBoundsException)) { - return false; - } - } - StackTraceElement trace[] = t.getStackTrace(); - for (StackTraceElement st : trace) { - if ("java.text.RuleBasedBreakIterator".equals(st.getClassName()) || - "sun.util.locale.provider.RuleBasedBreakIterator".equals(st.getClassName()) - && "lookupBackwardState".equals(st.getMethodName())) { - return true; - } - } - return false; + checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER); } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java index 6a7275b73e1..4b7119df01b 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java @@ -21,40 +21,27 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.Matchers.containsString; public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase { - - @Test public void testPatternCaptureTokenFilter() throws Exception { String json = "/org/elasticsearch/index/analysis/pattern_capture.json"; - Index index = new Index("test"); Settings settings = settingsBuilder() .put("path.home", createTempDir()) .loadFromStream(json, getClass().getResourceAsStream(json)) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class))) - .createChildInjector(parentInjector); - AnalysisService analysisService = injector.getInstance(AnalysisService.class); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); NamedAnalyzer analyzer1 = analysisService.analyzer("single"); @@ -68,11 +55,14 @@ public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase { assertTokenStreamContents(analyzer3.tokenStream("test", "foobarbaz"), new String[]{"foobar","foo"}); } - - - @Test(expected=IllegalArgumentException.class) + public void testNoPatterns() { - new PatternCaptureGroupTokenFilterFactory(new Index("test"), settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(), "pattern_capture", settingsBuilder().put("pattern", "foobar").build()); + try { + new PatternCaptureGroupTokenFilterFactory(IndexSettingsModule.newIndexSettings(new Index("test"), Settings.EMPTY), null, "pattern_capture", settingsBuilder().put("pattern", "foobar").build()); + fail ("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("required setting 'patterns' is missing")); + } } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactoryTests.java index c18e4fd7607..2cb8f99e7b8 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactoryTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -32,8 +31,6 @@ import static org.hamcrest.Matchers.not; * */ public class PreBuiltAnalyzerProviderFactoryTests extends ESTestCase { - - @Test public void testVersioningInFactoryProvider() throws Exception { PreBuiltAnalyzerProviderFactory factory = new PreBuiltAnalyzerProviderFactory("default", AnalyzerScope.INDEX, PreBuiltAnalyzers.STANDARD.getAnalyzer(Version.CURRENT)); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java index 77eb0cda1d6..fecb7e9b880 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -37,14 +36,15 @@ import java.util.List; import java.util.Locale; import static org.elasticsearch.test.VersionUtils.randomVersion; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; /** * */ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase { - - @Test public void testThatDefaultAndStandardAnalyzerAreTheSameInstance() { Analyzer currentStandardAnalyzer = PreBuiltAnalyzers.STANDARD.getAnalyzer(Version.CURRENT); Analyzer currentDefaultAnalyzer = PreBuiltAnalyzers.DEFAULT.getAnalyzer(Version.CURRENT); @@ -53,7 +53,6 @@ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase { assertThat(currentDefaultAnalyzer, is(currentStandardAnalyzer)); } - @Test public void testThatDefaultAndStandardAnalyzerChangedIn10Beta1() throws IOException { Analyzer currentStandardAnalyzer = PreBuiltAnalyzers.STANDARD.getAnalyzer(Version.V_1_0_0_Beta1); Analyzer currentDefaultAnalyzer = PreBuiltAnalyzers.DEFAULT.getAnalyzer(Version.V_1_0_0_Beta1); @@ -90,7 +89,6 @@ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase { } } - @Test public void testAnalyzerChangedIn10RC1() throws IOException { Analyzer pattern = PreBuiltAnalyzers.PATTERN.getAnalyzer(Version.V_1_0_0_RC1); Analyzer standardHtml = PreBuiltAnalyzers.STANDARD_HTML_STRIP.getAnalyzer(Version.V_1_0_0_RC1); @@ -125,13 +123,11 @@ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase { } } - @Test public void testThatInstancesAreTheSameAlwaysForKeywordAnalyzer() { assertThat(PreBuiltAnalyzers.KEYWORD.getAnalyzer(Version.CURRENT), is(PreBuiltAnalyzers.KEYWORD.getAnalyzer(Version.V_0_18_0))); } - @Test public void testThatInstancesAreCachedAndReused() { assertThat(PreBuiltAnalyzers.ARABIC.getAnalyzer(Version.CURRENT), is(PreBuiltAnalyzers.ARABIC.getAnalyzer(Version.CURRENT))); @@ -139,14 +135,12 @@ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase { is(PreBuiltAnalyzers.ARABIC.getAnalyzer(Version.V_0_18_0))); } - @Test public void testThatInstancesWithSameLuceneVersionAreReused() { // both are lucene 4.4 and should return the same instance assertThat(PreBuiltAnalyzers.CATALAN.getAnalyzer(Version.V_0_90_4), is(PreBuiltAnalyzers.CATALAN.getAnalyzer(Version.V_0_90_5))); } - @Test public void testThatAnalyzersAreUsedInMapping() throws IOException { int randomInt = randomInt(PreBuiltAnalyzers.values().length-1); PreBuiltAnalyzers randomPreBuiltAnalyzer = PreBuiltAnalyzers.values()[randomInt]; @@ -164,7 +158,7 @@ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase { FieldMapper fieldMapper = docMapper.mappers().getMapper("field"); assertThat(fieldMapper.fieldType().searchAnalyzer(), instanceOf(NamedAnalyzer.class)); - NamedAnalyzer fieldMapperNamedAnalyzer = (NamedAnalyzer) fieldMapper.fieldType().searchAnalyzer(); + NamedAnalyzer fieldMapperNamedAnalyzer = fieldMapper.fieldType().searchAnalyzer(); assertThat(fieldMapperNamedAnalyzer.analyzer(), is(namedAnalyzer.analyzer())); } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactoryTests.java index 863ed961980..39de728a484 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactoryTests.java @@ -23,26 +23,23 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.analysis.PreBuiltCharFilters; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; /** * */ public class PreBuiltCharFilterFactoryFactoryTests extends ESTestCase { - - @Test - public void testThatDifferentVersionsCanBeLoaded() { + public void testThatDifferentVersionsCanBeLoaded() throws IOException { PreBuiltCharFilterFactoryFactory factory = new PreBuiltCharFilterFactoryFactory(PreBuiltCharFilters.HTML_STRIP.getCharFilterFactory(Version.CURRENT)); - CharFilterFactory former090TokenizerFactory = factory.create("html_strip", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_0).build()); - CharFilterFactory former090TokenizerFactoryCopy = factory.create("html_strip", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_0).build()); - CharFilterFactory currentTokenizerFactory = factory.create("html_strip", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); + CharFilterFactory former090TokenizerFactory = factory.get(null, null, "html_strip", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_0).build()); + CharFilterFactory former090TokenizerFactoryCopy = factory.get(null, null, "html_strip", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_0).build()); + CharFilterFactory currentTokenizerFactory = factory.get(null, null, "html_strip", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); assertThat(currentTokenizerFactory, is(former090TokenizerFactory)); assertThat(currentTokenizerFactory, is(former090TokenizerFactoryCopy)); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactoryTests.java index c1cbc1267cf..670df069926 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenFilterFactoryFactoryTests.java @@ -23,37 +23,35 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.analysis.PreBuiltTokenFilters; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.hamcrest.CoreMatchers.*; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; /** * */ public class PreBuiltTokenFilterFactoryFactoryTests extends ESTestCase { - - @Test - public void testThatCachingWorksForCachingStrategyOne() { + public void testThatCachingWorksForCachingStrategyOne() throws IOException { PreBuiltTokenFilterFactoryFactory factory = new PreBuiltTokenFilterFactoryFactory(PreBuiltTokenFilters.WORD_DELIMITER.getTokenFilterFactory(Version.CURRENT)); - TokenFilterFactory former090TokenizerFactory = factory.create("word_delimiter", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_1).build()); - TokenFilterFactory former090TokenizerFactoryCopy = factory.create("word_delimiter", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_2).build()); - TokenFilterFactory currentTokenizerFactory = factory.create("word_delimiter", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); + TokenFilterFactory former090TokenizerFactory = factory.get(null, null, "word_delimiter", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_1).build()); + TokenFilterFactory former090TokenizerFactoryCopy = factory.get(null, null, "word_delimiter", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_2).build()); + TokenFilterFactory currentTokenizerFactory = factory.get(null, null, "word_delimiter", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); assertThat(currentTokenizerFactory, is(former090TokenizerFactory)); assertThat(currentTokenizerFactory, is(former090TokenizerFactoryCopy)); } - @Test - public void testThatDifferentVersionsCanBeLoaded() { + public void testThatDifferentVersionsCanBeLoaded() throws IOException { PreBuiltTokenFilterFactoryFactory factory = new PreBuiltTokenFilterFactoryFactory(PreBuiltTokenFilters.STOP.getTokenFilterFactory(Version.CURRENT)); - TokenFilterFactory former090TokenizerFactory = factory.create("stop", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_1).build()); - TokenFilterFactory former090TokenizerFactoryCopy = factory.create("stop", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_2).build()); - TokenFilterFactory currentTokenizerFactory = factory.create("stop", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); + TokenFilterFactory former090TokenizerFactory = factory.get(null, null, "stop", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_1).build()); + TokenFilterFactory former090TokenizerFactoryCopy = factory.get(null, null, "stop", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_2).build()); + TokenFilterFactory currentTokenizerFactory = factory.get(null, null, "stop", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); assertThat(currentTokenizerFactory, is(not(former090TokenizerFactory))); assertThat(former090TokenizerFactory, is(former090TokenizerFactoryCopy)); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactoryTests.java index a9e8f7b29f2..162dbb36424 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactoryTests.java @@ -23,27 +23,26 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.analysis.PreBuiltTokenizers; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.hamcrest.CoreMatchers.*; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; /** * */ public class PreBuiltTokenizerFactoryFactoryTests extends ESTestCase { - - @Test - public void testThatDifferentVersionsCanBeLoaded() { + public void testThatDifferentVersionsCanBeLoaded() throws IOException { PreBuiltTokenizerFactoryFactory factory = new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.STANDARD.getTokenizerFactory(Version.CURRENT)); // different es versions, same lucene version, thus cached - TokenizerFactory former090TokenizerFactory = factory.create("standard", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_1).build()); - TokenizerFactory former090TokenizerFactoryCopy = factory.create("standard", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_2).build()); - TokenizerFactory currentTokenizerFactory = factory.create("standard", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); + TokenizerFactory former090TokenizerFactory = factory.get(null, null, "standard", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_1).build()); + TokenizerFactory former090TokenizerFactoryCopy = factory.get(null, null, "standard", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_90_2).build()); + TokenizerFactory currentTokenizerFactory = factory.get(null, null, "standard", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); assertThat(currentTokenizerFactory, is(not(former090TokenizerFactory))); assertThat(currentTokenizerFactory, is(not(former090TokenizerFactoryCopy))); assertThat(former090TokenizerFactory, is(former090TokenizerFactoryCopy)); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java index 26883f562f5..2e2a45fab6a 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactoryTests.java @@ -21,12 +21,12 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; + import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -35,10 +35,8 @@ import static org.hamcrest.Matchers.instanceOf; @ThreadLeakScope(Scope.NONE) public class ShingleTokenFilterFactoryTests extends ESTokenStreamTestCase { - private static final String RESOURCE = "/org/elasticsearch/index/analysis/shingle_analysis.json"; - @Test public void testDefault() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle"); @@ -49,7 +47,6 @@ public class ShingleTokenFilterFactoryTests extends ESTokenStreamTestCase { assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testInverseMapping() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle_inverse"); @@ -61,7 +58,6 @@ public class ShingleTokenFilterFactoryTests extends ESTokenStreamTestCase { assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testInverseMappingNoShingles() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle_inverse"); @@ -73,7 +69,6 @@ public class ShingleTokenFilterFactoryTests extends ESTokenStreamTestCase { assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testFillerToken() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(createTempDir(), RESOURCE); TokenFilterFactory tokenFilter = analysisService.tokenFilter("shingle_filler"); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java index 22a7effdaac..737a991f0e0 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/StemmerTokenFilterFactoryTests.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.index.analysis; -import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.en.PorterStemFilter; import org.apache.lucene.analysis.snowball.SnowballFilter; @@ -27,7 +27,6 @@ import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.VersionUtils; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -40,8 +39,6 @@ import static org.hamcrest.Matchers.instanceOf; * */ public class StemmerTokenFilterFactoryTests extends ESTokenStreamTestCase { - - @Test public void testEnglishBackwardsCompatibility() throws IOException { int iters = scaledRandomIntBetween(20, 100); for (int i = 0; i < iters; i++) { @@ -75,7 +72,6 @@ public class StemmerTokenFilterFactoryTests extends ESTokenStreamTestCase { } - @Test public void testPorter2BackwardsCompatibility() throws IOException { int iters = scaledRandomIntBetween(20, 100); for (int i = 0; i < iters; i++) { diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java index 9265587929c..90e55e98d7e 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java @@ -21,40 +21,25 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import static org.elasticsearch.common.settings.Settings.settingsBuilder; public class StopAnalyzerTests extends ESTokenStreamTestCase { - - @Test public void testDefaultsCompoundAnalysis() throws Exception { String json = "/org/elasticsearch/index/analysis/stop.json"; - Index index = new Index("test"); Settings settings = settingsBuilder() .loadFromStream(json, getClass().getResourceAsStream(json)) .put("path.home", createTempDir().toString()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class))) - .createChildInjector(parentInjector); - - AnalysisService analysisService = injector.getInstance(AnalysisService.class); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); NamedAnalyzer analyzer1 = analysisService.analyzer("analyzer1"); @@ -64,5 +49,4 @@ public class StopAnalyzerTests extends ESTokenStreamTestCase { assertTokenStreamContents(analyzer2.tokenStream("test", "to be or not to be"), new String[0]); } - } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java index 2d52599c6c0..aa063a1d37e 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/StopTokenFilterTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.index.analysis; -import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.Lucene43StopFilter; import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.core.WhitespaceTokenizer; @@ -30,17 +30,15 @@ import org.elasticsearch.common.inject.ProvisionException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; public class StopTokenFilterTests extends ESTokenStreamTestCase { - - @Test(expected = ProvisionException.class) public void testPositionIncrementSetting() throws IOException { Builder builder = Settings.settingsBuilder().put("index.analysis.filter.my_stop.type", "stop") .put("index.analysis.filter.my_stop.enable_position_increments", false); @@ -49,11 +47,14 @@ public class StopTokenFilterTests extends ESTokenStreamTestCase { } builder.put("path.home", createTempDir().toString()); Settings settings = builder.build(); - AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); - analysisService.tokenFilter("my_stop"); + try { + AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("enable_position_increments is not supported anymore")); + } } - @Test public void testCorrectPositionIncrementSetting() throws IOException { Builder builder = Settings.settingsBuilder().put("index.analysis.filter.my_stop.type", "stop"); int thingToDo = random().nextInt(3); @@ -81,7 +82,6 @@ public class StopTokenFilterTests extends ESTokenStreamTestCase { } } - @Test public void testDeprecatedPositionIncrementSettingWithVersions() throws IOException { Settings settings = Settings.settingsBuilder() .put("index.analysis.filter.my_stop.type", "stop") @@ -98,7 +98,6 @@ public class StopTokenFilterTests extends ESTokenStreamTestCase { assertThat(create, instanceOf(Lucene43StopFilter.class)); } - @Test public void testThatSuggestStopFilterWorks() throws Exception { Settings settings = Settings.settingsBuilder() .put("index.analysis.filter.my_stop.type", "stop") diff --git a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java index d29b2ebae9e..54810028ae3 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -30,8 +29,6 @@ import java.io.StringReader; import static org.elasticsearch.common.settings.Settings.settingsBuilder; public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase { - - @Test public void testDefault() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -45,7 +42,6 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testCatenateWords() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -61,7 +57,6 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testCatenateNumbers() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -77,7 +72,6 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testCatenateAll() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -94,7 +88,6 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testSplitOnCaseChange() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -109,7 +102,6 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testPreserveOriginal() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -124,7 +116,6 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - @Test public void testStemEnglishPossessive() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -140,7 +131,6 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase } /** Correct offset order when doing both parts and concatenation: PowerShot is a synonym of Power */ - @Test public void testPartsAndCatenate() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) @@ -155,10 +145,9 @@ public class WordDelimiterTokenFilterFactoryTests extends ESTokenStreamTestCase tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } - - /** Back compat: + + /** Back compat: * old offset order when doing both parts and concatenation: PowerShot is a synonym of Shot */ - @Test public void testDeprecatedPartsAndCatenate() throws IOException { AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromSettings(settingsBuilder() .put("path.home", createTempDir().toString()) diff --git a/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java index c1bb7f8d3ef..52730dd2616 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/commongrams/CommonGramsTokenFilterFactoryTests.java @@ -22,14 +22,12 @@ package org.elasticsearch.index.analysis.commongrams; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.analysis.AnalysisTestsHelper; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.test.ESTokenStreamTestCase; import org.junit.Assert; -import org.junit.Test; import java.io.IOException; import java.io.InputStream; @@ -39,8 +37,6 @@ import java.nio.file.Path; import static org.hamcrest.Matchers.instanceOf; public class CommonGramsTokenFilterFactoryTests extends ESTokenStreamTestCase { - - @Test public void testDefault() throws IOException { Settings settings = Settings.settingsBuilder() .put("index.analysis.filter.common_grams_default.type", "common_grams") @@ -50,11 +46,12 @@ public class CommonGramsTokenFilterFactoryTests extends ESTokenStreamTestCase { try { AnalysisTestsHelper.createAnalysisServiceFromSettings(settings); Assert.fail("[common_words] or [common_words_path] is set"); - } catch (Exception e) { - assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } catch (IllegalArgumentException e) { + } catch (IOException e) { + fail("expected IAE"); } } - @Test + public void testWithoutCommonWordsMatch() throws IOException { { Settings settings = Settings.settingsBuilder().put("index.analysis.filter.common_grams_default.type", "common_grams") @@ -91,7 +88,6 @@ public class CommonGramsTokenFilterFactoryTests extends ESTokenStreamTestCase { } } - @Test public void testSettings() throws IOException { { Settings settings = Settings.settingsBuilder().put("index.analysis.filter.common_grams_1.type", "common_grams") @@ -136,7 +132,6 @@ public class CommonGramsTokenFilterFactoryTests extends ESTokenStreamTestCase { } } - @Test public void testCommonGramsAnalysis() throws IOException { String json = "/org/elasticsearch/index/analysis/commongrams/commongrams.json"; Settings settings = Settings.settingsBuilder() @@ -159,7 +154,6 @@ public class CommonGramsTokenFilterFactoryTests extends ESTokenStreamTestCase { } } - @Test public void testQueryModeSettings() throws IOException { { Settings settings = Settings.settingsBuilder().put("index.analysis.filter.common_grams_1.type", "common_grams") @@ -221,7 +215,6 @@ public class CommonGramsTokenFilterFactoryTests extends ESTokenStreamTestCase { } } - @Test public void testQueryModeCommonGramsAnalysis() throws IOException { String json = "/org/elasticsearch/index/analysis/commongrams/commongrams_query_mode.json"; Settings settings = Settings.settingsBuilder() diff --git a/core/src/test/java/org/elasticsearch/index/analysis/filter1/MyFilterTokenFilterFactory.java b/core/src/test/java/org/elasticsearch/index/analysis/filter1/MyFilterTokenFilterFactory.java index 47bf1bb8101..c7cd3cd625e 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/filter1/MyFilterTokenFilterFactory.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/filter1/MyFilterTokenFilterFactory.java @@ -21,17 +21,15 @@ package org.elasticsearch.index.analysis.filter1; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.StopAnalyzer; import org.apache.lucene.analysis.core.StopFilter; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.settings.IndexSettings; public class MyFilterTokenFilterFactory extends AbstractTokenFilterFactory { - @Inject - public MyFilterTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, String name) { - super(index, indexSettings, name, Settings.Builder.EMPTY_SETTINGS); + public MyFilterTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, Settings.Builder.EMPTY_SETTINGS); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java index f695b1b197e..3a6adca1c67 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java @@ -24,25 +24,19 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.lucene.all.AllEntries; import org.elasticsearch.common.lucene.all.AllTokenStream; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.analysis.AnalysisModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.MatcherAssert; -import org.junit.Test; import java.io.IOException; import java.io.InputStream; @@ -55,11 +49,9 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class SynonymsAnalysisTests extends ESTestCase { - protected final ESLogger logger = Loggers.getLogger(getClass()); private AnalysisService analysisService; - @Test public void testSynonymsAnalysis() throws IOException { InputStream synonyms = getClass().getResourceAsStream("synonyms.txt"); InputStream synonymsWordnet = getClass().getResourceAsStream("synonyms_wordnet.txt"); @@ -75,19 +67,9 @@ public class SynonymsAnalysisTests extends ESTestCase { .put("path.home", home) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - Index index = new Index("test"); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("index"), settings); + analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); - Injector parentInjector = new ModulesBuilder().add( - new SettingsModule(settings), - new EnvironmentModule(new Environment(settings))) - .createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class))) - .createChildInjector(parentInjector); - - analysisService = injector.getInstance(AnalysisService.class); match("synonymAnalyzer", "kimchy is the dude abides", "shay is the elasticsearch man!"); match("synonymAnalyzer_file", "kimchy is the dude abides", "shay is the elasticsearch man!"); diff --git a/core/src/test/java/org/elasticsearch/index/cache/IndexCacheModuleTests.java b/core/src/test/java/org/elasticsearch/index/cache/IndexCacheModuleTests.java deleted file mode 100644 index bd564744f20..00000000000 --- a/core/src/test/java/org/elasticsearch/index/cache/IndexCacheModuleTests.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache; - -import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.Weight; -import org.elasticsearch.common.inject.ModuleTestCase; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.query.QueryCache; -import org.elasticsearch.index.cache.query.index.IndexQueryCache; -import org.elasticsearch.index.cache.query.none.NoneQueryCache; - -import java.io.IOException; - -public class IndexCacheModuleTests extends ModuleTestCase { - - public void testCannotRegisterProvidedImplementations() { - IndexCacheModule module = new IndexCacheModule(Settings.EMPTY); - try { - module.registerQueryCache("index", IndexQueryCache.class); - } catch (IllegalArgumentException e) { - assertEquals(e.getMessage(), "Can't register the same [query_cache] more than once for [index]"); - } - - try { - module.registerQueryCache("none", NoneQueryCache.class); - } catch (IllegalArgumentException e) { - assertEquals(e.getMessage(), "Can't register the same [query_cache] more than once for [none]"); - } - } - - public void testRegisterCustomQueryCache() { - IndexCacheModule module = new IndexCacheModule( - Settings.builder().put(IndexCacheModule.QUERY_CACHE_TYPE, "custom").build() - ); - module.registerQueryCache("custom", CustomQueryCache.class); - try { - module.registerQueryCache("custom", CustomQueryCache.class); - } catch (IllegalArgumentException e) { - assertEquals(e.getMessage(), "Can't register the same [query_cache] more than once for [custom]"); - } - assertBinding(module, QueryCache.class, CustomQueryCache.class); - } - - public void testDefaultQueryCacheImplIsSelected() { - IndexCacheModule module = new IndexCacheModule(Settings.EMPTY); - assertBinding(module, QueryCache.class, IndexQueryCache.class); - } - - class CustomQueryCache implements QueryCache { - - @Override - public void clear(String reason) { - } - - @Override - public void close() throws IOException { - } - - @Override - public Index index() { - return new Index("test"); - } - - @Override - public Weight doCache(Weight weight, QueryCachingPolicy policy) { - return weight; - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java b/core/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java index c781a58b905..56bf966dd41 100644 --- a/core/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java +++ b/core/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java @@ -23,18 +23,10 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.LogByteSizeMergePolicy; -import org.apache.lucene.index.Term; +import org.apache.lucene.index.*; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BitSet; @@ -42,9 +34,11 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; @@ -54,6 +48,10 @@ import static org.hamcrest.Matchers.equalTo; public class BitSetFilterCacheTests extends ESTestCase { + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings(new Index("test"), Settings.EMPTY); + private final IndicesWarmer warmer = new IndicesWarmer(Settings.EMPTY, null); + + private static int matchCount(BitSetProducer producer, IndexReader reader) throws IOException { int count = 0; for (LeafReaderContext ctx : reader.leaves()) { @@ -65,7 +63,6 @@ public class BitSetFilterCacheTests extends ESTestCase { return count; } - @Test public void testInvalidateEntries() throws Exception { IndexWriter writer = new IndexWriter( new RAMDirectory(), @@ -89,8 +86,18 @@ public class BitSetFilterCacheTests extends ESTestCase { IndexReader reader = DirectoryReader.open(writer, false); IndexSearcher searcher = new IndexSearcher(reader); - BitsetFilterCache cache = new BitsetFilterCache(new Index("test"), Settings.EMPTY); - BitSetProducer filter = cache.getBitSetProducer(new QueryWrapperFilter(new TermQuery(new Term("field", "value")))); + BitsetFilterCache cache = new BitsetFilterCache(INDEX_SETTINGS, warmer, new BitsetFilterCache.Listener() { + @Override + public void onCache(ShardId shardId, Accountable accountable) { + + } + + @Override + public void onRemoval(ShardId shardId, Accountable accountable) { + + } + }); + BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value"))); assertThat(matchCount(filter, reader), equalTo(3)); // now cached @@ -132,8 +139,7 @@ public class BitSetFilterCacheTests extends ESTestCase { final AtomicInteger onCacheCalls = new AtomicInteger(); final AtomicInteger onRemoveCalls = new AtomicInteger(); - final BitsetFilterCache cache = new BitsetFilterCache(new Index("test"), Settings.EMPTY); - cache.setListener(new BitsetFilterCache.Listener() { + final BitsetFilterCache cache = new BitsetFilterCache(INDEX_SETTINGS, warmer, new BitsetFilterCache.Listener() { @Override public void onCache(ShardId shardId, Accountable accountable) { onCacheCalls.incrementAndGet(); @@ -160,7 +166,7 @@ public class BitSetFilterCacheTests extends ESTestCase { } } }); - BitSetProducer filter = cache.getBitSetProducer(new QueryWrapperFilter(new TermQuery(new Term("field", "value")))); + BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value"))); assertThat(matchCount(filter, reader), equalTo(1)); assertTrue(stats.get() > 0); assertEquals(1, onCacheCalls.get()); @@ -170,35 +176,12 @@ public class BitSetFilterCacheTests extends ESTestCase { assertEquals(0, stats.get()); } - public void testSetListenerTwice() { - final BitsetFilterCache cache = new BitsetFilterCache(new Index("test"), Settings.EMPTY); - cache.setListener(new BitsetFilterCache.Listener() { - - @Override - public void onCache(ShardId shardId, Accountable accountable) { - - } - - @Override - public void onRemoval(ShardId shardId, Accountable accountable) { - - } - }); + public void testSetNullListener() { try { - cache.setListener(new BitsetFilterCache.Listener() { - - @Override - public void onCache(ShardId shardId, Accountable accountable) { - - } - - @Override - public void onRemoval(ShardId shardId, Accountable accountable) { - - } - }); - fail("can't set it twice"); - } catch (IllegalStateException ex) { + new BitsetFilterCache(INDEX_SETTINGS, warmer, null); + fail("listener can't be null"); + } catch (IllegalArgumentException ex) { + assertEquals("listener must not be null", ex.getMessage()); // all is well } } diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 30a8e335fda..eae80418b0d 100644 --- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -31,6 +31,7 @@ import org.apache.lucene.codecs.lucene50.Lucene50Codec; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode; import org.apache.lucene.codecs.lucene53.Lucene53Codec; +import org.apache.lucene.codecs.lucene54.Lucene54Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -38,21 +39,32 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; +import java.io.IOException; +import java.util.Collections; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.instanceOf; @SuppressCodecs("*") // we test against default codec so never get a random one here! -public class CodecTests extends ESSingleNodeTestCase { - - @Test +public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene53Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene54Codec.class)); + assertThat(codecService.codec("Lucene53"), instanceOf(Lucene53Codec.class)); assertThat(codecService.codec("Lucene50"), instanceOf(Lucene50Codec.class)); assertThat(codecService.codec("Lucene410"), instanceOf(Lucene410Codec.class)); assertThat(codecService.codec("Lucene49"), instanceOf(Lucene49Codec.class)); @@ -62,17 +74,17 @@ public class CodecTests extends ESSingleNodeTestCase { assertThat(codecService.codec("Lucene41"), instanceOf(Lucene41Codec.class)); assertThat(codecService.codec("Lucene42"), instanceOf(Lucene42Codec.class)); } - + public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); assertCompressionEquals(Mode.BEST_SPEED, codec); } - + public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); assertCompressionEquals(Mode.BEST_COMPRESSION, codec); } - + // write some docs with it, inspect .si to see this was the used compression private void assertCompressionEquals(Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); @@ -91,13 +103,16 @@ public class CodecTests extends ESSingleNodeTestCase { dir.close(); } - private static CodecService createCodecService() { - return createCodecService(Settings.Builder.EMPTY_SETTINGS); - } - - private static CodecService createCodecService(Settings settings) { - IndexService indexService = createIndex("test", settings); - return indexService.getIndexServices().getCodecService(); + private static CodecService createCodecService() throws IOException { + Settings nodeSettings = settingsBuilder() + .put("path.home", createTempDir()) + .build(); + IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_na"), nodeSettings); + SimilarityService similarityService = new SimilarityService(settings, Collections.emptyMap()); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(settings); + MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()); + MapperService service = new MapperService(settings, analysisService, similarityService, mapperRegistry); + return new CodecService(service, ESLoggerFactory.getLogger("test")); } } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineIT.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineIT.java index bae6be5ad4f..76c07edcb0d 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineIT.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineIT.java @@ -26,15 +26,12 @@ import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.Collection; import java.util.HashSet; import java.util.Set; public class InternalEngineIT extends ESIntegTestCase { - - @Test public void testSetIndexCompoundOnFlush() { client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("number_of_replicas", 0).put("number_of_shards", 1)).get(); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java index 1ac7678c0a1..a844f971eac 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java @@ -25,22 +25,19 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; @ClusterScope(numDataNodes = 1, scope = Scope.SUITE) public class InternalEngineMergeIT extends ESIntegTestCase { - - @Test public void testMergesHappening() throws InterruptedException, IOException, ExecutionException { final int numOfShards = randomIntBetween(1,5); // some settings to keep num segments low diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 8d2017b26ac..7ee67645d43 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -28,16 +28,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; -import org.apache.lucene.index.LiveIndexWriterConfig; -import org.apache.lucene.index.LogByteSizeMergePolicy; -import org.apache.lucene.index.MergePolicy; -import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.SnapshotDeletionPolicy; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.*; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; @@ -62,24 +53,18 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.engine.Engine.Searcher; import org.elasticsearch.index.indexing.ShardIndexingService; -import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperForType; -import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.Mapper.BuilderContext; -import org.elasticsearch.index.mapper.MapperBuilders; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; @@ -90,15 +75,16 @@ import org.elasticsearch.index.store.DirectoryUtils; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; -import org.elasticsearch.index.translog.TranslogTests; import org.elasticsearch.indices.memory.IndexingMemoryController; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.threadpool.ThreadPool; import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.io.InputStream; @@ -106,13 +92,7 @@ import java.nio.charset.Charset; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -121,19 +101,14 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class InternalEngineTests extends ESTestCase { protected final ShardId shardId = new ShardId(new Index("index"), 1); + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings(new Index("index"), Settings.EMPTY); protected ThreadPool threadPool; @@ -143,7 +118,7 @@ public class InternalEngineTests extends ESTestCase { protected InternalEngine engine; protected InternalEngine replicaEngine; - private Settings defaultSettings; + private IndexSettings defaultSettings; private String codecName; private Path primaryTranslogDir; private Path replicaTranslogDir; @@ -153,7 +128,7 @@ public class InternalEngineTests extends ESTestCase { public void setUp() throws Exception { super.setUp(); - CodecService codecService = new CodecService(shardId.index()); + CodecService codecService = new CodecService(null, logger); String name = Codec.getDefault().getName(); if (Arrays.asList(codecService.availableCodecs()).contains(name)) { // some codecs are read only so we only take the ones that we have in the service and randomly @@ -162,12 +137,12 @@ public class InternalEngineTests extends ESTestCase { } else { codecName = "default"; } - defaultSettings = Settings.builder() + defaultSettings = IndexSettingsModule.newIndexSettings("test", Settings.builder() .put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, randomBoolean()) .put(EngineConfig.INDEX_GC_DELETES_SETTING, "1h") // make sure this doesn't kick in on us .put(EngineConfig.INDEX_CODEC_SETTING, codecName) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .build(); // TODO randomize more settings + .build()); // TODO randomize more settings threadPool = new ThreadPool(getClass().getName()); store = createStore(); storeReplica = createStore(); @@ -228,7 +203,7 @@ public class InternalEngineTests extends ESTestCase { } protected Store createStore(final Directory directory) throws IOException { - final DirectoryService directoryService = new DirectoryService(shardId, EMPTY_SETTINGS) { + final DirectoryService directoryService = new DirectoryService(shardId, INDEX_SETTINGS) { @Override public Directory newDirectory() throws IOException { return directory; @@ -239,7 +214,7 @@ public class InternalEngineTests extends ESTestCase { return 0; } }; - return new Store(shardId, EMPTY_SETTINGS, directoryService, new DummyShardLock(shardId)); + return new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); } protected Translog createTranslog() throws IOException { @@ -247,7 +222,7 @@ public class InternalEngineTests extends ESTestCase { } protected Translog createTranslog(Path translogPath) throws IOException { - TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, EMPTY_SETTINGS, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); + TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); return new Translog(translogConfig); } @@ -259,22 +234,22 @@ public class InternalEngineTests extends ESTestCase { return createEngine(defaultSettings, store, translogPath, new MergeSchedulerConfig(defaultSettings), newMergePolicy()); } - protected InternalEngine createEngine(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { + protected InternalEngine createEngine(IndexSettings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { return new InternalEngine(config(indexSettings, store, translogPath, mergeSchedulerConfig, mergePolicy), false); } - public EngineConfig config(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { + public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { IndexWriterConfig iwc = newIndexWriterConfig(); TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); - EngineConfig config = new EngineConfig(shardId, threadPool, new ShardIndexingService(shardId, indexSettings), indexSettings + EngineConfig config = new EngineConfig(shardId, threadPool, new ShardIndexingService(shardId, INDEX_SETTINGS), indexSettings , null, store, createSnapshotDeletionPolicy(), mergePolicy, mergeSchedulerConfig, - iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(shardId.index()), new Engine.FailedEngineListener() { + iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), new Engine.EventListener() { @Override - public void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable t) { + public void onFailedEngine(String reason, @Nullable Throwable t) { // we don't need to notify anybody in this test } - }, new TranslogHandler(shardId.index().getName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig); + }, new TranslogHandler(shardId.index().getName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5)); try { config.setCreate(Lucene.indexExists(store.directory()) == false); } catch (IOException e) { @@ -287,15 +262,14 @@ public class InternalEngineTests extends ESTestCase { protected static final BytesReference B_2 = new BytesArray(new byte[]{2}); protected static final BytesReference B_3 = new BytesArray(new byte[]{3}); - @Test public void testSegments() throws Exception { try (Store store = createStore(); - Engine engine = createEngine(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), NoMergePolicy.INSTANCE)) { + Engine engine = createEngine(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), NoMergePolicy.INSTANCE)) { List segments = engine.segments(false); assertThat(segments.isEmpty(), equalTo(true)); assertThat(engine.segmentsStats().getCount(), equalTo(0l)); assertThat(engine.segmentsStats().getMemoryInBytes(), equalTo(0l)); - final boolean defaultCompound = defaultSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, true); + final boolean defaultCompound = defaultSettings.getSettings().getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, true); // create a doc and refresh ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); @@ -438,11 +412,8 @@ public class InternalEngineTests extends ESTestCase { assertThat(segments.get(1).ramTree, notNullValue()); assertThat(segments.get(2).ramTree, notNullValue()); } - } - - @Test public void testSegmentsWithMergeFlag() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), new TieredMergePolicy())) { @@ -517,7 +488,6 @@ public class InternalEngineTests extends ESTestCase { assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY))); } - @Test public void testIndexSearcherWrapper() throws Exception { final AtomicInteger counter = new AtomicInteger(); IndexSearcherWrapper wrapper = new IndexSearcherWrapper() { @@ -529,7 +499,7 @@ public class InternalEngineTests extends ESTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { counter.incrementAndGet(); return searcher; } @@ -540,14 +510,12 @@ public class InternalEngineTests extends ESTestCase { engine.close(); engine = new InternalEngine(engine.config(), false); - Engine.Searcher searcher = wrapper.wrap(engine.config(), engine.acquireSearcher("test")); + Engine.Searcher searcher = wrapper.wrap(engine.acquireSearcher("test")); assertThat(counter.get(), equalTo(2)); searcher.close(); IOUtils.close(store, engine); } - @Test - /* */ public void testConcurrentGetAndFlush() throws Exception { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.index(new Engine.Index(newUid("1"), doc)); @@ -585,7 +553,6 @@ public class InternalEngineTests extends ESTestCase { latestGetResult.get().release(); } - @Test public void testSimpleOperations() throws Exception { Engine.Searcher searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); @@ -740,7 +707,6 @@ public class InternalEngineTests extends ESTestCase { searchResult.close(); } - @Test public void testSearchResultRelease() throws Exception { Engine.Searcher searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); @@ -803,6 +769,64 @@ public class InternalEngineTests extends ESTestCase { } } + public void testRenewSyncFlush() throws Exception { + final int iters = randomIntBetween(2, 5); // run this a couple of times to get some coverage + for (int i = 0; i < iters; i++) { + try (Store store = createStore(); + InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), + new LogDocMergePolicy()), false)) { + final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); + ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); + Engine.Index doc1 = new Engine.Index(newUid("1"), doc); + engine.index(doc1); + assertEquals(engine.getLastWriteNanos(), doc1.startTime()); + engine.flush(); + Engine.Index doc2 = new Engine.Index(newUid("2"), doc); + engine.index(doc2); + assertEquals(engine.getLastWriteNanos(), doc2.startTime()); + engine.flush(); + final boolean forceMergeFlushes = randomBoolean(); + if (forceMergeFlushes) { + engine.index(new Engine.Index(newUid("3"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime() - engine.engineConfig.getFlushMergesAfter().nanos())); + } else { + engine.index(new Engine.Index(newUid("3"), doc)); + } + Engine.CommitId commitID = engine.flush(); + assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), + Engine.SyncedFlushResult.SUCCESS); + assertEquals(3, engine.segments(false).size()); + + engine.forceMerge(false, 1, false, false, false); + if (forceMergeFlushes == false) { + engine.refresh("make all segments visible"); + assertEquals(4, engine.segments(false).size()); + assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); + assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); + assertTrue(engine.tryRenewSyncCommit()); + assertEquals(1, engine.segments(false).size()); + } else { + assertBusy(() -> assertEquals(1, engine.segments(false).size())); + } + assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); + assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); + + if (randomBoolean()) { + Engine.Index doc4 = new Engine.Index(newUid("4"), doc); + engine.index(doc4); + assertEquals(engine.getLastWriteNanos(), doc4.startTime()); + } else { + Engine.Delete delete = new Engine.Delete(doc1.type(), doc1.id(), doc1.uid()); + engine.delete(delete); + assertEquals(engine.getLastWriteNanos(), delete.startTime()); + } + assertFalse(engine.tryRenewSyncCommit()); + engine.flush(); + assertNull(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID)); + assertNull(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID)); + } + } + } + public void testSycnedFlushSurvivesEngineRestart() throws IOException { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); @@ -846,7 +870,6 @@ public class InternalEngineTests extends ESTestCase { assertNull("Sync ID must be gone since we have a document to replay", engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID)); } - @Test public void testVersioningNewCreate() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED); @@ -858,7 +881,6 @@ public class InternalEngineTests extends ESTestCase { assertThat(create.version(), equalTo(1l)); } - @Test public void testVersioningNewIndex() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -870,7 +892,6 @@ public class InternalEngineTests extends ESTestCase { assertThat(index.version(), equalTo(1l)); } - @Test public void testExternalVersioningNewIndex() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0); @@ -882,7 +903,6 @@ public class InternalEngineTests extends ESTestCase { assertThat(index.version(), equalTo(12l)); } - @Test public void testVersioningIndexConflict() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -911,7 +931,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testExternalVersioningIndexConflict() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0); @@ -931,7 +950,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testVersioningIndexConflictWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -962,7 +980,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testExternalVersioningIndexConflictWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0); @@ -1079,7 +1096,6 @@ public class InternalEngineTests extends ESTestCase { } - @Test public void testVersioningDeleteConflict() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -1130,7 +1146,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testVersioningDeleteConflictWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -1187,7 +1202,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testVersioningCreateExistsException() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0); @@ -1203,7 +1217,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testVersioningCreateExistsExceptionWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0); @@ -1221,7 +1234,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testVersioningReplicaConflict1() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -1257,7 +1269,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testVersioningReplicaConflict2() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -1306,8 +1317,6 @@ public class InternalEngineTests extends ESTestCase { } } - - @Test public void testBasicCreatedFlag() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -1322,7 +1331,6 @@ public class InternalEngineTests extends ESTestCase { assertTrue(engine.index(index)); } - @Test public void testCreatedFlagAfterFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); @@ -1367,7 +1375,6 @@ public class InternalEngineTests extends ESTestCase { // #5891: make sure IndexWriter's infoStream output is // sent to lucene.iw with log level TRACE: - @Test public void testIndexWriterInfoStream() { assumeFalse("who tests the tester?", VERBOSE); MockAppender mockAppender = new MockAppender(); @@ -1433,7 +1440,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testEnableGcDeletes() throws Exception { try (Store store = createStore(); Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), newMergePolicy()), false)) { @@ -1497,7 +1503,6 @@ public class InternalEngineTests extends ESTestCase { return new Term("_uid", id); } - @Test public void testExtractShardId() { try (Engine.Searcher test = this.engine.acquireSearcher("test")) { ShardId shardId = ShardUtils.extractShardId(test.getDirectoryReader()); @@ -1510,7 +1515,6 @@ public class InternalEngineTests extends ESTestCase { * Random test that throws random exception and ensures all references are * counted down / released and resources are closed. */ - @Test public void testFailStart() throws IOException { // this test fails if any reader, searcher or directory is not closed - MDW FTW final int iters = scaledRandomIntBetween(10, 100); @@ -1551,9 +1555,8 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testSettings() { - CodecService codecService = new CodecService(shardId.index()); + CodecService codecService = new CodecService(null, logger); LiveIndexWriterConfig currentIndexWriterConfig = engine.getCurrentIndexWriterConfig(); assertEquals(engine.config().getCodec().getName(), codecService.codec(codecName).getName()); @@ -1561,13 +1564,13 @@ public class InternalEngineTests extends ESTestCase { } // #10312 - @Test + // ncommit get this working again + /* public void testDeletesAloneCanTriggerRefresh() throws Exception { - Settings settings = Settings.builder() - .put(defaultSettings) - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "1kb").build(); + // nocommit need to set buffer up front again? try (Store store = createStore(); - Engine engine = new InternalEngine(config(settings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), newMergePolicy()), false)) { + Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), newMergePolicy()), false)) { + engine.config().setIndexingBufferSize(new ByteSizeValue(1, ByteSizeUnit.KB)); for (int i = 0; i < 100; i++) { String id = Integer.toString(i); ParsedDocument doc = testParsedDocument(id, id, "test", null, -1, -1, testDocument(), B_1, null); @@ -1580,17 +1583,12 @@ public class InternalEngineTests extends ESTestCase { // Make a shell of an IMC to check up on indexing buffer usage: IndexingMemoryController imc = new IndexingMemoryController(settings, threadPool, null) { @Override - protected IndexShard getShard(ShardId shardId) { - return null; - } - - @Override - protected List availableShards() { + protected List availableShards() { return Collections.singletonList(new ShardId("foo", 0)); } @Override - protected void refreshShardAsync(ShardId shardId) { + protected void refreshShardAsync(IndexShard shard) { engine.refresh("memory"); } @@ -1615,6 +1613,7 @@ public class InternalEngineTests extends ESTestCase { } } } + */ public void testMissingTranslog() throws IOException { // test that we can force start the engine , even if the translog is missing. @@ -1631,7 +1630,9 @@ public class InternalEngineTests extends ESTestCase { // expected } // now it should be OK. - Settings indexSettings = Settings.builder().put(defaultSettings).put(EngineConfig.INDEX_FORCE_NEW_TRANSLOG, true).build(); + IndexSettings indexSettings = new IndexSettings(defaultSettings.getIndexMetaData(), + Settings.builder().put(defaultSettings.getSettings()).put(EngineConfig.INDEX_FORCE_NEW_TRANSLOG, true).build(), + Collections.emptyList()); engine = createEngine(indexSettings, store, primaryTranslogDir, new MergeSchedulerConfig(indexSettings), newMergePolicy()); } @@ -1687,7 +1688,6 @@ public class InternalEngineTests extends ESTestCase { } } - @Test public void testSkipTranslogReplay() throws IOException { final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { @@ -1720,7 +1720,7 @@ public class InternalEngineTests extends ESTestCase { private Mapping dynamicUpdate() { BuilderContext context = new BuilderContext(Settings.EMPTY, new ContentPath()); final RootObjectMapper root = MapperBuilders.rootObject("some_type").build(context); - return new Mapping(Version.CURRENT, root, new MetadataFieldMapper[0], new Mapping.SourceTransform[0], emptyMap()); + return new Mapping(Version.CURRENT, root, new MetadataFieldMapper[0], emptyMap()); } public void testUpgradeOldIndex() throws IOException { @@ -1772,7 +1772,7 @@ public class InternalEngineTests extends ESTestCase { } CommitStats commitStats = engine.commitStats(); Map userData = commitStats.getUserData(); - assertTrue("userdata dosn't contain uuid",userData.containsKey(Translog.TRANSLOG_UUID_KEY)); + assertTrue("userdata dosn't contain uuid", userData.containsKey(Translog.TRANSLOG_UUID_KEY)); assertTrue("userdata doesn't contain generation key", userData.containsKey(Translog.TRANSLOG_GENERATION_KEY)); assertFalse("userdata contains legacy marker", userData.containsKey("translog_id")); } @@ -1915,13 +1915,14 @@ public class InternalEngineTests extends ESTestCase { Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); RootObjectMapper.Builder rootBuilder = new RootObjectMapper.Builder("test"); Index index = new Index(indexName); - AnalysisService analysisService = new AnalysisService(index, settings); - SimilarityService similarityService = new SimilarityService(index, settings); - MapperService mapperService = new MapperService(index, settings, analysisService, similarityService, null); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); + AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); + MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry(); + MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry); DocumentMapper.Builder b = new DocumentMapper.Builder(settings, rootBuilder, mapperService); - DocumentMapperParser parser = new DocumentMapperParser(settings, mapperService, analysisService, similarityService, null); + DocumentMapperParser parser = mapperService.documentMapperParser(); this.docMapper = b.build(mapperService, parser); - } @Override @@ -1957,7 +1958,7 @@ public class InternalEngineTests extends ESTestCase { Translog.TranslogGeneration generation = engine.getTranslog().getGeneration(); engine.close(); - Translog translog = new Translog(new TranslogConfig(shardId, createTempDir(), Settings.EMPTY, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool)); + Translog translog = new Translog(new TranslogConfig(shardId, createTempDir(), INDEX_SETTINGS, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool)); translog.add(new Translog.Index("test", "SomeBogusId", "{}".getBytes(Charset.forName("UTF-8")))); assertEquals(generation.translogFileGeneration, translog.currentFileGeneration()); translog.close(); @@ -1968,8 +1969,8 @@ public class InternalEngineTests extends ESTestCase { EngineConfig brokenConfig = new EngineConfig(shardId, threadPool, config.getIndexingService(), config.getIndexSettings() , null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getMergeSchedulerConfig(), - config.getAnalyzer(), config.getSimilarity(), new CodecService(shardId.index()), config.getFailedEngineListener() - , config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig); + config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener() + , config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5)); try { new InternalEngine(brokenConfig, false); diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index 7dadafb8a0b..3fe7a540bf8 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -37,8 +37,10 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.indexing.ShardIndexingService; import org.elasticsearch.index.mapper.Mapping; @@ -56,11 +58,11 @@ import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.threadpool.ThreadPool; import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.nio.file.Path; @@ -69,7 +71,6 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.hamcrest.Matchers.*; /** @@ -88,7 +89,7 @@ public class ShadowEngineTests extends ESTestCase { protected Engine primaryEngine; protected Engine replicaEngine; - private Settings defaultSettings; + private IndexSettings defaultSettings; private String codecName; private Path dirPath; @@ -96,7 +97,7 @@ public class ShadowEngineTests extends ESTestCase { @Before public void setUp() throws Exception { super.setUp(); - CodecService codecService = new CodecService(shardId.index()); + CodecService codecService = new CodecService(null, logger); String name = Codec.getDefault().getName(); if (Arrays.asList(codecService.availableCodecs()).contains(name)) { // some codecs are read only so we only take the ones that we have in the service and randomly @@ -105,12 +106,13 @@ public class ShadowEngineTests extends ESTestCase { } else { codecName = "default"; } - defaultSettings = Settings.builder() + defaultSettings = IndexSettingsModule.newIndexSettings("test", Settings.builder() .put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, randomBoolean()) .put(EngineConfig.INDEX_GC_DELETES_SETTING, "1h") // make sure this doesn't kick in on us .put(EngineConfig.INDEX_CODEC_SETTING, codecName) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .build(); // TODO randomize more settings + .build()); // TODO randomize more settings + threadPool = new ThreadPool(getClass().getName()); dirPath = createTempDir(); store = createStore(dirPath); @@ -168,8 +170,10 @@ public class ShadowEngineTests extends ESTestCase { return createStore(newMockFSDirectory(p)); } + protected Store createStore(final Directory directory) throws IOException { - final DirectoryService directoryService = new DirectoryService(shardId, EMPTY_SETTINGS) { + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.index(), Settings.EMPTY); + final DirectoryService directoryService = new DirectoryService(shardId, indexSettings) { @Override public Directory newDirectory() throws IOException { return directory; @@ -180,7 +184,7 @@ public class ShadowEngineTests extends ESTestCase { return 0; } }; - return new Store(shardId, EMPTY_SETTINGS, directoryService, new DummyShardLock(shardId)); + return new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId)); } protected SnapshotDeletionPolicy createSnapshotDeletionPolicy() { @@ -195,28 +199,28 @@ public class ShadowEngineTests extends ESTestCase { return createInternalEngine(defaultSettings, store, translogPath); } - protected ShadowEngine createShadowEngine(Settings indexSettings, Store store) { + protected ShadowEngine createShadowEngine(IndexSettings indexSettings, Store store) { return new ShadowEngine(config(indexSettings, store, null, new MergeSchedulerConfig(indexSettings), null)); } - protected InternalEngine createInternalEngine(Settings indexSettings, Store store, Path translogPath) { + protected InternalEngine createInternalEngine(IndexSettings indexSettings, Store store, Path translogPath) { return createInternalEngine(indexSettings, store, translogPath, newMergePolicy()); } - protected InternalEngine createInternalEngine(Settings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy) { + protected InternalEngine createInternalEngine(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy) { return new InternalEngine(config(indexSettings, store, translogPath, new MergeSchedulerConfig(indexSettings), mergePolicy), true); } - public EngineConfig config(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { + public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { IndexWriterConfig iwc = newIndexWriterConfig(); TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); EngineConfig config = new EngineConfig(shardId, threadPool, new ShardIndexingService(shardId, indexSettings), indexSettings , null, store, createSnapshotDeletionPolicy(), mergePolicy, mergeSchedulerConfig, - iwc.getAnalyzer(), iwc.getSimilarity() , new CodecService(shardId.index()), new Engine.FailedEngineListener() { + iwc.getAnalyzer(), iwc.getSimilarity() , new CodecService(null, logger), new Engine.EventListener() { @Override - public void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable t) { + public void onFailedEngine(String reason, @Nullable Throwable t) { // we don't need to notify anybody in this test - }}, null, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig); + }}, null, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5)); try { config.setCreate(Lucene.indexExists(store.directory()) == false); } catch (IOException e) { @@ -258,8 +262,6 @@ public class ShadowEngineTests extends ESTestCase { assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY))); } - - @Test public void testSegments() throws Exception { primaryEngine.close(); // recreate without merging primaryEngine = createInternalEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE); @@ -267,7 +269,7 @@ public class ShadowEngineTests extends ESTestCase { assertThat(segments.isEmpty(), equalTo(true)); assertThat(primaryEngine.segmentsStats().getCount(), equalTo(0l)); assertThat(primaryEngine.segmentsStats().getMemoryInBytes(), equalTo(0l)); - final boolean defaultCompound = defaultSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, true); + final boolean defaultCompound = defaultSettings.getSettings().getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, true); // create a doc and refresh ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); @@ -433,7 +435,6 @@ public class ShadowEngineTests extends ESTestCase { assertThat(segments.get(2).isCompound(), equalTo(true)); } - @Test public void testVerboseSegments() throws Exception { primaryEngine.close(); // recreate without merging primaryEngine = createInternalEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE); @@ -473,7 +474,6 @@ public class ShadowEngineTests extends ESTestCase { } - @Test public void testShadowEngineIgnoresWriteOperations() throws Exception { // create a document ParseContext.Document document = testDocumentWithTextField(); @@ -563,7 +563,6 @@ public class ShadowEngineTests extends ESTestCase { getResult.release(); } - @Test public void testSimpleOperations() throws Exception { Engine.Searcher searchResult = primaryEngine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); @@ -776,7 +775,6 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); } - @Test public void testSearchResultRelease() throws Exception { Engine.Searcher searchResult = replicaEngine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); @@ -827,7 +825,6 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); } - @Test public void testFailEngineOnCorruption() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); @@ -852,7 +849,6 @@ public class ShadowEngineTests extends ESTestCase { } } - @Test public void testExtractShardId() { try (Engine.Searcher test = replicaEngine.acquireSearcher("test")) { ShardId shardId = ShardUtils.extractShardId(test.getDirectoryReader()); @@ -865,7 +861,6 @@ public class ShadowEngineTests extends ESTestCase { * Random test that throws random exception and ensures all references are * counted down / released and resources are closed. */ - @Test public void testFailStart() throws IOException { // Need a commit point for this ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); @@ -910,13 +905,11 @@ public class ShadowEngineTests extends ESTestCase { } } - @Test public void testSettings() { - CodecService codecService = new CodecService(shardId.index()); + CodecService codecService = new CodecService(null, logger); assertEquals(replicaEngine.config().getCodec().getName(), codecService.codec(codecName).getName()); } - @Test public void testShadowEngineCreationRetry() throws Exception { final Path srDir = createTempDir(); final Store srStore = createStore(srDir); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java index ff072233ea6..743be637853 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java @@ -21,13 +21,18 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.search.MultiValueMode; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTestCase { @@ -63,7 +68,6 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes return 1; } - @Test public void testDeletedDocs() throws Exception { add2SingleValuedDocumentsAndDeleteOneOfThem(); IndexFieldData indexFieldData = getForField("value"); @@ -76,7 +80,6 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes } } - @Test public void testSingleValueAllSet() throws Exception { fillSingleValueAllSet(); IndexFieldData indexFieldData = getForField("value"); @@ -122,7 +125,7 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes } protected abstract void fillSingleValueWithMissing() throws Exception; - + public void assertValues(SortedBinaryDocValues values, int docId, BytesRef... actualValues) { values.setDocument(docId); assertThat(values.count(), equalTo(actualValues.length)); @@ -130,7 +133,7 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes assertThat(values.valueAt(i), equalTo(actualValues[i])); } } - + public void assertValues(SortedBinaryDocValues values, int docId, String... actualValues) { values.setDocument(docId); assertThat(values.count(), equalTo(actualValues.length)); @@ -139,8 +142,6 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes } } - - @Test public void testSingleValueWithMissing() throws Exception { fillSingleValueWithMissing(); IndexFieldData indexFieldData = getForField("value"); @@ -157,7 +158,6 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes protected abstract void fillMultiValueAllSet() throws Exception; - @Test public void testMultiValueAllSet() throws Exception { fillMultiValueAllSet(); IndexFieldData indexFieldData = getForField("value"); @@ -169,7 +169,7 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes assertValues(bytesValues, 0, two(), four()); assertValues(bytesValues, 1, one()); assertValues(bytesValues, 2, three()); - + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(3)); @@ -188,7 +188,6 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes protected abstract void fillMultiValueWithMissing() throws Exception; - @Test public void testMultiValueWithMissing() throws Exception { fillMultiValueWithMissing(); IndexFieldData indexFieldData = getForField("value"); @@ -223,7 +222,6 @@ public abstract class AbstractFieldDataImplTestCase extends AbstractFieldDataTes protected abstract void fillAllMissing() throws Exception; - @Test public void testSortMultiValuesFields() throws Exception { fillExtendedMvSet(); IndexFieldData indexFieldData = getForField("value"); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 9200873e1c8..bed9b480c01 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -24,8 +24,11 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.*; -import org.apache.lucene.search.Filter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.store.RAMDirectory; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; @@ -35,13 +38,18 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapperLegacy; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.VersionUtils; import org.junit.After; import org.junit.Before; +import java.io.IOException; + import static org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -72,7 +80,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase { public > IFD getForField(FieldDataType type, String fieldName, boolean docValues) { final MappedFieldType fieldType; - final BuilderContext context = new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1)); + final BuilderContext context = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); if (type.getType().equals("string")) { fieldType = MapperBuilders.stringField(fieldName).tokenized(false).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); } else if (type.getType().equals("float")) { @@ -88,7 +96,11 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase { } else if (type.getType().equals("byte")) { fieldType = MapperBuilders.byteField(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); } else if (type.getType().equals("geo_point")) { - fieldType = MapperBuilders.geoPointField(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); + if (indexService.getIndexSettings().getIndexVersionCreated().before(Version.V_2_2_0)) { + fieldType = new GeoPointFieldMapperLegacy.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); + } else { + fieldType = new GeoPointFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); + } } else if (type.getType().equals("_parent")) { fieldType = new ParentFieldMapper.Builder("_type").type(fieldName).build(context).fieldType(); } else if (type.getType().equals("binary")) { @@ -101,7 +113,9 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase { @Before public void setup() throws Exception { - Settings settings = Settings.builder().put("index.fielddata.cache", "none").build(); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.builder().put("index.fielddata.cache", "none") + .put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); indexService = createIndex("test", settings); mapperService = indexService.mapperService(); indicesFieldDataCache = getInstanceFromNode(IndicesFieldDataCache.class); @@ -130,9 +144,9 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase { writer.close(); } - protected Nested createNested(Filter parentFilter, Filter childFilter) { - BitsetFilterCache s = indexService.bitsetFilterCache(); - return new Nested(s.getBitSetProducer(parentFilter), childFilter); + protected Nested createNested(IndexSearcher searcher, Query parentFilter, Query childFilter) throws IOException { + BitsetFilterCache s = indexService.cache().bitsetFilterCache(); + return new Nested(s.getBitSetProducer(parentFilter), searcher.createNormalizedWeight(childFilter, false)); } public void testEmpty() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java new file mode 100644 index 00000000000..741ef3804b1 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.GeoPointField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.VersionUtils; + +import static org.elasticsearch.test.geo.RandomShapeGenerator.randomPoint; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public abstract class AbstractGeoFieldDataTestCase extends AbstractFieldDataImplTestCase { + @Override + protected abstract FieldDataType getFieldDataType(); + + protected Field randomGeoPointField(String fieldName, Field.Store store) { + GeoPoint point = randomPoint(random()); + if (indexService.getIndexSettings().getIndexVersionCreated().before(Version.V_2_2_0)) { + return new StringField(fieldName, point.lat()+","+point.lon(), store); + } + return new GeoPointField(fieldName, point.lon(), point.lat(), store); + } + + @Override + protected void fillAllMissing() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + writer.addDocument(d); + } + + @Override + public void testSortMultiValuesFields() { + assumeFalse("Only test on non geo_point fields", getFieldDataType().equals("geo_point")); + } + + protected void assertValues(MultiGeoPointValues values, int docId) { + assertValues(values, docId, false); + } + + protected void assertMissing(MultiGeoPointValues values, int docId) { + assertValues(values, docId, true); + } + + private void assertValues(MultiGeoPointValues values, int docId, boolean missing) { + values.setDocument(docId); + int docCount = values.count(); + if (missing) { + assertThat(docCount, equalTo(0)); + } else { + assertThat(docCount, greaterThan(0)); + for (int i = 0; i < docCount; ++i) { + final GeoPoint point = values.valueAt(i); + assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT_INCL), lessThanOrEqualTo(GeoUtils.MAX_LAT_INCL))); + assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON_INCL), lessThanOrEqualTo(GeoUtils.MAX_LON_INCL))); + } + } + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTestCase.java deleted file mode 100644 index 5c28a8f6c55..00000000000 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTestCase.java +++ /dev/null @@ -1,512 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.*; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.MultiValueMode; -import org.junit.Test; - -import java.util.Locale; - -import static org.hamcrest.Matchers.equalTo; - -/** - */ -public abstract class AbstractNumericFieldDataTestCase extends AbstractFieldDataImplTestCase { - - @Override - protected abstract FieldDataType getFieldDataType(); - - protected Settings.Builder getFieldDataSettings() { - Settings.Builder builder = Settings.builder(); - IndexFieldData.CommonSettings.MemoryStorageFormat[] formats = IndexFieldData.CommonSettings.MemoryStorageFormat.values(); - int i = randomInt(formats.length); - if (i < formats.length) { - builder.put(IndexFieldData.CommonSettings.SETTING_MEMORY_STORAGE_HINT, formats[i].name().toLowerCase(Locale.ROOT)); - } - return builder; - } - - @Test - public void testSingleValueAllSetNumber() throws Exception { - fillSingleValueAllSet(); - IndexNumericFieldData indexFieldData = getForField("value"); - AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); - - SortedNumericDocValues longValues = fieldData.getLongValues(); - - assertThat(FieldData.isMultiValued(longValues), equalTo(false)); - - longValues.setDocument(0); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(2l)); - - longValues.setDocument(1); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(1l)); - - longValues.setDocument(2); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(3l)); - - SortedNumericDoubleValues doubleValues = fieldData.getDoubleValues(); - - assertThat(FieldData.isMultiValued(doubleValues), equalTo(false)); - - doubleValues.setDocument(0); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(2d)); - - doubleValues.setDocument(1); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(1d)); - - doubleValues.setDocument(2); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(3d)); - - IndexSearcher searcher = new IndexSearcher(readerContext.reader()); - TopFieldDocs topDocs; - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); - } - - @Test - public void testSingleValueWithMissingNumber() throws Exception { - fillSingleValueWithMissing(); - IndexNumericFieldData indexFieldData = getForField("value"); - AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); - - SortedNumericDocValues longValues = fieldData.getLongValues(); - - assertThat(FieldData.isMultiValued(longValues), equalTo(false)); - - longValues.setDocument(0); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(2l)); - - longValues.setDocument(1); - assertThat(longValues.count(), equalTo(0)); - - longValues.setDocument(2); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(3l)); - - SortedNumericDoubleValues doubleValues = fieldData.getDoubleValues(); - - assertThat(FieldData.isMultiValued(doubleValues), equalTo(false)); - - doubleValues.setDocument(0); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(2d)); - - doubleValues.setDocument(1); - assertThat(0, equalTo(doubleValues.count())); - - doubleValues.setDocument(2); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(3d)); - - IndexSearcher searcher = new IndexSearcher(readerContext.reader()); - TopFieldDocs topDocs; - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null)))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(0)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MIN, null)))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MAX, null), true))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); - } - - @Test - public void testMultiValueAllSetNumber() throws Exception { - fillMultiValueAllSet(); - IndexNumericFieldData indexFieldData = getForField("value"); - AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); - - SortedNumericDocValues longValues = fieldData.getLongValues(); - - assertThat(FieldData.isMultiValued(longValues), equalTo(true)); - - longValues.setDocument(0); - assertThat(longValues.count(), equalTo(2)); - assertThat(longValues.valueAt(0), equalTo(2l)); - assertThat(longValues.valueAt(1), equalTo(4l)); - - longValues.setDocument(1); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(1l)); - - longValues.setDocument(2); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(3l)); - - SortedNumericDoubleValues doubleValues = fieldData.getDoubleValues(); - - assertThat(FieldData.isMultiValued(doubleValues), equalTo(true)); - - doubleValues.setDocument(0); - assertThat(2, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(2d)); - assertThat(doubleValues.valueAt(1), equalTo(4d)); - - doubleValues.setDocument(1); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(1d)); - - doubleValues.setDocument(2); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(3d)); - } - - @Test - public void testMultiValueWithMissingNumber() throws Exception { - fillMultiValueWithMissing(); - IndexNumericFieldData indexFieldData = getForField("value"); - AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); - - SortedNumericDocValues longValues = fieldData.getLongValues(); - - assertThat(FieldData.isMultiValued(longValues), equalTo(true)); - - longValues.setDocument(0); - assertThat(longValues.count(), equalTo(2)); - assertThat(longValues.valueAt(0), equalTo(2l)); - assertThat(longValues.valueAt(1), equalTo(4l)); - - longValues.setDocument(1); - assertThat(longValues.count(), equalTo(0)); - - longValues.setDocument(2); - assertThat(longValues.count(), equalTo(1)); - assertThat(longValues.valueAt(0), equalTo(3l)); - - SortedNumericDoubleValues doubleValues = fieldData.getDoubleValues(); - - assertThat(FieldData.isMultiValued(doubleValues), equalTo(true)); - - doubleValues.setDocument(0); - assertThat(2, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(2d)); - assertThat(doubleValues.valueAt(1), equalTo(4d)); - - doubleValues.setDocument(1); - assertThat(0, equalTo(doubleValues.count())); - - doubleValues.setDocument(2); - assertThat(1, equalTo(doubleValues.count())); - assertThat(doubleValues.valueAt(0), equalTo(3d)); - - } - - @Override - @Test - public void testMissingValueForAll() throws Exception { - fillAllMissing(); - IndexNumericFieldData indexFieldData = getForField("value"); - AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); - - // long values - - SortedNumericDocValues longValues = fieldData.getLongValues(); - - assertThat(FieldData.isMultiValued(longValues), equalTo(false)); - - for (int i = 0; i < 3; ++i) { - longValues.setDocument(0); - assertThat(longValues.count(), equalTo(0)); - } - - // double values - - SortedNumericDoubleValues doubleValues = fieldData.getDoubleValues(); - - assertThat(FieldData.isMultiValued(doubleValues), equalTo(false)); - - doubleValues.setDocument(0); - assertThat(0, equalTo(doubleValues.count())); - - doubleValues.setDocument(1); - assertThat(0, equalTo(doubleValues.count())); - - doubleValues.setDocument(2); - assertThat(0, equalTo(doubleValues.count())); - } - - - @Override - protected void fillAllMissing() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - writer.addDocument(d); - } - - @Override - @Test - public void testSortMultiValuesFields() throws Exception { - fillExtendedMvSet(); - IndexFieldData indexFieldData = getForField("value"); - - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(-10)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(2)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(3)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(4)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(4)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(6)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(6)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).intValue(), equalTo(8)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); -// assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); -// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(10)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(4)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(8)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(6)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(4)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(7)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).intValue(), equalTo(-8)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); -// assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); -// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); - - searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null)))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(-27)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(0)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(6)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(3)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(15)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(4)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(21)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(6)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).intValue(), equalTo(27)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); -// assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); -// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); - - searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null), true))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(27)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(4)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(21)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(15)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(6)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(7)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).intValue(), equalTo(-27)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); -// assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); -// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); - - searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null)))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(-9)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(3)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(5)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(4)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(7)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(6)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).intValue(), equalTo(9)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); -// assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); -// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); - - searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null), true))); // defaults to _last - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(9)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(4)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(7)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(5)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(3)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(7)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).intValue(), equalTo(-9)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); -// assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); -// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null)))); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(5)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(7)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(3)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(4)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(6)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true))); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(5)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(6)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(4)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(3)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("-9", MultiValueMode.MIN, null)))); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(5)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(3)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(4)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(6)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("9", MultiValueMode.MAX, null), true))); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(5)); - assertThat(topDocs.scoreDocs[3].doc, equalTo(4)); - assertThat(topDocs.scoreDocs[4].doc, equalTo(3)); - assertThat(topDocs.scoreDocs[5].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[6].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); - } - -} diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index 117ef2f4993..b1f9d73de73 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -26,17 +26,20 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.join.BitDocIdSetCachingWrapperFilter; +import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ToParentBlockJoinQuery; import org.apache.lucene.util.Accountable; @@ -51,7 +54,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.N import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; import org.elasticsearch.search.MultiValueMode; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -67,7 +69,6 @@ import static org.hamcrest.Matchers.sameInstance; /** */ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataImplTestCase { - private void addField(Document d, String name, String value) { d.add(new StringField(name, value, Field.Store.YES)); d.add(new SortedSetDocValuesField(name, new BytesRef(value))); @@ -402,11 +403,11 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI missingValue = new BytesRef(TestUtil.randomSimpleString(getRandom())); break; } - Filter parentFilter = new QueryWrapperFilter(new TermQuery(new Term("type", "parent"))); - Filter childFilter = new QueryWrapperFilter(Queries.not(parentFilter)); - Nested nested = createNested(parentFilter, childFilter); + Query parentFilter = new TermQuery(new Term("type", "parent")); + Query childFilter = Queries.not(parentFilter); + Nested nested = createNested(searcher, parentFilter, childFilter); BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested); - ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new FilteredQuery(new MatchAllDocsQuery(), childFilter), new BitDocIdSetCachingWrapperFilter(parentFilter), ScoreMode.None); + ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("text", nestedComparatorSource)); TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort); assertTrue(topDocs.scoreDocs.length > 0); @@ -464,7 +465,6 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI } } - @Test public void testGlobalOrdinals() throws Exception { fillExtendedMvSet(); refreshReader(); @@ -555,7 +555,6 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("!10")); } - @Test public void testTermsEnum() throws Exception { fillExtendedMvSet(); LeafReaderContext atomicReaderContext = refreshReader(); @@ -591,7 +590,6 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI assertThat(size, equalTo(3)); } - @Test public void testGlobalOrdinalsGetRemovedOnceIndexReaderCloses() throws Exception { fillExtendedMvSet(); refreshReader(); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java index bc63ed9f18b..73fdd79b108 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata; import com.carrotsearch.hppc.ObjectArrayList; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.settings.Settings; @@ -28,7 +29,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; @@ -36,13 +36,11 @@ import static org.hamcrest.Matchers.equalTo; * */ public class BinaryDVFieldDataTests extends AbstractFieldDataTestCase { - @Override protected boolean hasDocValues() { return true; } - @Test public void testDocValue() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("test") .startObject("properties") diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/DoubleFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/DoubleFieldDataTests.java deleted file mode 100644 index 6c93a2e5fdc..00000000000 --- a/core/src/test/java/org/elasticsearch/index/fielddata/DoubleFieldDataTests.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.DoubleField; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.Term; - -/** - */ -public class DoubleFieldDataTests extends AbstractNumericFieldDataTestCase { - - @Override - protected FieldDataType getFieldDataType() { - return new FieldDataType("double", getFieldDataSettings()); - } - - @Override - protected String one() { - return "1.0"; - } - - @Override - protected String two() { - return "2.0"; - } - - @Override - protected String three() { - return "3.0"; - } - - @Override - protected String four() { - return "4.0"; - } - - @Override - protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new DoubleField("value", 2.0d, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new DoubleField("value", 4.0d, Field.Store.NO)); - writer.addDocument(d); - - writer.commit(); - - writer.deleteDocuments(new Term("_id", "1")); - } - - @Override - protected void fillSingleValueAllSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new DoubleField("value", 2.0d, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new DoubleField("value", 1.0d, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new DoubleField("value", 3.0d, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillSingleValueWithMissing() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new DoubleField("value", 2.0d, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING.... - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new DoubleField("value", 3.0d, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillMultiValueAllSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new DoubleField("value", 2.0d, Field.Store.NO)); - d.add(new DoubleField("value", 4.0d, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new DoubleField("value", 1.0d, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new DoubleField("value", 3.0d, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillMultiValueWithMissing() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new DoubleField("value", 2.0d, Field.Store.NO)); - d.add(new DoubleField("value", 4.0d, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new DoubleField("value", 3.0f, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillExtendedMvSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new DoubleField("value", 2, Field.Store.NO)); - d.add(new DoubleField("value", 4, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new DoubleField("value", 3, Field.Store.NO)); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField("_id", "4", Field.Store.NO)); - d.add(new DoubleField("value", 4, Field.Store.NO)); - d.add(new DoubleField("value", 5, Field.Store.NO)); - d.add(new DoubleField("value", 6, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "5", Field.Store.NO)); - d.add(new DoubleField("value", 6, Field.Store.NO)); - d.add(new DoubleField("value", 7, Field.Store.NO)); - d.add(new DoubleField("value", 8, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "6", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "7", Field.Store.NO)); - d.add(new DoubleField("value", 8, Field.Store.NO)); - d.add(new DoubleField("value", 9, Field.Store.NO)); - d.add(new DoubleField("value", 10, Field.Store.NO)); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField("_id", "8", Field.Store.NO)); - d.add(new DoubleField("value", -8, Field.Store.NO)); - d.add(new DoubleField("value", -9, Field.Store.NO)); - d.add(new DoubleField("value", -10, Field.Store.NO)); - writer.addDocument(d); - } - -} diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java index f02c286c601..6c4673bbd0b 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; @@ -41,7 +42,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -54,26 +54,26 @@ import java.util.Map.Entry; import java.util.Random; import java.util.Set; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; public class DuelFieldDataTests extends AbstractFieldDataTestCase { - @Override protected FieldDataType getFieldDataType() { return null; } - @Test public void testDuelAllTypesSingleValue() throws Exception { final String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") - .startObject("bytes").field("type", "string").field("index", "not_analyzed").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("byte").field("type", "byte").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("short").field("type", "short").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("integer").field("type", "integer").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("long").field("type", "long").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("float").field("type", "float").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("double").field("type", "double").startObject("fielddata").field("format", "doc_values").endObject().endObject() + .startObject("bytes").field("type", "string").field("index", "not_analyzed").endObject() + .startObject("byte").field("type", "byte").endObject() + .startObject("short").field("type", "short").endObject() + .startObject("integer").field("type", "integer").endObject() + .startObject("long").field("type", "long").endObject() + .startObject("float").field("type", "float").endObject() + .startObject("double").field("type", "double").endObject() .endObject().endObject().endObject().string(); final DocumentMapper mapper = mapperService.documentMapperParser().parse(mapping); Random random = getRandom(); @@ -99,12 +99,6 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { LeafReaderContext context = refreshReader(); Map typeMap = new HashMap<>(); typeMap.put(new FieldDataType("string", Settings.builder().put("format", "paged_bytes")), Type.Bytes); - typeMap.put(new FieldDataType("byte", Settings.builder().put("format", "array")), Type.Integer); - typeMap.put(new FieldDataType("short", Settings.builder().put("format", "array")), Type.Integer); - typeMap.put(new FieldDataType("int", Settings.builder().put("format", "array")), Type.Integer); - typeMap.put(new FieldDataType("long", Settings.builder().put("format", "array")), Type.Long); - typeMap.put(new FieldDataType("double", Settings.builder().put("format", "array")), Type.Double); - typeMap.put(new FieldDataType("float", Settings.builder().put("format", "array")), Type.Float); typeMap.put(new FieldDataType("byte", Settings.builder().put("format", "doc_values")), Type.Integer); typeMap.put(new FieldDataType("short", Settings.builder().put("format", "doc_values")), Type.Integer); typeMap.put(new FieldDataType("int", Settings.builder().put("format", "doc_values")), Type.Integer); @@ -125,9 +119,9 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { } ifdService.clear(); - IndexFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT)); + IndexFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT), true); ifdService.clear(); - IndexFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT)); + IndexFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT), true); duelFieldDataBytes(random, context, leftFieldData, rightFieldData, pre); duelFieldDataBytes(random, context, rightFieldData, leftFieldData, pre); @@ -140,15 +134,13 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { } } - - @Test public void testDuelIntegers() throws Exception { final String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") - .startObject("byte").field("type", "byte").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("short").field("type", "short").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("integer").field("type", "integer").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("long").field("type", "long").startObject("fielddata").field("format", "doc_values").endObject().endObject() + .startObject("byte").field("type", "byte").endObject() + .startObject("short").field("type", "short").endObject() + .startObject("integer").field("type", "integer").endObject() + .startObject("long").field("type", "long").endObject() .endObject().endObject().endObject().string(); final DocumentMapper mapper = mapperService.documentMapperParser().parse(mapping); @@ -163,7 +155,7 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { for (int j = 0; j < numValues; ++j) { vals.add(randomByte()); } - + numValues = vals.size(); int upto = 0; for (Byte bb : vals) { @@ -189,10 +181,6 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { } LeafReaderContext context = refreshReader(); Map typeMap = new HashMap<>(); - typeMap.put(new FieldDataType("byte", Settings.builder().put("format", "array")), Type.Integer); - typeMap.put(new FieldDataType("short", Settings.builder().put("format", "array")), Type.Integer); - typeMap.put(new FieldDataType("int", Settings.builder().put("format", "array")), Type.Integer); - typeMap.put(new FieldDataType("long", Settings.builder().put("format", "array")), Type.Long); typeMap.put(new FieldDataType("byte", Settings.builder().put("format", "doc_values")), Type.Integer); typeMap.put(new FieldDataType("short", Settings.builder().put("format", "doc_values")), Type.Integer); typeMap.put(new FieldDataType("int", Settings.builder().put("format", "doc_values")), Type.Integer); @@ -208,9 +196,9 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { right = left = list.remove(0); } ifdService.clear(); - IndexNumericFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT)); + IndexNumericFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT), true); ifdService.clear(); - IndexNumericFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT)); + IndexNumericFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT), true); duelFieldDataLong(random, context, leftFieldData, rightFieldData); duelFieldDataLong(random, context, rightFieldData, leftFieldData); @@ -225,12 +213,11 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { } - @Test public void testDuelDoubles() throws Exception { final String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") - .startObject("float").field("type", "float").startObject("fielddata").field("format", "doc_values").endObject().endObject() - .startObject("double").field("type", "double").startObject("fielddata").field("format", "doc_values").endObject().endObject() + .startObject("float").field("type", "float").endObject() + .startObject("double").field("type", "double").endObject() .endObject().endObject().endObject().string(); final DocumentMapper mapper = mapperService.documentMapperParser().parse(mapping); @@ -275,8 +262,6 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { } LeafReaderContext context = refreshReader(); Map typeMap = new HashMap<>(); - typeMap.put(new FieldDataType("double", Settings.builder().put("format", "array")), Type.Double); - typeMap.put(new FieldDataType("float", Settings.builder().put("format", "array")), Type.Float); typeMap.put(new FieldDataType("double", Settings.builder().put("format", "doc_values")), Type.Double); typeMap.put(new FieldDataType("float", Settings.builder().put("format", "doc_values")), Type.Float); ArrayList> list = new ArrayList<>(typeMap.entrySet()); @@ -290,10 +275,10 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { right = left = list.remove(0); } ifdService.clear(); - IndexNumericFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT)); + IndexNumericFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT), true); ifdService.clear(); - IndexNumericFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT)); + IndexNumericFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT), true); duelFieldDataDouble(random, context, leftFieldData, rightFieldData); duelFieldDataDouble(random, context, rightFieldData, leftFieldData); @@ -308,8 +293,6 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { } - - @Test public void testDuelStrings() throws Exception { Random random = getRandom(); int atLeast = scaledRandomIntBetween(200, 1500); @@ -587,7 +570,7 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { for (int i = 0; i < numDocs; ++i) { leftValues.setDocument(i); final int numValues = leftValues.count(); - rightValues.setDocument(i);; + rightValues.setDocument(i); assertEquals(numValues, rightValues.count()); List leftPoints = new ArrayList<>(); List rightPoints = new ArrayList<>(); @@ -597,11 +580,14 @@ public class DuelFieldDataTests extends AbstractFieldDataTestCase { GeoPoint r = rightValues.valueAt(j); rightPoints.add(new GeoPoint(r.getLat(), r.getLon())); } - for (GeoPoint l : leftPoints) { - assertTrue("Couldn't find " + l + " among " + rightPoints, contains(l, rightPoints, precision)); - } - for (GeoPoint r : rightPoints) { - assertTrue("Couldn't find " + r + " among " + leftPoints, contains(r, leftPoints, precision)); + // missing values were treated as 0,0 which are valid geopoints, this now correctly tests for missing values + if (leftPoints.isEmpty() == false) { + for (GeoPoint l : leftPoints) { + assertTrue("Couldn't find " + l + " among " + rightPoints, contains(l, rightPoints, precision)); + } + for (GeoPoint r : rightPoints) { + assertTrue("Couldn't find " + r + " among " + leftPoints, contains(r, leftPoints, precision)); + } } } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java new file mode 100644 index 00000000000..59fc8952a67 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; +import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.FieldMaskingReader; + +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class FieldDataCacheTests extends ESTestCase { + + public void testLoadGlobal_neverCacheIfFieldIsMissing() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(null); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + IndexWriter iw = new IndexWriter(dir, iwc); + long numDocs = scaledRandomIntBetween(32, 128); + + for (int i = 1; i <= numDocs; i++) { + Document doc = new Document(); + doc.add(new SortedSetDocValuesField("field1", new BytesRef(String.valueOf(i)))); + doc.add(new StringField("field2", String.valueOf(i), Field.Store.NO)); + iw.addDocument(doc); + if (i % 24 == 0) { + iw.commit(); + } + } + iw.close(); + DirectoryReader ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId("_index", 0)); + + DummyAccountingFieldDataCache fieldDataCache = new DummyAccountingFieldDataCache(); + // Testing SortedSetDVOrdinalsIndexFieldData: + SortedSetDVOrdinalsIndexFieldData sortedSetDVOrdinalsIndexFieldData = createSortedDV("field1", fieldDataCache); + sortedSetDVOrdinalsIndexFieldData.loadGlobal(ir); + assertThat(fieldDataCache.cachedGlobally, equalTo(1)); + sortedSetDVOrdinalsIndexFieldData.loadGlobal(new FieldMaskingReader("field1", ir)); + assertThat(fieldDataCache.cachedGlobally, equalTo(1)); + + // Testing PagedBytesIndexFieldData + PagedBytesIndexFieldData pagedBytesIndexFieldData = createPagedBytes("field2", fieldDataCache); + pagedBytesIndexFieldData.loadGlobal(ir); + assertThat(fieldDataCache.cachedGlobally, equalTo(2)); + pagedBytesIndexFieldData.loadGlobal(new FieldMaskingReader("field2", ir)); + assertThat(fieldDataCache.cachedGlobally, equalTo(2)); + + ir.close(); + dir.close(); + } + + private SortedSetDVOrdinalsIndexFieldData createSortedDV(String fieldName, IndexFieldDataCache indexFieldDataCache) { + FieldDataType fieldDataType = new StringFieldMapper.StringFieldType().fieldDataType(); + MappedFieldType.Names names = new MappedFieldType.Names(fieldName); + return new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(), indexFieldDataCache, names, new NoneCircuitBreakerService(), fieldDataType); + } + + private PagedBytesIndexFieldData createPagedBytes(String fieldName, IndexFieldDataCache indexFieldDataCache) { + FieldDataType fieldDataType = new StringFieldMapper.StringFieldType().fieldDataType(); + MappedFieldType.Names names = new MappedFieldType.Names(fieldName); + return new PagedBytesIndexFieldData(createIndexSettings(), names, fieldDataType, indexFieldDataCache, new NoneCircuitBreakerService()); + } + + private IndexSettings createIndexSettings() { + Settings settings = Settings.EMPTY; + IndexMetaData indexMetaData = IndexMetaData.builder("_name") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(); + return new IndexSettings(indexMetaData, settings, Collections.emptyList()); + } + + private class DummyAccountingFieldDataCache implements IndexFieldDataCache { + + private int cachedGlobally = 0; + + @Override + public > FD load(LeafReaderContext context, IFD indexFieldData) throws Exception { + return indexFieldData.loadDirect(context); + } + + @Override + public > IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception { + cachedGlobally++; + return (IFD) indexFieldData.localGlobalDirect(indexReader); + } + + @Override + public void clear() { + } + + @Override + public void clear(String fieldName) { + } + } + +} diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataFilterIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataFilterIntegrationIT.java index 393f481e170..adb511f1c89 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataFilterIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataFilterIntegrationIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; @@ -42,7 +41,6 @@ public class FieldDataFilterIntegrationIT extends ESIntegTestCase { return 0; } - @Test public void testRegexpFilter() throws IOException { CreateIndexRequestBuilder builder = prepareCreate("test"); XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -78,7 +76,7 @@ public class FieldDataFilterIntegrationIT extends ESIntegTestCase { Terms nameAgg = aggs.get("name"); assertThat(nameAgg.getBuckets().size(), Matchers.equalTo(1)); assertThat(nameAgg.getBuckets().iterator().next().getKeyAsString(), Matchers.equalTo("bacon")); - + Terms notFilteredAgg = aggs.get("not_filtered"); assertThat(notFilteredAgg.getBuckets().size(), Matchers.equalTo(2)); assertThat(notFilteredAgg.getBuckets().get(0).getKeyAsString(), Matchers.isOneOf("bacon", "bastards")); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java index 12aeb70df21..fc8a830f9c5 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.fielddata; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -30,8 +29,6 @@ import static org.hamcrest.Matchers.greaterThan; /** */ public class FieldDataLoadingIT extends ESIntegTestCase { - - @Test public void testEagerFieldDataLoading() throws Exception { assertAcked(prepareCreate("test") .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties") @@ -49,7 +46,6 @@ public class FieldDataLoadingIT extends ESIntegTestCase { assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), greaterThan(0l)); } - @Test public void testEagerGlobalOrdinalsFieldDataLoading() throws Exception { assertAcked(prepareCreate("test") .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties") diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index 4b9d0e1805c..49cb414208d 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -24,21 +24,18 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomAccessOrds; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; import java.util.Random; import static org.hamcrest.Matchers.equalTo; public class FilterFieldDataTests extends AbstractFieldDataTestCase { - @Override protected FieldDataType getFieldDataType() { // TODO Auto-generated method stub return null; } - @Test public void testFilterByFrequency() throws Exception { Random random = getRandom(); for (int i = 0; i < 1000; i++) { @@ -61,7 +58,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { writer.forceMerge(1, true); LeafReaderContext context = refreshReader(); String[] formats = new String[] { "paged_bytes"}; - + for (String format : formats) { { ifdService.clear(); @@ -84,7 +81,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { assertThat(1L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5")); } - + { ifdService.clear(); // test # docs with value FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format) @@ -96,7 +93,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); } - + { ifdService.clear(); FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format) @@ -108,7 +105,7 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); } - + { ifdService.clear(); FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format) @@ -125,10 +122,8 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { } } - - @Test - public void testFilterByRegExp() throws Exception { + public void testFilterByRegExp() throws Exception { int hundred = 0; int ten = 0; int five = 0; @@ -182,6 +177,6 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase { @Override public void testEmpty() throws Exception { - // No need to test empty usage here + assumeTrue("No need to test empty usage here", false); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FloatFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FloatFieldDataTests.java deleted file mode 100644 index 2633673c9de..00000000000 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FloatFieldDataTests.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FloatField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.Term; - -/** - */ -public class FloatFieldDataTests extends AbstractNumericFieldDataTestCase { - - @Override - protected FieldDataType getFieldDataType() { - return new FieldDataType("float", getFieldDataSettings()); - } - - @Override - protected String one() { - return "1.0"; - } - - @Override - protected String two() { - return "2.0"; - } - - @Override - protected String three() { - return "3.0"; - } - - @Override - protected String four() { - return "4.0"; - } - - @Override - protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new FloatField("value", 2.0f, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new FloatField("value", 4.0f, Field.Store.NO)); - writer.addDocument(d); - - writer.commit(); - - writer.deleteDocuments(new Term("_id", "1")); - } - - @Override - protected void fillSingleValueAllSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new FloatField("value", 2.0f, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new FloatField("value", 1.0f, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new FloatField("value", 3.0f, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillSingleValueWithMissing() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new FloatField("value", 2.0f, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING.... - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new FloatField("value", 3.0f, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillMultiValueAllSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new FloatField("value", 2.0f, Field.Store.NO)); - d.add(new FloatField("value", 4.0f, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new FloatField("value", 1.0f, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new FloatField("value", 3.0f, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillMultiValueWithMissing() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new FloatField("value", 2.0f, Field.Store.NO)); - d.add(new FloatField("value", 4.0f, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new FloatField("value", 3.0f, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillExtendedMvSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new FloatField("value", 2, Field.Store.NO)); - d.add(new FloatField("value", 4, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new FloatField("value", 3, Field.Store.NO)); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField("_id", "4", Field.Store.NO)); - d.add(new FloatField("value", 4, Field.Store.NO)); - d.add(new FloatField("value", 5, Field.Store.NO)); - d.add(new FloatField("value", 6, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "5", Field.Store.NO)); - d.add(new FloatField("value", 6, Field.Store.NO)); - d.add(new FloatField("value", 7, Field.Store.NO)); - d.add(new FloatField("value", 8, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "6", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "7", Field.Store.NO)); - d.add(new FloatField("value", 8, Field.Store.NO)); - d.add(new FloatField("value", 9, Field.Store.NO)); - d.add(new FloatField("value", 10, Field.Store.NO)); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField("_id", "8", Field.Store.NO)); - d.add(new FloatField("value", -8, Field.Store.NO)); - d.add(new FloatField("value", -9, Field.Store.NO)); - d.add(new FloatField("value", -10, Field.Store.NO)); - writer.addDocument(d); - } - -} diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java new file mode 100644 index 00000000000..21780fdf1fa --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java @@ -0,0 +1,205 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.document.*; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.elasticsearch.index.fielddata.plain.AbstractAtomicGeoPointFieldData; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +/** + * Basic Unit Test for GeoPointField data + * todo include backcompat testing - see ISSUE #14562 + */ +public class GeoFieldDataTests extends AbstractGeoFieldDataTestCase { + private static String FIELD_NAME = "value"; + + @Override + protected FieldDataType getFieldDataType() { + return new FieldDataType("geo_point"); + } + + @Override + protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception { + Document d = new Document(); + + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.YES)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + writer.commit(); + + writer.deleteDocuments(new Term("_id", "1")); + } + + @Override + protected void fillMultiValueWithMissing() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + // missing + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillSingleValueAllSet() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillSingleValueWithMissing() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING.... + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillMultiValueAllSet() throws Exception { + Document d = new Document(); + d.add(new StringField("_id", "1", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "2", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + + d = new Document(); + d.add(new StringField("_id", "3", Field.Store.NO)); + d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); + writer.addDocument(d); + } + + @Override + protected void fillExtendedMvSet() throws Exception { + Document d; + final int maxDocs = randomInt(10); + for (int i=0; i fd = ifdService.getForField(stringMapper); - if (docValues) { - assertTrue(fd instanceof SortedSetDVOrdinalsIndexFieldData); - } else { - assertTrue(fd instanceof PagedBytesIndexFieldData); - } - - for (MappedFieldType mapper : Arrays.asList( - new ByteFieldMapper.Builder("int").docValues(docValues).build(ctx).fieldType(), - new ShortFieldMapper.Builder("int").docValues(docValues).build(ctx).fieldType(), - new IntegerFieldMapper.Builder("int").docValues(docValues).build(ctx).fieldType(), - new LongFieldMapper.Builder("long").docValues(docValues).build(ctx).fieldType() - )) { - ifdService.clear(); - fd = ifdService.getForField(mapper); - if (docValues) { - assertTrue(fd instanceof SortedNumericDVIndexFieldData); - } else { - assertTrue(fd instanceof PackedArrayIndexFieldData); - } - } - - final MappedFieldType floatMapper = new FloatFieldMapper.Builder("float").docValues(docValues).build(ctx).fieldType(); - ifdService.clear(); - fd = ifdService.getForField(floatMapper); - if (docValues) { - assertTrue(fd instanceof SortedNumericDVIndexFieldData); - } else { - assertTrue(fd instanceof FloatArrayIndexFieldData); - } - - final MappedFieldType doubleMapper = new DoubleFieldMapper.Builder("double").docValues(docValues).build(ctx).fieldType(); - ifdService.clear(); - fd = ifdService.getForField(doubleMapper); - if (docValues) { - assertTrue(fd instanceof SortedNumericDVIndexFieldData); - } else { - assertTrue(fd instanceof DoubleArrayIndexFieldData); - } - } - } - - @SuppressWarnings("unchecked") - public void testByPassDocValues() { - final IndexService indexService = createIndex("test"); - final IndexFieldDataService ifdService = indexService.fieldData(); - final BuilderContext ctx = new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1)); - final MappedFieldType stringMapper = MapperBuilders.stringField("string").tokenized(false).fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(Settings.builder().put("format", "disabled").build()).build(ctx).fieldType(); + final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); + final MappedFieldType stringMapper = new StringFieldMapper.Builder("string").tokenized(false).build(ctx).fieldType(); ifdService.clear(); IndexFieldData fd = ifdService.getForField(stringMapper); - assertTrue(fd instanceof DisabledIndexFieldData); + assertTrue(fd instanceof SortedSetDVOrdinalsIndexFieldData); - final Settings fdSettings = Settings.builder().put("format", "array").build(); for (MappedFieldType mapper : Arrays.asList( - new ByteFieldMapper.Builder("int").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx).fieldType(), - new ShortFieldMapper.Builder("int").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx).fieldType(), - new IntegerFieldMapper.Builder("int").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx).fieldType(), - new LongFieldMapper.Builder("long").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx).fieldType() + new ByteFieldMapper.Builder("int").build(ctx).fieldType(), + new ShortFieldMapper.Builder("int").build(ctx).fieldType(), + new IntegerFieldMapper.Builder("int").build(ctx).fieldType(), + new LongFieldMapper.Builder("long").build(ctx).fieldType() )) { ifdService.clear(); fd = ifdService.getForField(mapper); - assertTrue(fd instanceof PackedArrayIndexFieldData); + assertTrue(fd instanceof SortedNumericDVIndexFieldData); } - final MappedFieldType floatMapper = MapperBuilders.floatField("float").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx).fieldType(); + final MappedFieldType floatMapper = new FloatFieldMapper.Builder("float").build(ctx).fieldType(); ifdService.clear(); fd = ifdService.getForField(floatMapper); - assertTrue(fd instanceof FloatArrayIndexFieldData); + assertTrue(fd instanceof SortedNumericDVIndexFieldData); - final MappedFieldType doubleMapper = MapperBuilders.doubleField("double").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx).fieldType(); + final MappedFieldType doubleMapper = new DoubleFieldMapper.Builder("double").build(ctx).fieldType(); ifdService.clear(); fd = ifdService.getForField(doubleMapper); - assertTrue(fd instanceof DoubleArrayIndexFieldData); + assertTrue(fd instanceof SortedNumericDVIndexFieldData); } public void testChangeFieldDataFormat() throws Exception { final IndexService indexService = createIndex("test"); final IndexFieldDataService ifdService = indexService.fieldData(); - final BuilderContext ctx = new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1)); + final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); final MappedFieldType mapper1 = MapperBuilders.stringField("s").tokenized(false).docValues(true).fieldDataSettings(Settings.builder().put(FieldDataType.FORMAT_KEY, "paged_bytes").build()).build(ctx).fieldType(); final IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new KeywordAnalyzer())); Document doc = new Document(); @@ -169,10 +123,10 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { final IndexService indexService = createIndex("test"); IndexFieldDataService shardPrivateService = indexService.fieldData(); // copy the ifdService since we can set the listener only once. - final IndexFieldDataService ifdService = new IndexFieldDataService(shardPrivateService.index(), shardPrivateService.indexSettings(), + final IndexFieldDataService ifdService = new IndexFieldDataService(indexService.getIndexSettings(), getInstanceFromNode(IndicesFieldDataCache.class), getInstanceFromNode(CircuitBreakerService.class), indexService.mapperService()); - final BuilderContext ctx = new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1)); + final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); final MappedFieldType mapper1 = MapperBuilders.stringField("s").tokenized(false).docValues(true).fieldDataSettings(Settings.builder().put(FieldDataType.FORMAT_KEY, "paged_bytes").build()).build(ctx).fieldType(); final IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new KeywordAnalyzer())); Document doc = new Document(); @@ -238,4 +192,35 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { } } + private void doTestRequireDocValues(MappedFieldType ft) { + ThreadPool threadPool = new ThreadPool("random_threadpool_name"); + try { + IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null, threadPool); + IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings(new Index("test"), Settings.EMPTY), cache, null, null); + ft.setNames(new Names("some_long")); + ft.setHasDocValues(true); + ifds.getForField(ft); // no exception + ft.setHasDocValues(false); + try { + ifds.getForField(ft); + fail(); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), containsString("doc values")); + } + } finally { + threadPool.shutdown(); + } + } + + public void testRequireDocValuesOnLongs() { + doTestRequireDocValues(new LongFieldMapper.LongFieldType()); + } + + public void testRequireDocValuesOnDoubles() { + doTestRequireDocValues(new DoubleFieldMapper.DoubleFieldType()); + } + + public void testRequireDocValuesOnBools() { + doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/LongFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/LongFieldDataTests.java deleted file mode 100644 index f47b94d7081..00000000000 --- a/core/src/test/java/org/elasticsearch/index/fielddata/LongFieldDataTests.java +++ /dev/null @@ -1,431 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.cursors.LongCursor; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.LongField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.Term; -import org.joda.time.DateTimeZone; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThan; - -/** - * Tests for all integer types (byte, short, int, long). - */ -public class LongFieldDataTests extends AbstractNumericFieldDataTestCase { - - @Override - protected FieldDataType getFieldDataType() { - // we don't want to optimize the type so it will always be a long... - return new FieldDataType("long", getFieldDataSettings()); - } - - @Override - protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new LongField("value", 2, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new LongField("value", 4, Field.Store.NO)); - writer.addDocument(d); - - writer.commit(); - - writer.deleteDocuments(new Term("_id", "1")); - } - - @Test - public void testOptimizeTypeLong() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new LongField("value", Integer.MAX_VALUE + 1l, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new LongField("value", Integer.MIN_VALUE - 1l, Field.Store.NO)); - writer.addDocument(d); - - IndexNumericFieldData indexFieldData = getForField("value"); - AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); - assertThat(getFirst(fieldData.getLongValues(), 0), equalTo((long) Integer.MAX_VALUE + 1l)); - assertThat(getFirst(fieldData.getLongValues(), 1), equalTo((long) Integer.MIN_VALUE - 1l)); - } - - private static long getFirst(SortedNumericDocValues values, int docId) { - values.setDocument(docId); - final int numValues = values.count(); - assertThat(numValues, is(1)); - return values.valueAt(0); - } - - private static double getFirst(SortedNumericDoubleValues values, int docId) { - values.setDocument(docId); - final int numValues = values.count(); - assertThat(numValues, is(1)); - return values.valueAt(0); - } - - @Test - public void testDateScripts() throws Exception { - fillSingleValueAllSet(); - IndexNumericFieldData indexFieldData = getForField("value"); - AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); - - ScriptDocValues.Longs scriptValues = (ScriptDocValues.Longs) fieldData.getScriptValues(); - scriptValues.setNextDocId(0); - assertThat(scriptValues.getValue(), equalTo(2l)); - assertThat(scriptValues.getDate().getMillis(), equalTo(2l)); - assertThat(scriptValues.getDate().getZone(), equalTo(DateTimeZone.UTC)); - } - - @Override - protected void fillSingleValueAllSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new LongField("value", 2, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new LongField("value", 1, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new LongField("value", 3, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillSingleValueWithMissing() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new LongField("value", 2, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING.... - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new LongField("value", 3, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillMultiValueAllSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new LongField("value", 2, Field.Store.NO)); - d.add(new LongField("value", 4, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - d.add(new LongField("value", 1, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new LongField("value", 3, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillMultiValueWithMissing() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new LongField("value", 2, Field.Store.NO)); - d.add(new LongField("value", 4, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new LongField("value", 3, Field.Store.NO)); - writer.addDocument(d); - } - - @Override - protected void fillExtendedMvSet() throws Exception { - Document d = new Document(); - d.add(new StringField("_id", "1", Field.Store.NO)); - d.add(new LongField("value", 2, Field.Store.NO)); - d.add(new LongField("value", 4, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "2", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "3", Field.Store.NO)); - d.add(new LongField("value", 3, Field.Store.NO)); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField("_id", "4", Field.Store.NO)); - d.add(new LongField("value", 4, Field.Store.NO)); - d.add(new LongField("value", 5, Field.Store.NO)); - d.add(new LongField("value", 6, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "5", Field.Store.NO)); - d.add(new LongField("value", 6, Field.Store.NO)); - d.add(new LongField("value", 7, Field.Store.NO)); - d.add(new LongField("value", 8, Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "6", Field.Store.NO)); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField("_id", "7", Field.Store.NO)); - d.add(new LongField("value", 8, Field.Store.NO)); - d.add(new LongField("value", 9, Field.Store.NO)); - d.add(new LongField("value", 10, Field.Store.NO)); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField("_id", "8", Field.Store.NO)); - d.add(new LongField("value", -8, Field.Store.NO)); - d.add(new LongField("value", -9, Field.Store.NO)); - d.add(new LongField("value", -10, Field.Store.NO)); - writer.addDocument(d); - } - - private static final int SECONDS_PER_YEAR = 60 * 60 * 24 * 365; - - // TODO: use random() when migrating to Junit - public static enum Data { - SINGLE_VALUED_DENSE_ENUM { - @Override - public int numValues(Random r) { - return 1; - } - - @Override - public long nextValue(Random r) { - return 1 + r.nextInt(16); - } - }, - SINGLE_VALUED_DENSE_DATE { - @Override - public int numValues(Random r) { - return 1; - } - - @Override - public long nextValue(Random r) { - // somewhere in-between 2010 and 2012 - return 1000L * (40L * SECONDS_PER_YEAR + r.nextInt(2 * SECONDS_PER_YEAR)); - } - }, - MULTI_VALUED_DATE { - @Override - public int numValues(Random r) { - return r.nextInt(3); - } - - @Override - public long nextValue(Random r) { - // somewhere in-between 2010 and 2012 - return 1000L * (40L * SECONDS_PER_YEAR + r.nextInt(2 * SECONDS_PER_YEAR)); - } - }, - MULTI_VALUED_ENUM { - @Override - public int numValues(Random r) { - return r.nextInt(3); - } - - @Override - public long nextValue(Random r) { - return 3 + r.nextInt(8); - } - }, - SINGLE_VALUED_SPARSE_RANDOM { - @Override - public int numValues(Random r) { - return r.nextFloat() < 0.01 ? 1 : 0; - } - - @Override - public long nextValue(Random r) { - return r.nextLong(); - } - }, - MULTI_VALUED_SPARSE_RANDOM { - @Override - public int numValues(Random r) { - return r.nextFloat() < 0.01f ? 1 + r.nextInt(5) : 0; - } - - @Override - public long nextValue(Random r) { - return r.nextLong(); - } - }, - MULTI_VALUED_DENSE_RANDOM { - @Override - public int numValues(Random r) { - return 1 + r.nextInt(3); - } - - @Override - public long nextValue(Random r) { - return r.nextLong(); - } - }; - - public abstract int numValues(Random r); - - public abstract long nextValue(Random r); - } - - private void test(List values) throws Exception { - StringField id = new StringField("_id", "", Field.Store.NO); - - for (int i = 0; i < values.size(); ++i) { - Document doc = new Document(); - id.setStringValue("" + i); - doc.add(id); - final LongHashSet v = values.get(i); - for (LongCursor c : v) { - LongField value = new LongField("value", c.value, Field.Store.NO); - doc.add(value); - } - writer.addDocument(doc); - } - writer.forceMerge(1, true); - - final IndexNumericFieldData indexFieldData = getForField("value"); - final AtomicNumericFieldData atomicFieldData = indexFieldData.load(refreshReader()); - final SortedNumericDocValues data = atomicFieldData.getLongValues(); - final SortedNumericDoubleValues doubleData = atomicFieldData.getDoubleValues(); - final LongHashSet set = new LongHashSet(); - final LongHashSet doubleSet = new LongHashSet(); - for (int i = 0; i < values.size(); ++i) { - final LongHashSet v = values.get(i); - - data.setDocument(i); - assertThat(data.count() > 0, equalTo(!v.isEmpty())); - doubleData.setDocument(i); - assertThat(doubleData.count() > 0, equalTo(!v.isEmpty())); - - set.clear(); - data.setDocument(i); - int numValues = data.count(); - for (int j = 0; j < numValues; j++) { - set.add(data.valueAt(j)); - } - assertThat(set, equalTo(v)); - - final LongHashSet doubleV = new LongHashSet(); - for (LongCursor c : v) { - doubleV.add(Double.doubleToLongBits(c.value)); - } - doubleSet.clear(); - doubleData.setDocument(i); - numValues = doubleData.count(); - double prev = 0; - for (int j = 0; j < numValues; j++) { - double current = doubleData.valueAt(j); - doubleSet.add(Double.doubleToLongBits(current)); - if (j > 0) { - assertThat(prev, lessThan(current)); - } - prev = current; - } - assertThat(doubleSet, equalTo(doubleV)); - } - } - - private void test(Data data) throws Exception { - Random r = getRandom(); - final int numDocs = 1000 + r.nextInt(19000); - final List values = new ArrayList<>(numDocs); - for (int i = 0; i < numDocs; ++i) { - final int numValues = data.numValues(r); - final LongHashSet vals = new LongHashSet(numValues); - for (int j = 0; j < numValues; ++j) { - vals.add(data.nextValue(r)); - } - values.add(vals); - } - test(values); - } - - public void testSingleValuedDenseEnum() throws Exception { - test(Data.SINGLE_VALUED_DENSE_ENUM); - } - - public void testSingleValuedDenseDate() throws Exception { - test(Data.SINGLE_VALUED_DENSE_DATE); - } - - public void testSingleValuedSparseRandom() throws Exception { - test(Data.SINGLE_VALUED_SPARSE_RANDOM); - } - - public void testMultiValuedDate() throws Exception { - test(Data.MULTI_VALUED_DATE); - } - - public void testMultiValuedEnum() throws Exception { - test(Data.MULTI_VALUED_ENUM); - } - - public void testMultiValuedSparseRandom() throws Exception { - test(Data.MULTI_VALUED_SPARSE_RANDOM); - } - - public void testMultiValuedDenseRandom() throws Exception { - test(Data.MULTI_VALUED_DENSE_RANDOM); - } - -} diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java index 982f5e4d4e1..230330dbbf5 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java @@ -19,14 +19,13 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.Index; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.mapper.MappedFieldType.Names; import org.elasticsearch.search.MultiValueMode; -import org.junit.Test; /** Returns an implementation based on paged bytes which doesn't implement WithOrdinals in order to visit different paths in the code, * eg. BytesRefFieldComparatorSource makes decisions based on whether the field data implements WithOrdinals. */ @@ -79,9 +78,8 @@ public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTes return hideOrdinals(super.getForField(fieldName)); } - @Test @Override public void testTermsEnum() throws Exception { - // We can't test this, since the returned IFD instance doesn't implement IndexFieldData.WithOrdinals + assumeTrue("We can't test this, since the returned IFD instance doesn't implement IndexFieldData.WithOrdinals", false); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java index b265988c330..eefe8c89183 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java @@ -26,7 +26,12 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.compress.CompressedXContent; @@ -36,7 +41,6 @@ import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.search.MultiValueMode; import org.junit.Before; -import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -49,7 +53,6 @@ import static org.hamcrest.Matchers.nullValue; /** */ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { - private final String parentType = "parent"; private final String childType = "child"; private final String grandChildType = "grand-child"; @@ -118,7 +121,6 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { return new SortedDocValuesField(ParentFieldMapper.joinField(parentType), new BytesRef(id)); } - @Test public void testGetBytesValues() throws Exception { IndexFieldData indexFieldData = getForField(childType); AtomicFieldData fieldData = indexFieldData.load(refreshReader()); @@ -160,7 +162,6 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { assertThat(bytesValues.count(), equalTo(0)); } - @Test public void testSorting() throws Exception { IndexFieldData indexFieldData = getForField(childType); IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java index f855d3f9d7e..655483fa31a 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java @@ -25,10 +25,15 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; @@ -45,8 +50,6 @@ public class MultiOrdinalsTests extends ESTestCase { return builder.build(settings.build()); } - - @Test public void testRandomValues() throws IOException { Random random = getRandom(); int numDocs = 100 + random.nextInt(1000); @@ -182,7 +185,6 @@ public class MultiOrdinalsTests extends ESTestCase { } } - @Test public void testOrdinals() throws Exception { int maxDoc = 7; long maxOrds = 32; @@ -227,7 +229,6 @@ public class MultiOrdinalsTests extends ESTestCase { } } - @Test public void testMultiValuesDocsWithOverlappingStorageArrays() throws Exception { int maxDoc = 7; long maxOrds = 15; diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java index 59be0f9a4fe..3e0d5f6ad2f 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java @@ -23,19 +23,18 @@ import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; /** */ public class SingleOrdinalsTests extends ESTestCase { - - @Test public void testSvValues() throws IOException { int numDocs = 1000000; int numOrdinals = numDocs / 4; @@ -61,7 +60,6 @@ public class SingleOrdinalsTests extends ESTestCase { } } - @Test public void testMvOrdinalsTrigger() throws IOException { int numDocs = 1000000; OrdinalsBuilder builder = new OrdinalsBuilder(numDocs); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index f01df630ea7..966ea01e95c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -28,15 +29,21 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.core.DoubleFieldMapper; +import org.elasticsearch.index.mapper.core.FloatFieldMapper; import org.elasticsearch.index.mapper.core.IntegerFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.nullValue; public class DynamicMappingTests extends ESSingleNodeTestCase { @@ -407,4 +414,26 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { // expected } } + + public void testDefaultFloatingPointMappings() throws IOException { + DocumentMapper mapper = createIndex("test").mapperService().documentMapperWithAutoCreate("type").getDocumentMapper(); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.jsonBuilder()); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.yamlBuilder()); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.smileBuilder()); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.cborBuilder()); + } + + private void doTestDefaultFloatingPointMappings(DocumentMapper mapper, XContentBuilder builder) throws IOException { + BytesReference source = builder.startObject() + .field("foo", 3.2f) // float + .field("bar", 3.2d) // double + .field("baz", (double) 3.2f) // double that can be accurately represented as a float + .endObject().bytes(); + ParsedDocument parsedDocument = mapper.parse("index", "type", "id", source); + Mapping update = parsedDocument.dynamicMappingsUpdate(); + assertNotNull(update); + assertThat(update.root().getMapper("foo"), instanceOf(FloatFieldMapper.class)); + assertThat(update.root().getMapper("bar"), instanceOf(FloatFieldMapper.class)); + assertThat(update.root().getMapper("baz"), instanceOf(FloatFieldMapper.class)); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 6ab4ca38d40..5a31618f14e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -36,6 +37,8 @@ public class FieldTypeLookupTests extends ESTestCase { FieldTypeLookup lookup = new FieldTypeLookup(); assertNull(lookup.get("foo")); assertNull(lookup.getByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup.getTypes("foo")); + assertEquals(Collections.emptySet(), lookup.getTypesByIndexName("foo")); Collection names = lookup.simpleMatchToFullName("foo"); assertNotNull(names); assertTrue(names.isEmpty()); @@ -47,10 +50,20 @@ public class FieldTypeLookupTests extends ESTestCase { assertFalse(itr.hasNext()); } + public void testDefaultMapping() { + FieldTypeLookup lookup = new FieldTypeLookup(); + try { + lookup.copyAndAddAll(MapperService.DEFAULT_MAPPING, Collections.emptyList()); + fail(); + } catch (IllegalArgumentException expected) { + assertEquals("Default mappings should not be added to the lookup", expected.getMessage()); + } + } + public void testAddNewField() { FieldTypeLookup lookup = new FieldTypeLookup(); FakeFieldMapper f = new FakeFieldMapper("foo", "bar"); - FieldTypeLookup lookup2 = lookup.copyAndAddAll(newList(f)); + FieldTypeLookup lookup2 = lookup.copyAndAddAll("type", newList(f)); assertNull(lookup.get("foo")); assertNull(lookup.get("bar")); assertNull(lookup.getByIndexName("foo")); @@ -59,6 +72,14 @@ public class FieldTypeLookupTests extends ESTestCase { assertNull(lookup.get("bar")); assertEquals(f.fieldType(), lookup2.getByIndexName("bar")); assertNull(lookup.getByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup.getTypes("foo")); + assertEquals(Collections.emptySet(), lookup.getTypesByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup.getTypes("bar")); + assertEquals(Collections.emptySet(), lookup.getTypesByIndexName("bar")); + assertEquals(Collections.singleton("type"), lookup2.getTypes("foo")); + assertEquals(Collections.emptySet(), lookup2.getTypesByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup2.getTypes("bar")); + assertEquals(Collections.singleton("type"), lookup2.getTypesByIndexName("bar")); assertEquals(1, size(lookup2.iterator())); } @@ -67,8 +88,8 @@ public class FieldTypeLookupTests extends ESTestCase { MappedFieldType originalFieldType = f.fieldType(); FakeFieldMapper f2 = new FakeFieldMapper("foo", "foo"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f)); - FieldTypeLookup lookup2 = lookup.copyAndAddAll(newList(f2)); + lookup = lookup.copyAndAddAll("type1", newList(f)); + FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2)); assertNotSame(originalFieldType, f.fieldType()); assertSame(f.fieldType(), f2.fieldType()); @@ -82,8 +103,8 @@ public class FieldTypeLookupTests extends ESTestCase { FakeFieldMapper f2 = new FakeFieldMapper("bar", "foo"); MappedFieldType originalFieldType = f.fieldType(); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f)); - FieldTypeLookup lookup2 = lookup.copyAndAddAll(newList(f2)); + lookup = lookup.copyAndAddAll("type1", newList(f)); + FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2)); assertNotSame(originalFieldType, f.fieldType()); assertSame(f.fieldType(), f2.fieldType()); @@ -98,8 +119,8 @@ public class FieldTypeLookupTests extends ESTestCase { FakeFieldMapper f2 = new FakeFieldMapper("foo", "bar"); MappedFieldType originalFieldType = f.fieldType(); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f)); - FieldTypeLookup lookup2 = lookup.copyAndAddAll(newList(f2)); + lookup = lookup.copyAndAddAll("type1", newList(f)); + FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2)); assertNotSame(originalFieldType, f.fieldType()); assertSame(f.fieldType(), f2.fieldType()); @@ -113,18 +134,18 @@ public class FieldTypeLookupTests extends ESTestCase { FakeFieldMapper f = new FakeFieldMapper("foo", "foo"); FakeFieldMapper f2 = new FakeFieldMapper("bar", "bar"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f, f2)); + lookup = lookup.copyAndAddAll("type1", newList(f, f2)); try { FakeFieldMapper f3 = new FakeFieldMapper("foo", "bar"); - lookup.copyAndAddAll(newList(f3)); + lookup.copyAndAddAll("type2", newList(f3)); } catch (IllegalStateException e) { assertTrue(e.getMessage().contains("insane mappings")); } try { FakeFieldMapper f3 = new FakeFieldMapper("bar", "foo"); - lookup.copyAndAddAll(newList(f3)); + lookup.copyAndAddAll("type2", newList(f3)); } catch (IllegalStateException e) { assertTrue(e.getMessage().contains("insane mappings")); } @@ -133,25 +154,25 @@ public class FieldTypeLookupTests extends ESTestCase { public void testCheckCompatibilityNewField() { FakeFieldMapper f1 = new FakeFieldMapper("foo", "bar"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup.checkCompatibility(newList(f1), false); + lookup.checkCompatibility("type", newList(f1), false); } public void testCheckCompatibilityMismatchedTypes() { FieldMapper f1 = new FakeFieldMapper("foo", "bar"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f1)); + lookup = lookup.copyAndAddAll("type", newList(f1)); MappedFieldType ft2 = FakeFieldMapper.makeOtherFieldType("foo", "foo"); FieldMapper f2 = new FakeFieldMapper("foo", ft2); try { - lookup.checkCompatibility(newList(f2), false); + lookup.checkCompatibility("type2", newList(f2), false); fail("expected type mismatch"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("cannot be changed from type [faketype] to [otherfaketype]")); } // fails even if updateAllTypes == true try { - lookup.checkCompatibility(newList(f2), true); + lookup.checkCompatibility("type2", newList(f2), true); fail("expected type mismatch"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("cannot be changed from type [faketype] to [otherfaketype]")); @@ -161,31 +182,33 @@ public class FieldTypeLookupTests extends ESTestCase { public void testCheckCompatibilityConflict() { FieldMapper f1 = new FakeFieldMapper("foo", "bar"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f1)); + lookup = lookup.copyAndAddAll("type", newList(f1)); MappedFieldType ft2 = FakeFieldMapper.makeFieldType("foo", "bar"); ft2.setBoost(2.0f); FieldMapper f2 = new FakeFieldMapper("foo", ft2); try { - lookup.checkCompatibility(newList(f2), false); + // different type + lookup.checkCompatibility("type2", newList(f2), false); fail("expected conflict"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("to update [boost] across all types")); } - lookup.checkCompatibility(newList(f2), true); // boost is updateable, so ok if forcing + lookup.checkCompatibility("type", newList(f2), false); // boost is updateable, so ok since we are implicitly updating all types + lookup.checkCompatibility("type2", newList(f2), true); // boost is updateable, so ok if forcing // now with a non changeable setting MappedFieldType ft3 = FakeFieldMapper.makeFieldType("foo", "bar"); ft3.setStored(true); FieldMapper f3 = new FakeFieldMapper("foo", ft3); try { - lookup.checkCompatibility(newList(f3), false); + lookup.checkCompatibility("type2", newList(f3), false); fail("expected conflict"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("has different [store] values")); } // even with updateAllTypes == true, incompatible try { - lookup.checkCompatibility(newList(f3), true); + lookup.checkCompatibility("type2", newList(f3), true); fail("expected conflict"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("has different [store] values")); @@ -196,7 +219,7 @@ public class FieldTypeLookupTests extends ESTestCase { FakeFieldMapper f1 = new FakeFieldMapper("foo", "baz"); FakeFieldMapper f2 = new FakeFieldMapper("bar", "boo"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f1, f2)); + lookup = lookup.copyAndAddAll("type", newList(f1, f2)); Collection names = lookup.simpleMatchToIndexNames("b*"); assertTrue(names.contains("baz")); assertTrue(names.contains("boo")); @@ -206,7 +229,7 @@ public class FieldTypeLookupTests extends ESTestCase { FakeFieldMapper f1 = new FakeFieldMapper("foo", "baz"); FakeFieldMapper f2 = new FakeFieldMapper("bar", "boo"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f1, f2)); + lookup = lookup.copyAndAddAll("type", newList(f1, f2)); Collection names = lookup.simpleMatchToFullName("b*"); assertTrue(names.contains("foo")); assertTrue(names.contains("bar")); @@ -215,7 +238,7 @@ public class FieldTypeLookupTests extends ESTestCase { public void testIteratorImmutable() { FakeFieldMapper f1 = new FakeFieldMapper("foo", "bar"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup = lookup.copyAndAddAll(newList(f1)); + lookup = lookup.copyAndAddAll("type", newList(f1)); try { Iterator itr = lookup.iterator(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index a45348d530c..ca0cbf194d6 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -281,7 +281,7 @@ public abstract class FieldTypeTestCase extends ESTestCase { public void testCheckTypeName() { final MappedFieldType fieldType = createNamedDefaultFieldType(); List conflicts = new ArrayList<>(); - fieldType.checkTypeName(fieldType, conflicts); + fieldType.checkCompatibility(fieldType, conflicts, random().nextBoolean()); // no exception assertTrue(conflicts.toString(), conflicts.isEmpty()); MappedFieldType bogus = new MappedFieldType() { @@ -291,7 +291,7 @@ public abstract class FieldTypeTestCase extends ESTestCase { public String typeName() { return fieldType.typeName();} }; try { - fieldType.checkTypeName(bogus, conflicts); + fieldType.checkCompatibility(bogus, conflicts, random().nextBoolean()); fail("expected bad types exception"); } catch (IllegalStateException e) { assertTrue(e.getMessage().contains("Type names equal")); @@ -304,10 +304,13 @@ public abstract class FieldTypeTestCase extends ESTestCase { @Override public String typeName() { return "othertype";} }; - fieldType.checkTypeName(other, conflicts); - assertFalse(conflicts.isEmpty()); - assertTrue(conflicts.get(0).contains("cannot be changed from type")); - assertEquals(1, conflicts.size()); + try { + fieldType.checkCompatibility(other, conflicts, random().nextBoolean()); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("cannot be changed from type")); + } + assertTrue(conflicts.toString(), conflicts.isEmpty()); } public void testCheckCompatibility() { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index add7ee6a7f7..f4a7507a0b8 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -19,25 +19,30 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.IndexService; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Rule; -import org.junit.Test; import org.junit.rules.ExpectedException; import static org.elasticsearch.test.VersionUtils.getFirstVersion; import static org.elasticsearch.test.VersionUtils.getPreviousVersion; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.hasToString; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.concurrent.ExecutionException; + public class MapperServiceTests extends ESSingleNodeTestCase { @Rule public ExpectedException expectedException = ExpectedException.none(); - @Test public void testTypeNameStartsWithIllegalDot() { expectedException.expect(MapperParsingException.class); expectedException.expect(hasToString(containsString("mapping type name [.test-type] must not start with a '.'"))); @@ -53,7 +58,6 @@ public class MapperServiceTests extends ESSingleNodeTestCase { .actionGet(); } - @Test public void testThatLongTypeNameIsNotRejectedOnPreElasticsearchVersionTwo() { String index = "text-index"; String field = "field"; @@ -71,7 +75,6 @@ public class MapperServiceTests extends ESSingleNodeTestCase { assertNotNull(response); } - @Test public void testTypeNameTooLong() { String index = "text-index"; String field = "field"; @@ -87,4 +90,58 @@ public class MapperServiceTests extends ESSingleNodeTestCase { .execute() .actionGet(); } + + public void testTypes() throws Exception { + IndexService indexService1 = createIndex("index1"); + MapperService mapperService = indexService1.mapperService(); + assertEquals(Collections.emptySet(), mapperService.types()); + + mapperService.merge("type1", new CompressedXContent("{\"type1\":{}}"), true, false); + assertNull(mapperService.documentMapper(MapperService.DEFAULT_MAPPING)); + assertEquals(Collections.singleton("type1"), mapperService.types()); + + mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent("{\"_default_\":{}}"), true, false); + assertNotNull(mapperService.documentMapper(MapperService.DEFAULT_MAPPING)); + assertEquals(Collections.singleton("type1"), mapperService.types()); + + mapperService.merge("type2", new CompressedXContent("{\"type2\":{}}"), true, false); + assertNotNull(mapperService.documentMapper(MapperService.DEFAULT_MAPPING)); + assertEquals(new HashSet<>(Arrays.asList("type1", "type2")), mapperService.types()); + } + + public void testIndexIntoDefaultMapping() throws Throwable { + // 1. test implicit index creation + try { + client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{").execute().get(); + fail(); + } catch (Throwable t) { + if (t instanceof ExecutionException) { + t = ((ExecutionException) t).getCause(); + } + final Throwable throwable = ExceptionsHelper.unwrapCause(t); + if (throwable instanceof IllegalArgumentException) { + assertEquals("It is forbidden to index into the default mapping [_default_]", throwable.getMessage()); + } else { + throw t; + } + } + + // 2. already existing index + IndexService indexService = createIndex("index2"); + try { + client().prepareIndex("index2", MapperService.DEFAULT_MAPPING, "2").setSource().execute().get(); + fail(); + } catch (Throwable t) { + if (t instanceof ExecutionException) { + t = ((ExecutionException) t).getCause(); + } + final Throwable throwable = ExceptionsHelper.unwrapCause(t); + if (throwable instanceof IllegalArgumentException) { + assertEquals("It is forbidden to index into the default mapping [_default_]", throwable.getMessage()); + } else { + throw t; + } + } + assertFalse(indexService.mapperService().hasMapping(MapperService.DEFAULT_MAPPING)); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java b/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java index d6a5c9f5537..860c66863ff 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java @@ -20,13 +20,10 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; public class UidTests extends ESTestCase { - - @Test public void testCreateAndSplitId() { BytesRef createUid = Uid.createUidAsBytes("foo", "bar"); BytesRef[] splitUidIntoTypeAndId = Uid.splitUidIntoTypeAndId(createUid); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index 0e3a04aa699..bbba3432b66 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -43,29 +43,18 @@ import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.internal.AllFieldMapper; -import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; +import org.elasticsearch.test.ESSingleNodeTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class SimpleAllMapperTests extends ESSingleNodeTestCase { @@ -253,7 +242,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { if (randomBoolean()) { booleanOptionList.add(new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean())); } - Collections.shuffle(booleanOptionList, getRandom()); + Collections.shuffle(booleanOptionList, random()); for (Tuple option : booleanOptionList) { mappingBuilder.field(option.v1(), option.v2().booleanValue()); } @@ -349,24 +338,39 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { assertThat(allEntries.fields(), hasItem("foo.bar")); } - @Test(expected = MapperParsingException.class) public void testMisplacedTypeInRoot() throws IOException { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/misplaced_type_in_root.json"); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("test", mapping); + try { + createIndex("test").mapperService().documentMapperParser().parse("test", mapping); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Root mapping definition has unsupported parameters")); + assertThat(e.getMessage(), containsString("[type : string]")); + } } // related to https://github.com/elasticsearch/elasticsearch/issues/5864 - @Test(expected = MapperParsingException.class) public void testMistypedTypeInRoot() throws IOException { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mistyped_type_in_root.json"); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("test", mapping); + try { + createIndex("test").mapperService().documentMapperParser().parse("test", mapping); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Root mapping definition has unsupported parameters")); + assertThat(e.getMessage(), containsString("type=string")); + } } // issue https://github.com/elasticsearch/elasticsearch/issues/5864 - @Test(expected = MapperParsingException.class) public void testMisplacedMappingAsRoot() throws IOException { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/misplaced_mapping_key_in_root.json"); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("test", mapping); + try { + createIndex("test").mapperService().documentMapperParser().parse("test", mapping); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Root mapping definition has unsupported parameters")); + assertThat(e.getMessage(), containsString("type=string")); + } } // issue https://github.com/elasticsearch/elasticsearch/issues/5864 @@ -384,7 +388,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { } // issue https://github.com/elasticsearch/elasticsearch/issues/5864 - public void testRootMappersStillWorking() { + public void testMetadataMappersStillWorking() { String mapping = "{"; Map rootTypes = new HashMap<>(); //just pick some example from DocumentMapperParser.rootTypeParsers @@ -399,7 +403,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { mapping += "\"properties\":{}}" ; createIndex("test").mapperService().documentMapperParser().parse("test", mapping); } - + public void testDocValuesNotAllowed() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .startObject("_all") @@ -411,7 +415,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { } catch (MapperParsingException e) { assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values")); } - + mapping = jsonBuilder().startObject().startObject("type") .startObject("_all") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java index d18acfe56aa..05a0a03cc59 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java @@ -23,13 +23,10 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; public class CustomBoostMappingTests extends ESSingleNodeTestCase { - - @Test public void testCustomBoostValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("s_field").field("type", "string").endObject() diff --git a/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java b/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java index 5c5ce7bdaed..c9320e2da18 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java @@ -26,15 +26,12 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.closeTo; /** */ public class FieldLevelBoostTests extends ESSingleNodeTestCase { - - @Test public void testFieldLevelBoost() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("person").startObject("properties") .startObject("str_field").field("type", "string").endObject() @@ -85,7 +82,6 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase { assertThat((double) f.boost(), closeTo(9.0, 0.001)); } - @Test public void testInvalidFieldLevelBoost() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("person").startObject("properties") .startObject("str_field").field("type", "string").endObject() diff --git a/core/src/test/java/org/elasticsearch/index/mapper/camelcase/CamelCaseFieldNameTests.java b/core/src/test/java/org/elasticsearch/index/mapper/camelcase/CamelCaseFieldNameTests.java index 89e186445d1..1cfee0dd66e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/camelcase/CamelCaseFieldNameTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/camelcase/CamelCaseFieldNameTests.java @@ -24,14 +24,11 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; /** * */ public class CamelCaseFieldNameTests extends ESSingleNodeTestCase { - - @Test public void testCamelCaseFieldNameStaysAsIs() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java index 717823d9ffc..b81a3d6d40d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java @@ -18,25 +18,30 @@ */ package org.elasticsearch.index.mapper.completion; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.suggest.document.*; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; public class CompletionFieldMapperTests extends ESSingleNodeTestCase { - - @Test public void testDefaultConfiguration() throws IOException { String mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("completion") @@ -49,22 +54,66 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase { FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); - CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; - assertThat(completionFieldMapper.isStoringPayloads(), is(false)); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + + NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(true)); + + NamedAnalyzer searchAnalyzer = completionFieldType.searchAnalyzer(); + assertThat(searchAnalyzer.name(), equalTo("simple")); + assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(true)); } - @Test - public void testThatSerializationIncludesAllElements() throws Exception { + public void testCompletionAnalyzerSettings() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .field("analyzer", "simple") + .field("search_analyzer", "standard") + .field("preserve_separators", false) + .field("preserve_position_increments", true) + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); + + MappedFieldType completionFieldType = fieldMapper.fieldType(); + + NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(false)); + + NamedAnalyzer searchAnalyzer = completionFieldType.searchAnalyzer(); + assertThat(searchAnalyzer.name(), equalTo("standard")); + assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(false)); + + } + + public void testTypeParsing() throws Exception { String mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("completion") .field("type", "completion") .field("analyzer", "simple") .field("search_analyzer", "standard") - .field("payloads", true) .field("preserve_separators", false) .field("preserve_position_increments", true) .field("max_input_length", 14) - .endObject().endObject() .endObject().endObject().string(); @@ -75,46 +124,295 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase { CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; XContentBuilder builder = jsonBuilder().startObject(); - completionFieldMapper.toXContent(builder, null).endObject(); + completionFieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); builder.close(); - Map serializedMap; - try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) { - serializedMap = parser.map(); - } + Map serializedMap = JsonXContent.jsonXContent.createParser(builder.bytes()).map(); Map configMap = (Map) serializedMap.get("completion"); assertThat(configMap.get("analyzer").toString(), is("simple")); assertThat(configMap.get("search_analyzer").toString(), is("standard")); - assertThat(Boolean.valueOf(configMap.get("payloads").toString()), is(true)); assertThat(Boolean.valueOf(configMap.get("preserve_separators").toString()), is(false)); assertThat(Boolean.valueOf(configMap.get("preserve_position_increments").toString()), is(true)); assertThat(Integer.valueOf(configMap.get("max_input_length").toString()), is(14)); } - @Test - public void testThatSerializationCombinesToOneAnalyzerFieldIfBothAreEqual() throws Exception { + public void testParsingMinimal() throws Exception { String mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("completion") .field("type", "completion") - .field("analyzer", "simple") - .field("search_analyzer", "simple") .endObject().endObject() .endObject().endObject().string(); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); - assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); - - CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; - XContentBuilder builder = jsonBuilder().startObject(); - completionFieldMapper.toXContent(builder, null).endObject(); - builder.close(); - Map serializedMap; - try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) { - serializedMap = parser.map(); - } - Map configMap = (Map) serializedMap.get("completion"); - assertThat(configMap.get("analyzer").toString(), is("simple")); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", "suggestion") + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 1); } + public void testParsingMultiValued() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .array("completion", "suggestion1", "suggestion2") + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 2); + } + + public void testParsingWithWeight() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("completion") + .field("input", "suggestion") + .field("weight", 2) + .endObject() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 1); + } + + public void testParsingMultiValueWithWeight() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("completion") + .array("input", "suggestion1", "suggestion2", "suggestion3") + .field("weight", 2) + .endObject() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 3); + } + + public void testParsingFull() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion1") + .field("weight", 3) + .endObject() + .startObject() + .field("input", "suggestion2") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion3") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 3); + } + + public void testParsingMixed() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .array("input", "suggestion1", "suggestion2") + .field("weight", 3) + .endObject() + .startObject() + .field("input", "suggestion3") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion4", "suggestion5", "suggestion6") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 6); + } + + public void testNonContextEnabledParsingWithContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("field1") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("field1") + .field("input", "suggestion1") + .startObject("contexts") + .field("ctx", "ctx2") + .endObject() + .field("weight", 3) + .endObject() + .endObject() + .bytes()); + fail("Supplying contexts to a non context-enabled field should error"); + } catch (MapperParsingException e) { + assertThat(e.getRootCause().getMessage(), containsString("field1")); + } + } + + public void testFieldValueValidation() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + CharsRefBuilder charsRefBuilder = new CharsRefBuilder(); + charsRefBuilder.append("sugg"); + charsRefBuilder.setCharAt(2, '\u001F'); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", charsRefBuilder.get().toString()) + .endObject() + .bytes()); + fail("No error indexing value with reserved character [0x1F]"); + } catch (MapperParsingException e) { + Throwable cause = e.unwrapCause().getCause(); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("[0x1f]")); + } + + charsRefBuilder.setCharAt(2, '\u0000'); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", charsRefBuilder.get().toString()) + .endObject() + .bytes()); + fail("No error indexing value with reserved character [0x0]"); + } catch (MapperParsingException e) { + Throwable cause = e.unwrapCause().getCause(); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("[0x0]")); + } + + charsRefBuilder.setCharAt(2, '\u001E'); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", charsRefBuilder.get().toString()) + .endObject() + .bytes()); + fail("No error indexing value with reserved character [0x1E]"); + } catch (MapperParsingException e) { + Throwable cause = e.unwrapCause().getCause(); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("[0x1e]")); + } + } + + public void testPrefixQueryType() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; + Query prefixQuery = completionFieldMapper.fieldType().prefixQuery(new BytesRef("co")); + assertThat(prefixQuery, instanceOf(PrefixCompletionQuery.class)); + } + + public void testFuzzyQueryType() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; + Query prefixQuery = completionFieldMapper.fieldType().fuzzyQuery("co", + Fuzziness.fromEdits(FuzzyCompletionQuery.DEFAULT_MAX_EDITS), FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX, + FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH, Operations.DEFAULT_MAX_DETERMINIZED_STATES, + FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS, FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE); + assertThat(prefixQuery, instanceOf(FuzzyCompletionQuery.class)); + } + + public void testRegexQueryType() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; + Query prefixQuery = completionFieldMapper.fieldType() + .regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + assertThat(prefixQuery, instanceOf(RegexCompletionQuery.class)); + } + + private static void assertSuggestFields(IndexableField[] fields, int expected) { + int actualFieldCount = 0; + for (IndexableField field : fields) { + if (field instanceof SuggestField) { + actualFieldCount++; + } + } + assertThat(actualFieldCount, equalTo(expected)); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java b/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java index fa2e1a12375..4dc017aa6bd 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java @@ -23,14 +23,11 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; public class CompoundTypesTests extends ESSingleNodeTestCase { - - @Test public void testStringType() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java index 5919c52bc31..4a010747624 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java @@ -27,19 +27,16 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; /** */ public class CopyToMapperIntegrationIT extends ESIntegTestCase { - - - @Test public void testDynamicTemplateCopyTo() throws Exception { assertAcked( client().admin().indices().prepareCreate("test-idx") @@ -56,7 +53,7 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase { client().admin().indices().prepareRefresh("test-idx").execute().actionGet(); SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values()); - + SearchResponse response = client().prepareSearch("test-idx") .setQuery(QueryBuilders.termQuery("even", true)) .addAggregation(AggregationBuilders.terms("test").field("test_field").size(recordCount * 2) @@ -72,6 +69,25 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase { } + public void testDynamicObjectCopyTo() throws Exception { + String mapping = jsonBuilder().startObject().startObject("doc").startObject("properties") + .startObject("foo") + .field("type", "string") + .field("copy_to", "root.top.child") + .endObject() + .endObject().endObject().endObject().string(); + assertAcked( + client().admin().indices().prepareCreate("test-idx") + .addMapping("doc", mapping) + ); + client().prepareIndex("test-idx", "doc", "1") + .setSource("foo", "bar") + .get(); + client().admin().indices().prepareRefresh("test-idx").execute().actionGet(); + SearchResponse response = client().prepareSearch("test-idx") + .setQuery(QueryBuilders.termQuery("root.top.child", "bar")).get(); + assertThat(response.getHits().totalHits(), equalTo(1L)); + } private XContentBuilder createDynamicTemplateMapping() throws IOException { return XContentFactory.jsonBuilder().startObject().startObject("doc") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java index 419dde456de..d94ae2b6735 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.core.LongFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.List; @@ -54,9 +53,7 @@ import static org.hamcrest.Matchers.startsWith; * */ public class CopyToMapperTests extends ESSingleNodeTestCase { - @SuppressWarnings("unchecked") - @Test public void testCopyToFieldsParsing() throws Exception { String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("copy_test") @@ -135,8 +132,6 @@ public class CopyToMapperTests extends ESSingleNodeTestCase { assertThat(fieldMapper, instanceOf(LongFieldMapper.class)); } - @SuppressWarnings("unchecked") - @Test public void testCopyToFieldsInnerObjectParsing() throws Exception { String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -172,35 +167,130 @@ public class CopyToMapperTests extends ESSingleNodeTestCase { } - @SuppressWarnings("unchecked") - @Test - public void testCopyToFieldsNonExistingInnerObjectParsing() throws Exception { - String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties") - + public void testCopyToDynamicInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties") .startObject("copy_test") - .field("type", "string") - .field("copy_to", "very.inner.field") + .field("type", "string") + .field("copy_to", "very.inner.field") .endObject() - - .endObject().endObject().endObject().string(); + .endObject() + .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); BytesReference json = jsonBuilder().startObject() .field("copy_test", "foo") + .field("new_field", "bar") .endObject().bytes(); + ParseContext.Document doc = docMapper.parse("test", "type1", "1", json).rootDoc(); + assertThat(doc.getFields("copy_test").length, equalTo(1)); + assertThat(doc.getFields("copy_test")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("very.inner.field").length, equalTo(1)); + assertThat(doc.getFields("very.inner.field")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("new_field").length, equalTo(1)); + assertThat(doc.getFields("new_field")[0].stringValue(), equalTo("bar")); + } + + public void testCopyToDynamicInnerInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.far.inner.field") + .endObject() + .startObject("very") + .field("type", "object") + .startObject("properties") + .startObject("far") + .field("type", "object") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .field("new_field", "bar") + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("test", "type1", "1", json).rootDoc(); + assertThat(doc.getFields("copy_test").length, equalTo(1)); + assertThat(doc.getFields("copy_test")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("very.far.inner.field").length, equalTo(1)); + assertThat(doc.getFields("very.far.inner.field")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("new_field").length, equalTo(1)); + assertThat(doc.getFields("new_field")[0].stringValue(), equalTo("bar")); + } + + public void testCopyToStrictDynamicInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .field("dynamic", "strict") + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.inner.field") + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .endObject().bytes(); + try { docMapper.parse("test", "type1", "1", json).rootDoc(); fail(); } catch (MapperParsingException ex) { - assertThat(ex.getMessage(), startsWith("attempt to copy value to non-existing object")); + assertThat(ex.getMessage(), startsWith("mapping set to strict, dynamic introduction of [very] within [type1] is not allowed")); } } - @Test - public void testCopyToFieldMerge() throws Exception { + public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.far.field") + .endObject() + .startObject("very") + .field("type", "object") + .startObject("properties") + .startObject("far") + .field("type", "object") + .field("dynamic", "strict") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .endObject().bytes(); + + try { + docMapper.parse("test", "type1", "1", json).rootDoc(); + fail(); + } catch (MapperParsingException ex) { + assertThat(ex.getMessage(), startsWith("mapping set to strict, dynamic introduction of [field] within [very.far] is not allowed")); + } + } + + public void testCopyToFieldMerge() throws Exception { String mappingBefore = jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("copy_test") @@ -346,6 +436,41 @@ public class CopyToMapperTests extends ESSingleNodeTestCase { } } + public void testCopyToDynamicNestedObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startArray("dynamic_templates") + .startObject() + .startObject("objects") + .field("match_mapping_type", "object") + .startObject("mapping") + .field("type", "nested") + .endObject() + .endObject() + .endObject() + .endArray() + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.inner.field") + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .field("new_field", "bar") + .endObject().bytes(); + + try { + docMapper.parse("test", "type1", "1", json).rootDoc(); + fail(); + } catch (MapperParsingException ex) { + assertThat(ex.getMessage(), startsWith("It is forbidden to create dynamic nested objects ([very]) through `copy_to`")); + } + } + private void assertFieldValue(Document doc, String field, Number... expected) { IndexableField[] values = doc.getFields(field); if (values == null) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/CompletionFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/CompletionFieldTypeTests.java index 55dd7f8f7c9..7ec1814a59b 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/CompletionFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/CompletionFieldTypeTests.java @@ -20,20 +20,16 @@ package org.elasticsearch.index.mapper.core; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.search.suggest.completion.AnalyzingCompletionLookupProvider; -import org.elasticsearch.search.suggest.context.ContextBuilder; -import org.elasticsearch.search.suggest.context.ContextMapping; +import org.elasticsearch.search.suggest.completion.context.ContextBuilder; +import org.elasticsearch.search.suggest.completion.context.ContextMappings; import org.junit.Before; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.Arrays; public class CompletionFieldTypeTests extends FieldTypeTestCase { @Override protected MappedFieldType createDefaultFieldType() { - CompletionFieldMapper.CompletionFieldType ft = new CompletionFieldMapper.CompletionFieldType(); - ft.setProvider(new AnalyzingCompletionLookupProvider(true, false, true, false)); - return ft; + return new CompletionFieldMapper.CompletionFieldType(); } @Before @@ -42,30 +38,22 @@ public class CompletionFieldTypeTests extends FieldTypeTestCase { @Override public void modify(MappedFieldType ft) { CompletionFieldMapper.CompletionFieldType cft = (CompletionFieldMapper.CompletionFieldType)ft; - cft.setProvider(new AnalyzingCompletionLookupProvider(false, false, true, false)); + cft.setPreserveSep(false); } }); addModifier(new Modifier("preserve_position_increments", false, true) { @Override public void modify(MappedFieldType ft) { CompletionFieldMapper.CompletionFieldType cft = (CompletionFieldMapper.CompletionFieldType)ft; - cft.setProvider(new AnalyzingCompletionLookupProvider(true, false, false, false)); + cft.setPreservePositionIncrements(false); } }); - addModifier(new Modifier("payload", false, true) { + addModifier(new Modifier("context_mappings", false, true) { @Override public void modify(MappedFieldType ft) { CompletionFieldMapper.CompletionFieldType cft = (CompletionFieldMapper.CompletionFieldType)ft; - cft.setProvider(new AnalyzingCompletionLookupProvider(true, false, true, true)); - } - }); - addModifier(new Modifier("context_mapping", false, true) { - @Override - public void modify(MappedFieldType ft) { - CompletionFieldMapper.CompletionFieldType cft = (CompletionFieldMapper.CompletionFieldType)ft; - SortedMap contextMapping = new TreeMap<>(); - contextMapping.put("foo", ContextBuilder.location("foo").build()); - cft.setContextMapping(contextMapping); + ContextMappings contextMappings = new ContextMappings(Arrays.asList(ContextBuilder.category("foo").build(), ContextBuilder.geo("geo").build())); + cft.setContextMappings(contextMappings); } }); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/MultiFieldCopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/MultiFieldCopyToMapperTests.java new file mode 100644 index 00000000000..821eaeb8365 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/MultiFieldCopyToMapperTests.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.elasticsearch.index.mapper.core; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.core.IsEqual.equalTo; + +public class MultiFieldCopyToMapperTests extends ESTestCase { + + public void testExceptionForCopyToInMultiFields() throws IOException { + XContentBuilder mapping = createMappinmgWithCopyToInMultiField(); + Tuple, List> versionsWithAndWithoutExpectedExceptions = versionsWithAndWithoutExpectedExceptions(); + + // first check that for newer versions we throw exception if copy_to is found withing multi field + Version indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v1()); + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build()); + try { + mapperService.parse("type", new CompressedXContent(mapping.string()), true); + fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), equalTo("copy_to in multi fields is not allowed. Found the copy_to in field [c] which is within a multi field.")); + } + + // now test that with an older version the pasring just works + indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v2()); + mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build()); + DocumentMapper documentMapper = mapperService.parse("type", new CompressedXContent(mapping.string()), true); + assertFalse(documentMapper.mapping().toString().contains("copy_to")); + } + + private static XContentBuilder createMappinmgWithCopyToInMultiField() throws IOException { + XContentBuilder mapping = jsonBuilder(); + mapping.startObject() + .startObject("type") + .startObject("properties") + .startObject("a") + .field("type", "string") + .endObject() + .startObject("b") + .field("type", "string") + .startObject("fields") + .startObject("c") + .field("type", "string") + .field("copy_to", "a") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + return mapping; + } + + // returs a tuple where + // v1 is a list of versions for which we expect an excpetion when a copy_to in multi fields is found and + // v2 is older versions where we throw no exception and we just log a warning + private static Tuple, List> versionsWithAndWithoutExpectedExceptions() { + List versionsWithException = new ArrayList<>(); + List versionsWithoutException = new ArrayList<>(); + for (Version version : VersionUtils.allVersions()) { + if (version.after(Version.V_2_1_0) || + (version.after(Version.V_2_0_1) && version.before(Version.V_2_1_0))) { + versionsWithException.add(version); + } else { + versionsWithoutException.add(version); + } + } + return new Tuple<>(versionsWithException, versionsWithoutException); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java index 613cdde97e6..abca5595537 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.core; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -31,7 +32,6 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -39,7 +39,10 @@ import java.util.Arrays; import java.util.List; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase { @ParametersFactory @@ -65,8 +68,7 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase { /** * It is possible to get the token count in a search response. */ - @Test - public void searchReturnsTokenCount() throws IOException { + public void testSearchReturnsTokenCount() throws IOException { init(); assertSearchReturns(searchById("single"), "single"); @@ -80,8 +82,7 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase { /** * It is possible to search by token count. */ - @Test - public void searchByTokenCount() throws IOException { + public void testSearchByTokenCount() throws IOException { init(); assertSearchReturns(searchByNumericRange(4, 4).get(), "single"); @@ -94,8 +95,7 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase { /** * It is possible to search by token count. */ - @Test - public void facetByTokenCount() throws IOException { + public void testFacetByTokenCount() throws IOException { init(); String facetField = randomFrom(Arrays.asList( diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java index 5a644e56f48..ba9303e8b58 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java @@ -19,17 +19,12 @@ package org.elasticsearch.index.mapper.core; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.CannedTokenStream; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.*; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; @@ -41,7 +36,6 @@ import static org.hamcrest.Matchers.equalTo; * Test for {@link TokenCountFieldMapper}. */ public class TokenCountFieldMapperTests extends ESSingleNodeTestCase { - @Test public void testMerge() throws IOException { String stage1Mapping = XContentFactory.jsonBuilder().startObject() .startObject("person") @@ -77,7 +71,6 @@ public class TokenCountFieldMapperTests extends ESSingleNodeTestCase { assertThat(((TokenCountFieldMapper) stage1.mappers().smartNameFieldMapper("tc")).analyzer(), equalTo("standard")); } - @Test public void testCountPositions() throws IOException { // We're looking to make sure that we: Token t1 = new Token(); // Don't count tokens without an increment @@ -88,7 +81,7 @@ public class TokenCountFieldMapperTests extends ESSingleNodeTestCase { t2.setPositionIncrement(2); // Count funny tokens with more than one increment int finalTokenIncrement = 4; // Count the final token increment on the rare token streams that have them Token[] tokens = new Token[] {t1, t2, t3}; - Collections.shuffle(Arrays.asList(tokens), getRandom()); + Collections.shuffle(Arrays.asList(tokens), random()); final TokenStream tokenStream = new CannedTokenStream(finalTokenIncrement, 0, tokens); // TODO: we have no CannedAnalyzer? Analyzer analyzer = new Analyzer() { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/date/DateBackwardsCompatibilityTests.java b/core/src/test/java/org/elasticsearch/index/mapper/date/DateBackwardsCompatibilityTests.java index 4e906ae82fa..8ddfc3a2ae7 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/date/DateBackwardsCompatibilityTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/date/DateBackwardsCompatibilityTests.java @@ -40,6 +40,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; /** @@ -174,7 +175,8 @@ public class DateBackwardsCompatibilityTests extends ESSingleNodeTestCase { createIndex(Version.CURRENT, mapping); fail("Expected a MapperParsingException, but did not happen"); } catch (MapperParsingException e) { - assertThat(e.getMessage(), is("mapping [" + type + "]")); + assertThat(e.getMessage(), containsString("Failed to parse mapping [" + type + "]")); + assertThat(e.getMessage(), containsString("Epoch [epoch_seconds] is not supported as dynamic date format")); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/genericstore/GenericStoreDynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/genericstore/GenericStoreDynamicTemplateTests.java index 76383408ed8..d07e6177814 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/genericstore/GenericStoreDynamicTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/genericstore/GenericStoreDynamicTemplateTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; @@ -37,8 +36,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class GenericStoreDynamicTemplateTests extends ESSingleNodeTestCase { - - @Test public void testSimple() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-mapping.json"); IndexService index = createIndex("test"); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java index 34c855f4f2e..829730e68cd 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java @@ -24,10 +24,9 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; @@ -37,8 +36,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class PathMatchDynamicTemplateTests extends ESSingleNodeTestCase { - - @Test public void testSimple() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/pathmatch/test-mapping.json"); IndexService index = createIndex("test"); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/simple/SimpleDynamicTemplatesTests.java b/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/simple/SimpleDynamicTemplatesTests.java index 09358b5280c..014f0295808 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/simple/SimpleDynamicTemplatesTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/dynamictemplate/simple/SimpleDynamicTemplatesTests.java @@ -25,11 +25,13 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; @@ -39,8 +41,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class SimpleDynamicTemplatesTests extends ESSingleNodeTestCase { - - @Test public void testMatchTypeOnly() throws Exception { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject().startObject("person").startArray("dynamic_templates").startObject().startObject("test") @@ -66,8 +66,6 @@ public class SimpleDynamicTemplatesTests extends ESSingleNodeTestCase { } - - @Test public void testSimple() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-mapping.json"); IndexService index = createIndex("test"); @@ -124,7 +122,6 @@ public class SimpleDynamicTemplatesTests extends ESSingleNodeTestCase { assertNotNull(fieldMapper); } - @Test public void testSimpleWithXContentTraverse() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-mapping.json"); IndexService index = createIndex("test"); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java index 3d2134f3664..e5d08db8d9f 100755 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java @@ -20,11 +20,13 @@ package org.elasticsearch.index.mapper.externalvalues; import com.spatial4j.core.shape.Point; + import org.apache.lucene.document.Field; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.ContentPath; @@ -32,12 +34,13 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.BinaryFieldMapper; import org.elasticsearch.index.mapper.core.BooleanFieldMapper; +import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapperLegacy; import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; import java.io.IOException; @@ -72,6 +75,7 @@ public class ExternalMapper extends FieldMapper { private BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN); private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL); private GeoPointFieldMapper.Builder pointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT); + private GeoPointFieldMapperLegacy.Builder legacyPointBuilder = new GeoPointFieldMapperLegacy.Builder(Names.FIELD_POINT); private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); private Mapper.Builder stringBuilder; private String generatedValue; @@ -98,7 +102,8 @@ public class ExternalMapper extends FieldMapper { context.path().add(name); BinaryFieldMapper binMapper = binBuilder.build(context); BooleanFieldMapper boolMapper = boolBuilder.build(context); - GeoPointFieldMapper pointMapper = pointBuilder.build(context); + BaseGeoPointFieldMapper pointMapper = (context.indexCreatedVersion().before(Version.V_2_2_0)) ? + legacyPointBuilder.build(context) : pointBuilder.build(context); GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context); FieldMapper stringMapper = (FieldMapper)stringBuilder.build(context); context.path().remove(); @@ -164,13 +169,13 @@ public class ExternalMapper extends FieldMapper { private final BinaryFieldMapper binMapper; private final BooleanFieldMapper boolMapper; - private final GeoPointFieldMapper pointMapper; + private final BaseGeoPointFieldMapper pointMapper; private final GeoShapeFieldMapper shapeMapper; private final FieldMapper stringMapper; public ExternalMapper(String simpleName, MappedFieldType fieldType, String generatedValue, String mapperName, - BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, + BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, BaseGeoPointFieldMapper pointMapper, GeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, new ExternalFieldType(), indexSettings, multiFields, copyTo); this.generatedValue = generatedValue; @@ -196,7 +201,7 @@ public class ExternalMapper extends FieldMapper { pointMapper.parse(context.createExternalValueContext(point)); // Let's add a Dummy Shape - Point shape = ShapeBuilder.newPoint(-100, 45).build(); + Point shape = ShapeBuilders.newPoint(-100, 45).build(); shapeMapper.parse(context.createExternalValueContext(shape)); context = context.createExternalValueContext(generatedValue); @@ -214,7 +219,7 @@ public class ExternalMapper extends FieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { // ignore this for now } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java index d9cee69f8b0..863e0c25fb0 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java @@ -19,14 +19,15 @@ package org.elasticsearch.index.mapper.externalvalues; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - public class ExternalMapperPlugin extends Plugin { + + public static final String EXTERNAL = "external"; + public static final String EXTERNAL_BIS = "external_bis"; + public static final String EXTERNAL_UPPER = "external_upper"; + @Override public String name() { return "external-mappers"; @@ -37,8 +38,11 @@ public class ExternalMapperPlugin extends Plugin { return "External Mappers Plugin"; } - @Override - public Collection indexModules(Settings indexSettings) { - return Collections.singletonList(new ExternalIndexModule()); + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMetadataMapper(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()); + indicesModule.registerMapper(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo")); + indicesModule.registerMapper(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar")); + indicesModule.registerMapper(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR")); } -} + +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java index 1cda8eee239..dae8bc67fda 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -67,7 +66,7 @@ public class ExternalMetadataMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { if (!(mergeWith instanceof ExternalMetadataMapper)) { mergeResult.addConflict("Trying to merge " + mergeWith + " with " + this); } @@ -110,12 +109,17 @@ public class ExternalMetadataMapper extends MetadataFieldMapper { } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { return new Builder(); } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new ExternalMetadataMapper(indexSettings); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java index 6d28f2daaff..4cf7b405217 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java @@ -21,32 +21,29 @@ package org.elasticsearch.index.mapper.externalvalues; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collection; import static org.hamcrest.Matchers.equalTo; public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(ExternalMapperPlugin.class); } - @Test public void testExternalValues() throws Exception { prepareCreate("test-idx").addMapping("type", XContentFactory.jsonBuilder().startObject().startObject("type") .startObject(ExternalMetadataMapper.CONTENT_TYPE) .endObject() .startObject("properties") - .startObject("field").field("type", RegisterExternalTypes.EXTERNAL).endObject() + .startObject("field").field("type", ExternalMapperPlugin.EXTERNAL).endObject() .endObject() .endObject().endObject()).execute().get(); ensureYellow("test-idx"); @@ -72,7 +69,7 @@ public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase { assertThat(response.getHits().totalHits(), equalTo((long) 1)); response = client().prepareSearch("test-idx") - .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", ShapeBuilder.newPoint(-100, 45)).relation(ShapeRelation.WITHIN)) + .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", ShapeBuilders.newPoint(-100, 45)).relation(ShapeRelation.WITHIN)) .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo((long) 1)); @@ -84,12 +81,11 @@ public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase { assertThat(response.getHits().totalHits(), equalTo((long) 1)); } - @Test public void testExternalValuesWithMultifield() throws Exception { prepareCreate("test-idx").addMapping("doc", XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties") .startObject("f") - .field("type", RegisterExternalTypes.EXTERNAL_UPPER) + .field("type", ExternalMapperPlugin.EXTERNAL_UPPER) .startObject("fields") .startObject("f") .field("type", "string") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java deleted file mode 100755 index 8fb1814a60e..00000000000 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.externalvalues; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; - -public class RegisterExternalTypes extends AbstractIndexComponent { - public static final String EXTERNAL = "external"; - public static final String EXTERNAL_BIS = "external_bis"; - public static final String EXTERNAL_UPPER = "external_upper"; - - @Inject - public RegisterExternalTypes(Index index, @IndexSettings Settings indexSettings, MapperService mapperService) { - super(index, indexSettings); - - mapperService.documentMapperParser().putRootTypeParser(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()); - mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo")); - mapperService.documentMapperParser().putTypeParser(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar")); - mapperService.documentMapperParser().putTypeParser(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR")); - } -} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java index bc808ab5b03..24449015a21 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java @@ -19,12 +19,24 @@ package org.elasticsearch.index.mapper.externalvalues; +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; +import org.elasticsearch.test.VersionUtils; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -33,15 +45,17 @@ import static org.hamcrest.Matchers.notNullValue; */ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { - @Test public void testExternalValues() throws Exception { - MapperService mapperService = createIndex("test").mapperService(); - mapperService.documentMapperParser().putRootTypeParser(ExternalMetadataMapper.CONTENT_TYPE, - new ExternalMetadataMapper.TypeParser()); - mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL, - new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo")); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + IndexService indexService = createIndex("test", settings); + MapperRegistry mapperRegistry = new MapperRegistry( + Collections.singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")), + Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser())); - DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + DocumentMapper documentMapper = parser.parse( XContentFactory.jsonBuilder().startObject().startObject("type") .startObject(ExternalMetadataMapper.CONTENT_TYPE) .endObject() @@ -61,7 +75,11 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); assertThat(doc.rootDoc().getField("field.point"), notNullValue()); - assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoUtils.mortonHash(51.0, 42.0))); + } assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); @@ -72,16 +90,22 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { } - @Test public void testExternalValuesWithMultifield() throws Exception { - MapperService mapperService = createIndex("test").mapperService(); - mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL, - new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo")); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + IndexService indexService = createIndex("test", settings); + Map mapperParsers = new HashMap<>(); + mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); + mapperParsers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); + MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap()); - DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + + DocumentMapper documentMapper = parser.parse( XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("field") - .field("type", RegisterExternalTypes.EXTERNAL) + .field("type", ExternalMapperPlugin.EXTERNAL) .startObject("fields") .startObject("field") .field("type", "string") @@ -109,7 +133,11 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); assertThat(doc.rootDoc().getField("field.point"), notNullValue()); - assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoUtils.mortonHash(51.0, 42.0))); + } assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); @@ -120,25 +148,29 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("field.field.raw").stringValue(), is("foo")); } - @Test public void testExternalValuesWithMultifieldTwoLevels() throws Exception { - MapperService mapperService = createIndex("test").mapperService(); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + IndexService indexService = createIndex("test", settings); + Map mapperParsers = new HashMap<>(); + mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); + mapperParsers.put(ExternalMapperPlugin.EXTERNAL_BIS, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "bar")); + mapperParsers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); + MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap()); - mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL, - new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo")); - mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL_BIS, - new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL_BIS, "bar")); + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); - DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + DocumentMapper documentMapper = parser.parse( XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("field") - .field("type", RegisterExternalTypes.EXTERNAL) + .field("type", ExternalMapperPlugin.EXTERNAL) .startObject("fields") .startObject("field") .field("type", "string") .startObject("fields") .startObject("generated") - .field("type", RegisterExternalTypes.EXTERNAL_BIS) + .field("type", ExternalMapperPlugin.EXTERNAL_BIS) .endObject() .startObject("raw") .field("type", "string") @@ -163,7 +195,11 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); assertThat(doc.rootDoc().getField("field.point"), notNullValue()); - assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoUtils.mortonHash(51.0, 42.0))); + } assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoEncodingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoEncodingTests.java index 08b6e0d7169..89dcbf89600 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoEncodingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoEncodingTests.java @@ -37,7 +37,7 @@ public class GeoEncodingTests extends ESTestCase { final double lat = randomDouble() * 180 - 90; final double lon = randomDouble() * 360 - 180; final Distance precision = new Distance(1+(randomDouble() * 9), randomFrom(Arrays.asList(DistanceUnit.MILLIMETERS, DistanceUnit.METERS, DistanceUnit.KILOMETERS))); - final GeoPointFieldMapper.Encoding encoding = GeoPointFieldMapper.Encoding.of(precision); + final GeoPointFieldMapperLegacy.Encoding encoding = GeoPointFieldMapperLegacy.Encoding.of(precision); assertThat(encoding.precision().convert(DistanceUnit.METERS).value, lessThanOrEqualTo(precision.convert(DistanceUnit.METERS).value)); final GeoPoint geoPoint = encoding.decode(encoding.encodeCoordinate(lat), encoding.encodeCoordinate(lon), new GeoPoint()); final double error = GeoDistance.PLANE.calculate(lat, lon, geoPoint.lat(), geoPoint.lon(), DistanceUnit.METERS); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java index 1c5a847c93b..93fd71599c4 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java @@ -18,23 +18,26 @@ */ package org.elasticsearch.index.mapper.geo; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; +import org.apache.lucene.util.GeoUtils; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.VersionUtils; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -43,18 +46,22 @@ import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isIn; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { - @Test public void testLatLonValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -62,21 +69,29 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .endObject() .bytes()); + boolean indexCreatedBefore22 = version.before(Version.V_2_2_0); assertThat(doc.rootDoc().getField("point.lat"), notNullValue()); - assertThat(doc.rootDoc().getField("point.lat").fieldType().stored(), is(false)); + final boolean stored = indexCreatedBefore22 == false; + assertThat(doc.rootDoc().getField("point.lat").fieldType().stored(), is(stored)); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); - assertThat(doc.rootDoc().getField("point.lon").fieldType().stored(), is(false)); + assertThat(doc.rootDoc().getField("point.lon").fieldType().stored(), is(stored)); assertThat(doc.rootDoc().getField("point.geohash"), nullValue()); - assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + if (indexCreatedBefore22 == true) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testLatLonValuesWithGeohash() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true).endObject().endObject() + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("geohash", true).endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -86,16 +101,17 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lat"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); - assertThat(doc.rootDoc().get("point.geohash"), equalTo(XGeoHashUtils.stringEncode(1.3, 1.2))); + assertThat(doc.rootDoc().get("point.geohash"), equalTo(GeoHashUtils.stringEncode(1.3, 1.2))); } - @Test public void testLatLonInOneValueWithGeohash() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true).endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("geohash", true).endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -105,39 +121,41 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lat"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); - assertThat(doc.rootDoc().get("point.geohash"), equalTo(XGeoHashUtils.stringEncode(1.3, 1.2))); + assertThat(doc.rootDoc().get("point.geohash"), equalTo(GeoHashUtils.stringEncode(1.3, 1.2))); } - @Test public void testGeoHashIndexValue() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true).endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("geohash", true).endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() - .field("point", XGeoHashUtils.stringEncode(1.3, 1.2)) + .field("point", GeoHashUtils.stringEncode(1.3, 1.2)) .endObject() .bytes()); assertThat(doc.rootDoc().getField("point.lat"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); - assertThat(doc.rootDoc().get("point.geohash"), equalTo(XGeoHashUtils.stringEncode(1.3, 1.2))); + assertThat(doc.rootDoc().get("point.geohash"), equalTo(GeoHashUtils.stringEncode(1.3, 1.2))); } - @Test public void testGeoHashValue() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() - .field("point", XGeoHashUtils.stringEncode(1.3, 1.2)) + .field("point", GeoHashUtils.stringEncode(1.3, 1.2)) .endObject() .bytes()); @@ -146,15 +164,18 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("point"), notNullValue()); } - @Test public void testNormalizeLatLonValuesDefault() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); // default to normalize - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("coerce", true) - .field("ignore_malformed", true).endObject().endObject() - .endObject().endObject().string(); + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("point").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + mapping.field("coerce", true); + } + mapping.field("ignore_malformed", true).endObject().endObject().endObject().endObject(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping.string()); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -162,7 +183,11 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .endObject() .bytes()); - assertThat(doc.rootDoc().get("point"), equalTo("89.0,1.0")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("89.0,1.0")); + } else { + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoUtils.mortonHash(1.0, 89.0))); + } doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -170,7 +195,11 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .endObject() .bytes()); - assertThat(doc.rootDoc().get("point"), equalTo("-89.0,-1.0")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("-89.0,-1.0")); + } else { + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoUtils.mortonHash(-1.0, -89.0))); + } doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -178,20 +207,26 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .endObject() .bytes()); - assertThat(doc.rootDoc().get("point"), equalTo("-1.0,-179.0")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("-1.0,-179.0")); + } else { + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoUtils.mortonHash(-179.0, -1.0))); + } } - @Test public void testValidateLatLonValues() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("coerce", false) - .field("ignore_malformed", false).endObject().endObject() - .endObject().endObject().string(); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true); + if (version.before(Version.V_2_2_0)) { + mapping.field("coerce", false); + } + mapping.field("ignore_malformed", false).endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping.string()); - - ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() .startObject("point").field("lat", 90).field("lon", 1.3).endObject() .endObject() @@ -242,17 +277,19 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { } } - @Test public void testNoValidateLatLonValues() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("coerce", false) - .field("ignore_malformed", true).endObject().endObject() - .endObject().endObject().string(); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true); + if (version.before(Version.V_2_2_0)) { + mapping.field("coerce", false); + } + mapping.field("ignore_malformed", true).endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping.string()); - - ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() .startObject("point").field("lat", 90).field("lon", 1.3).endObject() .endObject() @@ -283,13 +320,14 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .bytes()); } - @Test public void testLatLonValuesStored() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes").endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("store", "yes").endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -302,16 +340,21 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3)); assertThat(doc.rootDoc().getField("point.geohash"), nullValue()); - assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testArrayLatLonValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes").endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("store", "yes").endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -326,19 +369,28 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2)); assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2)); assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3)); - assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5)); - assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoUtils.mortonHash(1.5, 1.4))); + } } - @Test public void testLatLonInOneValue() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -348,16 +400,21 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lat"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); - assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testLatLonInOneValueStored() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes").endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -369,16 +426,21 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2)); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3)); - assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testLatLonInOneValueArray() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes").endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("store", "yes").endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -393,19 +455,28 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2)); assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2)); assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3)); - assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5)); - assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoUtils.mortonHash(1.5, 1.4))); + } } - @Test public void testLonLatArray() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -415,18 +486,22 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lat"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); - assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testLonLatArrayDynamic() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startArray("dynamic_templates").startObject() - .startObject("point").field("match", "point*").startObject("mapping").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endArray() - .endObject().endObject().string(); + .startObject("point").field("match", "point*").startObject("mapping").field("type", "geo_point") + .field("lat_lon", true).endObject().endObject().endObject().endArray().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -436,16 +511,21 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lat"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); - assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testLonLatArrayStored() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes").endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("store", "yes").endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -457,16 +537,21 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2)); assertThat(doc.rootDoc().getField("point.lon"), notNullValue()); assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3)); - assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testLonLatArrayArrayStored() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes").endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("store", "yes").endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -481,19 +566,28 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2)); assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2)); assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3)); - assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5)); - assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5")); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoUtils.mortonHash(1.5, 1.4))); + } } /** * Test that expected exceptions are thrown when creating a new index with deprecated options */ - @Test public void testOptionDeprecation() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapperParser parser = createIndex("test", settings).mapperService().documentMapperParser(); // test deprecation exceptions on newly created indexes try { String validateMapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -566,7 +660,6 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { /** * Test backward compatibility */ - @Test public void testBackwardCompatibleOptions() throws Exception { // backward compatibility testing Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, @@ -618,35 +711,31 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(parser.parse(mapping).mapping().toString(), containsString("\"coerce\":true")); } - @Test public void testGeoPointMapperMerge() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); String stage1Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true) - .field("coerce", true).endObject().endObject() - .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - DocumentMapper stage1 = parser.parse(stage1Mapping); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("geohash", true).endObject().endObject().endObject().endObject().string(); + MapperService mapperService = createIndex("test", settings).mapperService(); + DocumentMapper stage1 = mapperService.merge("type", new CompressedXContent(stage1Mapping), true, false); String stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false).field("geohash", true) - .field("coerce", false).endObject().endObject() - .endObject().endObject().string(); - DocumentMapper stage2 = parser.parse(stage2Mapping); - - MergeResult mergeResult = stage1.merge(stage2.mapping(), false, false); - assertThat(mergeResult.hasConflicts(), equalTo(true)); - assertThat(mergeResult.buildConflicts().length, equalTo(2)); - // todo better way of checking conflict? - assertThat("mapper [point] has different [lat_lon]", isIn(new ArrayList<>(Arrays.asList(mergeResult.buildConflicts())))); - assertThat("mapper [point] has different [coerce]", isIn(new ArrayList<>(Arrays.asList(mergeResult.buildConflicts())))); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false) + .field("geohash", false).endObject().endObject().endObject().endObject().string(); + try { + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [point] has different [lat_lon]")); + assertThat(e.getMessage(), containsString("mapper [point] has different [geohash]")); + assertThat(e.getMessage(), containsString("mapper [point] has different [geohash_precision]")); + } // correct mapping and ensure no failures stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true).field("geohash", true) - .field("coerce", true).endObject().endObject() - .endObject().endObject().string(); - stage2 = parser.parse(stage2Mapping); - mergeResult = stage1.merge(stage2.mapping(), false, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) + .field("geohash", true).endObject().endObject().endObject().endObject().string(); + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); } public void testGeoHashSearch() throws Exception { @@ -657,7 +746,10 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .endObject().endObject().endObject().string(); // create index and add a test point (dr5regy6rc6z) - CreateIndexRequestBuilder mappingRequest = client().admin().indices().prepareCreate("test").addMapping("pin", mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + CreateIndexRequestBuilder mappingRequest = client().admin().indices().prepareCreate("test").setSettings(settings) + .addMapping("pin", mapping); mappingRequest.execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); client().prepareIndex("test", "pin", "1").setSource(jsonBuilder().startObject().startObject("location").field("lat", 40.7143528) @@ -679,7 +771,10 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .endObject().endObject().endObject().endObject().string(); // create index and add a test point (dr5regy6rc6z) - CreateIndexRequestBuilder mappingRequest = client().admin().indices().prepareCreate("test").addMapping("pin", mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + CreateIndexRequestBuilder mappingRequest = client().admin().indices().prepareCreate("test").setSettings(settings) + .addMapping("pin", mapping); mappingRequest.execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); client().prepareIndex("test", "pin", "1").setSource(jsonBuilder().startObject().startObject("location").field("lat", 40.7143528) diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldTypeTests.java index e3b18831bba..19eb536e32e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldTypeTests.java @@ -27,7 +27,7 @@ import org.junit.Before; public class GeoPointFieldTypeTests extends FieldTypeTestCase { @Override protected MappedFieldType createDefaultFieldType() { - return new GeoPointFieldMapper.GeoPointFieldType(); + return new BaseGeoPointFieldMapper.GeoPointFieldType(); } @Before @@ -35,13 +35,13 @@ public class GeoPointFieldTypeTests extends FieldTypeTestCase { addModifier(new Modifier("geohash", false, true) { @Override public void modify(MappedFieldType ft) { - ((GeoPointFieldMapper.GeoPointFieldType)ft).setGeohashEnabled(new StringFieldMapper.StringFieldType(), 1, true); + ((BaseGeoPointFieldMapper.GeoPointFieldType)ft).setGeoHashEnabled(new StringFieldMapper.StringFieldType(), 1, true); } }); addModifier(new Modifier("lat_lon", false, true) { @Override public void modify(MappedFieldType ft) { - ((GeoPointFieldMapper.GeoPointFieldType)ft).setLatLonEnabled(new DoubleFieldMapper.DoubleFieldType(), new DoubleFieldMapper.DoubleFieldType()); + ((BaseGeoPointFieldMapper.GeoPointFieldType)ft).setLatLonEnabled(new DoubleFieldMapper.DoubleFieldType(), new DoubleFieldMapper.DoubleFieldType()); } }); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java index 26f7129e0a9..54e9e96f8ad 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java @@ -22,27 +22,27 @@ import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.isIn; public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { - - @Test public void testDefaultConfiguration() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -135,7 +135,6 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(coerce, equalTo(false)); } - @Test public void testGeohashConfiguration() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -158,7 +157,6 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); } - @Test public void testQuadtreeConfiguration() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -182,8 +180,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); assertThat(strategy.isPointsOnly(), equalTo(true)); } - - @Test + public void testLevelPrecisionConfiguration() throws IOException { DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); @@ -198,7 +195,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { .endObject().endObject() .endObject().endObject().string(); - + DocumentMapper defaultMapper = parser.parse(mapping); FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); @@ -209,7 +206,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); } { @@ -237,7 +234,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { // 70m is less precise so it loses assertThat(strategy.getGrid().getMaxLevels(), equalTo(26)); } - + { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -259,9 +256,9 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); } - + { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -282,9 +279,9 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); } - + { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -305,11 +302,10 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); } } - @Test public void testPointsOnlyOption() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -330,7 +326,6 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.isPointsOnly(), equalTo(true)); } - @Test public void testLevelDefaults() throws IOException { DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); { @@ -342,7 +337,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { .endObject().endObject() .endObject().endObject().string(); - + DocumentMapper defaultMapper = parser.parse(mapping); FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); @@ -353,9 +348,9 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); } - + { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -375,33 +370,30 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); } } - @Test public void testGeoShapeMapperMerge() throws Exception { String stage1Mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("shape").field("type", "geo_shape").field("tree", "geohash").field("strategy", "recursive") .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01).field("orientation", "ccw") .endObject().endObject().endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - DocumentMapper stage1 = parser.parse(stage1Mapping); + MapperService mapperService = createIndex("test").mapperService(); + DocumentMapper stage1 = mapperService.merge("type", new CompressedXContent(stage1Mapping), true, false); String stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("shape").field("type", "geo_shape").field("tree", "quadtree") .field("strategy", "term").field("precision", "1km").field("tree_levels", 26).field("distance_error_pct", 26) .field("orientation", "cw").endObject().endObject().endObject().endObject().string(); - DocumentMapper stage2 = parser.parse(stage2Mapping); - - MergeResult mergeResult = stage1.merge(stage2.mapping(), false, false); - // check correct conflicts - assertThat(mergeResult.hasConflicts(), equalTo(true)); - assertThat(mergeResult.buildConflicts().length, equalTo(4)); - ArrayList conflicts = new ArrayList<>(Arrays.asList(mergeResult.buildConflicts())); - assertThat("mapper [shape] has different [strategy]", isIn(conflicts)); - assertThat("mapper [shape] has different [tree]", isIn(conflicts)); - assertThat("mapper [shape] has different [tree_levels]", isIn(conflicts)); - assertThat("mapper [shape] has different [precision]", isIn(conflicts)); + try { + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); + } // verify nothing changed FieldMapper fieldMapper = stage1.mappers().getMapper("shape"); @@ -420,11 +412,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("shape").field("type", "geo_shape").field("precision", "1m") .field("tree_levels", 8).field("distance_error_pct", 0.001).field("orientation", "cw").endObject().endObject().endObject().endObject().string(); - stage2 = parser.parse(stage2Mapping); - mergeResult = stage1.merge(stage2.mapping(), false, false); - - // verify mapping changes, and ensure no failures - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); fieldMapper = stage1.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java index 6338f4b924c..c3a92e8846d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java @@ -19,29 +19,37 @@ package org.elasticsearch.index.mapper.geo; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.VersionUtils; import org.hamcrest.MatcherAssert; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * */ public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase { - - @Test public void testLatLonValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false).endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false) + .endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -49,18 +57,23 @@ public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase { .endObject() .bytes()); - MatcherAssert.assertThat(doc.rootDoc().getField("point.lat"), nullValue()); - MatcherAssert.assertThat(doc.rootDoc().getField("point.lon"), nullValue()); - MatcherAssert.assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + assertThat(doc.rootDoc().getField("point.lat"), nullValue()); + assertThat(doc.rootDoc().getField("point.lon"), nullValue()); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testLatLonInOneValue() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false).endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -68,62 +81,72 @@ public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase { .endObject() .bytes()); - MatcherAssert.assertThat(doc.rootDoc().getField("point.lat"), nullValue()); - MatcherAssert.assertThat(doc.rootDoc().getField("point.lon"), nullValue()); - MatcherAssert.assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + assertThat(doc.rootDoc().getField("point.lat"), nullValue()); + assertThat(doc.rootDoc().getField("point.lon"), nullValue()); + if (version.before(Version.V_2_2_0)) { + assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); + } else { + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoUtils.mortonHash(1.3, 1.2))); + } } - @Test public void testGeoHashValue() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true).endObject().endObject() - .endObject().endObject().string(); + .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true) + .endObject().endObject().endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() - .field("point", XGeoHashUtils.stringEncode(1.3, 1.2)) + .field("point", GeoHashUtils.stringEncode(1.3, 1.2)) .endObject() .bytes()); - MatcherAssert.assertThat(doc.rootDoc().getField("point.lat"), nullValue()); - MatcherAssert.assertThat(doc.rootDoc().getField("point.lon"), nullValue()); - MatcherAssert.assertThat(doc.rootDoc().get("point.geohash"), equalTo(XGeoHashUtils.stringEncode(1.3, 1.2))); - MatcherAssert.assertThat(doc.rootDoc().get("point"), notNullValue()); + assertThat(doc.rootDoc().getField("point.lat"), nullValue()); + assertThat(doc.rootDoc().getField("point.lon"), nullValue()); + assertThat(doc.rootDoc().get("point.geohash"), equalTo(GeoHashUtils.stringEncode(1.3, 1.2))); + assertThat(doc.rootDoc().get("point"), notNullValue()); } - @Test public void testGeoHashPrecisionAsInteger() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true).field("geohash_precision", 10).endObject().endObject() - .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true) + .field("geohash_precision", 10).endObject().endObject().endObject().endObject().string(); + + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("point"); - assertThat(mapper, instanceOf(GeoPointFieldMapper.class)); - GeoPointFieldMapper geoPointFieldMapper = (GeoPointFieldMapper) mapper; - assertThat(geoPointFieldMapper.fieldType().geohashPrecision(), is(10)); + assertThat(mapper, instanceOf(BaseGeoPointFieldMapper.class)); + BaseGeoPointFieldMapper geoPointFieldMapper = (BaseGeoPointFieldMapper) mapper; + assertThat(geoPointFieldMapper.fieldType().geoHashPrecision(), is(10)); } - @Test public void testGeoHashPrecisionAsLength() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("geohash", true).field("geohash_precision", "5m").endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("point"); - assertThat(mapper, instanceOf(GeoPointFieldMapper.class)); - GeoPointFieldMapper geoPointFieldMapper = (GeoPointFieldMapper) mapper; - assertThat(geoPointFieldMapper.fieldType().geohashPrecision(), is(10)); + assertThat(mapper, instanceOf(BaseGeoPointFieldMapper.class)); + BaseGeoPointFieldMapper geoPointFieldMapper = (BaseGeoPointFieldMapper) mapper; + assertThat(geoPointFieldMapper.fieldType().geoHashPrecision(), is(10)); } - @Test public void testNullValue() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + DocumentMapper defaultMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() diff --git a/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java index bfb01476957..40cf05c4d6a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java @@ -57,7 +57,7 @@ public class IndexTypeMapperTests extends ESSingleNodeTestCase { .startObject("_index").field("enabled", false).endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse(mapping); - IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.class); + IndexFieldMapper indexMapper = docMapper.metadataMapper(IndexFieldMapper.class); assertThat(indexMapper.enabled(), equalTo(false)); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -74,7 +74,7 @@ public class IndexTypeMapperTests extends ESSingleNodeTestCase { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.class); + IndexFieldMapper indexMapper = docMapper.metadataMapper(IndexFieldMapper.class); assertThat(indexMapper.enabled(), equalTo(false)); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -128,7 +128,7 @@ public class IndexTypeMapperTests extends ESSingleNodeTestCase { .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse(mapping); - IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.class); + IndexFieldMapper indexMapper = docMapper.metadataMapper(IndexFieldMapper.class); assertThat(indexMapper.enabled(), equalTo(true)); assertThat(indexMapper.fieldType().stored(), equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java index 4753f903cbf..3f3c5702e8c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java @@ -19,17 +19,31 @@ package org.elasticsearch.index.mapper.internal; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.io.IOException; import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; @@ -68,7 +82,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().hasDocValues()); assertEquals(IndexOptions.DOCS, fieldNamesMapper.fieldType().indexOptions()); assertFalse(fieldNamesMapper.fieldType().tokenized()); @@ -88,16 +102,16 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .endObject() .endObject() .bytes()); - + assertFieldNames(set("a", "b", "b.c", "_uid", "_type", "_version", "_source", "_all"), doc); } - + public void testExplicitEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_field_names").field("enabled", true).endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertTrue(fieldNamesMapper.fieldType().isEnabled()); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -114,7 +128,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .startObject("_field_names").field("enabled", false).endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().isEnabled()); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -122,18 +136,18 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .field("field", "value") .endObject() .bytes()); - + assertNull(doc.rootDoc().get("_field_names")); } - + public void testPre13Disabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_2_4.id).build(); DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().isEnabled()); } - + public void testDisablingBackcompat() throws Exception { // before 1.5, disabling happened by setting index:no String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -142,7 +156,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().isEnabled()); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -161,7 +175,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertTrue(fieldNamesMapper.fieldType().stored()); } @@ -173,14 +187,111 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .startObject("_field_names").field("enabled", false).endObject() .endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - + DocumentMapper mapperEnabled = parser.parse(enabledMapping); DocumentMapper mapperDisabled = parser.parse(disabledMapping); mapperEnabled.merge(mapperDisabled.mapping(), false, false); - assertFalse(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); + assertFalse(mapperEnabled.metadataMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); mapperEnabled = parser.parse(enabledMapping); mapperDisabled.merge(mapperEnabled.mapping(), false, false); - assertTrue(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); + assertTrue(mapperEnabled.metadataMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); + } + + private static class DummyMetadataFieldMapper extends MetadataFieldMapper { + + public static class TypeParser implements MetadataFieldMapper.TypeParser { + + @Override + public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + return new MetadataFieldMapper.Builder("_dummy", FIELD_TYPE) { + @Override + public DummyMetadataFieldMapper build(BuilderContext context) { + return new DummyMetadataFieldMapper(context.indexSettings()); + } + }; + } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new DummyMetadataFieldMapper(indexSettings); + } + + } + + private static class DummyFieldType extends MappedFieldType { + + public DummyFieldType() { + super(); + } + + private DummyFieldType(MappedFieldType other) { + super(other); + } + + @Override + public MappedFieldType clone() { + return new DummyFieldType(this); + } + + @Override + public String typeName() { + return "_dummy"; + } + + } + + private static final MappedFieldType FIELD_TYPE = new DummyFieldType(); + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setNames(new MappedFieldType.Names("_dummy")); + FIELD_TYPE.freeze(); + } + + protected DummyMetadataFieldMapper(Settings indexSettings) { + super("_dummy", FIELD_TYPE, FIELD_TYPE, indexSettings); + } + + @Override + public void preParse(ParseContext context) throws IOException { + } + + @Override + public void postParse(ParseContext context) throws IOException { + context.doc().add(new Field("_dummy", "dummy", FIELD_TYPE)); + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + } + + @Override + protected String contentType() { + return "_dummy"; + } + + } + + public void testSeesFieldsFromPlugins() throws IOException { + IndexService indexService = createIndex("test"); + IndicesModule indicesModule = new IndicesModule(); + indicesModule.registerMetadataMapper("_dummy", new DummyMetadataFieldMapper.TypeParser()); + final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); + MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry); + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService, + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); + DocumentMapper mapper = parser.parse(mapping); + ParsedDocument parsedDocument = mapper.parse("index", "type", "id", new BytesArray("{}")); + IndexableField[] fields = parsedDocument.rootDoc().getFields(FieldNamesFieldMapper.NAME); + boolean found = false; + for (IndexableField f : fields) { + if ("_dummy".equals(f.stringValue())) { + found = true; + break; + } + } + assertTrue("Could not find the dummy field among " + Arrays.toString(fields), found); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java new file mode 100644 index 00000000000..105b3b446ce --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.internal; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.test.ESSingleNodeTestCase; + +public class TypeFieldMapperTests extends ESSingleNodeTestCase { + + public void testDocValues() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class); + assertTrue(typeMapper.fieldType().hasDocValues()); + } + + public void testDocValuesPre21() throws Exception { + // between 2.0 and 2.1, doc values was disabled for _type + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); + Settings bwcSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0_beta1.id).build(); + + DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse(mapping); + TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class); + assertFalse(typeMapper.fieldType().hasDocValues()); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ip/SimpleIpMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ip/SimpleIpMappingTests.java index 07d0940b3b3..4245641fd82 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ip/SimpleIpMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ip/SimpleIpMappingTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.containsString; @@ -38,8 +37,6 @@ import static org.hamcrest.Matchers.nullValue; * */ public class SimpleIpMappingTests extends ESSingleNodeTestCase { - - @Test public void testSimpleMapping() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("ip").field("type", "ip").endObject().endObject() @@ -57,12 +54,10 @@ public class SimpleIpMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("ip"), is("2130706433")); } - @Test public void testThatValidIpCanBeConvertedToLong() throws Exception { assertThat(IpFieldMapper.ipToLong("127.0.0.1"), is(2130706433L)); } - @Test public void testThatInvalidIpThrowsException() throws Exception { try { IpFieldMapper.ipToLong("127.0.011.1111111"); @@ -72,7 +67,6 @@ public class SimpleIpMappingTests extends ESSingleNodeTestCase { } } - @Test public void testThatIpv6AddressThrowsException() throws Exception { try { IpFieldMapper.ipToLong("2001:db8:0:8d3:0:8a2e:70:7344"); @@ -82,7 +76,6 @@ public class SimpleIpMappingTests extends ESSingleNodeTestCase { } } - @Test public void testIgnoreMalformedOption() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("field1") .field("type", "ip").field("ignore_malformed", true).endObject().startObject("field2").field("type", "ip") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/lucene/DoubleIndexingDocTests.java b/core/src/test/java/org/elasticsearch/index/mapper/lucene/DoubleIndexingDocTests.java index 9aade61c5b0..656599c5036 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/lucene/DoubleIndexingDocTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/lucene/DoubleIndexingDocTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; @@ -38,8 +37,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class DoubleIndexingDocTests extends ESSingleNodeTestCase { - - @Test public void testDoubleIndexingSameDoc() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), Lucene.STANDARD_ANALYZER)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java b/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java index 380e8e34d0f..d67b97c3d85 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.HashSet; @@ -47,8 +46,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class StoredNumericValuesTests extends ESSingleNodeTestCase { - - @Test public void testBytesAndNumericRepresentation() throws Exception { IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java index dc14ebecd5e..58fa8fd69b0 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java @@ -21,6 +21,9 @@ package org.elasticsearch.index.mapper.multifield; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -28,48 +31,35 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.index.mapper.core.DateFieldMapper; -import org.elasticsearch.index.mapper.core.LongFieldMapper; -import org.elasticsearch.index.mapper.core.StringFieldMapper; -import org.elasticsearch.index.mapper.core.TokenCountFieldMapper; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.core.*; +import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; +import org.elasticsearch.test.VersionUtils; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.TreeMap; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.mapper.MapperBuilders.*; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.mapper.MapperBuilders.doc; -import static org.elasticsearch.index.mapper.MapperBuilders.rootObject; -import static org.elasticsearch.index.mapper.MapperBuilders.stringField; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.*; /** * */ public class MultiFieldTests extends ESSingleNodeTestCase { - - @Test - public void testMultiField_multiFieldType() throws Exception { + public void testMultiFieldMultiFieldType() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/test-multi-field-type.json"); testMultiField(mapping); } - @Test - public void testMultiField_multiFields() throws Exception { + public void testMultiFieldMultiFields() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/test-multi-fields.json"); testMultiField(mapping); } @@ -145,10 +135,9 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertThat(docMapper.mappers().getMapper("object1.multi1.string").fieldType().tokenized(), equalTo(false)); } - @Test public void testBuildThenParse() throws Exception { IndexService indexService = createIndex("test"); - Settings settings = indexService.settingsService().getSettings(); + Settings settings = indexService.getIndexSettings().getSettings(); DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser(); DocumentMapper builderDocMapper = doc(settings, rootObject("person").add( @@ -186,7 +175,6 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertEquals(IndexOptions.NONE, f.fieldType().indexOptions()); } - @Test public void testConvertMultiFieldNoDefaultField() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/test-multi-field-type-no-default-field.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); @@ -256,10 +244,12 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertThat(docMapper.mappers().getMapper("age.stored").fieldType().tokenized(), equalTo(false)); } - @Test public void testConvertMultiFieldGeoPoint() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + boolean indexCreatedBefore22 = version.before(Version.V_2_2_0); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/test-multi-field-type-geo_point.json"); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + DocumentMapper docMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); assertThat(docMapper.mappers().getMapper("a"), notNullValue()); assertThat(docMapper.mappers().getMapper("a"), instanceOf(StringFieldMapper.class)); @@ -268,10 +258,13 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertThat(docMapper.mappers().getMapper("a").fieldType().tokenized(), equalTo(false)); assertThat(docMapper.mappers().getMapper("a.b"), notNullValue()); - assertThat(docMapper.mappers().getMapper("a.b"), instanceOf(GeoPointFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("a.b"), instanceOf(BaseGeoPointFieldMapper.class)); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("a.b").fieldType().indexOptions()); - assertThat(docMapper.mappers().getMapper("a.b").fieldType().stored(), equalTo(false)); + final boolean stored = indexCreatedBefore22 == false; + assertThat(docMapper.mappers().getMapper("a.b").fieldType().stored(), equalTo(stored)); assertThat(docMapper.mappers().getMapper("a.b").fieldType().tokenized(), equalTo(false)); + final boolean hasDocValues = indexCreatedBefore22 == false; + assertThat(docMapper.mappers().getMapper("a.b").fieldType().hasDocValues(), equalTo(hasDocValues)); BytesReference json = jsonBuilder().startObject() .field("a", "-1,-1") @@ -288,15 +281,20 @@ public class MultiFieldTests extends ESSingleNodeTestCase { f = doc.getField("a.b"); assertThat(f, notNullValue()); assertThat(f.name(), equalTo("a.b")); - assertThat(f.stringValue(), equalTo("-1.0,-1.0")); - assertThat(f.fieldType().stored(), equalTo(false)); + if (indexCreatedBefore22 == true) { + assertThat(f.stringValue(), equalTo("-1.0,-1.0")); + } else { + assertThat(Long.parseLong(f.stringValue()), equalTo(GeoUtils.mortonHash(-1.0, -1.0))); + } + assertThat(f.fieldType().stored(), equalTo(stored)); assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(docMapper.mappers().getMapper("b"), notNullValue()); - assertThat(docMapper.mappers().getMapper("b"), instanceOf(GeoPointFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("b"), instanceOf(BaseGeoPointFieldMapper.class)); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("b").fieldType().indexOptions()); - assertThat(docMapper.mappers().getMapper("b").fieldType().stored(), equalTo(false)); + assertThat(docMapper.mappers().getMapper("b").fieldType().stored(), equalTo(stored)); assertThat(docMapper.mappers().getMapper("b").fieldType().tokenized(), equalTo(false)); + assertThat(docMapper.mappers().getMapper("b").fieldType().hasDocValues(), equalTo(hasDocValues)); assertThat(docMapper.mappers().getMapper("b.a"), notNullValue()); assertThat(docMapper.mappers().getMapper("b.a"), instanceOf(StringFieldMapper.class)); @@ -312,8 +310,12 @@ public class MultiFieldTests extends ESSingleNodeTestCase { f = doc.getField("b"); assertThat(f, notNullValue()); assertThat(f.name(), equalTo("b")); - assertThat(f.stringValue(), equalTo("-1.0,-1.0")); - assertThat(f.fieldType().stored(), equalTo(false)); + if (indexCreatedBefore22 == true) { + assertThat(f.stringValue(), equalTo("-1.0,-1.0")); + } else { + assertThat(Long.parseLong(f.stringValue()), equalTo(GeoUtils.mortonHash(-1.0, -1.0))); + } + assertThat(f.fieldType().stored(), equalTo(stored)); assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); f = doc.getField("b.a"); @@ -331,15 +333,23 @@ public class MultiFieldTests extends ESSingleNodeTestCase { f = doc.getFields("b")[0]; assertThat(f, notNullValue()); assertThat(f.name(), equalTo("b")); - assertThat(f.stringValue(), equalTo("-1.0,-1.0")); - assertThat(f.fieldType().stored(), equalTo(false)); + if (indexCreatedBefore22 == true) { + assertThat(f.stringValue(), equalTo("-1.0,-1.0")); + } else { + assertThat(Long.parseLong(f.stringValue()), equalTo(GeoUtils.mortonHash(-1.0, -1.0))); + } + assertThat(f.fieldType().stored(), equalTo(stored)); assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); f = doc.getFields("b")[1]; assertThat(f, notNullValue()); assertThat(f.name(), equalTo("b")); - assertThat(f.stringValue(), equalTo("-2.0,-2.0")); - assertThat(f.fieldType().stored(), equalTo(false)); + if (indexCreatedBefore22 == true) { + assertThat(f.stringValue(), equalTo("-2.0,-2.0")); + } else { + assertThat(Long.parseLong(f.stringValue()), equalTo(GeoUtils.mortonHash(-2.0, -2.0))); + } + assertThat(f.fieldType().stored(), equalTo(stored)); assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); f = doc.getField("b.a"); @@ -353,7 +363,6 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); } - @Test public void testConvertMultiFieldCompletion() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/test-multi-field-type-completion.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); @@ -421,7 +430,6 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); } - @Test // The underlying order of the fields in multi fields in the mapping source should always be consistent, if not this // can to unnecessary re-syncing of the mappings between the local instance and cluster state public void testMultiFieldsInConsistentOrder() throws Exception { @@ -451,12 +459,11 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertThat(field, equalTo(multiFieldNames[i++])); } } - - @Test + // The fielddata settings need to be the same after deserializing/re-serialsing, else unneccesary mapping sync's can be triggered public void testMultiFieldsFieldDataSettingsInConsistentOrder() throws Exception { final String MY_MULTI_FIELD = "multi_field"; - + // Possible fielddata settings Map possibleSettings = new TreeMap(); possibleSettings.put("filter.frequency.min", 1); @@ -466,18 +473,18 @@ public class MultiFieldTests extends ESSingleNodeTestCase { possibleSettings.put("foo", "bar"); possibleSettings.put("zetting", "zValue"); possibleSettings.put("aSetting", "aValue"); - + // Generate a mapping with the a random subset of possible fielddata settings XContentBuilder builder = jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("my_field").field("type", "string").startObject("fields").startObject(MY_MULTI_FIELD) .field("type", "string").startObject("fielddata"); String[] keys = possibleSettings.keySet().toArray(new String[]{}); - Collections.shuffle(Arrays.asList(keys)); + Collections.shuffle(Arrays.asList(keys), random()); for(int i = randomIntBetween(0, possibleSettings.size()-1); i >= 0; --i) builder.field(keys[i], possibleSettings.get(keys[i])); builder.endObject().endObject().endObject().endObject().endObject().endObject().endObject(); - - // Check the mapping remains identical when deserialed/re-serialsed + + // Check the mapping remains identical when deserialed/re-serialsed final DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(builder.string()); DocumentMapper docMapper2 = parser.parse(docMapper.mappingSource().string()); @@ -509,4 +516,30 @@ public class MultiFieldTests extends ESSingleNodeTestCase { assertTrue(e.getMessage().contains("cannot be used in multi field")); } } + + public void testMultiFieldWithDot() throws IOException { + XContentBuilder mapping = jsonBuilder(); + mapping.startObject() + .startObject("my_type") + .startObject("properties") + .startObject("city") + .field("type", "string") + .startObject("fields") + .startObject("raw.foo") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + MapperService mapperService = createIndex("test").mapperService(); + try { + mapperService.documentMapperParser().parse(mapping.string()); + fail("this should throw an exception because one field contains a dot"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), equalTo("Field name [raw.foo] which is a multi field of [city] cannot contain '.'")); + } + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java index f3636bf4c47..0c26324ac6c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper.multifield; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.unit.DistanceUnit; @@ -28,22 +27,22 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Map; -import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery; import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** */ public class MultiFieldsIntegrationIT extends ESIntegTestCase { - - @Test public void testMultiFields() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") @@ -101,7 +100,6 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); } - @Test public void testGeoPointMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") @@ -124,15 +122,14 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { assertThat(bField.get("index").toString(), equalTo("not_analyzed")); client().prepareIndex("my-index", "my-type", "1").setSource("a", "51,19").setRefresh(true).get(); - CountResponse countResponse = client().prepareCount("my-index") + SearchResponse countResponse = client().prepareSearch("my-index").setSize(0) .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS))) .get(); - assertThat(countResponse.getCount(), equalTo(1l)); - countResponse = client().prepareCount("my-index").setQuery(matchQuery("a.b", "51,19")).get(); - assertThat(countResponse.getCount(), equalTo(1l)); + assertThat(countResponse.getHits().totalHits(), equalTo(1l)); + countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "51,19")).get(); + assertThat(countResponse.getHits().totalHits(), equalTo(1l)); } - @Test public void testTokenCountMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") @@ -167,11 +164,10 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { assertThat(bField.get("index").toString(), equalTo("not_analyzed")); client().prepareIndex("my-index", "my-type", "1").setSource("a", "my tokens").setRefresh(true).get(); - CountResponse countResponse = client().prepareCount("my-index").setQuery(matchQuery("a.b", "my tokens")).get(); - assertThat(countResponse.getCount(), equalTo(1l)); + SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "my tokens")).get(); + assertThat(countResponse.getHits().totalHits(), equalTo(1l)); } - @Test public void testCompletionMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") @@ -183,7 +179,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { assertThat(mappingMetaData, not(nullValue())); Map mappingSource = mappingMetaData.sourceAsMap(); Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); - assertThat(aField.size(), equalTo(7)); + assertThat(aField.size(), equalTo(6)); assertThat(aField.get("type").toString(), equalTo("completion")); assertThat(aField.get("fields"), notNullValue()); @@ -193,11 +189,10 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { assertThat(bField.get("index").toString(), equalTo("not_analyzed")); client().prepareIndex("my-index", "my-type", "1").setSource("a", "complete me").setRefresh(true).get(); - CountResponse countResponse = client().prepareCount("my-index").setQuery(matchQuery("a.b", "complete me")).get(); - assertThat(countResponse.getCount(), equalTo(1l)); + SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "complete me")).get(); + assertThat(countResponse.getHits().totalHits(), equalTo(1l)); } - @Test public void testIpMultiField() throws Exception { assertAcked( client().admin().indices().prepareCreate("my-index") @@ -219,8 +214,8 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { assertThat(bField.get("index").toString(), equalTo("not_analyzed")); client().prepareIndex("my-index", "my-type", "1").setSource("a", "127.0.0.1").setRefresh(true).get(); - CountResponse countResponse = client().prepareCount("my-index").setQuery(matchQuery("a.b", "127.0.0.1")).get(); - assertThat(countResponse.getCount(), equalTo(1l)); + SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "127.0.0.1")).get(); + assertThat(countResponse.getHits().totalHits(), equalTo(1l)); } private XContentBuilder createMappingSource(String fieldType) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java index eec0002a6ef..30890dcd22a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java @@ -22,25 +22,27 @@ package org.elasticsearch.index.mapper.multifield.merge; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.util.Arrays; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * */ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { - - @Test public void testMergeMultiField() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping1.json"); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); @@ -112,12 +114,11 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { assertThat(docMapper.mappers().getMapper("name.not_indexed3"), notNullValue()); } - @Test public void testUpgradeFromMultiFieldTypeToMultiFields() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping1.json"); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper docMapper = parser.parse(mapping); + DocumentMapper docMapper = mapperService.merge("person", new CompressedXContent(mapping), true, false); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); assertThat(docMapper.mappers().getMapper("name.indexed"), nullValue()); @@ -131,12 +132,7 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade1.json"); - DocumentMapper docMapper2 = parser.parse(mapping); - - MergeResult mergeResult = docMapper.merge(docMapper2.mapping(), true, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); - - docMapper.merge(docMapper2.mapping(), false, false); + mapperService.merge("person", new CompressedXContent(mapping), false, false); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); @@ -153,12 +149,7 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { assertThat(f, notNullValue()); mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade2.json"); - DocumentMapper docMapper3 = parser.parse(mapping); - - mergeResult = docMapper.merge(docMapper3.mapping(), true, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); - - docMapper.merge(docMapper3.mapping(), false, false); + mapperService.merge("person", new CompressedXContent(mapping), false, false); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); @@ -170,24 +161,19 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade3.json"); - DocumentMapper docMapper4 = parser.parse(mapping); - mergeResult = docMapper.merge(docMapper4.mapping(), true, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(true)); - assertThat(mergeResult.buildConflicts()[0], equalTo("mapper [name] has different [index] values")); - assertThat(mergeResult.buildConflicts()[1], equalTo("mapper [name] has different [store] values")); + try { + mapperService.merge("person", new CompressedXContent(mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [name] has different [index] values")); + assertThat(e.getMessage(), containsString("mapper [name] has different [store] values")); + } - mergeResult = docMapper.merge(docMapper4.mapping(), false, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(true)); - - assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); - assertThat(mergeResult.buildConflicts()[0], equalTo("mapper [name] has different [index] values")); - assertThat(mergeResult.buildConflicts()[1], equalTo("mapper [name] has different [store] values")); - - // There are conflicts, but the `name.not_indexed3` has been added, b/c that field has no conflicts + // There are conflicts, so the `name.not_indexed3` has not been added assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue()); assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue()); assertThat(docMapper.mappers().getMapper("name.not_indexed2"), notNullValue()); - assertThat(docMapper.mappers().getMapper("name.not_indexed3"), notNullValue()); + assertThat(docMapper.mappers().getMapper("name.not_indexed3"), nullValue()); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java index 5ae90bd5a0c..be27e9f83fb 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java @@ -26,15 +26,12 @@ import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.ObjectMapper.Dynamic; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public class NestedMappingTests extends ESSingleNodeTestCase { - - @Test - public void emptyNested() throws Exception { + public void testEmptyNested() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject().string(); @@ -60,8 +57,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase { assertThat(doc.docs().size(), equalTo(1)); } - @Test - public void singleNested() throws Exception { + public void testSingleNested() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject().string(); @@ -108,8 +104,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase { assertThat(doc.docs().get(2).get("field"), equalTo("value")); } - @Test - public void multiNested() throws Exception { + public void testMultiNested() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").startObject("properties") .startObject("nested2").field("type", "nested") @@ -160,8 +155,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase { assertThat(doc.docs().get(6).get("nested1.nested2.field2"), nullValue()); } - @Test - public void multiObjectAndNested1() throws Exception { + public void testMultiObjectAndNested1() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").startObject("properties") .startObject("nested2").field("type", "nested").field("include_in_parent", true) @@ -212,8 +206,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase { assertThat(doc.docs().get(6).get("nested1.nested2.field2"), nullValue()); } - @Test - public void multiObjectAndNested2() throws Exception { + public void testMultiObjectAndNested2() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").field("include_in_parent", true).startObject("properties") .startObject("nested2").field("type", "nested").field("include_in_parent", true) @@ -264,8 +257,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase { assertThat(doc.docs().get(6).getFields("nested1.nested2.field2").length, equalTo(4)); } - @Test - public void multiRootAndNested1() throws Exception { + public void testMultiRootAndNested1() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").startObject("properties") .startObject("nested2").field("type", "nested").field("include_in_root", true) @@ -316,8 +308,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase { assertThat(doc.docs().get(6).getFields("nested1.nested2.field2").length, equalTo(4)); } - @Test - public void nestedArray_strict() throws Exception { + public void testNestedArrayStrict() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").field("dynamic", "strict").startObject("properties") .startObject("field1").field("type", "string") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/null_value/NullValueTests.java b/core/src/test/java/org/elasticsearch/index/mapper/null_value/NullValueTests.java index d7aa84c2dfe..fedb2d83d5d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/null_value/NullValueTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/null_value/NullValueTests.java @@ -22,19 +22,16 @@ package org.elasticsearch.index.mapper.null_value; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; /** */ public class NullValueTests extends ESSingleNodeTestCase { - - @Test - public void testNullNull_Value() throws Exception { + public void testNullNullValue() throws Exception { IndexService indexService = createIndex("test", Settings.settingsBuilder().build()); String[] typesToTest = {"integer", "long", "double", "float", "short", "date", "ip", "string", "boolean", "byte"}; @@ -57,9 +54,6 @@ public class NullValueTests extends ESSingleNodeTestCase { } catch (MapperParsingException e) { assertThat(e.getMessage(), equalTo("Property [null_value] cannot be null.")); } - } - - } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java b/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java index 728152afdb3..d93ae9b6787 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java @@ -24,6 +24,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexService; @@ -39,12 +41,13 @@ import org.elasticsearch.index.mapper.core.NumberFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.string.SimpleStringMappingTests; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; +import java.util.Arrays; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -52,8 +55,6 @@ import static org.hamcrest.Matchers.nullValue; /** */ public class SimpleNumericTests extends ESSingleNodeTestCase { - - @Test public void testNumericDetectionEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .field("numeric_detection", true) @@ -79,7 +80,6 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertThat(mapper, instanceOf(DoubleFieldMapper.class)); } - @Test public void testNumericDetectionDefault() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); @@ -104,7 +104,6 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertThat(mapper, instanceOf(StringFieldMapper.class)); } - @Test public void testIgnoreMalformedOption() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -168,7 +167,6 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { } } - @Test public void testCoerceOption() throws Exception { String [] nonFractionNumericFieldTypes={"integer","long","short"}; //Test co-ercion policies on all non-fraction numerics @@ -201,7 +199,7 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("noErrorCoerceField"), notNullValue()); //Default is ignore_malformed=true and coerce=true assertThat(doc.rootDoc().getField("errorDefaultCoerce"), notNullValue()); - + //Test valid case of numbers passed as numbers int validNumber=1; doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -214,7 +212,7 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertEquals(validNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); assertEquals(validNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); assertEquals(validNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); - + //Test valid case of negative numbers passed as numbers int validNegativeNumber=-1; doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -227,7 +225,7 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); assertEquals(validNegativeNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); - + try { defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -238,8 +236,8 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { } catch (MapperParsingException e) { assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); } - - + + //Test questionable case of floats passed to ints float invalidJsonForInteger=1.9f; int coercedFloatValue=1; //This is what the JSON parser will do to a float - truncate not round @@ -254,7 +252,7 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertEquals(coercedFloatValue,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); //Default is ignore_malformed=true and coerce=true assertEquals(coercedFloatValue,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); - + try { defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -266,8 +264,8 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { } } } - - + + public void testDocValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -346,9 +344,8 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertEquals(DocValuesType.SORTED_NUMERIC, SimpleStringMappingTests.docValuesType(doc, "nested.double")); } } - + /** Test default precision step for autodetected numeric types */ - @Test public void testPrecisionStepDefaultsDetected() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .field("numeric_detection", true) @@ -364,17 +361,16 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { .field("date", "2010-01-01") .endObject() .bytes()); - + assertEquals(1, doc.docs().size()); Document luceneDoc = doc.docs().get(0); - + assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("long")); assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("double")); assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("date")); } - + /** Test default precision step for numeric types */ - @Test public void testPrecisionStepDefaultsMapped() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -402,12 +398,12 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { .startObject("ip") .field("type", "ip") .endObject() - + .endObject() .endObject().endObject().string(); DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - + ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("int", "100") @@ -420,24 +416,23 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { .field("ip", "255.255.255.255") .endObject() .bytes()); - + assertEquals(1, doc.docs().size()); Document luceneDoc = doc.docs().get(0); - + assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("long")); assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("double")); assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("date")); assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("ip")); - + assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_32_BIT, luceneDoc.getField("int")); assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_32_BIT, luceneDoc.getField("float")); - + assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_16_BIT, luceneDoc.getField("short")); assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_8_BIT, luceneDoc.getField("byte")); } - + /** Test precision step set to silly explicit values */ - @Test public void testPrecisionStepExplicit() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -473,12 +468,12 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { .field("type", "ip") .field("precision_step", "2") .endObject() - + .endObject() .endObject().endObject().string(); DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - + ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("int", "100") @@ -491,10 +486,10 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { .field("ip", "255.255.255.255") .endObject() .bytes()); - + assertEquals(1, doc.docs().size()); Document luceneDoc = doc.docs().get(0); - + assertPrecisionStepEquals(1, luceneDoc.getField("int")); assertPrecisionStepEquals(2, luceneDoc.getField("float")); assertPrecisionStepEquals(1, luceneDoc.getField("long")); @@ -505,18 +500,76 @@ public class SimpleNumericTests extends ESSingleNodeTestCase { assertPrecisionStepEquals(2, luceneDoc.getField("ip")); } - + /** checks precisionstep on both the fieldtype and the tokenstream */ private static void assertPrecisionStepEquals(int expected, IndexableField field) throws IOException { assertNotNull(field); assertThat(field, instanceOf(Field.class)); - + // check fieldtype's precisionstep assertEquals(expected, ((Field)field).fieldType().numericPrecisionStep()); - + // check the tokenstream actually used by the indexer TokenStream ts = field.tokenStream(null, null); - assertThat(ts, instanceOf(NumericTokenStream.class)); + assertThat(ts, instanceOf(NumericTokenStream.class)); assertEquals(expected, ((NumericTokenStream)ts).getPrecisionStep()); } + + public void testTermVectorsBackCompat() throws Exception { + for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) { + doTestTermVectorsBackCompat(type); + } + } + + private void doTestTermVectorsBackCompat(String type) throws Exception { + DocumentMapperParser parser = createIndex("index-" + type).mapperService().documentMapperParser(); + String mappingWithTV = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("foo") + .field("type", type) + .field("term_vector", "yes") + .endObject() + .endObject().endObject().endObject().string(); + try { + parser.parse(mappingWithTV); + fail(); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Mapping definition for [foo] has unsupported parameters: [term_vector : yes]")); + } + + Settings oldIndexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_1_0) + .build(); + parser = createIndex("index2-" + type, oldIndexSettings).mapperService().documentMapperParser(); + parser.parse(mappingWithTV); // no exception + } + + public void testAnalyzerBackCompat() throws Exception { + for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) { + doTestAnalyzerBackCompat(type); + } + } + + private void doTestAnalyzerBackCompat(String type) throws Exception { + DocumentMapperParser parser = createIndex("index-" + type).mapperService().documentMapperParser(); + String mappingWithTV = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("foo") + .field("type", type) + .field("analyzer", "keyword") + .endObject() + .endObject().endObject().endObject().string(); + try { + parser.parse(mappingWithTV); + fail(); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Mapping definition for [foo] has unsupported parameters: [analyzer : keyword]")); + } + + Settings oldIndexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_1_0) + .build(); + parser = createIndex("index2-" + type, oldIndexSettings).mapperService().documentMapperParser(); + parser.parse(mappingWithTV); // no exception + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/object/NullValueObjectMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/object/NullValueObjectMappingTests.java index f2b0b19af57..b13fcc8ed91 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/object/NullValueObjectMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/object/NullValueObjectMappingTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; @@ -33,8 +32,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class NullValueObjectMappingTests extends ESSingleNodeTestCase { - - @Test public void testNullValueObject() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("obj1").field("type", "object").endObject().endObject() diff --git a/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java index ee604e006bf..917ee9806ed 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java @@ -24,13 +24,12 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; + +import static org.hamcrest.Matchers.containsString; /** */ public class SimpleObjectMappingTests extends ESSingleNodeTestCase { - - @Test public void testDifferentInnerObjectTokenFailure() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); @@ -56,7 +55,6 @@ public class SimpleObjectMappingTests extends ESSingleNodeTestCase { } } - @Test public void testEmptyArrayProperties() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startArray("properties").endArray() @@ -64,8 +62,7 @@ public class SimpleObjectMappingTests extends ESSingleNodeTestCase { createIndex("test").mapperService().documentMapperParser().parse(mapping); } - @Test - public void emptyFieldsArrayMultiFieldsTest() throws Exception { + public void testEmptyFieldsArrayMultiFields() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("tweet") @@ -83,8 +80,7 @@ public class SimpleObjectMappingTests extends ESSingleNodeTestCase { createIndex("test").mapperService().documentMapperParser().parse(mapping); } - @Test(expected = MapperParsingException.class) - public void fieldsArrayMultiFieldsShouldThrowExceptionTest() throws Exception { + public void testFieldsArrayMultiFieldsShouldThrowException() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("tweet") @@ -101,11 +97,16 @@ public class SimpleObjectMappingTests extends ESSingleNodeTestCase { .endObject() .endObject() .string(); - createIndex("test").mapperService().documentMapperParser().parse(mapping); + try { + createIndex("test").mapperService().documentMapperParser().parse(mapping); + fail("Expected MapperParsingException"); + } catch(MapperParsingException e) { + assertThat(e.getMessage(), containsString("expected map for property [fields]")); + assertThat(e.getMessage(), containsString("but got a class java.util.ArrayList")); + } } - @Test - public void emptyFieldsArrayTest() throws Exception { + public void testEmptyFieldsArray() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("tweet") @@ -119,8 +120,7 @@ public class SimpleObjectMappingTests extends ESSingleNodeTestCase { createIndex("test").mapperService().documentMapperParser().parse(mapping); } - @Test(expected = MapperParsingException.class) - public void fieldsWithFilledArrayShouldThrowExceptionTest() throws Exception { + public void testFieldsWithFilledArrayShouldThrowException() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("tweet") @@ -133,11 +133,15 @@ public class SimpleObjectMappingTests extends ESSingleNodeTestCase { .endObject() .endObject() .string(); - createIndex("test").mapperService().documentMapperParser().parse(mapping); + try { + createIndex("test").mapperService().documentMapperParser().parse(mapping); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Expected map for property [fields]")); + } } - @Test - public void fieldPropertiesArrayTest() throws Exception { + public void testFieldPropertiesArray() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("tweet") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/path/PathMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/path/PathMapperTests.java index 06ef922e94d..2582562c039 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/path/PathMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/path/PathMapperTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.path; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; @@ -33,8 +32,6 @@ import static org.hamcrest.Matchers.nullValue; * */ public class PathMapperTests extends ESSingleNodeTestCase { - - @Test public void testPathMapping() throws IOException { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/path/test-mapping.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java index d89ae84e3a2..0e8c74aee89 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java @@ -19,31 +19,35 @@ package org.elasticsearch.index.mapper.simple; -import java.nio.charset.StandardCharsets; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.*; -import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; +import java.nio.charset.StandardCharsets; + +import static org.elasticsearch.index.mapper.MapperBuilders.doc; +import static org.elasticsearch.index.mapper.MapperBuilders.object; +import static org.elasticsearch.index.mapper.MapperBuilders.rootObject; +import static org.elasticsearch.index.mapper.MapperBuilders.stringField; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; -import static org.elasticsearch.index.mapper.MapperBuilders.*; import static org.hamcrest.Matchers.equalTo; /** * */ public class SimpleMapperTests extends ESSingleNodeTestCase { - - @Test public void testSimpleMapper() throws Exception { IndexService indexService = createIndex("test"); - Settings settings = indexService.settingsService().getSettings(); + Settings settings = indexService.getIndexSettings().getSettings(); DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser(); DocumentMapper docMapper = doc(settings, rootObject("person") @@ -54,31 +58,22 @@ public class SimpleMapperTests extends ESSingleNodeTestCase { Document doc = docMapper.parse("test", "person", "1", json).rootDoc(); assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()), equalTo("shay")); -// System.out.println("Document: " + doc); -// System.out.println("Json: " + docMapper.sourceMapper().value(doc)); doc = docMapper.parse("test", "person", "1", json).rootDoc(); -// System.out.println("Document: " + doc); -// System.out.println("Json: " + docMapper.sourceMapper().value(doc)); } - @Test public void testParseToJsonAndParse() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json"); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); String builtMapping = docMapper.mappingSource().string(); -// System.out.println(builtMapping); // reparse it DocumentMapper builtDocMapper = parser.parse(builtMapping); BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json")); Document doc = builtDocMapper.parse("test", "person", "1", json).rootDoc(); assertThat(doc.get(docMapper.uidMapper().fieldType().names().indexName()), equalTo(Uid.createUid("person", "1"))); assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()), equalTo("shay")); -// System.out.println("Document: " + doc); -// System.out.println("Json: " + docMapper.sourceMapper().value(doc)); } - @Test public void testSimpleParser() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); @@ -89,11 +84,8 @@ public class SimpleMapperTests extends ESSingleNodeTestCase { Document doc = docMapper.parse("test", "person", "1", json).rootDoc(); assertThat(doc.get(docMapper.uidMapper().fieldType().names().indexName()), equalTo(Uid.createUid("person", "1"))); assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()), equalTo("shay")); -// System.out.println("Document: " + doc); -// System.out.println("Json: " + docMapper.sourceMapper().value(doc)); } - @Test public void testSimpleParserNoTypeNoId() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); @@ -101,11 +93,8 @@ public class SimpleMapperTests extends ESSingleNodeTestCase { Document doc = docMapper.parse("test", "person", "1", json).rootDoc(); assertThat(doc.get(docMapper.uidMapper().fieldType().names().indexName()), equalTo(Uid.createUid("person", "1"))); assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()), equalTo("shay")); -// System.out.println("Document: " + doc); -// System.out.println("Json: " + docMapper.sourceMapper().value(doc)); } - @Test public void testAttributes() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json"); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); @@ -118,10 +107,9 @@ public class SimpleMapperTests extends ESSingleNodeTestCase { assertThat((String) builtDocMapper.meta().get("param1"), equalTo("value1")); } - @Test public void testNoDocumentSent() throws Exception { IndexService indexService = createIndex("test"); - Settings settings = indexService.settingsService().getSettings(); + Settings settings = indexService.getIndexSettings().getSettings(); DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser(); DocumentMapper docMapper = doc(settings, rootObject("person") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/source/CompressSourceMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/source/CompressSourceMappingTests.java deleted file mode 100644 index 56c188b7aa3..00000000000 --- a/core/src/test/java/org/elasticsearch/index/mapper/source/CompressSourceMappingTests.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.source; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.compress.CompressorFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; - -import static org.hamcrest.Matchers.equalTo; - -/** - * - */ -public class CompressSourceMappingTests extends ESSingleNodeTestCase { - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - - @Test - public void testCompressDisabled() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_source").field("compress", false).endObject() - .endObject().endObject().string(); - - DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); - - ParsedDocument doc = documentMapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject() - .field("field1", "value1") - .field("field2", "value2") - .endObject().bytes()); - BytesRef bytes = doc.rootDoc().getBinaryValue("_source"); - assertThat(CompressorFactory.isCompressed(new BytesArray(bytes)), equalTo(false)); - } - - @Test - public void testCompressEnabled() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_source").field("compress", true).endObject() - .endObject().endObject().string(); - - DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); - - ParsedDocument doc = documentMapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject() - .field("field1", "value1") - .field("field2", "value2") - .endObject().bytes()); - - BytesRef bytes = doc.rootDoc().getBinaryValue("_source"); - assertThat(CompressorFactory.isCompressed(new BytesArray(bytes)), equalTo(true)); - } - - @Test - public void testCompressThreshold() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_source").field("compress_threshold", "200b").endObject() - .endObject().endObject().string(); - - DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); - - ParsedDocument doc = documentMapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject() - .field("field1", "value1") - .endObject().bytes()); - - BytesRef bytes = doc.rootDoc().getBinaryValue("_source"); - assertThat(CompressorFactory.isCompressed(new BytesArray(bytes)), equalTo(false)); - - doc = documentMapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject() - .field("field1", "value1") - .field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz") - .field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz") - .field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz") - .field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz") - .endObject().bytes()); - - bytes = doc.rootDoc().getBinaryValue("_source"); - assertThat(CompressorFactory.isCompressed(new BytesArray(bytes)), equalTo(true)); - } -} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java index 4ec0ff5211e..364e9f2063f 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.*; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.ArrayList; @@ -63,51 +64,16 @@ public class DefaultSourceMappingTests extends ESSingleNodeTestCase { assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.SMILE)); } - public void testJsonFormat() throws Exception { + public void testFormatBackCompat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").field("format", "json").endObject() .endObject().endObject().string(); + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_2_0)) + .build(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - DocumentMapper documentMapper = parser.parse(mapping); - ParsedDocument doc = documentMapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject() - .field("field", "value") - .endObject().bytes()); - - assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.JSON)); - - documentMapper = parser.parse(mapping); - doc = documentMapper.parse("test", "type", "1", XContentFactory.smileBuilder().startObject() - .field("field", "value") - .endObject().bytes()); - - assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.JSON)); - } - - public void testJsonFormatCompressedBackcompat() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_source").field("format", "json").field("compress", true).endObject() - .endObject().endObject().string(); - - Settings backcompatSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - DocumentMapperParser parser = createIndex("test", backcompatSettings).mapperService().documentMapperParser(); - DocumentMapper documentMapper = parser.parse(mapping); - ParsedDocument doc = documentMapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject() - .field("field", "value") - .endObject().bytes()); - - assertThat(CompressorFactory.isCompressed(doc.source()), equalTo(true)); - byte[] uncompressed = CompressorFactory.uncompressIfNeeded(doc.source()).toBytes(); - assertThat(XContentFactory.xContentType(uncompressed), equalTo(XContentType.JSON)); - - documentMapper = parser.parse(mapping); - doc = documentMapper.parse("test", "type", "1", XContentFactory.smileBuilder().startObject() - .field("field", "value") - .endObject().bytes()); - - assertThat(CompressorFactory.isCompressed(doc.source()), equalTo(true)); - uncompressed = CompressorFactory.uncompressIfNeeded(doc.source()).toBytes(); - assertThat(XContentFactory.xContentType(uncompressed), equalTo(XContentType.JSON)); + DocumentMapperParser parser = createIndex("test", settings).mapperService().documentMapperParser(); + parser.parse(mapping); // no exception } public void testIncludes() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java index a54b63d7531..9ac039a49fb 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -47,7 +48,6 @@ import org.elasticsearch.index.mapper.core.StringFieldMapper.Builder; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.VersionUtils; import org.junit.Before; -import org.junit.Test; import java.util.Arrays; import java.util.Map; @@ -61,7 +61,6 @@ import static org.hamcrest.Matchers.nullValue; /** */ public class SimpleStringMappingTests extends ESSingleNodeTestCase { - private static Settings DOC_VALUES_SETTINGS = Settings.builder().put(FieldDataType.FORMAT_KEY, FieldDataType.DOC_VALUES_FORMAT_VALUE).build(); IndexService indexService; @@ -73,7 +72,6 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { parser = indexService.mapperService().documentMapperParser(); } - @Test public void testLimit() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field").field("type", "string").field("ignore_above", 5).endObject().endObject() @@ -134,7 +132,6 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { assertEquals(expected, doc.rootDoc().getField("field").fieldType()); } - @Test public void testDefaultsForAnalyzed() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field").field("type", "string").endObject().endObject() @@ -153,7 +150,6 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { assertParseIdemPotent(fieldType, defaultMapper); } - @Test public void testDefaultsForNotAnalyzed() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field").field("type", "string").field("index", "not_analyzed").endObject().endObject() @@ -218,7 +214,6 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { assertParseIdemPotent(fieldType, defaultMapper); } - @Test public void testSearchQuoteAnalyzerSerialization() throws Exception { // Cases where search_quote_analyzer should not be added to the mapping. String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -294,7 +289,6 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { return result; } - @Test public void testTermVectors() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -372,7 +366,7 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { public void testDocValuesFielddata() throws Exception { IndexService indexService = createIndex("index"); DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - final BuilderContext ctx = new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1)); + final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); assertFalse(new Builder("anything").index(false).build(ctx).fieldType().hasDocValues()); assertTrue(new Builder("anything").index(false).fieldDataSettings(DOC_VALUES_SETTINGS).build(ctx).fieldType().hasDocValues()); @@ -411,7 +405,7 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { public void testDocValues() throws Exception { // doc values only work on non-analyzed content - final BuilderContext ctx = new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1)); + final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); try { new StringFieldMapper.Builder("anything").docValues(true).build(ctx); fail(); @@ -480,13 +474,12 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { return DocValuesType.NONE; } - @Test public void testDisableNorms() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field").field("type", "string").endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = parser.parse(mapping); + DocumentMapper defaultMapper = indexService.mapperService().merge("type", new CompressedXContent(mapping), true, false); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -515,10 +508,12 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { updatedMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field").field("type", "string").startObject("norms").field("enabled", true).endObject() .endObject().endObject().endObject().endObject().string(); - mergeResult = defaultMapper.merge(parser.parse(updatedMapping).mapping(), true, false); - assertTrue(mergeResult.hasConflicts()); - assertEquals(1, mergeResult.buildConflicts().length); - assertTrue(mergeResult.buildConflicts()[0].contains("different [omit_norms]")); + try { + defaultMapper.merge(parser.parse(updatedMapping).mapping(), true, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("different [omit_norms]")); + } } /** diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index e51b6a61d50..53a3bf7bb6e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -37,10 +37,16 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MergeResult; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -55,14 +61,20 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isIn; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; /** */ public class TimestampMappingTests extends ESSingleNodeTestCase { Settings BWC_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - @Test public void testSimpleDisabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); @@ -76,7 +88,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("_timestamp"), equalTo(null)); } - @Test public void testEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", "yes").endObject() @@ -94,7 +105,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("_timestamp").tokenStream(docMapper.mappers().indexAnalyzer(), null), notNullValue()); } - @Test public void testDefaultValues() throws Exception { for (Version version : Arrays.asList(V_1_5_0, V_2_0_0_beta1, randomVersion(random()))) { for (String mapping : Arrays.asList( @@ -114,7 +124,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { } } - @Test public void testBackcompatSetValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -132,7 +141,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(docMapper.timestampFieldMapper().fieldType().hasDocValues(), equalTo(true)); } - @Test public void testThatDisablingDuringMergeIsWorking() throws Exception { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).endObject() @@ -150,7 +158,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(enabledMapper.timestampFieldMapper().enabled(), is(false)); } - @Test // issue 3174 + // issue 3174 public void testThatSerializationWorksCorrectlyForIndexField() throws Exception { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).field("store", "yes").field("index", "no").endObject() @@ -171,7 +179,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(timestampConfiguration.get("index").toString(), is("no")); } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testBackcompatPathMissingDefaultValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -199,7 +207,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { } } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testTimestampDefaultValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -225,7 +233,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(delay, lessThanOrEqualTo(60000L)); } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testBackcompatPathMissingDefaultToEpochValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -251,7 +259,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"), Version.CURRENT))); } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testTimestampMissingDefaultToEpochValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -276,7 +284,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"), Version.CURRENT))); } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testBackcompatPathMissingNowDefaultValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -305,7 +313,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(delay, lessThanOrEqualTo(60000L)); } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testTimestampMissingNowDefaultValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -333,7 +341,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(delay, lessThanOrEqualTo(60000L)); } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testPathMissingWithForcedNullDefaultShouldFail() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -350,7 +358,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { } } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testBackcompatPathMissingShouldFail() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -378,7 +386,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { } } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testTimestampMissingWithForcedNullDefaultShouldFail() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -395,7 +403,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { } } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testTimestampDefaultAndIgnore() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -413,7 +421,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { } } - @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] public void testTimestampMissingShouldNotFail() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -440,7 +448,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(delay, lessThanOrEqualTo(60000L)); } - @Test public void testDefaultTimestampStream() throws IOException { // Testing null value for default timestamp { @@ -494,7 +501,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { } } - @Test public void testMergingFielddataLoadingWorks() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "yes").endObject() @@ -515,7 +521,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(docMapper.timestampFieldMapper().fieldType().fieldDataType().getFormat(indexSettings), equalTo("array")); } - @Test public void testParsingNotDefaultTwiceDoesNotChangeMapping() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -529,7 +534,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(docMapper.mappingSource().string(), equalTo(mapping)); } - @Test public void testBackcompatParsingTwiceDoesNotChangeTokenizeValue() throws Exception { String[] index_options = {"no", "analyzed", "not_analyzed"}; String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -551,11 +555,9 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(tokenized, equalTo(docMapper.timestampFieldMapper().fieldType().tokenized())); } - @Test public void testMergingConflicts() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true) - .startObject("fielddata").field("format", "doc_values").endObject() .field("store", "yes") .field("index", "analyzed") .field("path", "foo") @@ -563,9 +565,9 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { .endObject() .endObject().endObject().string(); Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - DocumentMapperParser parser = createIndex("test", indexSettings).mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test", indexSettings).mapperService(); - DocumentMapper docMapper = parser.parse(mapping); + DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(mapping), true, false); assertThat(docMapper.timestampFieldMapper().fieldType().fieldDataType().getLoading(), equalTo(MappedFieldType.Loading.LAZY)); mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", false) @@ -577,23 +579,34 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { .endObject() .endObject().endObject().string(); - MergeResult mergeResult = docMapper.merge(parser.parse(mapping).mapping(), true, false); - List expectedConflicts = new ArrayList<>(Arrays.asList( - "mapper [_timestamp] has different [index] values", - "mapper [_timestamp] has different [store] values", - "Cannot update default in _timestamp value. Value is 1970-01-01 now encountering 1970-01-02", - "Cannot update path in _timestamp value. Value is foo path in merged mapping is bar")); - - for (String conflict : mergeResult.buildConflicts()) { - assertTrue("found unexpected conflict [" + conflict + "]", expectedConflicts.remove(conflict)); + try { + mapperService.merge("type", new CompressedXContent(mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [_timestamp] has different [index] values")); + assertThat(e.getMessage(), containsString("mapper [_timestamp] has different [store] values")); } - assertTrue("missing conflicts: " + Arrays.toString(expectedConflicts.toArray()), expectedConflicts.isEmpty()); + assertThat(docMapper.timestampFieldMapper().fieldType().fieldDataType().getLoading(), equalTo(MappedFieldType.Loading.LAZY)); assertTrue(docMapper.timestampFieldMapper().enabled()); - assertThat(docMapper.timestampFieldMapper().fieldType().fieldDataType().getFormat(indexSettings), equalTo("doc_values")); + + mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp").field("enabled", true) + .field("store", "yes") + .field("index", "analyzed") + .field("path", "bar") + .field("default", "1970-01-02") + .endObject() + .endObject().endObject().string(); + try { + mapperService.merge("type", new CompressedXContent(mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Cannot update default in _timestamp value. Value is 1970-01-01 now encountering 1970-01-02")); + assertThat(e.getMessage(), containsString("Cannot update path in _timestamp value. Value is foo path in merged mapping is bar")); + } } - @Test public void testBackcompatMergingConflictsForIndexValues() throws Exception { List indexValues = new ArrayList<>(); indexValues.add("analyzed"); @@ -633,7 +646,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { /** * Test for issue #9223 */ - @Test public void testInitMappers() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject() .startObject("type") @@ -646,7 +658,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { new MappingMetaData(new CompressedXContent(mapping)); } - @Test public void testBackcompatMergePaths() throws Exception { String[] possiblePathValues = {"some_path", "anotherPath", null}; DocumentMapperParser parser = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser(); @@ -681,7 +692,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { assertThat(mergeResult.buildConflicts()[0], containsString(conflict)); } } - + public void testBackcompatDocValuesSerialization() throws Exception { // default String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -726,7 +737,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { .endObject().endObject().endObject().string(); assertDocValuesSerialization(mapping); } - + void assertDocValuesSerialization(String mapping) throws Exception { DocumentMapperParser parser = createIndex("test_doc_values", BWC_SETTINGS).mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java index b9f7a988788..efe07615532 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java @@ -31,11 +31,15 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.*; -import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MergeResult; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; @@ -45,7 +49,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; public class TTLMappingTests extends ESSingleNodeTestCase { - @Test public void testSimpleDisabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); @@ -59,7 +62,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("_ttl"), equalTo(null)); } - @Test public void testEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_ttl").field("enabled", "yes").endObject() @@ -77,7 +79,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getField("_ttl").tokenStream(docMapper.mappers().indexAnalyzer(), null), notNullValue()); } - @Test public void testDefaultValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); @@ -86,8 +87,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(docMapper.TTLFieldMapper().fieldType().indexOptions(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.indexOptions())); } - - @Test public void testSetValuesBackcompat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_ttl") @@ -101,7 +100,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { } - @Test public void testThatEnablingTTLFieldOnMergeWorks() throws Exception { String mappingWithoutTtl = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").field("field").startObject().field("type", "string").endObject().endObject() @@ -124,7 +122,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(mapperWithoutTtl.TTLFieldMapper().enabled(), equalTo(true)); } - @Test public void testThatChangingTTLKeepsMapperEnabled() throws Exception { String mappingWithTtl = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_ttl") @@ -150,7 +147,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(initialMapper.TTLFieldMapper().enabled(), equalTo(true)); } - @Test public void testThatDisablingTTLReportsConflict() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String mappingWithTtlDisabled = getMappingWithTtlDisabled().string(); @@ -164,7 +160,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(initialMapper.TTLFieldMapper().enabled(), equalTo(true)); } - @Test public void testThatDisablingTTLReportsConflictOnCluster() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String mappingWithTtlDisabled = getMappingWithTtlDisabled().string(); @@ -173,14 +168,13 @@ public class TTLMappingTests extends ESSingleNodeTestCase { try { client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtlDisabled).setType("type").get(); fail(); - } catch (MergeMappingException mme) { - assertThat(mme.getDetailedMessage(), containsString("_ttl cannot be disabled once it was enabled.")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("_ttl cannot be disabled once it was enabled.")); } GetMappingsResponse mappingsAfterUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get(); assertThat(mappingsBeforeUpdateResponse.getMappings().get("testindex").get("type").source(), equalTo(mappingsAfterUpdateResponse.getMappings().get("testindex").get("type").source())); } - @Test public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String withTtlDisabled = getMappingWithTtlDisabled().string(); @@ -192,7 +186,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(mappingsAfterUpdateResponse.getMappings().get("testindex").get("type").sourceAsMap().get("_ttl").toString(), equalTo("{enabled=true}")); } - @Test public void testNoConflictIfNothingSetAndDisabledLater() throws Exception { IndexService indexService = createIndex("testindex", Settings.settingsBuilder().build(), "type"); XContentBuilder mappingWithTtlDisabled = getMappingWithTtlDisabled("7d"); @@ -200,7 +193,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertFalse(mergeResult.hasConflicts()); } - @Test public void testNoConflictIfNothingSetAndEnabledLater() throws Exception { IndexService indexService = createIndex("testindex", Settings.settingsBuilder().build(), "type"); XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d"); @@ -208,7 +200,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertFalse(mergeResult.hasConflicts()); } - @Test public void testMergeWithOnlyDefaultSet() throws Exception { XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d"); IndexService indexService = createIndex("testindex", Settings.settingsBuilder().build(), "type", mappingWithTtlEnabled); @@ -219,7 +210,6 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(mappingAfterMerge, equalTo(new CompressedXContent("{\"type\":{\"_ttl\":{\"enabled\":true,\"default\":360000},\"properties\":{\"field\":{\"type\":\"string\"}}}}"))); } - @Test public void testMergeWithOnlyDefaultSetTtlDisabled() throws Exception { XContentBuilder mappingWithTtlEnabled = getMappingWithTtlDisabled("7d"); IndexService indexService = createIndex("testindex", Settings.settingsBuilder().build(), "type", mappingWithTtlEnabled); @@ -232,9 +222,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase { assertThat(mappingAfterMerge, equalTo(new CompressedXContent("{\"type\":{\"_ttl\":{\"enabled\":false},\"properties\":{\"field\":{\"type\":\"string\"}}}}"))); } - @Test public void testThatSimulatedMergingLeavesStateUntouched() throws Exception { - //check if default ttl changed when simulate set to true XContentBuilder mappingWithTtl = getMappingWithTtlEnabled("6d"); IndexService indexService = createIndex("testindex", Settings.settingsBuilder().build(), "type", mappingWithTtl); @@ -306,7 +294,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase { request.process(MetaData.builder().build(), mappingMetaData, true, "test"); // _ttl in a document never worked, so backcompat is ignoring the field - assertEquals(-1, request.ttl()); + assertNull(request.ttl()); assertNull(docMapper.parse("test", "type", "1", doc.bytes()).rootDoc().get("_ttl")); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseDocumentTypeLevelsTests.java b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseDocumentTypeLevelsTests.java index 21ee96522c1..26d710b137f 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseDocumentTypeLevelsTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseDocumentTypeLevelsTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; @@ -31,8 +30,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { - - @Test public void testNoLevel() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -51,7 +48,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("inner.inner_field"), equalTo("inner_value")); } - @Test public void testTypeLevel() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -70,7 +66,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("type.inner.inner_field"), equalTo("inner_value")); } - @Test public void testNoLevelWithFieldTypeAsValue() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -91,7 +86,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("inner.inner_field"), equalTo("inner_value")); } - @Test public void testTypeLevelWithFieldTypeAsValue() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -112,7 +106,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("type.inner.inner_field"), equalTo("inner_value")); } - @Test public void testNoLevelWithFieldTypeAsObject() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -133,7 +126,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("test2"), equalTo("value2")); } - @Test public void testTypeLevelWithFieldTypeAsObject() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -154,7 +146,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("type.inner.inner_field"), equalTo("inner_value")); } - @Test public void testNoLevelWithFieldTypeAsValueNotFirst() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -175,7 +166,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("type.inner.inner_field"), equalTo("inner_value")); } - @Test public void testTypeLevelWithFieldTypeAsValueNotFirst() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -196,7 +186,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("type.inner.inner_field"), equalTo("inner_value")); } - @Test public void testNoLevelWithFieldTypeAsObjectNotFirst() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); @@ -218,7 +207,6 @@ public class ParseDocumentTypeLevelsTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().get("inner.inner_field"), equalTo("inner_value")); } - @Test public void testTypeLevelWithFieldTypeAsObjectNotFirst() throws Exception { String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java index f67547586b2..d99efee6824 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java @@ -23,14 +23,11 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; // TODO: move this test...it doesn't need to be by itself public class ParseMappingTypeLevelTests extends ESSingleNodeTestCase { - - @Test public void testTypeLevel() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).endObject() diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java index 4ae039a3610..35034dfd911 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java @@ -24,37 +24,31 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.HashMap; -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class UpdateMappingOnClusterIT extends ESIntegTestCase { - private static final String INDEX = "index"; private static final String TYPE = "type"; - - @Test - public void test_all_enabled() throws Exception { + public void testAllEnabled() throws Exception { XContentBuilder mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").field("enabled", "false").endObject().endObject().endObject().endObject(); XContentBuilder mappingUpdate = jsonBuilder().startObject().startObject("_all").field("enabled", "true").endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); String errorMessage = "[_all] enabled is false now encountering true"; testConflict(mapping.string(), mappingUpdate.string(), errorMessage); } - @Test - public void test_all_conflicts() throws Exception { + public void testAllConflicts() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_create_index.json"); String mappingUpdate = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json"); - String[] errorMessage = {"[_all] enabled is true now encountering false", + String[] errorMessage = { "[_all] has different [omit_norms] values", "[_all] has different [store] values", "[_all] has different [store_term_vector] values", @@ -67,9 +61,14 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase { testConflict(mapping, mappingUpdate, errorMessage); } + public void testAllDisabled() throws Exception { + XContentBuilder mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").field("enabled", true).endObject().endObject().endObject().endObject(); + XContentBuilder mappingUpdate = jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); + String errorMessage = "[_all] enabled is true now encountering false"; + testConflict(mapping.string(), mappingUpdate.string(), errorMessage); + } - @Test - public void test_all_with_default() throws Exception { + public void testAllWithDefault() throws Exception { String defaultMapping = jsonBuilder().startObject().startObject("_default_") .startObject("_all") .field("enabled", false) @@ -115,8 +114,7 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase { } - @Test - public void test_doc_valuesInvalidMapping() throws Exception { + public void testDocValuesInvalidMapping() throws Exception { String mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().string(); try { prepareCreate(INDEX).setSource(mapping).get(); @@ -126,8 +124,7 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase { } } - @Test - public void test_doc_valuesInvalidMappingOnUpdate() throws Exception { + public void testDocValuesInvalidMappingOnUpdate() throws Exception { String mapping = jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject().string(); prepareCreate(INDEX).addMapping(TYPE, mapping).get(); String mappingUpdate = jsonBuilder().startObject().startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().string(); @@ -143,7 +140,6 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase { } // checks if the setting for timestamp and size are kept even if disabled - @Test public void testDisabledSizeTimestampIndexDoNotLooseMappings() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); prepareCreate(INDEX).addMapping(TYPE, mapping).get(); @@ -160,9 +156,9 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase { try { client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mappingUpdate).get(); fail(); - } catch (MergeMappingException e) { + } catch (IllegalArgumentException e) { for (String errorMessage : errorMessages) { - assertThat(e.getDetailedMessage(), containsString(errorMessage)); + assertThat(e.getMessage(), containsString(errorMessage)); } } compareMappingOnNodes(mappingsBeforeUpdateResponse); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java index c10dc3b1dea..abf5f4819cd 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java @@ -26,53 +26,48 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MergeResult; +import org.elasticsearch.index.mapper.core.LongFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; import java.util.LinkedHashMap; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; public class UpdateMappingTests extends ESSingleNodeTestCase { - - @Test - public void test_all_enabled_after_disabled() throws Exception { + public void testAllEnabledAfterDisabled() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject(); XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate); } - @Test - public void test_all_disabled_after_enabled() throws Exception { + public void testAllDisabledAfterEnabled() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject(); XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate); } - @Test - public void test_all_disabled_after_default_enabled() throws Exception { + public void testAllDisabledAfterDefaultEnabled() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("some_text").field("type", "string").endObject().endObject().endObject(); XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate); } - @Test - public void test_all_enabled_after_enabled() throws Exception { + public void testAllEnabledAfterEnabled() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject(); XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject().endObject(); testNoConflictWhileMergingAndMappingChanged(mapping, mappingUpdate, expectedMapping); } - @Test - public void test_all_disabled_after_disabled() throws Exception { + public void testAllDisabledAfterDisabled() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject(); XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject().endObject(); @@ -114,7 +109,99 @@ public class UpdateMappingTests extends ESSingleNodeTestCase { assertThat(mappingAfterUpdate, equalTo(mappingBeforeUpdate)); } - @Test + public void testConflictSameType() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("foo").field("type", "long").endObject() + .endObject().endObject().endObject(); + MapperService mapperService = createIndex("test", Settings.settingsBuilder().build(), "type", mapping).mapperService(); + + XContentBuilder update = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("foo").field("type", "double").endObject() + .endObject().endObject().endObject(); + + try { + mapperService.merge("type", new CompressedXContent(update.string()), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [foo] cannot be changed from type [long] to [double]")); + } + + try { + mapperService.merge("type", new CompressedXContent(update.string()), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [foo] cannot be changed from type [long] to [double]")); + } + + assertTrue(mapperService.documentMapper("type").mapping().root().getMapper("foo") instanceof LongFieldMapper); + } + + public void testConflictNewType() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("foo").field("type", "long").endObject() + .endObject().endObject().endObject(); + MapperService mapperService = createIndex("test", Settings.settingsBuilder().build(), "type1", mapping).mapperService(); + + XContentBuilder update = XContentFactory.jsonBuilder().startObject().startObject("type2") + .startObject("properties").startObject("foo").field("type", "double").endObject() + .endObject().endObject().endObject(); + + try { + mapperService.merge("type2", new CompressedXContent(update.string()), false, false); + fail(); + } catch (IllegalArgumentException e) { + // expected + assertTrue(e.getMessage(), e.getMessage().contains("mapper [foo] cannot be changed from type [long] to [double]")); + } + + try { + mapperService.merge("type2", new CompressedXContent(update.string()), false, false); + fail(); + } catch (IllegalArgumentException e) { + // expected + assertTrue(e.getMessage(), e.getMessage().contains("mapper [foo] cannot be changed from type [long] to [double]")); + } + + assertTrue(mapperService.documentMapper("type1").mapping().root().getMapper("foo") instanceof LongFieldMapper); + assertNull(mapperService.documentMapper("type2")); + } + + // same as the testConflictNewType except that the mapping update is on an existing type + public void testConflictNewTypeUpdate() throws Exception { + XContentBuilder mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("foo").field("type", "long").endObject() + .endObject().endObject().endObject(); + XContentBuilder mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type2").endObject().endObject(); + MapperService mapperService = createIndex("test", Settings.settingsBuilder().build()).mapperService(); + + mapperService.merge("type1", new CompressedXContent(mapping1.string()), false, false); + mapperService.merge("type2", new CompressedXContent(mapping2.string()), false, false); + + XContentBuilder update = XContentFactory.jsonBuilder().startObject().startObject("type2") + .startObject("properties").startObject("foo").field("type", "double").endObject() + .endObject().endObject().endObject(); + + try { + mapperService.merge("type2", new CompressedXContent(update.string()), false, false); + fail(); + } catch (IllegalArgumentException e) { + // expected + assertTrue(e.getMessage(), e.getMessage().contains("mapper [foo] cannot be changed from type [long] to [double]")); + } + + try { + mapperService.merge("type2", new CompressedXContent(update.string()), false, false); + fail(); + } catch (IllegalArgumentException e) { + // expected + assertTrue(e.getMessage(), e.getMessage().contains("mapper [foo] cannot be changed from type [long] to [double]")); + } + + assertTrue(mapperService.documentMapper("type1").mapping().root().getMapper("foo") instanceof LongFieldMapper); + assertNotNull(mapperService.documentMapper("type2")); + assertNull(mapperService.documentMapper("type2").mapping().root().getMapper("foo")); + } + public void testIndexFieldParsingBackcompat() throws IOException { IndexService indexService = createIndex("test", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build()); XContentBuilder indexMapping = XContentFactory.jsonBuilder(); @@ -132,7 +219,6 @@ public class UpdateMappingTests extends ESSingleNodeTestCase { assertThat(documentMapper.indexMapper().enabled(), equalTo(enabled)); } - @Test public void testTimestampParsing() throws IOException { IndexService indexService = createIndex("test", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build()); XContentBuilder indexMapping = XContentFactory.jsonBuilder(); @@ -158,7 +244,6 @@ public class UpdateMappingTests extends ESSingleNodeTestCase { assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); } - @Test public void testSizeTimestampIndexParsing() throws IOException { IndexService indexService = createIndex("test", Settings.settingsBuilder().build()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); @@ -168,7 +253,6 @@ public class UpdateMappingTests extends ESSingleNodeTestCase { assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); } - @Test public void testDefaultApplied() throws IOException { createIndex("test1", Settings.settingsBuilder().build()); createIndex("test2", Settings.settingsBuilder().build()); diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java index f8a66730853..aa97d722737 100644 --- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java @@ -20,9 +20,15 @@ package org.elasticsearch.index.query; import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.io.JsonStringEncoder; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.spans.SpanBoostQuery; +import org.apache.lucene.util.Accountable; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.get.GetRequest; @@ -36,6 +42,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; @@ -49,37 +57,36 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.*; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.analysis.AnalysisModule; -import org.elasticsearch.index.cache.IndexCacheModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.functionscore.ScoreFunctionParser; import org.elasticsearch.index.query.functionscore.ScoreFunctionParserMapper; import org.elasticsearch.index.query.support.QueryParsers; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.index.similarity.SimilarityModule; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.script.MockScriptEngine; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptContextRegistry; -import org.elasticsearch.script.ScriptEngineService; -import org.elasticsearch.script.ScriptModule; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.*; +import org.elasticsearch.script.Script.ScriptParseException; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.TestSearchContext; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.cluster.TestClusterService; @@ -91,22 +98,15 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ExecutionException; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.*; public abstract class AbstractQueryTestCase> extends ESTestCase { @@ -124,12 +124,21 @@ public abstract class AbstractQueryTestCase> BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_SHAPE_FIELD_NAME }; protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[] { STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, GEO_POINT_FIELD_NAME }; + private static final int NUMBER_OF_TESTQUERIES = 20; private static Injector injector; - private static IndexQueryParserService queryParserService; + private static IndicesQueriesRegistry indicesQueriesRegistry; + private static QueryShardContext queryShardContext; + private static IndexFieldDataService indexFieldDataService; + private static int queryNameId = 0; - protected static IndexQueryParserService queryParserService() { - return queryParserService; + + protected static QueryShardContext queryShardContext() { + return queryShardContext; + } + + protected static IndexFieldDataService indexFieldDataService() { + return indexFieldDataService; } private static Index index; @@ -159,10 +168,12 @@ public abstract class AbstractQueryTestCase> Settings settings = Settings.settingsBuilder() .put("name", AbstractQueryTestCase.class.toString()) .put("path.home", createTempDir()) + .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING, false) .build(); Settings indexSettings = Settings.settingsBuilder() .put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); index = new Index(randomAsciiOfLengthBetween(1, 10)); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); final TestClusterService clusterService = new TestClusterService(); clusterService.setState(new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder().put( new IndexMetaData.Builder(index.name()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0)))); @@ -172,13 +183,14 @@ public abstract class AbstractQueryTestCase> clientInvocationHandler); injector = new ModulesBuilder().add( new EnvironmentModule(new Environment(settings)), - new SettingsModule(settings), + new SettingsModule(settings, new SettingsFilter(settings)), new ThreadPoolModule(new ThreadPool(settings)), - new IndicesModule(settings) { + new IndicesModule() { @Override public void configure() { // skip services bindQueryParsersExtension(); + bindMapperExtension(); } }, new ScriptModule(settings) { @@ -192,15 +204,8 @@ public abstract class AbstractQueryTestCase> MockScriptEngine mockScriptEngine = new MockScriptEngine(); Multibinder multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class); multibinder.addBinding().toInstance(mockScriptEngine); - try { - Class.forName("com.github.mustachejava.Mustache"); - } catch(ClassNotFoundException e) { - throw new IllegalStateException("error while loading mustache", e); - } - MustacheScriptEngineService mustacheScriptEngineService = new MustacheScriptEngineService(settings); Set engines = new HashSet<>(); engines.add(mockScriptEngine); - engines.add(mustacheScriptEngineService); List customContexts = new ArrayList<>(); bind(ScriptContextRegistry.class).toInstance(new ScriptContextRegistry(customContexts)); try { @@ -214,11 +219,7 @@ public abstract class AbstractQueryTestCase> } }, new IndexSettingsModule(index, indexSettings), - new IndexCacheModule(indexSettings), - new AnalysisModule(indexSettings, new IndicesAnalysisService(indexSettings)), - new SimilarityModule(index, indexSettings), - new IndexNameModule(index), - new AbstractModule() { + new AbstractModule() { @Override protected void configure() { bind(Client.class).toInstance(proxy); @@ -230,9 +231,25 @@ public abstract class AbstractQueryTestCase> } } ).createInjector(); - queryParserService = injector.getInstance(IndexQueryParserService.class); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + ScriptService scriptService = injector.getInstance(ScriptService.class); + SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); + MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class); + MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry); + indexFieldDataService = new IndexFieldDataService(idxSettings, injector.getInstance(IndicesFieldDataCache.class), injector.getInstance(CircuitBreakerService.class), mapperService); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new IndicesWarmer(idxSettings.getNodeSettings(), null), new BitsetFilterCache.Listener() { + @Override + public void onCache(ShardId shardId, Accountable accountable) { - MapperService mapperService = queryParserService.mapperService; + } + + @Override + public void onRemoval(ShardId shardId, Accountable accountable) { + + } + }); + indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class); + queryShardContext = new QueryShardContext(idxSettings, proxy, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry); //create some random type with some default field, those types will stick around for all of the subclasses currentTypes = new String[randomIntBetween(0, 5)]; for (int i = 0; i < currentTypes.length; i++) { @@ -261,10 +278,12 @@ public abstract class AbstractQueryTestCase> terminate(injector.getInstance(ThreadPool.class)); injector = null; index = null; - queryParserService = null; + queryShardContext = null; currentTypes = null; namedWriteableRegistry = null; randomTypes = null; + indicesQueriesRegistry = null; + indexFieldDataService = null; } @Before @@ -295,12 +314,21 @@ public abstract class AbstractQueryTestCase> query.boost(2.0f / randomIntBetween(1, 20)); } if (randomBoolean()) { - query.queryName(randomAsciiOfLengthBetween(1, 10)); + query.queryName(createUniqueRandomName()); } } return query; } + /** + * make sure query names are unique by suffixing them with increasing counter + */ + private static String createUniqueRandomName() { + String queryName = randomAsciiOfLengthBetween(1, 10) + queryNameId; + queryNameId++; + return queryName; + } + /** * Create the query that is being tested */ @@ -310,12 +338,70 @@ public abstract class AbstractQueryTestCase> * Generic test that creates new query from the test query and checks both for equality * and asserts equality on the two queries. */ - @Test public void testFromXContent() throws IOException { - QB testQuery = createTestQueryBuilder(); - assertParsedQuery(testQuery.toString(), testQuery); - for (Map.Entry alternateVersion : getAlternateVersions().entrySet()) { - assertParsedQuery(alternateVersion.getKey(), alternateVersion.getValue()); + for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) { + QB testQuery = createTestQueryBuilder(); + XContentBuilder builder = toXContent(testQuery, randomFrom(XContentType.values())); + assertParsedQuery(builder.bytes(), testQuery); + for (Map.Entry alternateVersion : getAlternateVersions().entrySet()) { + String queryAsString = alternateVersion.getKey(); + assertParsedQuery(new BytesArray(queryAsString), alternateVersion.getValue(), ParseFieldMatcher.EMPTY); + } + } + } + + protected static XContentBuilder toXContent(QueryBuilder query, XContentType contentType) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (randomBoolean()) { + builder.prettyPrint(); + } + query.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + /** + * Test that unknown field trigger ParsingException. + * To find the right position in the root query, we add a marker as `queryName` which + * all query builders support. The added bogus field after that should trigger the exception. + * Queries that allow arbitrary field names at this level need to override this test. + */ + public void testUnknownField() throws IOException { + String marker = "#marker#"; + QB testQuery; + do { + testQuery = createTestQueryBuilder(); + } while (testQuery.toString().contains(marker)); + testQuery.queryName(marker); // to find root query to add additional bogus field there + String queryAsString = testQuery.toString().replace("\"" + marker + "\"", "\"" + marker + "\", \"bogusField\" : \"someValue\""); + try { + parseQuery(queryAsString); + fail("ParsingException expected."); + } catch (ParsingException e) { + // we'd like to see the offending field name here + assertThat(e.getMessage(), containsString("bogusField")); + } + } + + /** + * Test that adding additional object into otherwise correct query string + * should always trigger some kind of Parsing Exception. + */ + public void testUnknownObjectException() throws IOException { + String validQuery = createTestQueryBuilder().toString(); + assertThat(validQuery, containsString("{")); + for (int insertionPosition = 0; insertionPosition < validQuery.length(); insertionPosition++) { + if (validQuery.charAt(insertionPosition) == '{') { + String testQuery = validQuery.substring(0, insertionPosition) + "{ \"newField\" : " + validQuery.substring(insertionPosition) + "}"; + try { + parseQuery(testQuery); + fail("some parsing exception expected for query: " + testQuery); + } catch (ParsingException | ScriptParseException | ElasticsearchParseException e) { + // different kinds of exception wordings depending on location + // of mutation, so no simple asserts possible here + } catch (JsonParseException e) { + // mutation produced invalid json + } + } } } @@ -341,6 +427,20 @@ public abstract class AbstractQueryTestCase> assertEquals(expectedQuery.hashCode(), newQuery.hashCode()); } + /** + * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder} + */ + protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException { + assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT); + } + + protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { + QueryBuilder newQuery = parseQuery(queryAsBytes, matcher); + assertNotSame(newQuery, expectedQuery); + assertEquals(expectedQuery, newQuery); + assertEquals(expectedQuery.hashCode(), newQuery.hashCode()); + } + protected final QueryBuilder parseQuery(String queryAsString) throws IOException { return parseQuery(queryAsString, ParseFieldMatcher.STRICT); } @@ -350,67 +450,75 @@ public abstract class AbstractQueryTestCase> return parseQuery(parser, matcher); } - protected final QueryBuilder parseQuery(BytesReference query) throws IOException { - XContentParser parser = XContentFactory.xContent(query).createParser(query); - return parseQuery(parser, ParseFieldMatcher.STRICT); + protected final QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException { + return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT); } - protected final QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException { + protected final QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException { + XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes); + return parseQuery(parser, matcher); + } + + private QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException { QueryParseContext context = createParseContext(); context.reset(parser); context.parseFieldMatcher(matcher); - return context.parseInnerQueryBuilder(); + QueryBuilder parseInnerQueryBuilder = context.parseInnerQueryBuilder(); + assertTrue(parser.nextToken() == null); + return parseInnerQueryBuilder; } /** * Test creates the {@link Query} from the {@link QueryBuilder} under test and delegates the * assertions being made on the result to the implementing subclass. */ - @Test public void testToQuery() throws IOException { - QueryShardContext context = createShardContext(); - context.setAllowUnmappedFields(true); - - QB firstQuery = createTestQueryBuilder(); - QB controlQuery = copyQuery(firstQuery); - setSearchContext(randomTypes); // only set search context for toQuery to be more realistic - Query firstLuceneQuery = firstQuery.toQuery(context); - assertLuceneQuery(firstQuery, firstLuceneQuery, context); - SearchContext.removeCurrent(); // remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well - assertTrue("query is not equal to its copy after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, - firstQuery.equals(controlQuery)); - assertTrue("equals is not symmetric after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, - controlQuery.equals(firstQuery)); - assertThat("query copy's hashcode is different from original hashcode after calling toQuery, firstQuery: " + firstQuery - + ", secondQuery: " + controlQuery, controlQuery.hashCode(), equalTo(firstQuery.hashCode())); - - - QB secondQuery = copyQuery(firstQuery); - //query _name never should affect the result of toQuery, we randomly set it to make sure - if (randomBoolean()) { - secondQuery.queryName(secondQuery.queryName() == null ? randomAsciiOfLengthBetween(1, 30) : secondQuery.queryName() + randomAsciiOfLengthBetween(1, 10)); - } - setSearchContext(randomTypes); // only set search context for toQuery to be more realistic - Query secondLuceneQuery = secondQuery.toQuery(context); - assertLuceneQuery(secondQuery, secondLuceneQuery, context); - SearchContext.removeCurrent(); // remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well - - assertThat("two equivalent query builders lead to different lucene queries", secondLuceneQuery, equalTo(firstLuceneQuery)); - - //if the initial lucene query is null, changing its boost won't have any effect, we shouldn't test that - if (firstLuceneQuery != null && supportsBoostAndQueryName()) { - secondQuery.boost(firstQuery.boost() + 1f + randomFloat()); + for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + QB firstQuery = createTestQueryBuilder(); + QB controlQuery = copyQuery(firstQuery); setSearchContext(randomTypes); // only set search context for toQuery to be more realistic - Query thirdLuceneQuery = secondQuery.toQuery(context); + Query firstLuceneQuery = firstQuery.toQuery(context); + assertLuceneQuery(firstQuery, firstLuceneQuery, context); + SearchContext.removeCurrent(); // remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well + assertTrue( + "query is not equal to its copy after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, + firstQuery.equals(controlQuery)); + assertTrue("equals is not symmetric after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, + controlQuery.equals(firstQuery)); + assertThat("query copy's hashcode is different from original hashcode after calling toQuery, firstQuery: " + firstQuery + + ", secondQuery: " + controlQuery, controlQuery.hashCode(), equalTo(firstQuery.hashCode())); + + QB secondQuery = copyQuery(firstQuery); + // query _name never should affect the result of toQuery, we randomly set it to make sure + if (randomBoolean()) { + secondQuery.queryName(secondQuery.queryName() == null ? randomAsciiOfLengthBetween(1, 30) : secondQuery.queryName() + + randomAsciiOfLengthBetween(1, 10)); + } + setSearchContext(randomTypes); + Query secondLuceneQuery = secondQuery.toQuery(context); + assertLuceneQuery(secondQuery, secondLuceneQuery, context); SearchContext.removeCurrent(); - assertThat("modifying the boost doesn't affect the corresponding lucene query", firstLuceneQuery, not(equalTo(thirdLuceneQuery))); + + assertThat("two equivalent query builders lead to different lucene queries", secondLuceneQuery, equalTo(firstLuceneQuery)); + + // if the initial lucene query is null, changing its boost won't have any effect, we shouldn't test that + if (firstLuceneQuery != null && supportsBoostAndQueryName()) { + secondQuery.boost(firstQuery.boost() + 1f + randomFloat()); + setSearchContext(randomTypes); + Query thirdLuceneQuery = secondQuery.toQuery(context); + SearchContext.removeCurrent(); + assertThat("modifying the boost doesn't affect the corresponding lucene query", firstLuceneQuery, + not(equalTo(thirdLuceneQuery))); + } } } /** * Few queries allow you to set the boost and queryName on the java api, although the corresponding parser doesn't parse them as they are not supported. * This method allows to disable boost and queryName related tests for those queries. Those queries are easy to identify: their parsers - * don't parse `boost` and `_name` as they don't apply to the specific query: filter query, wrapper query and match_none + * don't parse `boost` and `_name` as they don't apply to the specific query: wrapper query and match_none */ protected boolean supportsBoostAndQueryName() { return true; @@ -427,33 +535,53 @@ public abstract class AbstractQueryTestCase> assertThat(namedQuery, equalTo(query)); } if (query != null) { - assertBoost(queryBuilder, query); + if (queryBuilder.boost() != AbstractQueryBuilder.DEFAULT_BOOST) { + assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(SpanBoostQuery.class))); + if (query instanceof SpanBoostQuery) { + SpanBoostQuery spanBoostQuery = (SpanBoostQuery) query; + assertThat(spanBoostQuery.getBoost(), equalTo(queryBuilder.boost())); + query = spanBoostQuery.getQuery(); + } else { + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(queryBuilder.boost())); + query = boostQuery.getQuery(); + } + } } doAssertLuceneQuery(queryBuilder, query, context); } - /** - * Allows to override boost assertions for queries that don't have the default behaviour - */ - protected void assertBoost(QB queryBuilder, Query query) throws IOException { - // workaround https://bugs.openjdk.java.net/browse/JDK-8056984 - float boost = queryBuilder.boost(); - assertThat(query.getBoost(), equalTo(boost)); - } - /** * Checks the result of {@link QueryBuilder#toQuery(QueryShardContext)} given the original {@link QueryBuilder} and {@link QueryShardContext}. * Contains the query specific checks to be implemented by subclasses. */ protected abstract void doAssertLuceneQuery(QB queryBuilder, Query query, QueryShardContext context) throws IOException; + protected static void assertTermOrBoostQuery(Query query, String field, String value, float fieldBoost) { + if (fieldBoost != AbstractQueryBuilder.DEFAULT_BOOST) { + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(fieldBoost)); + query = boostQuery.getQuery(); + } + assertTermQuery(query, field, value); + } + + protected static void assertTermQuery(Query query, String field, String value) { + assertThat(query, instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) query; + assertThat(termQuery.getTerm().field(), equalTo(field)); + assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(value.toLowerCase(Locale.ROOT))); + } + /** * Test serialization and deserialization of the test query. */ - @Test public void testSerialization() throws IOException { - QB testQuery = createTestQueryBuilder(); - assertSerialization(testQuery); + for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) { + QB testQuery = createTestQueryBuilder(); + assertSerialization(testQuery); + } } /** @@ -474,41 +602,42 @@ public abstract class AbstractQueryTestCase> } } - @Test public void testEqualsAndHashcode() throws IOException { - QB firstQuery = createTestQueryBuilder(); - assertFalse("query is equal to null", firstQuery.equals(null)); - assertFalse("query is equal to incompatible type", firstQuery.equals("")); - assertTrue("query is not equal to self", firstQuery.equals(firstQuery)); - assertThat("same query's hashcode returns different values if called multiple times", firstQuery.hashCode(), equalTo(firstQuery.hashCode())); + for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) { + QB firstQuery = createTestQueryBuilder(); + assertFalse("query is equal to null", firstQuery.equals(null)); + assertFalse("query is equal to incompatible type", firstQuery.equals("")); + assertTrue("query is not equal to self", firstQuery.equals(firstQuery)); + assertThat("same query's hashcode returns different values if called multiple times", firstQuery.hashCode(), + equalTo(firstQuery.hashCode())); - QB secondQuery = copyQuery(firstQuery); - assertTrue("query is not equal to self", secondQuery.equals(secondQuery)); - assertTrue("query is not equal to its copy", firstQuery.equals(secondQuery)); - assertTrue("equals is not symmetric", secondQuery.equals(firstQuery)); - assertThat("query copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(firstQuery.hashCode())); + QB secondQuery = copyQuery(firstQuery); + assertTrue("query is not equal to self", secondQuery.equals(secondQuery)); + assertTrue("query is not equal to its copy", firstQuery.equals(secondQuery)); + assertTrue("equals is not symmetric", secondQuery.equals(firstQuery)); + assertThat("query copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(firstQuery.hashCode())); - QB thirdQuery = copyQuery(secondQuery); - assertTrue("query is not equal to self", thirdQuery.equals(thirdQuery)); - assertTrue("query is not equal to its copy", secondQuery.equals(thirdQuery)); - assertThat("query copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(thirdQuery.hashCode())); - assertTrue("equals is not transitive", firstQuery.equals(thirdQuery)); - assertThat("query copy's hashcode is different from original hashcode", firstQuery.hashCode(), equalTo(thirdQuery.hashCode())); - assertTrue("equals is not symmetric", thirdQuery.equals(secondQuery)); - assertTrue("equals is not symmetric", thirdQuery.equals(firstQuery)); + QB thirdQuery = copyQuery(secondQuery); + assertTrue("query is not equal to self", thirdQuery.equals(thirdQuery)); + assertTrue("query is not equal to its copy", secondQuery.equals(thirdQuery)); + assertThat("query copy's hashcode is different from original hashcode", secondQuery.hashCode(), equalTo(thirdQuery.hashCode())); + assertTrue("equals is not transitive", firstQuery.equals(thirdQuery)); + assertThat("query copy's hashcode is different from original hashcode", firstQuery.hashCode(), equalTo(thirdQuery.hashCode())); + assertTrue("equals is not symmetric", thirdQuery.equals(secondQuery)); + assertTrue("equals is not symmetric", thirdQuery.equals(firstQuery)); - if (randomBoolean()) { - secondQuery.queryName(secondQuery.queryName() == null ? randomAsciiOfLengthBetween(1, 30) : secondQuery.queryName() - + randomAsciiOfLengthBetween(1, 10)); - } else { - secondQuery.boost(firstQuery.boost() + 1f + randomFloat()); + if (randomBoolean()) { + secondQuery.queryName(secondQuery.queryName() == null ? randomAsciiOfLengthBetween(1, 30) : secondQuery.queryName() + + randomAsciiOfLengthBetween(1, 10)); + } else { + secondQuery.boost(firstQuery.boost() + 1f + randomFloat()); + } + assertThat("different queries should not be equal", secondQuery, not(equalTo(firstQuery))); } - assertThat("different queries should not be equal", secondQuery, not(equalTo(firstQuery))); - assertThat("different queries should have different hashcode", secondQuery.hashCode(), not(equalTo(firstQuery.hashCode()))); } private QueryParser queryParser(String queryId) { - return queryParserService.indicesQueriesRegistry().queryParsers().get(queryId); + return indicesQueriesRegistry.queryParsers().get(queryId); } //we use the streaming infra to create a copy of the query provided as argument @@ -528,7 +657,7 @@ public abstract class AbstractQueryTestCase> * @return a new {@link QueryShardContext} based on the base test index and queryParserService */ protected static QueryShardContext createShardContext() { - QueryShardContext queryCreationContext = new QueryShardContext(index, queryParserService); + QueryShardContext queryCreationContext = new QueryShardContext(queryShardContext); queryCreationContext.reset(); queryCreationContext.parseFieldMatcher(ParseFieldMatcher.STRICT); return queryCreationContext; @@ -538,7 +667,7 @@ public abstract class AbstractQueryTestCase> * @return a new {@link QueryParseContext} based on the base test index and queryParserService */ protected static QueryParseContext createParseContext() { - QueryParseContext queryParseContext = new QueryParseContext(queryParserService.indicesQueriesRegistry()); + QueryParseContext queryParseContext = new QueryParseContext(indicesQueriesRegistry); queryParseContext.reset(null); queryParseContext.parseFieldMatcher(ParseFieldMatcher.STRICT); return queryParseContext; @@ -699,21 +828,21 @@ public abstract class AbstractQueryTestCase> AbstractQueryTestCase delegate; @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - if (method.equals(Client.class.getDeclaredMethod("get", GetRequest.class))) { + if (method.equals(Client.class.getMethod("get", GetRequest.class))) { return new PlainActionFuture() { @Override public GetResponse get() throws InterruptedException, ExecutionException { return delegate.executeGet((GetRequest) args[0]); } }; - } else if (method.equals(Client.class.getDeclaredMethod("multiTermVectors", MultiTermVectorsRequest.class))) { + } else if (method.equals(Client.class.getMethod("multiTermVectors", MultiTermVectorsRequest.class))) { return new PlainActionFuture() { @Override public MultiTermVectorsResponse get() throws InterruptedException, ExecutionException { return delegate.executeMultiTermVectors((MultiTermVectorsRequest) args[0]); } }; - } else if (method.equals(Object.class.getDeclaredMethod("toString"))) { + } else if (method.equals(Object.class.getMethod("toString"))) { return "MockClient"; } throw new UnsupportedOperationException("this test can't handle calls to: " + method); @@ -735,4 +864,54 @@ public abstract class AbstractQueryTestCase> throw new UnsupportedOperationException("this test can't handle MultiTermVector requests"); } + /** + * Call this method to check a valid json string representing the query under test against + * it's generated json. + * + * Note: By the time of this writing (Nov 2015) all queries are taken from the query dsl + * reference docs mirroring examples there. Here's how the queries were generated: + * + *

      + *
    • Take a reference documentation example. + *
    • Stick it into the createParseableQueryJson method of the respective query test. + *
    • Manually check that what the QueryBuilder generates equals the input json ignoring default options. + *
    • Put the manual checks into the asserQueryParsedFromJson method. + *
    • Now copy the generated json including default options into createParseableQueryJso + *
    • By now the roundtrip check for the json should be happy. + *
    + **/ + public static void checkGeneratedJson(String expected, QueryBuilder source) throws IOException { + // now assert that we actually generate the same JSON + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + source.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals( + msg(expected, builder.string()), + expected.replaceAll("\\s+",""), + builder.string().replaceAll("\\s+","")); + } + + private static String msg(String left, String right) { + int size = Math.min(left.length(), right.length()); + StringBuilder builder = new StringBuilder("size: " + left.length() + " vs. " + right.length()); + builder.append(" content: <<"); + for (int i = 0; i < size; i++) { + if (left.charAt(i) == right.charAt(i)) { + builder.append(left.charAt(i)); + } else { + builder.append(">> ").append("until offset: ").append(i) + .append(" [").append(left.charAt(i)).append(" vs.").append(right.charAt(i)) + .append("] [").append((int)left.charAt(i) ).append(" vs.").append((int)right.charAt(i)).append(']'); + return builder.toString(); + } + } + if (left.length() != right.length()) { + int leftEnd = Math.max(size, left.length()) - 1; + int rightEnd = Math.max(size, right.length()) - 1; + builder.append(">> ").append("until offset: ").append(size) + .append(" [").append(left.charAt(leftEnd)).append(" vs.").append(right.charAt(rightEnd)) + .append("] [").append((int)left.charAt(leftEnd)).append(" vs.").append((int)right.charAt(rightEnd)).append(']'); + return builder.toString(); + } + return ""; + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java index adab170127e..161e7582bb8 100644 --- a/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java @@ -21,13 +21,10 @@ package org.elasticsearch.index.query; import com.fasterxml.jackson.core.io.JsonStringEncoder; -import org.junit.Test; - import java.util.HashMap; import java.util.Map; public abstract class AbstractTermQueryTestCase> extends AbstractQueryTestCase { - @Override protected final QB doCreateTestQueryBuilder() { String fieldName = null; @@ -75,7 +72,6 @@ public abstract class AbstractTermQueryTestCase { - @Override protected BoolQueryBuilder doCreateTestQueryBuilder() { BoolQueryBuilder query = new BoolQueryBuilder(); @@ -156,7 +148,6 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase { - /** * @return a {@link ConstantScoreQueryBuilder} with random boost between 0.1f and 2.0f */ @@ -54,13 +53,16 @@ public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase { - /** * @return a {@link DisMaxQueryBuilder} with random inner queries */ @@ -84,7 +83,6 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase innerQueryBuilder = parseQuery(queryString); @@ -103,7 +100,6 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase 0); String queryAsString = "{\n" + @@ -138,9 +133,45 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase disjuncts = disjunctionMaxQuery.getDisjuncts(); assertThat(disjuncts.size(), equalTo(1)); - PrefixQuery firstQ = (PrefixQuery) disjuncts.get(0); + assertThat(disjuncts.get(0), instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) disjuncts.get(0); + assertThat((double) boostQuery.getBoost(), closeTo(1.2, 0.00001)); + assertThat(boostQuery.getQuery(), instanceOf(PrefixQuery.class)); + PrefixQuery firstQ = (PrefixQuery) boostQuery.getQuery(); // since age is automatically registered in data, we encode it as numeric assertThat(firstQ.getPrefix(), equalTo(new Term(STRING_FIELD_NAME, "sh"))); - assertThat((double) firstQ.getBoost(), closeTo(1.2, 0.00001)); + + } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"dis_max\" : {\n" + + " \"tie_breaker\" : 0.7,\n" + + " \"queries\" : [ {\n" + + " \"term\" : {\n" + + " \"age\" : {\n" + + " \"value\" : 34,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " }, {\n" + + " \"term\" : {\n" + + " \"age\" : {\n" + + " \"value\" : 35,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " } ],\n" + + " \"boost\" : 1.2\n" + + " }\n" + + "}"; + + DisMaxQueryBuilder parsed = (DisMaxQueryBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + + assertEquals(json, 1.2, parsed.boost(), 0.0001); + assertEquals(json, 0.7, parsed.tieBreaker(), 0.0001); + assertEquals(json, 2, parsed.innerQueries().size()); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java index 92523bb99f3..0dc3a5ff90e 100644 --- a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.junit.Test; import java.io.IOException; import java.util.Collection; @@ -34,7 +33,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; public class ExistsQueryBuilderTests extends AbstractQueryTestCase { - @Override protected ExistsQueryBuilder doCreateTestQueryBuilder() { String fieldPattern; @@ -80,7 +78,6 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase { - @Override protected FieldMaskingSpanQueryBuilder doCreateTestQueryBuilder() { String fieldName; @@ -56,7 +54,6 @@ public class FieldMaskingSpanQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -120,23 +117,24 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + " \"fuzzy\":{\n" + " \"" + INT_FIELD_NAME + "\":{\n" + " \"value\":12,\n" + - " \"fuzziness\":5,\n" + - " \"boost\":2.0\n" + + " \"fuzziness\":5\n" + " }\n" + " }\n" + "}\n"; @@ -146,4 +144,24 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase { /** Randomly generate either NaN or one of the two infinity values. */ private static Double[] brokenDoubles = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; - + @Override protected GeoBoundingBoxQueryBuilder doCreateTestQueryBuilder() { GeoBoundingBoxQueryBuilder builder = new GeoBoundingBoxQueryBuilder(GEO_POINT_FIELD_NAME); @@ -48,7 +51,7 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); super.testToQuery(); } - - @Test(expected = QueryShardException.class) + public void testExceptionOnMissingTypes() throws IOException { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length == 0); - super.testToQuery(); + try { + super.testToQuery(); + fail("Expected IllegalArgumentException"); + } catch (QueryShardException e) { + assertThat(e.getMessage(), is("failed to find geo_point field [mapped_geo_point]")); + } } - @Test public void testBrokenCoordinateCannotBeSet() { PointTester[] testers = { new TopTester(), new LeftTester(), new BottomTester(), new RightTester() }; @@ -128,7 +144,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); GeoBoundingBoxQueryBuilder qb = createTestQueryBuilder(); @@ -243,18 +257,21 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -333,7 +349,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -353,7 +368,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -367,7 +381,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -381,7 +394,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -395,7 +407,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -410,14 +421,47 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase { @@ -84,7 +84,7 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); super.testToQuery(); @@ -160,6 +167,15 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -188,10 +218,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -200,10 +229,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -212,10 +240,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -224,10 +251,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -240,10 +266,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -256,10 +281,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -271,16 +295,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -292,16 +309,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -314,10 +324,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -330,10 +339,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -345,10 +353,9 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -361,17 +368,45 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase 0); Query parsedQuery = parseQuery(query).toQuery(createShardContext()); - GeoDistanceRangeQuery filter = (GeoDistanceRangeQuery) parsedQuery; - assertThat(filter.fieldName(), equalTo(GEO_POINT_FIELD_NAME)); - assertThat(filter.lat(), closeTo(40, 0.00001)); - assertThat(filter.lon(), closeTo(-70, 0.00001)); - assertThat(filter.minInclusiveDistance(), equalTo(Double.NEGATIVE_INFINITY)); - assertThat(filter.maxInclusiveDistance(), closeTo(DistanceUnit.DEFAULT.convert(12, DistanceUnit.MILES), 0.00001)); + Version version = queryShardContext().indexVersionCreated(); + if (version.before(Version.V_2_2_0)) { + GeoDistanceRangeQuery q = (GeoDistanceRangeQuery) parsedQuery; + assertThat(q.fieldName(), equalTo(GEO_POINT_FIELD_NAME)); + assertThat(q.lat(), closeTo(lat, 1E-5D)); + assertThat(q.lon(), closeTo(lon, 1E-5D)); + assertThat(q.minInclusiveDistance(), equalTo(Double.NEGATIVE_INFINITY)); + assertThat(q.maxInclusiveDistance(), closeTo(distanceUnit.convert(distance, DistanceUnit.MILES), 1E-5D)); + } else { + GeoPointDistanceQuery q = (GeoPointDistanceQuery) parsedQuery; + assertThat(q.getField(), equalTo(GEO_POINT_FIELD_NAME)); + assertThat(q.getCenterLat(), closeTo(lat, 1E-5D)); + assertThat(q.getCenterLon(), closeTo(lon, 1E-5D)); + assertThat(q.getRadiusMeters(), closeTo(distanceUnit.convert(distance, DistanceUnit.MILES), 1E-5D)); + } + } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"geo_distance\" : {\n" + + " \"pin.location\" : [ -70.0, 40.0 ],\n" + + " \"distance\" : 12000.0,\n" + + " \"distance_type\" : \"sloppy_arc\",\n" + + " \"optimize_bbox\" : \"memory\",\n" + + " \"validation_method\" : \"STRICT\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + GeoDistanceQueryBuilder parsed = (GeoDistanceQueryBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + assertEquals(json, -70.0, parsed.point().getLon(), 0.0001); + assertEquals(json, 40.0, parsed.point().getLat(), 0.0001); + assertEquals(json, 12000.0, parsed.distance(), 0.0001); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java index 19e48aa41a7..6a952cebc0c 100644 --- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java @@ -19,65 +19,77 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.GeoPointDistanceRangeQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.SloppyMath; +import org.elasticsearch.Version; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery; -import org.junit.Test; +import org.elasticsearch.test.geo.RandomGeoGenerator; import java.io.IOException; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; public class GeoDistanceRangeQueryTests extends AbstractQueryTestCase { @Override protected GeoDistanceRangeQueryBuilder doCreateTestQueryBuilder() { + Version version = queryShardContext().indexVersionCreated(); GeoDistanceRangeQueryBuilder builder; + GeoPoint randomPoint = RandomGeoGenerator.randomPointIn(random(), -180.0, -89.9, 180.0, 89.9); if (randomBoolean()) { - builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, randomGeohash(1, 12)); + builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, randomPoint.geohash()); } else { - double lat = randomDouble() * 180 - 90; - double lon = randomDouble() * 360 - 180; if (randomBoolean()) { - builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, new GeoPoint(lat, lon)); + builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, randomPoint); } else { - builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, lat, lon); + builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, randomPoint.lat(), randomPoint.lon()); } } - int fromValue = randomInt(1000000); - int toValue = randomIntBetween(fromValue, 1000000); - String fromToUnits = randomFrom(DistanceUnit.values()).toString(); + GeoPoint point = builder.point(); + final double maxRadius = GeoUtils.maxRadialDistance(point); + final int fromValueMeters = randomInt((int)(maxRadius*0.5)); + final int toValueMeters = randomIntBetween(fromValueMeters + 1, (int)maxRadius); + DistanceUnit fromToUnits = randomFrom(DistanceUnit.values()); + final String fromToUnitsStr = fromToUnits.toString(); + final double fromValue = DistanceUnit.convert(fromValueMeters, DistanceUnit.DEFAULT, fromToUnits); + final double toValue = DistanceUnit.convert(toValueMeters, DistanceUnit.DEFAULT, fromToUnits); + if (randomBoolean()) { int branch = randomInt(2); + fromToUnits = DistanceUnit.DEFAULT; switch (branch) { case 0: - builder.from(fromValue); + builder.from(fromValueMeters); break; case 1: - builder.to(toValue); + builder.to(toValueMeters); break; case 2: - builder.from(fromValue); - builder.to(toValue); + builder.from(fromValueMeters); + builder.to(toValueMeters); break; } } else { int branch = randomInt(2); switch (branch) { case 0: - builder.from(fromValue + fromToUnits); + builder.from(fromValue + fromToUnitsStr); break; case 1: - builder.to(toValue + fromToUnits); + builder.to(toValue + fromToUnitsStr); break; case 2: - builder.from(fromValue + fromToUnits); - builder.to(toValue + fromToUnits); + builder.from(fromValue + fromToUnitsStr); + builder.to(toValue + fromToUnitsStr); break; } } @@ -90,12 +102,10 @@ public class GeoDistanceRangeQueryTests extends AbstractQueryTestCase 0); super.testToQuery(); } - @Test(expected=IllegalArgumentException.class) public void testNullFieldName() { - if (randomBoolean()) { - new GeoDistanceRangeQueryBuilder(null, new GeoPoint()); - } else { - new GeoDistanceRangeQueryBuilder("", new GeoPoint()); + try { + if (randomBoolean()) { + new GeoDistanceRangeQueryBuilder(null, new GeoPoint()); + } else { + new GeoDistanceRangeQueryBuilder("", new GeoPoint()); + } + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("fieldName must not be null")); } } - @Test(expected=IllegalArgumentException.class) public void testNoPoint() { - if (randomBoolean()) { - new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, (GeoPoint) null); - } else { - new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, (String) null); + try { + if (randomBoolean()) { + new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, (GeoPoint) null); + } else { + new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, (String) null); + } + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("point must not be null")); } } - @Test(expected=IllegalArgumentException.class) public void testInvalidFrom() { GeoDistanceRangeQueryBuilder builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, new GeoPoint()); - if (randomBoolean()) { - builder.from((String) null); - } else { - builder.from((Number) null); + try { + if (randomBoolean()) { + builder.from((String) null); + } else { + builder.from((Number) null); + } + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("[from] must not be null")); } } - @Test(expected=IllegalArgumentException.class) public void testInvalidTo() { GeoDistanceRangeQueryBuilder builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, new GeoPoint()); - if (randomBoolean()) { - builder.to((String) null); - } else { - builder.to((Number) null); + try { + if (randomBoolean()) { + builder.to((String) null); + } else { + builder.to((Number) null); + } + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("[to] must not be null")); } } - @Test(expected=IllegalArgumentException.class) public void testInvalidOptimizeBBox() { GeoDistanceRangeQueryBuilder builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, new GeoPoint()); if (randomBoolean()) { - builder.optimizeBbox(null); + try { + builder.optimizeBbox(null); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("optimizeBbox must not be null")); + } } else { - builder.optimizeBbox("foo"); + try { + builder.optimizeBbox("foo"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("optimizeBbox must be one of [none, memory, indexed]")); + } } } - @Test(expected=IllegalArgumentException.class) public void testInvalidGeoDistance() { GeoDistanceRangeQueryBuilder builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, new GeoPoint()); - builder.geoDistance(null); + try { + builder.geoDistance(null); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("geoDistance calculation mode must not be null")); + } } - @Test(expected=IllegalArgumentException.class) public void testInvalidDistanceUnit() { GeoDistanceRangeQueryBuilder builder = new GeoDistanceRangeQueryBuilder(GEO_POINT_FIELD_NAME, new GeoPoint()); - builder.unit(null); + try { + builder.unit(null); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("distance unit must not be null")); + } + } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"geo_distance_range\" : {\n" + + " \"pin.location\" : [ -70.0, 40.0 ],\n" + + " \"from\" : \"200km\",\n" + + " \"to\" : \"400km\",\n" + + " \"include_lower\" : true,\n" + + " \"include_upper\" : true,\n" + + " \"unit\" : \"m\",\n" + + " \"distance_type\" : \"sloppy_arc\",\n" + + " \"optimize_bbox\" : \"memory\",\n" + + " \"validation_method\" : \"STRICT\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + GeoDistanceRangeQueryBuilder parsed = (GeoDistanceRangeQueryBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + assertEquals(json, -70.0, parsed.point().lon(), 0.0001); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java index e49d15d751d..4a34ef5ffb0 100644 --- a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java @@ -22,7 +22,9 @@ package org.elasticsearch.index.query; import com.spatial4j.core.shape.jts.JtsGeometry; import com.vividsolutions.jts.geom.Coordinate; +import org.apache.lucene.search.GeoPointInPolygonQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; @@ -32,7 +34,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.search.geo.GeoPolygonQuery; import org.elasticsearch.test.geo.RandomShapeGenerator; import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -42,9 +43,9 @@ import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase { - @Override protected GeoPolygonQueryBuilder doCreateTestQueryBuilder() { List polygon = randomPolygon(randomIntBetween(4, 50)); @@ -57,6 +58,15 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase queryBuilderPoints = queryBuilder.points(); + double[] lats = geoQuery.getLats(); + double[] lons = geoQuery.getLons(); + assertThat(lats.length, equalTo(queryBuilderPoints.size())); + assertThat(lons.length, equalTo(queryBuilderPoints.size())); + for (int i=0; i < queryBuilderPoints.size(); ++i) { + final GeoPoint queryBuilderPoint = queryBuilderPoints.get(i); + final GeoPoint pointCopy = new GeoPoint(queryBuilderPoint); + GeoUtils.normalizePoint(pointCopy); + assertThat(lats[i], closeTo(pointCopy.getLat(), 1E-5D)); + assertThat(lons[i], closeTo(pointCopy.getLon(), 1E-5D)); + } + } + /** * Overridden here to ensure the test is only run if at least one type is * present in the mappings. Geo queries do not execute if the field is not @@ -105,36 +133,51 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase()); - } else { - new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, null); + try { + new GeoPolygonQueryBuilder(null, randomPolygon(5)); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("fieldName must not be null")); + } + } + + public void testEmptyPolygon() { + try { + if (randomBoolean()) { + new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, new ArrayList()); + } else { + new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, null); + } + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("polygon must not be null or empty")); } } - @Test(expected=IllegalArgumentException.class) public void testInvalidClosedPolygon() { List points = new ArrayList<>(); points.add(new GeoPoint(0, 90)); points.add(new GeoPoint(90, 90)); points.add(new GeoPoint(0, 90)); - new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, points); - + try { + new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, points); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("too few points defined for geo_polygon query")); + } } - @Test(expected=IllegalArgumentException.class) public void testInvalidOpenPolygon() { List points = new ArrayList<>(); points.add(new GeoPoint(0, 90)); points.add(new GeoPoint(90, 90)); - new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, points); + try { + new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, points); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("too few points defined for geo_polygon query")); + } } public void testDeprecatedXContent() throws IOException { @@ -160,7 +203,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -197,7 +238,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -223,7 +263,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -240,7 +279,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -258,15 +296,51 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase { @@ -55,8 +57,10 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase 0); super.testToQuery(); } - @Test(expected = IllegalArgumentException.class) public void testNoFieldName() throws Exception { ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(getRandom(), null); - new GeoShapeQueryBuilder(null, shape); + try { + new GeoShapeQueryBuilder(null, shape); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("fieldName is required")); + } } - @Test public void testNoShape() throws IOException { try { new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, (ShapeBuilder) null); @@ -156,24 +167,35 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase { @@ -81,28 +85,65 @@ public class GeohashCellQueryBuilderTests extends AbstractQueryTestCase super.testToQuery(); } - @Test(expected=IllegalArgumentException.class) public void testNullField() { - if (randomBoolean()) { - new Builder(null, new GeoPoint()); - } else { - new Builder("", new GeoPoint()); + try { + if (randomBoolean()) { + new Builder(null, new GeoPoint()); + } else { + new Builder("", new GeoPoint()); + } + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("fieldName must not be null")); } } - @Test(expected=IllegalArgumentException.class) public void testNullGeoPoint() { - if (randomBoolean()) { - new Builder(GEO_POINT_FIELD_NAME, (GeoPoint) null); - } else { - new Builder(GEO_POINT_FIELD_NAME, ""); + try { + if (randomBoolean()) { + new Builder(GEO_POINT_FIELD_NAME, (GeoPoint) null); + } else { + new Builder(GEO_POINT_FIELD_NAME, ""); + } + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("geohash or point must be defined")); } } - @Test(expected=IllegalArgumentException.class) public void testInvalidPrecision() { GeohashCellQuery.Builder builder = new Builder(GEO_POINT_FIELD_NAME, new GeoPoint()); - builder.precision(-1); + try { + builder.precision(-1); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("precision must be greater than 0")); + } } + public void testLocationParsing() throws IOException { + Point point = RandomShapeGenerator.xRandomPoint(getRandom()); + Builder pointTestBuilder = new GeohashCellQuery.Builder("pin", new GeoPoint(point.getY(), point.getX())); + String pointTest1 = "{\"geohash_cell\": {\"pin\": {\"lat\": " + point.getY() + ",\"lon\": " + point.getX() + "}}}"; + assertParsedQuery(pointTest1, pointTestBuilder); + String pointTest2 = "{\"geohash_cell\": {\"pin\": \"" + point.getY() + "," + point.getX() + "\"}}"; + assertParsedQuery(pointTest2, pointTestBuilder); + String pointTest3 = "{\"geohash_cell\": {\"pin\": [" + point.getX() + "," + point.getY() + "]}}"; + assertParsedQuery(pointTest3, pointTestBuilder); + } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"geohash_cell\" : {\n" + + " \"neighbors\" : true,\n" + + " \"precision\" : 3,\n" + + " \"pin\" : \"t4mk70fgk067\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + GeohashCellQuery.Builder parsed = (GeohashCellQuery.Builder) parseQuery(json); + checkGeneratedJson(json, parsed); + assertEquals(json, 3, parsed.precision().intValue()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java index c567ba3a0f3..51da0fc3996 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java @@ -20,11 +20,16 @@ package org.elasticsearch.index.query; import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import com.fasterxml.jackson.core.JsonParseException; + import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.*; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -35,6 +40,7 @@ import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.query.support.QueryInnerHits; +import org.elasticsearch.script.Script.ScriptParseException; import org.elasticsearch.search.fetch.innerhits.InnerHitsBuilder; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; import org.elasticsearch.search.internal.SearchContext; @@ -44,7 +50,8 @@ import org.elasticsearch.test.TestSearchContext; import java.io.IOException; import java.util.Collections; -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.containsString; + import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -52,9 +59,10 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase> ").append("until offset: ").append(i) - .append(" [").append(left.charAt(i)).append(" vs.").append(right.charAt(i)) - .append("] [").append((int)left.charAt(i) ).append(" vs.").append((int)right.charAt(i)).append(']'); - return builder.toString(); - } - } - if (left.length() != right.length()) { - int leftEnd = Math.max(size, left.length()) - 1; - int rightEnd = Math.max(size, right.length()) - 1; - builder.append(">> ").append("until offset: ").append(size) - .append(" [").append(left.charAt(leftEnd)).append(" vs.").append(right.charAt(rightEnd)) - .append("] [").append((int)left.charAt(leftEnd)).append(" vs.").append((int)right.charAt(rightEnd)).append(']'); - return builder.toString(); - } - return ""; } - public void testToQueryInnerQueryType() throws IOException { String[] searchTypes = new String[]{PARENT_TYPE}; QueryShardContext.setTypes(searchTypes); @@ -303,4 +284,33 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase { - /** - * check that parser throws exception on missing values field + * Check that parser throws exception on missing values field. */ - @Test(expected=ParsingException.class) public void testIdsNotProvided() throws IOException { String noIdsFieldQuery = "{\"ids\" : { \"type\" : \"my_type\" }"; - parseQuery(noIdsFieldQuery); + try { + parseQuery(noIdsFieldQuery); + fail("Expected ParsingException"); + } catch (ParsingException e) { + assertThat(e.getMessage(), containsString("no ids values provided")); + } } @Override @@ -137,4 +141,30 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase //all good } } + + // see #7686. + public void testIdsQueryWithInvalidValues() throws Exception { + String query = "{ \"ids\": { \"values\": [[1]] } }"; + try { + parseQuery(query); + fail("Expected ParsingException"); + } catch (ParsingException e) { + assertThat(e.getMessage(), is("Illegal value for id, expecting a string or number, got: START_ARRAY")); + } + } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"ids\" : {\n" + + " \"type\" : [ \"my_type\" ],\n" + + " \"values\" : [ \"1\", \"100\", \"4\" ],\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + assertEquals(json, 3, parsed.ids().size()); + assertEquals(json, "my_type", parsed.types()[0]); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java index 8db4317a589..834dce857a6 100644 --- a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.Query; -import org.junit.Test; import java.io.IOException; @@ -57,18 +56,9 @@ public class IndicesQueryBuilderTests extends AbstractQueryTestCase { @Override - protected boolean supportsBoostAndQueryName() { - return false; - } - - @Override - protected AbstractQueryBuilder doCreateTestQueryBuilder() { + protected MatchNoneQueryBuilder doCreateTestQueryBuilder() { return new MatchNoneQueryBuilder(); } @Override - protected void doAssertLuceneQuery(AbstractQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { + protected void doAssertLuceneQuery(MatchNoneQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery booleanQuery = (BooleanQuery) query; assertThat(booleanQuery.clauses().size(), equalTo(0)); } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"match_none\" : {\n" + + " \"boost\" : 1.2\n" + + " }\n" + + "}"; + MatchNoneQueryBuilder parsed = (MatchNoneQueryBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + assertEquals(json, 1.2, parsed.boost(), 0.0001); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java index f9da80d97d6..894c20d1d27 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java @@ -26,18 +26,17 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.search.MatchQuery; import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery; -import org.junit.Test; import java.io.IOException; import java.util.Locale; import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; public class MatchQueryBuilderTests extends AbstractQueryTestCase { - @Override protected MatchQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomFrom(STRING_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME, @@ -45,12 +44,12 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase 0); } - Object value = ""; + Object value; if (fieldName.equals(STRING_FIELD_NAME)) { int terms = randomIntBetween(0, 3); StringBuilder builder = new StringBuilder(); for (int i = 0; i < terms; i++) { - builder.append(randomAsciiOfLengthBetween(1, 10) + " "); + builder.append(randomAsciiOfLengthBetween(1, 10)).append(" "); } value = builder.toString().trim(); } else { @@ -134,8 +133,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase { - - @Override - protected MissingQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomBoolean() ? randomFrom(MAPPED_FIELD_NAMES) : randomAsciiOfLengthBetween(1, 10); - Boolean existence = randomBoolean(); - Boolean nullValue = randomBoolean(); - if (existence == false && nullValue == false) { - if (randomBoolean()) { - existence = true; - } else { - nullValue = true; - } - } - return new MissingQueryBuilder(fieldName, nullValue, existence); - } - - @Override - protected void doAssertLuceneQuery(MissingQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - //too many mapping dependent cases to test, we don't want to end up duplication the toQuery method - } - - @Test - public void testIllegalArguments() { - try { - if (randomBoolean()) { - new MissingQueryBuilder("", true, true); - } else { - new MissingQueryBuilder(null, true, true); - } - fail("must not be null or empty"); - } catch (IllegalArgumentException e) { - // expected - } - - try { - new MissingQueryBuilder("fieldname", false, false); - fail("existence and nullValue cannot both be false"); - } catch (IllegalArgumentException e) { - // expected - } - - try { - new MissingQueryBuilder("fieldname", MissingQueryBuilder.DEFAULT_NULL_VALUE, false); - fail("existence and nullValue cannot both be false"); - } catch (IllegalArgumentException e) { - // expected - } - } - - @Test(expected = QueryShardException.class) - public void testBothNullValueAndExistenceFalse() throws IOException { - QueryShardContext context = createShardContext(); - context.setAllowUnmappedFields(true); - MissingQueryBuilder.newFilter(context, "field", false, false); - } -} diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java index 7e9f97d76ac..246cece2b90 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -42,7 +42,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; @@ -244,19 +243,26 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase 0); String unsupportedField = randomFrom(INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME); @@ -270,7 +276,6 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase { @@ -119,14 +120,13 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase 0); QueryShardContext shardContext = createShardContext(); MultiMatchQueryBuilder multiMatchQueryBuilder = new MultiMatchQueryBuilder("test"); - multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5); + multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5f); Query query = multiMatchQueryBuilder.toQuery(shardContext); - assertThat(query, instanceOf(TermQuery.class)); - assertThat(query.getBoost(), equalTo(5f)); + assertTermOrBoostQuery(query, STRING_FIELD_NAME, "test", 5f); multiMatchQueryBuilder = new MultiMatchQueryBuilder("test"); - multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5); - multiMatchQueryBuilder.boost(2); + multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5f); + multiMatchQueryBuilder.boost(2f); query = multiMatchQueryBuilder.toQuery(shardContext); - assertThat(query, instanceOf(TermQuery.class)); - assertThat(query.getBoost(), equalTo(10f)); + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(2f)); + assertTermOrBoostQuery(boostQuery.getQuery(), STRING_FIELD_NAME, "test", 5f); } - @Test public void testToQueryMultipleTermsBooleanQuery() throws Exception { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); Query query = multiMatchQuery("test1 test2").field(STRING_FIELD_NAME).useDisMax(false).toQuery(createShardContext()); @@ -191,7 +185,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase 0); Query query = multiMatchQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(false).toQuery(createShardContext()); @@ -202,18 +195,18 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase 0); Query query = multiMatchQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(true).toQuery(createShardContext()); assertThat(query, instanceOf(DisjunctionMaxQuery.class)); DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; List disjuncts = disMaxQuery.getDisjuncts(); + assertThat(disjuncts.get(0), instanceOf(TermQuery.class)); assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test"))); + assertThat(disjuncts.get(1), instanceOf(TermQuery.class)); assertThat(((TermQuery) disjuncts.get(1)).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test"))); } - @Test public void testToQueryFieldsWildcard() throws Exception { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); Query query = multiMatchQuery("test").field("mapped_str*").useDisMax(false).toQuery(createShardContext()); @@ -223,4 +216,30 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase { - - /** - * @return a NotQueryBuilder with random limit between 0 and 20 - */ - @Override - protected NotQueryBuilder doCreateTestQueryBuilder() { - return new NotQueryBuilder(RandomQueryBuilder.createQuery(random())); - } - - @Override - protected void doAssertLuceneQuery(NotQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - Query filter = queryBuilder.innerQuery().toQuery(context); - if (filter == null) { - assertThat(query, nullValue()); - } else { - assertThat(query, instanceOf(BooleanQuery.class)); - BooleanQuery booleanQuery = (BooleanQuery) query; - assertThat(booleanQuery.clauses().size(), equalTo(2)); - assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST)); - assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(MatchAllDocsQuery.class)); - assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST_NOT)); - assertThat(booleanQuery.clauses().get(1).getQuery(), instanceOf(filter.getClass())); - } - } - - @Test(expected=ParsingException.class) - public void testMissingFilterSection() throws IOException { - String queryString = "{ \"not\" : {}"; - parseQuery(queryString); - } - - @Override - protected Map getAlternateVersions() { - Map alternateVersions = new HashMap<>(); - QueryBuilder innerQuery = createTestQueryBuilder().innerQuery(); - //not doesn't support empty query when query/filter element is not specified - if (innerQuery != EmptyQueryBuilder.PROTOTYPE) { - NotQueryBuilder testQuery2 = new NotQueryBuilder(innerQuery); - String contentString2 = "{\n" + - " \"not\" : " + testQuery2.innerQuery().toString() + "\n}"; - alternateVersions.put(contentString2, testQuery2); - } - - return alternateVersions; - } - - - public void testDeprecatedXContent() throws IOException { - String deprecatedJson = "{\n" + - " \"not\" : {\n" + - " \"filter\" : " + EmptyQueryBuilder.PROTOTYPE.toString() + "\n" + - " }\n" + - "}"; - try { - parseQuery(deprecatedJson); - fail("filter is deprecated"); - } catch (IllegalArgumentException ex) { - assertEquals("Deprecated field [filter] used, expected [query] instead", ex.getMessage()); - } - - NotQueryBuilder queryBuilder = (NotQueryBuilder) parseQuery(deprecatedJson, ParseFieldMatcher.EMPTY); - assertEquals(EmptyQueryBuilder.PROTOTYPE, queryBuilder.innerQuery()); - } - - @Test - public void testValidate() { - try { - new NotQueryBuilder(null); - fail("cannot be null"); - } catch (IllegalArgumentException e) { - // expected - } - } -} diff --git a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java index 7d7a3a4cfeb..2c7881452a1 100644 --- a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; @@ -54,7 +53,6 @@ public class PrefixQueryBuilderTests extends AbstractQueryTestCase points = new ArrayList(); + points.add(new GeoPoint(40, -70)); + points.add(new GeoPoint(30, -80)); + points.add(new GeoPoint(20, -90)); + geoPolygonQuery("pin.location", points); + } + + public void testGeoShape() throws IOException { + GeoShapeQueryBuilder qb = geoShapeQuery( + "pin.location", + ShapeBuilders.newMultiPoint() + .point(0, 0) + .point(0, 10) + .point(10, 10) + .point(10, 0) + .point(0, 0)); + qb.relation(ShapeRelation.WITHIN); + + qb = geoShapeQuery( + "pin.location", + "DEU", + "countries"); + qb.relation(ShapeRelation.WITHIN) + .indexedShapeIndex("shapes") + .indexedShapePath("location"); + } + + public void testGeoHashCell() { + geoHashCellQuery("pin.location", + new GeoPoint(13.4080, 52.5186)) + .neighbors(true) + .precision(3); + } + + public void testHasChild() { + hasChildQuery( + "blog_tag", + termQuery("tag","something") + ); + } + + public void testHasParent() { + hasParentQuery( + "blog", + termQuery("tag","something") + ); + } + + public void testIds() { + idsQuery("my_type", "type2") + .addIds("1", "4", "100"); + + idsQuery().addIds("1", "4", "100"); + } + + public void testIndices() { + indicesQuery( + termQuery("tag", "wow"), + "index1", "index2" + ).noMatchQuery(termQuery("tag", "kow")); + + indicesQuery( + termQuery("tag", "wow"), + "index1", "index2" + ).noMatchQuery("all"); + } + + public void testMatchAll() { + matchAllQuery(); + } + + public void testMatch() { + matchQuery("name", "kimchy elasticsearch"); + } + + public void testMLT() { + String[] fields = {"name.first", "name.last"}; + String[] texts = {"text like this one"}; + Item[] items = null; + + moreLikeThisQuery(fields, texts, items) + .minTermFreq(1) + .maxQueryTerms(12); + } + + public void testMultiMatch() { + multiMatchQuery("kimchy elasticsearch", "user", "message"); + } + + public void testNested() { + nestedQuery( + "obj1", + boolQuery() + .must(matchQuery("obj1.name", "blue")) + .must(rangeQuery("obj1.count").gt(5)) + ) + .scoreMode(ScoreMode.Avg); + } + + public void testPrefix() { + prefixQuery("brand", "heine"); + } + + public void testQueryString() { + queryStringQuery("+kimchy -elasticsearch"); + } + + public void testRange() { + rangeQuery("price") + .from(5) + .to(10) + .includeLower(true) + .includeUpper(false); + + rangeQuery("age") + .gte("10") + .lt("20"); + } + + public void testRegExp() { + regexpQuery("name.first", "s.*y"); + } + + public void testScript() { + scriptQuery( + new Script("doc['num1'].value > 1") + ); + + Map parameters = new HashMap<>(); + parameters.put("param1", 5); + scriptQuery( + new Script( + "mygroovyscript", + ScriptType.FILE, + "groovy", + parameters) + ); + + } + + public void testSimpleQueryString() { + simpleQueryStringQuery("+kimchy -elasticsearch"); + } + + public void testSpanContaining() { + spanContainingQuery( + spanNearQuery(spanTermQuery("field1","bar"), 5) + .clause(spanTermQuery("field1","baz")) + .inOrder(true), + spanTermQuery("field1","foo")); + } + + public void testSpanFirst() { + spanFirstQuery( + spanTermQuery("user", "kimchy"), + 3 + ); + } + + public void testSpanMultiTerm() { + spanMultiTermQueryBuilder(prefixQuery("user", "ki")); + } + + public void testSpanNear() { + spanNearQuery(spanTermQuery("field","value1"), 12) + .clause(spanTermQuery("field","value2")) + .clause(spanTermQuery("field","value3")) + .inOrder(false) + .collectPayloads(false); + } + + public void testSpanNot() { + spanNotQuery(spanTermQuery("field","value1"), + spanTermQuery("field","value2")); + } + + public void testSpanOr() { + spanOrQuery(spanTermQuery("field","value1")) + .clause(spanTermQuery("field","value2")) + .clause(spanTermQuery("field","value3")); + } + + public void testSpanTerm() { + spanTermQuery("user", "kimchy"); + } + + public void testSpanWithin() { + spanWithinQuery( + spanNearQuery(spanTermQuery("field1", "bar"), 5) + .clause(spanTermQuery("field1", "baz")) + .inOrder(true), + spanTermQuery("field1", "foo")); + } + + public void testTemplate() { + templateQuery( + "gender_template", + ScriptType.INDEXED, + new HashMap<>()); + } + + public void testTerm() { + termQuery("name", "kimchy"); + } + + public void testTerms() { + termsQuery("tags", "blue", "pill"); + } + + public void testType() { + typeQuery("my_type"); + } + + public void testWildcard() { + wildcardQuery("user", "k?mch*"); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryFilterBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryFilterBuilderTests.java deleted file mode 100644 index 15075b30921..00000000000 --- a/core/src/test/java/org/elasticsearch/index/query/QueryFilterBuilderTests.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Query; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.*; - -@SuppressWarnings("deprecation") -public class QueryFilterBuilderTests extends AbstractQueryTestCase { - - @Override - protected QueryFilterBuilder doCreateTestQueryBuilder() { - QueryBuilder innerQuery = RandomQueryBuilder.createQuery(random()); - return new QueryFilterBuilder(innerQuery); - } - - @Override - protected void doAssertLuceneQuery(QueryFilterBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - Query innerQuery = queryBuilder.innerQuery().toQuery(context); - if (innerQuery == null) { - assertThat(query, nullValue()); - } else { - assertThat(query, instanceOf(ConstantScoreQuery.class)); - ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query; - assertThat(constantScoreQuery.getQuery(), equalTo(innerQuery)); - } - } - - @Override - protected boolean supportsBoostAndQueryName() { - return false; - } - - /** - * test that wrapping an inner filter that returns null also returns null to pass on upwards - */ - @Test - public void testInnerQueryReturnsNull() throws IOException { - // create inner filter - String queryString = "{ \"constant_score\" : { \"filter\" : {} } }"; - QueryBuilder innerQuery = parseQuery(queryString); - // check that when wrapping this filter, toQuery() returns null - QueryFilterBuilder queryFilterQuery = new QueryFilterBuilder(innerQuery); - assertNull(queryFilterQuery.toQuery(createShardContext())); - } - - @Test - public void testValidate() { - try { - new QueryFilterBuilder(null); - fail("cannot be null"); - } catch (IllegalArgumentException e) { - // expected - } - } -} diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 1b85a26c5ab..ef9f4a619af 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -25,7 +25,6 @@ import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.common.lucene.all.AllTermQuery; import org.hamcrest.Matchers; import org.joda.time.DateTimeZone; -import org.junit.Test; import java.io.IOException; import java.util.List; @@ -34,8 +33,8 @@ import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery; import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; public class QueryStringQueryBuilderTests extends AbstractQueryTestCase { @@ -145,7 +144,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); Query query = queryStringQuery("test").defaultField(STRING_FIELD_NAME).toQuery(createShardContext()); @@ -170,7 +166,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); Query query = queryStringQuery("\"term1 term2\"").defaultField(STRING_FIELD_NAME).phraseSlop(3).toQuery(createShardContext()); @@ -185,7 +180,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); QueryShardContext shardContext = createShardContext(); @@ -193,13 +187,17 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); Query query = queryStringQuery("test1 test2").field(STRING_FIELD_NAME).useDisMax(false).toQuery(createShardContext()); @@ -232,7 +229,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); Query query = queryStringQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(false).toQuery(createShardContext()); @@ -243,7 +239,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); Query query = queryStringQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(true).toQuery(createShardContext()); @@ -254,7 +249,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); Query query = queryStringQuery("test").field("mapped_str*").useDisMax(false).toQuery(createShardContext()); @@ -265,20 +259,16 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); Query query = queryStringQuery("test").field(STRING_FIELD_NAME, 2.2f).field(STRING_FIELD_NAME_2).useDisMax(true).toQuery(createShardContext()); assertThat(query, instanceOf(DisjunctionMaxQuery.class)); DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query; List disjuncts = disMaxQuery.getDisjuncts(); - assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test"))); - assertThat((double) disjuncts.get(0).getBoost(), closeTo(2.2, 0.01)); - assertThat(((TermQuery) disjuncts.get(1)).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test"))); - assertThat((double) disjuncts.get(1).getBoost(), closeTo(1, 0.01)); + assertTermOrBoostQuery(disjuncts.get(0), STRING_FIELD_NAME, "test", 2.2f); + assertTermOrBoostQuery(disjuncts.get(1), STRING_FIELD_NAME_2, "test", 1.0f); } - @Test public void testToQueryRegExpQuery() throws Exception { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); Query query = queryStringQuery("/foo*bar/").defaultField(STRING_FIELD_NAME).maxDeterminizedStates(5000).toQuery(createShardContext()); @@ -287,23 +277,25 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); - queryStringQuery("/[ac]*a[ac]{50,200}/").defaultField(STRING_FIELD_NAME).toQuery(createShardContext()); + try { + queryStringQuery("/[ac]*a[ac]{50,200}/").defaultField(STRING_FIELD_NAME).toQuery(createShardContext()); + fail("Expected TooComplexToDeterminizeException"); + } catch (TooComplexToDeterminizeException e) { + assertThat(e.getMessage(), containsString("Determinizing [ac]*")); + assertThat(e.getMessage(), containsString("would result in more than 10000 states")); + } } - @Test public void testToQueryNumericRangeQuery() throws Exception { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); Query query = queryStringQuery("12~0.2").defaultField(INT_FIELD_NAME).toQuery(createShardContext()); NumericRangeQuery fuzzyQuery = (NumericRangeQuery) query; assertThat(fuzzyQuery.getMin().longValue(), equalTo(12l)); assertThat(fuzzyQuery.getMax().longValue(), equalTo(12l)); - } - @Test public void testTimezone() throws Exception { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); String queryAsString = "{\n" + @@ -330,4 +322,87 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0); + int numBoosts = randomIntBetween(2, 10); + float[] boosts = new float[numBoosts + 1]; + String queryStringPrefix = ""; + String queryStringSuffix = ""; + for (int i = 0; i < boosts.length - 1; i++) { + float boost = 2.0f / randomIntBetween(3, 20); + boosts[i] = boost; + queryStringPrefix += "("; + queryStringSuffix += ")^" + boost; + } + String queryString = queryStringPrefix + "foo bar" + queryStringSuffix; + + float mainBoost = 2.0f / randomIntBetween(3, 20); + boosts[boosts.length - 1] = mainBoost; + QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(queryString).field(STRING_FIELD_NAME) + .minimumShouldMatch("2").boost(mainBoost); + Query query = queryStringQueryBuilder.toQuery(createShardContext()); + + for (int i = boosts.length - 1; i >= 0; i--) { + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(boosts[i])); + query = boostQuery.getQuery(); + } + + assertThat(query, instanceOf(BooleanQuery.class)); + BooleanQuery booleanQuery = (BooleanQuery) query; + assertThat(booleanQuery.getMinimumNumberShouldMatch(), equalTo(2)); + assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + assertThat(booleanQuery.clauses().get(0).getQuery(), equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "foo")))); + assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + assertThat(booleanQuery.clauses().get(1).getQuery(), equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "bar")))); + } + + public void testToQueryPhraseQueryBoostAndSlop() throws IOException { + assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); + QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder("\"test phrase\"~2").field(STRING_FIELD_NAME, 5f); + Query query = queryStringQueryBuilder.toQuery(createShardContext()); + assertThat(query, instanceOf(DisjunctionMaxQuery.class)); + DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; + assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(1)); + assertThat(disjunctionMaxQuery.getDisjuncts().get(0), instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) disjunctionMaxQuery.getDisjuncts().get(0); + assertThat(boostQuery.getBoost(), equalTo(5f)); + assertThat(boostQuery.getQuery(), instanceOf(PhraseQuery.class)); + PhraseQuery phraseQuery = (PhraseQuery) boostQuery.getQuery(); + assertThat(phraseQuery.getSlop(), Matchers.equalTo(2)); + assertThat(phraseQuery.getTerms().length, equalTo(2)); + } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"query_string\" : {\n" + + " \"query\" : \"this AND that OR thus\",\n" + + " \"default_field\" : \"content\",\n" + + " \"fields\" : [ ],\n" + + " \"use_dis_max\" : true,\n" + + " \"tie_breaker\" : 0.0,\n" + + " \"default_operator\" : \"or\",\n" + + " \"auto_generated_phrase_queries\" : false,\n" + + " \"max_determined_states\" : 10000,\n" + + " \"lowercase_expanded_terms\" : true,\n" + + " \"enable_position_increment\" : true,\n" + + " \"fuzziness\" : \"AUTO\",\n" + + " \"fuzzy_prefix_length\" : 0,\n" + + " \"fuzzy_max_expansions\" : 50,\n" + + " \"phrase_slop\" : 0,\n" + + " \"locale\" : \"und\",\n" + + " \"escape\" : false,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + + QueryStringQueryBuilder parsed = (QueryStringQueryBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + + assertEquals(json, "this AND that OR thus", parsed.queryString()); + assertEquals(json, "content", parsed.defaultField()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java index 2b173bdb1fe..147d21576c6 100644 --- a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java +++ b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java @@ -61,27 +61,38 @@ public class RandomQueryBuilder { public static MultiTermQueryBuilder createMultiTermQuery(Random r) { // for now, only use String Rangequeries for MultiTerm test, numeric and date makes little sense // see issue #12123 for discussion + MultiTermQueryBuilder multiTermQueryBuilder; switch(RandomInts.randomIntBetween(r, 0, 5)) { case 0: RangeQueryBuilder stringRangeQuery = new RangeQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME); stringRangeQuery.from("a" + RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); stringRangeQuery.to("z" + RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); - return stringRangeQuery; + multiTermQueryBuilder = stringRangeQuery; + break; case 1: RangeQueryBuilder numericRangeQuery = new RangeQueryBuilder(AbstractQueryTestCase.INT_FIELD_NAME); numericRangeQuery.from(RandomInts.randomIntBetween(r, 1, 100)); numericRangeQuery.to(RandomInts.randomIntBetween(r, 101, 200)); - return numericRangeQuery; + multiTermQueryBuilder = numericRangeQuery; + break; case 2: - return new FuzzyQueryBuilder(AbstractQueryTestCase.INT_FIELD_NAME, RandomInts.randomInt(r, 1000)); + multiTermQueryBuilder = new FuzzyQueryBuilder(AbstractQueryTestCase.INT_FIELD_NAME, RandomInts.randomInt(r, 1000)); + break; case 3: - return new FuzzyQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME, RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); + multiTermQueryBuilder = new FuzzyQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME, RandomStrings.randomAsciiOfLengthBetween(r, 1, 10)); + break; case 4: - return new PrefixQueryBuilderTests().createTestQueryBuilder(); + multiTermQueryBuilder = new PrefixQueryBuilderTests().createTestQueryBuilder(); + break; case 5: - return new WildcardQueryBuilderTests().createTestQueryBuilder(); + multiTermQueryBuilder = new WildcardQueryBuilderTests().createTestQueryBuilder(); + break; default: throw new UnsupportedOperationException(); } + if (r.nextBoolean()) { + multiTermQueryBuilder.boost(2.0f / RandomInts.randomIntBetween(r, 1, 20)); + } + return multiTermQueryBuilder; } } diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java index 14c1d4f3f49..4df799e9f37 100644 --- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java @@ -23,17 +23,22 @@ import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermRangeQuery; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.lucene.BytesRefs; +import org.hamcrest.core.IsEqual; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.junit.Test; import java.io.IOException; import java.util.HashMap; import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class RangeQueryBuilderTests extends AbstractQueryTestCase { @@ -55,7 +60,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase 0); Query parsedQuery = rangeQuery(INT_FIELD_NAME).from(23).to(54).includeLower(true).includeUpper(false).toQuery(createShardContext()); @@ -198,7 +208,6 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase 0); // We test 01/01/2012 from gte and 2030 for lt @@ -240,7 +249,6 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase 0); String query = "{\n" + @@ -284,7 +292,6 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase 0); long startDate = System.currentTimeMillis(); @@ -326,4 +333,64 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase { - @Override protected ScriptQueryBuilder doCreateTestQueryBuilder() { String script = "5"; @@ -45,7 +43,6 @@ public class ScriptQueryBuilderTests extends AbstractQueryTestCase { @@ -83,7 +89,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase 0 || shardContext.indexQueryParserService().getIndexCreatedVersion().before(Version.V_1_4_0_Beta1)) { + if (getCurrentTypes().length > 0 || shardContext.indexVersionCreated().before(Version.V_1_4_0_Beta1)) { Query luceneQuery = queryBuilder.toQuery(shardContext); assertThat(luceneQuery, instanceOf(TermQuery.class)); TermQuery termQuery = (TermQuery) luceneQuery; @@ -250,10 +265,9 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase 1) { - assertTrue("Query should have been BooleanQuery but was " + query.getClass().getName(), query instanceof BooleanQuery); - + assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery boolQuery = (BooleanQuery) query; if (queryBuilder.lowercaseExpandedTerms()) { for (BooleanClause clause : boolQuery.clauses()) { @@ -263,44 +277,26 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase fields = queryBuilder.fields().keySet().iterator(); + Iterator> fieldsIterator = queryBuilder.fields().entrySet().iterator(); for (BooleanClause booleanClause : boolQuery) { - assertThat(booleanClause.getQuery(), instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) booleanClause.getQuery(); - assertThat(termQuery.getTerm().field(), equalTo(fields.next())); - assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(queryBuilder.value().toLowerCase(Locale.ROOT))); + Map.Entry field = fieldsIterator.next(); + assertTermOrBoostQuery(booleanClause.getQuery(), field.getKey(), queryBuilder.value(), field.getValue()); } - if (queryBuilder.minimumShouldMatch() != null) { assertThat(boolQuery.getMinimumNumberShouldMatch(), greaterThan(0)); } - } else if (queryBuilder.fields().size() <= 1) { - assertTrue("Query should have been TermQuery but was " + query.getClass().getName(), query instanceof TermQuery); - - TermQuery termQuery = (TermQuery) query; - String field; - if (queryBuilder.fields().size() == 0) { - field = MetaData.ALL; - } else { - field = queryBuilder.fields().keySet().iterator().next(); - } - assertThat(termQuery.getTerm().field(), equalTo(field)); - assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(queryBuilder.value().toLowerCase(Locale.ROOT))); + } else if (queryBuilder.fields().size() == 1) { + Map.Entry field = queryBuilder.fields().entrySet().iterator().next(); + assertTermOrBoostQuery(query, field.getKey(), queryBuilder.value(), field.getValue()); + } else if (queryBuilder.fields().size() == 0) { + assertTermQuery(query, MetaData.ALL, queryBuilder.value()); } else { fail("Encountered lucene query type we do not have a validation implementation for in our " + SimpleQueryStringBuilderTests.class.getSimpleName()); } } - @Override - protected void assertBoost(SimpleQueryStringBuilder queryBuilder, Query query) throws IOException { - //boost may get parsed from the random query, we then combine the main boost with that one coming from lucene - //instead of trying to reparse the query and guess what the boost should be, we delegate boost checks to specific boost tests below - } - - - private int shouldClauses(BooleanQuery query) { + private static int shouldClauses(BooleanQuery query) { int result = 0; for (BooleanClause c : query.clauses()) { if (c.getOccur() == BooleanClause.Occur.SHOULD) { @@ -310,21 +306,61 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase 0); QueryShardContext shardContext = createShardContext(); SimpleQueryStringBuilder simpleQueryStringBuilder = new SimpleQueryStringBuilder("test"); simpleQueryStringBuilder.field(STRING_FIELD_NAME, 5); Query query = simpleQueryStringBuilder.toQuery(shardContext); - assertThat(query, instanceOf(TermQuery.class)); - assertThat(query.getBoost(), equalTo(5f)); + assertThat(query, instanceOf(BoostQuery.class)); + BoostQuery boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(5f)); + assertThat(boostQuery.getQuery(), instanceOf(TermQuery.class)); simpleQueryStringBuilder = new SimpleQueryStringBuilder("test"); simpleQueryStringBuilder.field(STRING_FIELD_NAME, 5); simpleQueryStringBuilder.boost(2); query = simpleQueryStringBuilder.toQuery(shardContext); - assertThat(query, instanceOf(TermQuery.class)); - assertThat(query.getBoost(), equalTo(10f)); + boostQuery = (BoostQuery) query; + assertThat(boostQuery.getBoost(), equalTo(2f)); + assertThat(boostQuery.getQuery(), instanceOf(BoostQuery.class)); + boostQuery = (BoostQuery) boostQuery.getQuery(); + assertThat(boostQuery.getBoost(), equalTo(5f)); + assertThat(boostQuery.getQuery(), instanceOf(TermQuery.class)); + } + + public void testNegativeFlags() throws IOException { + String query = "{\"simple_query_string\": {\"query\": \"foo bar\", \"flags\": -1}}"; + SimpleQueryStringBuilder builder = new SimpleQueryStringBuilder("foo bar"); + builder.flags(SimpleQueryStringFlag.ALL); + assertParsedQuery(query, builder); + SimpleQueryStringBuilder otherBuilder = new SimpleQueryStringBuilder("foo bar"); + otherBuilder.flags(-1); + assertThat(builder, equalTo(otherBuilder)); + } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"simple_query_string\" : {\n" + + " \"query\" : \"\\\"fried eggs\\\" +(eggplant | potato) -frittata\",\n" + + " \"fields\" : [ \"_all^1.0\", \"body^5.0\" ],\n" + + " \"analyzer\" : \"snowball\",\n" + + " \"flags\" : -1,\n" + + " \"default_operator\" : \"and\",\n" + + " \"lowercase_expanded_terms\" : true,\n" + + " \"lenient\" : false,\n" + + " \"analyze_wildcard\" : false,\n" + + " \"locale\" : \"und\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + + SimpleQueryStringBuilder parsed = (SimpleQueryStringBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + + assertEquals(json, "\"fried eggs\" +(eggplant | potato) -frittata", parsed.value()); + assertEquals(json, 2, parsed.fields().size()); + assertEquals(json, "snowball", parsed.analyzer()); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java index ff5882a6fa9..79089b57fb4 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java @@ -21,14 +21,12 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.Query; import org.apache.lucene.search.spans.SpanContainingQuery; -import org.junit.Test; import java.io.IOException; import static org.hamcrest.CoreMatchers.instanceOf; public class SpanContainingQueryBuilderTests extends AbstractQueryTestCase { - @Override protected SpanContainingQueryBuilder doCreateTestQueryBuilder() { SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(2); @@ -40,7 +38,6 @@ public class SpanContainingQueryBuilderTests extends AbstractQueryTestCase { - @Override protected SpanFirstQueryBuilder doCreateTestQueryBuilder() { SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(1); @@ -47,9 +45,7 @@ public class SpanFirstQueryBuilderTests extends AbstractQueryTestCase { - @Override protected SpanMultiTermQueryBuilder doCreateTestQueryBuilder() { MultiTermQueryBuilder multiTermQueryBuilder = RandomQueryBuilder.createMultiTermQuery(random()); @@ -39,14 +40,24 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase((MultiTermQuery)multiTermQuery).getWrappedQuery())); } - @Test public void testIllegalArgument() { try { new SpanMultiTermQueryBuilder(null); @@ -62,7 +73,6 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase { - @Override protected SpanNearQueryBuilder doCreateTestQueryBuilder() { SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(randomIntBetween(1, 6)); @@ -57,7 +55,6 @@ public class SpanNearQueryBuilderTests extends AbstractQueryTestCase { - @Override protected SpanNotQueryBuilder doCreateTestQueryBuilder() { SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(2); @@ -60,7 +60,6 @@ public class SpanNotQueryBuilderTests extends AbstractQueryTestCase { - @Override protected SpanOrQueryBuilder doCreateTestQueryBuilder() { SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(randomIntBetween(1, 6)); @@ -53,7 +51,6 @@ public class SpanOrQueryBuilderTests extends AbstractQueryTestCase { - @Override protected SpanWithinQueryBuilder doCreateTestQueryBuilder() { SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(2); @@ -40,7 +38,6 @@ public class SpanWithinQueryBuilderTests extends AbstractQueryTestCase vars = new HashMap<>(); vars.put("template", "filled"); @@ -84,7 +95,27 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase params = new HashMap<>(); + params.put("template", "all"); + QueryBuilder expectedBuilder = new TemplateQueryBuilder(new Template(expectedTemplateString, ScriptType.INLINE, null, null, + params)); + assertParsedQuery(query, expectedBuilder); + } + + public void testRawTemplate() throws IOException { + String expectedTemplateString = "{\"match_{{template}}\":{}}"; + String query = "{\"template\": {\"query\": {\"match_{{template}}\": {}},\"params\" : {\"template\" : \"all\"}}}"; + Map params = new HashMap<>(); + params.put("template", "all"); + QueryBuilder expectedBuilder = new TemplateQueryBuilder(new Template(expectedTemplateString, ScriptType.INLINE, null, + XContentType.JSON, params)); + assertParsedQuery(query, expectedBuilder); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java index f79e249ac96..6aba2691785 100644 --- a/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java @@ -25,15 +25,14 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.MappedFieldType; -import org.junit.Test; import java.io.IOException; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.is; public class TermQueryBuilderTests extends AbstractTermQueryTestCase { - /** * @return a TermQuery with random field name and value, optional random boost and queryname */ @@ -56,13 +55,34 @@ public class TermQueryBuilderTests extends AbstractTermQueryTestCase { - private List randomTerms; private String termsPath; @@ -128,21 +128,28 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase { @@ -48,7 +49,6 @@ public class TypeQueryBuilderTests extends AbstractQueryTestCase innerQuery = contextCopy.parseContext().parseInnerQueryBuilder(); Query expected = innerQuery.toQuery(context); @@ -64,12 +68,6 @@ public class WrapperQueryBuilderTests extends AbstractQueryTestCase { @@ -180,7 +197,6 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase)null); @@ -274,7 +290,6 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase 0); String queryString = jsonBuilder().startObject() @@ -515,7 +525,6 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase> nodePlugins() { return pluginList(DummyQueryParserPlugin.class); } + @Override @Before public void setUp() throws Exception { super.setUp(); @@ -59,39 +58,35 @@ public class CustomQueryParserIT extends ESIntegTestCase { return cluster().numDataNodes(); } - @Test public void testCustomDummyQuery() { assertHitCount(client().prepareSearch("index").setQuery(new DummyQueryParserPlugin.DummyQueryBuilder()).get(), 1l); } - @Test public void testCustomDummyQueryWithinBooleanQuery() { assertHitCount(client().prepareSearch("index").setQuery(new BoolQueryBuilder().must(new DummyQueryParserPlugin.DummyQueryBuilder())).get(), 1l); } - private static IndexQueryParserService queryParser() { + private static QueryShardContext queryShardContext() { IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class); - return indicesService.indexServiceSafe("index").queryParserService(); + return indicesService.indexServiceSafe("index").getQueryShardContext(); } - @Test //see #11120 + //see #11120 public void testConstantScoreParsesFilter() throws Exception { - IndexQueryParserService queryParser = queryParser(); - Query q = constantScoreQuery(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryParser.getShardContext()); + Query q = constantScoreQuery(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryShardContext()); Query inner = ((ConstantScoreQuery) q).getQuery(); assertThat(inner, instanceOf(DummyQueryParserPlugin.DummyQuery.class)); assertEquals(true, ((DummyQueryParserPlugin.DummyQuery) inner).isFilter); } - @Test //see #11120 + //see #11120 public void testBooleanParsesFilter() throws Exception { - IndexQueryParserService queryParser = queryParser(); // single clause, serialized as inner object Query q = boolQuery() .should(new DummyQueryParserPlugin.DummyQueryBuilder()) .must(new DummyQueryParserPlugin.DummyQueryBuilder()) .filter(new DummyQueryParserPlugin.DummyQueryBuilder()) - .mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryParser.getShardContext()); + .mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryShardContext()); assertThat(q, instanceOf(BooleanQuery.class)); BooleanQuery bq = (BooleanQuery) q; assertEquals(4, bq.clauses().size()); @@ -116,7 +111,7 @@ public class CustomQueryParserIT extends ESIntegTestCase { .should(new DummyQueryParserPlugin.DummyQueryBuilder()).should(new DummyQueryParserPlugin.DummyQueryBuilder()) .must(new DummyQueryParserPlugin.DummyQueryBuilder()).must(new DummyQueryParserPlugin.DummyQueryBuilder()) .filter(new DummyQueryParserPlugin.DummyQueryBuilder()).filter(new DummyQueryParserPlugin.DummyQueryBuilder()) - .mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryParser.getShardContext()); + .mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryShardContext()); assertThat(q, instanceOf(BooleanQuery.class)); bq = (BooleanQuery) q; assertEquals(8, bq.clauses().size()); diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java index 432c833aef2..c72470c0cce 100644 --- a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java +++ b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java @@ -27,7 +27,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryParser; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; @@ -64,17 +67,12 @@ public class DummyQueryParserPlugin extends Plugin { @Override protected DummyQueryBuilder doReadFrom(StreamInput in) throws IOException { - return null; + return new DummyQueryBuilder(); } @Override protected void doWriteTo(StreamOutput out) throws IOException { - - } - - @Override - protected boolean doEquals(DummyQueryBuilder other) { - return false; + // Do Nothing } @Override @@ -82,6 +80,11 @@ public class DummyQueryParserPlugin extends Plugin { return 0; } + @Override + protected boolean doEquals(DummyQueryBuilder other) { + return true; + } + @Override public String getWriteableName() { return NAME; diff --git a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 3c3f1b44951..1aa0978e6c8 100644 --- a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -22,11 +22,9 @@ package org.elasticsearch.index.search; import org.apache.lucene.index.Term; import org.apache.lucene.search.*; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -39,7 +37,6 @@ import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; public class MultiMatchQueryTests extends ESSingleNodeTestCase { - private IndexQueryParserService queryParser; private IndexService indexService; @Before @@ -64,11 +61,10 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { "}"; mapperService.merge("person", new CompressedXContent(mapping), true, false); this.indexService = indexService; - queryParser = indexService.queryParserService(); } public void testCrossFieldMultiMatchQuery() throws IOException { - QueryShardContext queryShardContext = new QueryShardContext(new Index("test"), queryParser); + QueryShardContext queryShardContext = indexService.getShard(0).getQueryShardContext(); queryShardContext.setAllowUnmappedFields(true); Query parsedQuery = multiMatchQuery("banon").field("name.first", 2).field("name.last", 3).field("foobar").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).toQuery(queryShardContext); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { diff --git a/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java b/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java index 191d2e74037..0eddb0ab9d4 100644 --- a/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java @@ -19,8 +19,7 @@ package org.elasticsearch.index.search.geo; - -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; @@ -28,58 +27,82 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.geo.RandomGeoGenerator; import java.io.IOException; -import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.is; public class GeoPointParsingTests extends ESTestCase { + static double TOLERANCE = 1E-5; - // mind geohash precision and error - private static final double ERROR = 0.00001d; - - @Test public void testGeoPointReset() throws IOException { double lat = 1 + randomDouble() * 89; double lon = 1 + randomDouble() * 179; GeoPoint point = new GeoPoint(0, 0); - assertCloseTo(point, 0, 0); + GeoPoint point2 = new GeoPoint(0, 0); + assertPointsEqual(point, point2); - assertCloseTo(point.reset(lat, lon), lat, lon); - assertCloseTo(point.reset(0, 0), 0, 0); - assertCloseTo(point.resetLat(lat), lat, 0); - assertCloseTo(point.resetLat(0), 0, 0); - assertCloseTo(point.resetLon(lon), 0, lon); - assertCloseTo(point.resetLon(0), 0, 0); - assertCloseTo(point.resetFromGeoHash(XGeoHashUtils.stringEncode(lon, lat)), lat, lon); - assertCloseTo(point.reset(0, 0), 0, 0); - assertCloseTo(point.resetFromString(Double.toString(lat) + ", " + Double.toHexString(lon)), lat, lon); - assertCloseTo(point.reset(0, 0), 0, 0); + assertPointsEqual(point.reset(lat, lon), point2.reset(lat, lon)); + assertPointsEqual(point.reset(0, 0), point2.reset(0, 0)); + assertPointsEqual(point.resetLat(lat), point2.reset(lat, 0)); + assertPointsEqual(point.resetLat(0), point2.reset(0, 0)); + assertPointsEqual(point.resetLon(lon), point2.reset(0, lon)); + assertPointsEqual(point.resetLon(0), point2.reset(0, 0)); + assertCloseTo(point.resetFromGeoHash(GeoHashUtils.stringEncode(lon, lat)), lat, lon); + assertPointsEqual(point.reset(0, 0), point2.reset(0, 0)); + assertPointsEqual(point.resetFromString(Double.toString(lat) + ", " + Double.toHexString(lon)), point2.reset(lat, lon)); + assertPointsEqual(point.reset(0, 0), point2.reset(0, 0)); } - - @Test + + public void testEqualsHashCodeContract() { + // generate a random geopoint + final GeoPoint x = RandomGeoGenerator.randomPoint(random()); + final GeoPoint y = new GeoPoint(x.lat(), x.lon()); + final GeoPoint z = new GeoPoint(y.lat(), y.lon()); + // GeoPoint doesn't care about coordinate system bounds, this simply validates inequality + final GeoPoint a = new GeoPoint(x.lat() + randomIntBetween(1, 5), x.lon() + randomIntBetween(1, 5)); + + /** equality test */ + // reflexive + assertTrue(x.equals(x)); + // symmetry + assertTrue(x.equals(y)); + // transitivity + assertTrue(y.equals(z)); + assertTrue(x.equals(z)); + // inequality + assertFalse(x.equals(a)); + + /** hashCode test */ + // symmetry + assertTrue(x.hashCode() == y.hashCode()); + // transitivity + assertTrue(y.hashCode() == z.hashCode()); + assertTrue(x.hashCode() == z.hashCode()); + // inequality + assertFalse(x.hashCode() == a.hashCode()); + } + public void testGeoPointParsing() throws IOException { - double lat = randomDouble() * 180 - 90; - double lon = randomDouble() * 360 - 180; - - GeoPoint point = GeoUtils.parseGeoPoint(objectLatLon(lat, lon)); - assertCloseTo(point, lat, lon); - - GeoUtils.parseGeoPoint(arrayLatLon(lat, lon), point); - assertCloseTo(point, lat, lon); + GeoPoint randomPt = RandomGeoGenerator.randomPoint(random()); - GeoUtils.parseGeoPoint(geohash(lat, lon), point); - assertCloseTo(point, lat, lon); + GeoPoint point = GeoUtils.parseGeoPoint(objectLatLon(randomPt.lat(), randomPt.lon())); + assertPointsEqual(point, randomPt); - GeoUtils.parseGeoPoint(stringLatLon(lat, lon), point); - assertCloseTo(point, lat, lon); + GeoUtils.parseGeoPoint(arrayLatLon(randomPt.lat(), randomPt.lon()), point); + assertPointsEqual(point, randomPt); + + GeoUtils.parseGeoPoint(geohash(randomPt.lat(), randomPt.lon()), point); + assertCloseTo(point, randomPt.lat(), randomPt.lon()); + + GeoUtils.parseGeoPoint(stringLatLon(randomPt.lat(), randomPt.lon()), point); + assertCloseTo(point, randomPt.lat(), randomPt.lon()); } - // Based on issue5390 - @Test(expected = ElasticsearchParseException.class) + // Based on #5390 public void testInvalidPointEmbeddedObject() throws IOException { XContentBuilder content = JsonXContent.contentBuilder(); content.startObject(); @@ -91,36 +114,48 @@ public class GeoPointParsingTests extends ESTestCase { XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes()); parser.nextToken(); - GeoUtils.parseGeoPoint(parser); + try { + GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]")); + } } - @Test(expected = ElasticsearchParseException.class) public void testInvalidPointLatHashMix() throws IOException { XContentBuilder content = JsonXContent.contentBuilder(); content.startObject(); - content.field("lat", 0).field("geohash", XGeoHashUtils.stringEncode(0, 0)); + content.field("lat", 0).field("geohash", GeoHashUtils.stringEncode(0d, 0d)); content.endObject(); XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes()); parser.nextToken(); - GeoUtils.parseGeoPoint(parser); + try { + GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("field must be either lat/lon or geohash")); + } } - @Test(expected = ElasticsearchParseException.class) public void testInvalidPointLonHashMix() throws IOException { XContentBuilder content = JsonXContent.contentBuilder(); content.startObject(); - content.field("lon", 0).field("geohash", XGeoHashUtils.stringEncode(0, 0)); + content.field("lon", 0).field("geohash", GeoHashUtils.stringEncode(0d, 0d)); content.endObject(); XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes()); parser.nextToken(); - GeoUtils.parseGeoPoint(parser); + try { + GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("field must be either lat/lon or geohash")); + } } - @Test(expected = ElasticsearchParseException.class) public void testInvalidField() throws IOException { XContentBuilder content = JsonXContent.contentBuilder(); content.startObject(); @@ -130,7 +165,12 @@ public class GeoPointParsingTests extends ESTestCase { XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes()); parser.nextToken(); - GeoUtils.parseGeoPoint(parser); + try { + GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]")); + } } private static XContentParser objectLatLon(double lat, double lon) throws IOException { @@ -161,15 +201,19 @@ public class GeoPointParsingTests extends ESTestCase { private static XContentParser geohash(double lat, double lon) throws IOException { XContentBuilder content = JsonXContent.contentBuilder(); - content.value(XGeoHashUtils.stringEncode(lon, lat)); + content.value(GeoHashUtils.stringEncode(lon, lat)); XContentParser parser = JsonXContent.jsonXContent.createParser(content.bytes()); parser.nextToken(); return parser; } - - public static void assertCloseTo(GeoPoint point, double lat, double lon) { - assertThat(point.lat(), closeTo(lat, ERROR)); - assertThat(point.lon(), closeTo(lon, ERROR)); + + public static void assertPointsEqual(final GeoPoint point1, final GeoPoint point2) { + assertEquals(point1, point2); + assertEquals(point1.hashCode(), point2.hashCode()); } + public static void assertCloseTo(final GeoPoint point, final double lat, final double lon) { + assertEquals(point.lat(), lat, TOLERANCE); + assertEquals(point.lon(), lon, TOLERANCE); + } } diff --git a/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java b/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java index bf5b7f92607..c283150c30d 100644 --- a/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/geo/GeoUtilsTests.java @@ -21,10 +21,11 @@ package org.elasticsearch.index.search.geo; import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.distance.DistanceUtils; + import org.apache.lucene.spatial.prefix.tree.Cell; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoPoint; @@ -33,21 +34,25 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; public class GeoUtilsTests extends ESTestCase { - private static final char[] BASE_32 = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; private static final double MAX_ACCEPTABLE_ERROR = 0.000000001; - @Test public void testGeohashCellWidth() { double equatorialDistance = 2 * Math.PI * 6378137.0; assertThat(GeoUtils.geoHashCellWidth(0), equalTo(equatorialDistance)); @@ -65,7 +70,6 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.geoHashCellWidth(12), equalTo(equatorialDistance / 1073741824)); } - @Test public void testGeohashCellHeight() { double polarDistance = Math.PI * 6356752.314245; assertThat(GeoUtils.geoHashCellHeight(0), equalTo(polarDistance)); @@ -83,7 +87,6 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.geoHashCellHeight(12), equalTo(polarDistance / 1073741824)); } - @Test public void testGeohashCellSize() { double equatorialDistance = 2 * Math.PI * 6378137.0; double polarDistance = Math.PI * 6356752.314245; @@ -112,7 +115,6 @@ public class GeoUtilsTests extends ESTestCase { equalTo(Math.sqrt(Math.pow(polarDistance / 1073741824, 2) + Math.pow(equatorialDistance / 1073741824, 2)))); } - @Test public void testGeoHashLevelsForPrecision() { for (int i = 0; i < 100; i++) { double precision = randomDouble() * 100; @@ -121,7 +123,6 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test public void testGeoHashLevelsForPrecision_String() { for (int i = 0; i < 100; i++) { double precision = randomDouble() * 100; @@ -131,7 +132,6 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test public void testQuadTreeCellWidth() { double equatorialDistance = 2 * Math.PI * 6378137.0; assertThat(GeoUtils.quadTreeCellWidth(0), equalTo(equatorialDistance)); @@ -149,7 +149,6 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.quadTreeCellWidth(12), equalTo(equatorialDistance / 4096)); } - @Test public void testQuadTreeCellHeight() { double polarDistance = Math.PI * 6356752.314245; assertThat(GeoUtils.quadTreeCellHeight(0), equalTo(polarDistance)); @@ -167,7 +166,6 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.quadTreeCellHeight(12), equalTo(polarDistance / 4096)); } - @Test public void testQuadTreeCellSize() { double equatorialDistance = 2 * Math.PI * 6378137.0; double polarDistance = Math.PI * 6356752.314245; @@ -192,7 +190,6 @@ public class GeoUtilsTests extends ESTestCase { equalTo(Math.sqrt(Math.pow(polarDistance / 4096, 2) + Math.pow(equatorialDistance / 4096, 2)))); } - @Test public void testQuadTreeLevelsForPrecision() { for (int i = 0; i < 100; i++) { double precision = randomDouble() * 100; @@ -201,8 +198,7 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testQuadTreeLevelsForPrecision_String() { + public void testQuadTreeLevelsForPrecisionString() { for (int i = 0; i < 100; i++) { double precision = randomDouble() * 100; String precisionString = precision + "m"; @@ -211,16 +207,14 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testNormalizeLat_inNormalRange() { + public void testNormalizeLatInNormalRange() { for (int i = 0; i < 100; i++) { double testValue = (randomDouble() * 180.0) - 90.0; assertThat(GeoUtils.normalizeLat(testValue), closeTo(testValue, MAX_ACCEPTABLE_ERROR)); } } - @Test - public void testNormalizeLat_outsideNormalRange() { + public void testNormalizeLatOutsideNormalRange() { for (int i = 0; i < 100; i++) { double normalisedValue = (randomDouble() * 180.0) - 90.0; int shift = (randomBoolean() ? 1 : -1) * randomIntBetween(1, 10000); @@ -230,8 +224,7 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testNormalizeLat_Huge() { + public void testNormalizeLatHuge() { assertThat(GeoUtils.normalizeLat(-18000000000091.0), equalTo(GeoUtils.normalizeLat(-091.0))); assertThat(GeoUtils.normalizeLat(-18000000000090.0), equalTo(GeoUtils.normalizeLat(-090.0))); assertThat(GeoUtils.normalizeLat(-18000000000089.0), equalTo(GeoUtils.normalizeLat(-089.0))); @@ -246,8 +239,7 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.normalizeLat(+18000000000091.0), equalTo(GeoUtils.normalizeLat(+091.0))); } - @Test - public void testNormalizeLat_edgeCases() { + public void testNormalizeLatEdgeCases() { assertThat(GeoUtils.normalizeLat(Double.POSITIVE_INFINITY), equalTo(Double.NaN)); assertThat(GeoUtils.normalizeLat(Double.NEGATIVE_INFINITY), equalTo(Double.NaN)); assertThat(GeoUtils.normalizeLat(Double.NaN), equalTo(Double.NaN)); @@ -260,16 +252,14 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.normalizeLat(90.0), equalTo(90.0)); } - @Test - public void testNormalizeLon_inNormalRange() { + public void testNormalizeLonInNormalRange() { for (int i = 0; i < 100; i++) { double testValue = (randomDouble() * 360.0) - 180.0; assertThat(GeoUtils.normalizeLon(testValue), closeTo(testValue, MAX_ACCEPTABLE_ERROR)); } } - @Test - public void testNormalizeLon_outsideNormalRange() { + public void testNormalizeLonOutsideNormalRange() { for (int i = 0; i < 100; i++) { double normalisedValue = (randomDouble() * 360.0) - 180.0; double testValue = normalisedValue + ((randomBoolean() ? 1 : -1) * 360.0 * randomIntBetween(1, 10000)); @@ -277,8 +267,7 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testNormalizeLon_Huge() { + public void testNormalizeLonHuge() { assertThat(GeoUtils.normalizeLon(-36000000000181.0), equalTo(GeoUtils.normalizeLon(-181.0))); assertThat(GeoUtils.normalizeLon(-36000000000180.0), equalTo(GeoUtils.normalizeLon(-180.0))); assertThat(GeoUtils.normalizeLon(-36000000000179.0), equalTo(GeoUtils.normalizeLon(-179.0))); @@ -293,8 +282,7 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.normalizeLon(+36000000000181.0), equalTo(GeoUtils.normalizeLon(+181.0))); } - @Test - public void testNormalizeLon_edgeCases() { + public void testNormalizeLonEdgeCases() { assertThat(GeoUtils.normalizeLon(Double.POSITIVE_INFINITY), equalTo(Double.NaN)); assertThat(GeoUtils.normalizeLon(Double.NEGATIVE_INFINITY), equalTo(Double.NaN)); assertThat(GeoUtils.normalizeLon(Double.NaN), equalTo(Double.NaN)); @@ -307,8 +295,7 @@ public class GeoUtilsTests extends ESTestCase { assertThat(GeoUtils.normalizeLon(180.0), equalTo(180.0)); } - @Test - public void testNormalizePoint_inNormalRange() { + public void testNormalizePointInNormalRange() { for (int i = 0; i < 100; i++) { double testLat = (randomDouble() * 180.0) - 90.0; double testLon = (randomDouble() * 360.0) - 180.0; @@ -317,8 +304,7 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testNormalizePoint_outsideNormalRange() { + public void testNormalizePointOutsideNormalRange() { for (int i = 0; i < 100; i++) { double normalisedLat = (randomDouble() * 180.0) - 90.0; double normalisedLon = (randomDouble() * 360.0) - 180.0; @@ -337,8 +323,7 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testNormalizePoint_outsideNormalRange_withOptions() { + public void testNormalizePointOutsideNormalRange_withOptions() { for (int i = 0; i < 100; i++) { boolean normalize = randomBoolean(); double normalisedLat = (randomDouble() * 180.0) - 90.0; @@ -367,8 +352,7 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testNormalizePoint_Huge() { + public void testNormalizePointHuge() { assertNormalizedPoint(new GeoPoint(-18000000000091.0, -36000000000181.0), new GeoPoint(-089.0, -001.0)); assertNormalizedPoint(new GeoPoint(-18000000000090.0, -36000000000180.0), new GeoPoint(-090.0, +180.0)); assertNormalizedPoint(new GeoPoint(-18000000000089.0, -36000000000179.0), new GeoPoint(-089.0, -179.0)); @@ -384,8 +368,7 @@ public class GeoUtilsTests extends ESTestCase { } - @Test - public void testNormalizePoint_edgeCases() { + public void testNormalizePointEdgeCases() { assertNormalizedPoint(new GeoPoint(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY), new GeoPoint(Double.NaN, Double.NaN)); assertNormalizedPoint(new GeoPoint(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY), new GeoPoint(Double.NaN, Double.NaN)); assertNormalizedPoint(new GeoPoint(Double.NaN, Double.NaN), new GeoPoint(Double.NaN, Double.NaN)); @@ -394,11 +377,10 @@ public class GeoUtilsTests extends ESTestCase { assertNormalizedPoint(new GeoPoint(0.0, 0.0), new GeoPoint(0.0, 0.0)); assertNormalizedPoint(new GeoPoint(-180.0, -360.0), new GeoPoint(0.0, 180.0)); assertNormalizedPoint(new GeoPoint(180.0, 360.0), new GeoPoint(0.0, 180.0)); - assertNormalizedPoint(new GeoPoint(-90.0, -180.0), new GeoPoint(-90.0, 180.0)); + assertNormalizedPoint(new GeoPoint(-90.0, -180.0), new GeoPoint(-90.0, -180.0)); assertNormalizedPoint(new GeoPoint(90.0, 180.0), new GeoPoint(90.0, 180.0)); } - @Test public void testParseGeoPoint() throws IOException { for (int i = 0; i < 100; i++) { double lat = randomDouble() * 180 - 90 + randomIntBetween(-1000, 1000) * 180; @@ -430,10 +412,9 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test - public void testParseGeoPoint_geohash() throws IOException { + public void testParseGeoPointGeohash() throws IOException { for (int i = 0; i < 100; i++) { - int geoHashLength = randomIntBetween(1, XGeoHashUtils.PRECISION); + int geoHashLength = randomIntBetween(1, GeoHashUtils.PRECISION); StringBuilder geohashBuilder = new StringBuilder(geoHashLength); for (int j = 0; j < geoHashLength; j++) { geohashBuilder.append(BASE_32[randomInt(BASE_32.length - 1)]); @@ -455,62 +436,85 @@ public class GeoUtilsTests extends ESTestCase { } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_geohashWrongType() throws IOException { - BytesReference jsonBytes = jsonBuilder().startObject().field("geohash", 1.0).endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - parser.nextToken(); + public void testParseGeoPointGeohashWrongType() throws IOException { + BytesReference jsonBytes = jsonBuilder().startObject().field("geohash", 1.0).endObject().bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + parser.nextToken(); + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("geohash must be a string")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_LatNoLon() throws IOException { - double lat = 0.0; - BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - parser.nextToken(); + public void testParseGeoPointLatNoLon() throws IOException { + double lat = 0.0; + BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).endObject().bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + parser.nextToken(); + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("field [lon] missing")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_LonNoLat() throws IOException { - double lon = 0.0; - BytesReference jsonBytes = jsonBuilder().startObject().field("lon", lon).endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - parser.nextToken(); + public void testParseGeoPointLonNoLat() throws IOException { + double lon = 0.0; + BytesReference jsonBytes = jsonBuilder().startObject().field("lon", lon).endObject().bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + parser.nextToken(); + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("field [lat] missing")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_LonWrongType() throws IOException { - double lat = 0.0; - BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", false).endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - parser.nextToken(); + public void testParseGeoPointLonWrongType() throws IOException { + double lat = 0.0; + BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", false).endObject().bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + parser.nextToken(); + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("longitude must be a number")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_LatWrongType() throws IOException { - double lon = 0.0; - BytesReference jsonBytes = jsonBuilder().startObject().field("lat", false).field("lon", lon).endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - parser.nextToken(); + public void testParseGeoPointLatWrongType() throws IOException { + double lon = 0.0; + BytesReference jsonBytes = jsonBuilder().startObject().field("lat", false).field("lon", lon).endObject().bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + parser.nextToken(); + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("latitude must be a number")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_ExtraField() throws IOException { + public void testParseGeoPointExtraField() throws IOException { double lat = 0.0; double lon = 0.0; - BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", lon).field("foo", true).endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - parser.nextToken(); + BytesReference jsonBytes = jsonBuilder().startObject().field("lat", lat).field("lon", lon).field("foo", true).endObject().bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + parser.nextToken(); + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("field must be either [lat], [lon] or [geohash]")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_LonLatGeoHash() throws IOException { + public void testParseGeoPointLonLatGeoHash() throws IOException { double lat = 0.0; double lon = 0.0; String geohash = "abcd"; @@ -518,45 +522,62 @@ public class GeoUtilsTests extends ESTestCase { .bytes(); XContentParser parser = XContentHelper.createParser(jsonBytes); parser.nextToken(); - GeoUtils.parseGeoPoint(parser); - } - - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_ArrayTooManyValues() throws IOException { - double lat = 0.0; - double lon = 0.0; - double elev = 0.0; - BytesReference jsonBytes = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).value(elev).endArray().endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - while (parser.currentToken() != Token.START_ARRAY) { - parser.nextToken(); - } + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), containsString("field must be either lat/lon or geohash")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_ArrayWrongType() throws IOException { - double lat = 0.0; - boolean lon = false; - BytesReference jsonBytes = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).endArray().endObject().bytes(); - XContentParser parser = XContentHelper.createParser(jsonBytes); - while (parser.currentToken() != Token.START_ARRAY) { - parser.nextToken(); - } + public void testParseGeoPointArrayTooManyValues() throws IOException { + double lat = 0.0; + double lon = 0.0; + double elev = 0.0; + BytesReference jsonBytes = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).value(elev).endArray().endObject() + .bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + while (parser.currentToken() != Token.START_ARRAY) { + parser.nextToken(); + } + try { GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("only two values allowed")); + } } - @Test(expected=ElasticsearchParseException.class) - public void testParseGeoPoint_InvalidType() throws IOException { + public void testParseGeoPointArrayWrongType() throws IOException { + double lat = 0.0; + boolean lon = false; + BytesReference jsonBytes = jsonBuilder().startObject().startArray("foo").value(lon).value(lat).endArray().endObject().bytes(); + XContentParser parser = XContentHelper.createParser(jsonBytes); + while (parser.currentToken() != Token.START_ARRAY) { + parser.nextToken(); + } + try { + GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("numeric value expected")); + } + } + + public void testParseGeoPointInvalidType() throws IOException { BytesReference jsonBytes = jsonBuilder().startObject().field("foo", 5).endObject().bytes(); XContentParser parser = XContentHelper.createParser(jsonBytes); while (parser.currentToken() != Token.VALUE_NUMBER) { parser.nextToken(); } - GeoUtils.parseGeoPoint(parser); + try { + GeoUtils.parseGeoPoint(parser); + fail("Expected ElasticsearchParseException"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), is("geo_point expected")); + } } - @Test public void testPrefixTreeCellSizes() { assertThat(GeoUtils.EARTH_SEMI_MAJOR_AXIS, equalTo(DistanceUtils.EARTH_EQUATORIAL_RADIUS_KM * 1000)); assertThat(GeoUtils.quadTreeCellWidth(0), lessThanOrEqualTo(GeoUtils.EARTH_EQUATOR)); diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java b/core/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java index 1581693b00a..fa27323c13a 100644 --- a/core/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java @@ -24,19 +24,16 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.join.BitDocIdSetCachingWrapperFilter; +import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ToParentBlockJoinQuery; import org.elasticsearch.common.lucene.search.Queries; @@ -45,7 +42,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.search.MultiValueMode; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -57,122 +53,126 @@ import static org.hamcrest.Matchers.equalTo; */ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldDataTestCase { - @Test + @Override + protected boolean hasDocValues() { + return true; + } + public void testNestedSorting() throws Exception { List docs = new ArrayList<>(); Document document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 1, Field.Store.NO)); + document.add(createField("field1", 1)); docs.add(document); writer.addDocuments(docs); writer.commit(); docs.clear(); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 2, Field.Store.NO)); + document.add(createField("field2", 2)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 2, Field.Store.NO)); + document.add(createField("field1", 2)); docs.add(document); writer.addDocuments(docs); docs.clear(); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 1, Field.Store.NO)); + document.add(createField("field2", 1)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 3, Field.Store.NO)); + document.add(createField("field1", 3)); docs.add(document); writer.addDocuments(docs); docs.clear(); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 4, Field.Store.NO)); + document.add(createField("field2", 4)); document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 4, Field.Store.NO)); + document.add(createField("field1", 4)); docs.add(document); writer.addDocuments(docs); writer.commit(); docs.clear(); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 5, Field.Store.NO)); + document.add(createField("field2", 5)); document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 5, Field.Store.NO)); + document.add(createField("field1", 5)); docs.add(document); writer.addDocuments(docs); writer.commit(); docs.clear(); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 6, Field.Store.NO)); + document.add(createField("field2", 6)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 6, Field.Store.NO)); + document.add(createField("field1", 6)); docs.add(document); writer.addDocuments(docs); writer.commit(); @@ -180,26 +180,26 @@ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldD // This doc will not be included, because it doesn't have nested docs document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 7, Field.Store.NO)); + document.add(createField("field1", 7)); writer.addDocument(document); writer.commit(); docs.clear(); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 3, Field.Store.NO)); + document.add(createField("field2", 3)); document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(createField("field2", 7, Field.Store.NO)); + document.add(createField("field2", 7)); document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); document.add(new StringField("__type", "parent", Field.Store.NO)); - document.add(createField("field1", 8, Field.Store.NO)); + document.add(createField("field1", 8)); docs.add(document); writer.addDocuments(docs); writer.commit(); @@ -217,10 +217,10 @@ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldD MultiValueMode sortMode = MultiValueMode.SUM; IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); - Filter parentFilter = new QueryWrapperFilter(new TermQuery(new Term("__type", "parent"))); - Filter childFilter = new QueryWrapperFilter(Queries.not(parentFilter)); - XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, null, createNested(parentFilter, childFilter)); - ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new FilteredQuery(new MatchAllDocsQuery(), childFilter), new BitDocIdSetCachingWrapperFilter(parentFilter), ScoreMode.None); + Query parentFilter = new TermQuery(new Term("__type", "parent")); + Query childFilter = Queries.not(parentFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, null, createNested(searcher, parentFilter, childFilter)); + ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopFieldDocs topDocs = searcher.search(query, 5, sort); @@ -252,11 +252,11 @@ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldD assertThat(topDocs.scoreDocs[4].doc, equalTo(3)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9)); - childFilter = new QueryWrapperFilter(new TermQuery(new Term("filter_1", "T"))); - nestedComparatorSource = createFieldComparator("field2", sortMode, null, createNested(parentFilter, childFilter)); + childFilter = new TermQuery(new Term("filter_1", "T")); + nestedComparatorSource = createFieldComparator("field2", sortMode, null, createNested(searcher, parentFilter, childFilter)); query = new ToParentBlockJoinQuery( - new FilteredQuery(new MatchAllDocsQuery(), childFilter), - new BitDocIdSetCachingWrapperFilter(parentFilter), + new ConstantScoreQuery(childFilter), + new QueryBitSetProducer(parentFilter), ScoreMode.None ); sort = new Sort(new SortField("field2", nestedComparatorSource, true)); @@ -289,7 +289,7 @@ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldD assertThat(topDocs.scoreDocs[4].doc, equalTo(3)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9)); - nestedComparatorSource = createFieldComparator("field2", sortMode, 127, createNested(parentFilter, childFilter)); + nestedComparatorSource = createFieldComparator("field2", sortMode, 127, createNested(searcher, parentFilter, childFilter)); sort = new Sort(new SortField("field2", nestedComparatorSource, true)); topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort); assertThat(topDocs.totalHits, equalTo(8)); @@ -305,7 +305,7 @@ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldD assertThat(topDocs.scoreDocs[4].doc, equalTo(7)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(8)); - nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(parentFilter, childFilter)); + nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(searcher, parentFilter, childFilter)); sort = new Sort(new SortField("field2", nestedComparatorSource)); topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort); assertThat(topDocs.totalHits, equalTo(8)); @@ -327,11 +327,11 @@ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldD searcher.getIndexReader().close(); } - protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException { + protected void assertAvgScoreMode(Query parentFilter, IndexSearcher searcher) throws IOException { MultiValueMode sortMode = MultiValueMode.AVG; - Filter childFilter = new QueryWrapperFilter(Queries.not(parentFilter)); - XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(parentFilter, childFilter)); - Query query = new ToParentBlockJoinQuery(new FilteredQuery(new MatchAllDocsQuery(), childFilter), new BitDocIdSetCachingWrapperFilter(parentFilter), ScoreMode.None); + Query childFilter = Queries.not(parentFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(searcher, parentFilter, childFilter)); + Query query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopDocs topDocs = searcher.search(query, 5, sort); assertThat(topDocs.totalHits, equalTo(7)); @@ -348,7 +348,7 @@ public abstract class AbstractNumberNestedSortingTestCase extends AbstractFieldD assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(4)); } - protected abstract IndexableField createField(String name, int value, Field.Store store); + protected abstract IndexableField createField(String name, int value); protected abstract IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested); diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java index fde2aa51777..847cd3536e7 100644 --- a/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java @@ -18,29 +18,26 @@ */ package org.elasticsearch.index.search.nested; -import org.apache.lucene.document.DoubleField; -import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.join.BitDocIdSetCachingWrapperFilter; +import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ToParentBlockJoinQuery; +import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; -import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -58,21 +55,21 @@ public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTestCas @Override protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) { - DoubleArrayIndexFieldData fieldData = getForField(fieldName); + IndexNumericFieldData fieldData = getForField(fieldName); return new DoubleValuesComparatorSource(fieldData, missingValue, sortMode, nested); } @Override - protected IndexableField createField(String name, int value, Field.Store store) { - return new DoubleField(name, value, store); + protected IndexableField createField(String name, int value) { + return new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value)); } @Override - protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException { + protected void assertAvgScoreMode(Query parentFilter, IndexSearcher searcher) throws IOException { MultiValueMode sortMode = MultiValueMode.AVG; - Filter childFilter = new QueryWrapperFilter(Queries.not(parentFilter)); - XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(parentFilter, childFilter)); - Query query = new ToParentBlockJoinQuery(new FilteredQuery(new MatchAllDocsQuery(), childFilter), new BitDocIdSetCachingWrapperFilter(parentFilter), ScoreMode.None); + Query childFilter = Queries.not(parentFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(searcher, parentFilter, childFilter)); + Query query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopDocs topDocs = searcher.search(query, 5, sort); assertThat(topDocs.totalHits, equalTo(7)); diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java index 12cd10a2cd2..e63dde2d7fe 100644 --- a/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java @@ -18,29 +18,26 @@ */ package org.elasticsearch.index.search.nested; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.join.BitDocIdSetCachingWrapperFilter; +import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ToParentBlockJoinQuery; +import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; -import org.elasticsearch.index.fielddata.plain.FloatArrayIndexFieldData; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -58,20 +55,20 @@ public class FloatNestedSortingTests extends DoubleNestedSortingTests { @Override protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) { - FloatArrayIndexFieldData fieldData = getForField(fieldName); + IndexNumericFieldData fieldData = getForField(fieldName); return new FloatValuesComparatorSource(fieldData, missingValue, sortMode, nested); } @Override - protected IndexableField createField(String name, int value, Field.Store store) { - return new FloatField(name, value, store); + protected IndexableField createField(String name, int value) { + return new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value)); } - protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException { + protected void assertAvgScoreMode(Query parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException { MultiValueMode sortMode = MultiValueMode.AVG; - Filter childFilter = new QueryWrapperFilter(Queries.not(parentFilter)); - XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(parentFilter, childFilter)); - Query query = new ToParentBlockJoinQuery(new FilteredQuery(new MatchAllDocsQuery(), childFilter), new BitDocIdSetCachingWrapperFilter(parentFilter), ScoreMode.None); + Query childFilter = Queries.not(parentFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(searcher, parentFilter, childFilter)); + Query query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopDocs topDocs = searcher.search(query, 5, sort); assertThat(topDocs.totalHits, equalTo(7)); diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java index 9113222e94a..da7aca4297f 100644 --- a/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java @@ -18,14 +18,13 @@ */ package org.elasticsearch.index.search.nested; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.LongField; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexableField; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; -import org.elasticsearch.index.fielddata.plain.PackedArrayIndexFieldData; import org.elasticsearch.search.MultiValueMode; /** @@ -39,13 +38,13 @@ public class LongNestedSortingTests extends AbstractNumberNestedSortingTestCase @Override protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) { - PackedArrayIndexFieldData fieldData = getForField(fieldName); + IndexNumericFieldData fieldData = getForField(fieldName); return new LongValuesComparatorSource(fieldData, missingValue, sortMode, nested); } @Override - protected IndexableField createField(String name, int value, Field.Store store) { - return new LongField(name, value, store); + protected IndexableField createField(String name, int value) { + return new SortedNumericDocValuesField(name, value); } } diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index 36981fe4a52..49af5f0b82e 100644 --- a/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -28,18 +28,14 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.join.BitDocIdSetCachingWrapperFilter; +import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ToParentBlockJoinQuery; import org.apache.lucene.util.BytesRef; @@ -54,7 +50,6 @@ import org.elasticsearch.index.fielddata.NoOrdinalsStringFieldDataTests; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.search.MultiValueMode; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -66,13 +61,11 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class NestedSortingTests extends AbstractFieldDataTestCase { - @Override protected FieldDataType getFieldDataType() { return new FieldDataType("string", Settings.builder().put("format", "paged_bytes")); } - @Test public void testDuel() throws Exception { final int numDocs = scaledRandomIntBetween(100, 1000); for (int i = 0; i < numDocs; ++i) { @@ -118,15 +111,14 @@ public class NestedSortingTests extends AbstractFieldDataTestCase { } private TopDocs getTopDocs(IndexSearcher searcher, IndexFieldData indexFieldData, String missingValue, MultiValueMode sortMode, int n, boolean reverse) throws IOException { - Filter parentFilter = new QueryWrapperFilter(new TermQuery(new Term("__type", "parent"))); - Filter childFilter = new QueryWrapperFilter(new TermQuery(new Term("__type", "child"))); - XFieldComparatorSource nestedComparatorSource = indexFieldData.comparatorSource(missingValue, sortMode, createNested(parentFilter, childFilter)); + Query parentFilter = new TermQuery(new Term("__type", "parent")); + Query childFilter = new TermQuery(new Term("__type", "child")); + XFieldComparatorSource nestedComparatorSource = indexFieldData.comparatorSource(missingValue, sortMode, createNested(searcher, parentFilter, childFilter)); Query query = new ConstantScoreQuery(parentFilter); Sort sort = new Sort(new SortField("f", nestedComparatorSource, reverse)); return searcher.search(query, n, sort); } - @Test public void testNestedSorting() throws Exception { List docs = new ArrayList<>(); Document document = new Document(); @@ -284,10 +276,10 @@ public class NestedSortingTests extends AbstractFieldDataTestCase { MultiValueMode sortMode = MultiValueMode.MIN; IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); PagedBytesIndexFieldData indexFieldData = getForField("field2"); - Filter parentFilter = new QueryWrapperFilter(new TermQuery(new Term("__type", "parent"))); - Filter childFilter = new QueryWrapperFilter(Queries.not(parentFilter)); - BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, createNested(parentFilter, childFilter)); - ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new FilteredQuery(new MatchAllDocsQuery(), childFilter), new BitDocIdSetCachingWrapperFilter(parentFilter), ScoreMode.None); + Query parentFilter = new TermQuery(new Term("__type", "parent")); + Query childFilter = Queries.not(parentFilter); + BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, createNested(searcher, parentFilter, childFilter)); + ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopFieldDocs topDocs = searcher.search(query, 5, sort); @@ -305,7 +297,7 @@ public class NestedSortingTests extends AbstractFieldDataTestCase { assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("i")); sortMode = MultiValueMode.MAX; - nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, createNested(parentFilter, childFilter)); + nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, createNested(searcher, parentFilter, childFilter)); sort = new Sort(new SortField("field2", nestedComparatorSource, true)); topDocs = searcher.search(query, 5, sort); assertThat(topDocs.totalHits, equalTo(7)); @@ -325,11 +317,11 @@ public class NestedSortingTests extends AbstractFieldDataTestCase { BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(parentFilter, Occur.MUST_NOT); bq.add(new TermQuery(new Term("filter_1", "T")), Occur.MUST); - childFilter = new QueryWrapperFilter(bq.build()); - nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, createNested(parentFilter, childFilter)); + childFilter = bq.build(); + nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, createNested(searcher, parentFilter, childFilter)); query = new ToParentBlockJoinQuery( - new FilteredQuery(new MatchAllDocsQuery(), childFilter), - new BitDocIdSetCachingWrapperFilter(parentFilter), + new ConstantScoreQuery(childFilter), + new QueryBitSetProducer(parentFilter), ScoreMode.None ); sort = new Sort(new SortField("field2", nestedComparatorSource, true)); diff --git a/core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java b/core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java index 3ac624c25e4..8821f0b9e77 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java @@ -19,12 +19,11 @@ package org.elasticsearch.index.shard; -import java.nio.charset.StandardCharsets; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import static org.hamcrest.Matchers.equalTo; @@ -34,10 +33,8 @@ import static org.hamcrest.Matchers.nullValue; * */ public class CommitPointsTests extends ESTestCase { - private final ESLogger logger = Loggers.getLogger(CommitPointsTests.class); - @Test public void testCommitPointXContent() throws Exception { ArrayList indexFiles = new ArrayList<>(); indexFiles.add(new CommitPoint.FileInfo("file1", "file1_p", 100, "ck1")); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java index e8a7a75b1d6..76f20afc696 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java @@ -24,16 +24,12 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.*; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.test.ESTestCase; @@ -45,7 +41,6 @@ import java.util.concurrent.atomic.AtomicInteger; /** */ public class IndexSearcherWrapperTests extends ESTestCase { - private static final EngineConfig ENGINE_CONFIG = new EngineConfig(null, null, null, Settings.EMPTY, null, null, null, null, null, null, new DefaultSimilarity(), null, null, null, null, QueryCachingPolicy.ALWAYS_CACHE, null); public void testReaderCloseListenerIsCalled() throws IOException { Directory dir = newDirectory(); @@ -66,7 +61,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } @@ -75,7 +70,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { final AtomicInteger count = new AtomicInteger(); final AtomicInteger outerCount = new AtomicInteger(); try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - final Engine.Searcher wrap = wrapper.wrap(ENGINE_CONFIG, engineSearcher); + final Engine.Searcher wrap = wrapper.wrap(engineSearcher); assertEquals(1, wrap.reader().getRefCount()); ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> { if (reader == open) { @@ -117,13 +112,13 @@ public class IndexSearcherWrapperTests extends ESTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; final ConcurrentHashMap cache = new ConcurrentHashMap<>(); try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - try (final Engine.Searcher wrap = wrapper.wrap(ENGINE_CONFIG, engineSearcher)) { + try (final Engine.Searcher wrap = wrapper.wrap(engineSearcher)) { ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> { cache.remove(reader.getCoreCacheKey()); }); @@ -153,7 +148,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { searcher.setSimilarity(iwc.getSimilarity()); IndexSearcherWrapper wrapper = new IndexSearcherWrapper(); try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - final Engine.Searcher wrap = wrapper.wrap(ENGINE_CONFIG, engineSearcher); + final Engine.Searcher wrap = wrapper.wrap(engineSearcher); assertSame(wrap, engineSearcher); } IOUtils.close(open, writer, dir); @@ -179,7 +174,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { }; try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { try { - wrapper.wrap(ENGINE_CONFIG, engineSearcher); + wrapper.wrap(engineSearcher); fail("reader must delegate cache key"); } catch (IllegalStateException ex) { // all is well @@ -193,7 +188,7 @@ public class IndexSearcherWrapperTests extends ESTestCase { }; try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { try { - wrapper.wrap(ENGINE_CONFIG, engineSearcher); + wrapper.wrap(engineSearcher); fail("reader must delegate cache key"); } catch (IllegalStateException ex) { // all is well diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 83966a32172..6159147a399 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -20,8 +20,13 @@ package org.elasticsearch.index.shard; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Constants; @@ -42,7 +47,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.*; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -59,9 +63,8 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexServicesProvider; +import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.fielddata.FieldDataStats; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -70,7 +73,6 @@ import org.elasticsearch.index.indexing.IndexingOperationListener; import org.elasticsearch.index.indexing.ShardIndexingService; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.snapshots.IndexShardRepository; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; @@ -78,27 +80,23 @@ import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoveryState; -import org.elasticsearch.test.DummyShardLock; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.VersionUtils; -import org.junit.Test; +import org.elasticsearch.test.*; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.cluster.metadata.IndexMetaData.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.equalTo; @@ -135,17 +133,18 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardId id = new ShardId("foo", 1); long version = between(1, Integer.MAX_VALUE / 2); boolean primary = randomBoolean(); - ShardStateMetaData state1 = new ShardStateMetaData(version, primary, "foo"); + AllocationId allocationId = randomAllocationId(); + ShardStateMetaData state1 = new ShardStateMetaData(version, primary, "foo", allocationId); write(state1, env.availableShardPaths(id)); ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(id)); assertEquals(shardStateMetaData, state1); - ShardStateMetaData state2 = new ShardStateMetaData(version, primary, "foo"); + ShardStateMetaData state2 = new ShardStateMetaData(version, primary, "foo", allocationId); write(state2, env.availableShardPaths(id)); shardStateMetaData = load(logger, env.availableShardPaths(id)); assertEquals(shardStateMetaData, state1); - ShardStateMetaData state3 = new ShardStateMetaData(version + 1, primary, "foo"); + ShardStateMetaData state3 = new ShardStateMetaData(version + 1, primary, "foo", allocationId); write(state3, env.availableShardPaths(id)); shardStateMetaData = load(logger, env.availableShardPaths(id)); assertEquals(shardStateMetaData, state3); @@ -153,7 +152,6 @@ public class IndexShardTests extends ESSingleNodeTestCase { } } - @Test public void testLockTryingToDelete() throws Exception { createIndex("test"); ensureGreen(); @@ -162,7 +160,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { logger.info("--> paths: [{}]", (Object)shardPaths); // Should not be able to acquire the lock because it's already open try { - NodeEnvironment.acquireFSLockForPaths(Settings.EMPTY, shardPaths); + NodeEnvironment.acquireFSLockForPaths(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), shardPaths); fail("should not have been able to acquire the lock"); } catch (LockObtainFailedException e) { assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock")); @@ -172,7 +170,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { // we're green to delete the shard's directory) ShardLock sLock = new DummyShardLock(new ShardId("test", 0)); try { - env.deleteShardDirectoryUnderLock(sLock, Settings.builder().build()); + env.deleteShardDirectoryUnderLock(sLock, IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); fail("should not have been able to delete the directory"); } catch (LockObtainFailedException e) { assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock")); @@ -193,39 +191,38 @@ public class IndexShardTests extends ESSingleNodeTestCase { shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals(shardStateMetaData, getShardStateMetadata(shard)); - assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID))); + assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings().getUUID(), routing.allocationId())); routing = new ShardRouting(shard.shardRouting, shard.shardRouting.version() + 1); shard.updateRoutingEntry(routing, true); shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals(shardStateMetaData, getShardStateMetadata(shard)); - assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID))); + assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings().getUUID(), routing.allocationId())); routing = new ShardRouting(shard.shardRouting, shard.shardRouting.version() + 1); shard.updateRoutingEntry(routing, true); shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals(shardStateMetaData, getShardStateMetadata(shard)); - assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID))); + assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings().getUUID(), routing.allocationId())); // test if we still write it even if the shard is not active ShardRouting inactiveRouting = TestShardRouting.newShardRouting(shard.shardRouting.index(), shard.shardRouting.shardId().id(), shard.shardRouting.currentNodeId(), null, null, true, ShardRoutingState.INITIALIZING, shard.shardRouting.version() + 1); shard.persistMetadata(inactiveRouting, shard.shardRouting); shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals("inactive shard state shouldn't be persisted", shardStateMetaData, getShardStateMetadata(shard)); - assertEquals("inactive shard state shouldn't be persisted", shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID))); - + assertEquals("inactive shard state shouldn't be persisted", shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings().getUUID(), routing.allocationId())); shard.updateRoutingEntry(new ShardRouting(shard.shardRouting, shard.shardRouting.version() + 1), false); shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertFalse("shard state persisted despite of persist=false", shardStateMetaData.equals(getShardStateMetadata(shard))); - assertEquals("shard state persisted despite of persist=false", shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID))); + assertEquals("shard state persisted despite of persist=false", shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings().getUUID(), routing.allocationId())); routing = new ShardRouting(shard.shardRouting, shard.shardRouting.version() + 1); shard.updateRoutingEntry(routing, true); shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals(shardStateMetaData, getShardStateMetadata(shard)); - assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID))); + assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings().getUUID(), routing.allocationId())); } public void testDeleteShardState() throws IOException { @@ -278,29 +275,37 @@ public class IndexShardTests extends ESSingleNodeTestCase { if (shardRouting == null) { return null; } else { - return new ShardStateMetaData(shardRouting.version(), shardRouting.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID)); + return new ShardStateMetaData(shardRouting.version(), shardRouting.primary(), shard.indexSettings().getUUID(), shardRouting.allocationId()); } } + private AllocationId randomAllocationId() { + AllocationId allocationId = AllocationId.newInitializing(); + if (randomBoolean()) { + allocationId = AllocationId.newRelocation(allocationId); + } + return allocationId; + } + public void testShardStateMetaHashCodeEquals() { - ShardStateMetaData meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10)); + ShardStateMetaData meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomAllocationId()); - assertEquals(meta, new ShardStateMetaData(meta.version, meta.primary, meta.indexUUID)); - assertEquals(meta.hashCode(), new ShardStateMetaData(meta.version, meta.primary, meta.indexUUID).hashCode()); + assertEquals(meta, new ShardStateMetaData(meta.version, meta.primary, meta.indexUUID, meta.allocationId)); + assertEquals(meta.hashCode(), new ShardStateMetaData(meta.version, meta.primary, meta.indexUUID, meta.allocationId).hashCode()); - assertFalse(meta.equals(new ShardStateMetaData(meta.version, !meta.primary, meta.indexUUID))); - assertFalse(meta.equals(new ShardStateMetaData(meta.version + 1, meta.primary, meta.indexUUID))); - assertFalse(meta.equals(new ShardStateMetaData(meta.version, !meta.primary, meta.indexUUID + "foo"))); + assertFalse(meta.equals(new ShardStateMetaData(meta.version, !meta.primary, meta.indexUUID, meta.allocationId))); + assertFalse(meta.equals(new ShardStateMetaData(meta.version + 1, meta.primary, meta.indexUUID, meta.allocationId))); + assertFalse(meta.equals(new ShardStateMetaData(meta.version, !meta.primary, meta.indexUUID + "foo", meta.allocationId))); + assertFalse(meta.equals(new ShardStateMetaData(meta.version, !meta.primary, meta.indexUUID + "foo", randomAllocationId()))); Set hashCodes = new HashSet<>(); for (int i = 0; i < 30; i++) { // just a sanity check that we impl hashcode - meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10)); + meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomAllocationId()); hashCodes.add(meta.hashCode()); } assertTrue("more than one unique hashcode expected but got: " + hashCodes.size(), hashCodes.size() > 1); } - @Test public void testDeleteIndexDecreasesCounter() throws InterruptedException, ExecutionException, IOException { assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get()); ensureGreen("test"); @@ -317,7 +322,6 @@ public class IndexShardTests extends ESSingleNodeTestCase { } } - @Test public void testIndexShardCounter() throws InterruptedException, ExecutionException, IOException { assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get()); ensureGreen("test"); @@ -334,22 +338,19 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertEquals(0, indexShard.getOperationsCount()); } - @Test public void testMarkAsInactiveTriggersSyncedFlush() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); client().prepareIndex("test", "test").setSource("{}").get(); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); - // force the shard to become idle now: indicesService.indexService("test").getShardOrNull(0).checkIdle(0); - assertBusy(new Runnable() { // should be very very quick - @Override - public void run() { - IndexStats indexStats = client().admin().indices().prepareStats("test").clear().get().getIndex("test"); - assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID)); + assertBusy(() -> { + IndexStats indexStats = client().admin().indices().prepareStats("test").clear().get().getIndex("test"); + assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID)); + indicesService.indexService("test").getShardOrNull(0).checkIdle(0); } - }); + ); IndexStats indexStats = client().admin().indices().prepareStats("test").get().getIndex("test"); assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID)); } @@ -419,10 +420,10 @@ public class IndexShardTests extends ESSingleNodeTestCase { public void testUpdatePriority() { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(IndexMetaData.SETTING_PRIORITY, 200)); - IndexSettingsService indexSettingsService = getInstanceFromNode(IndicesService.class).indexService("test").settingsService(); - assertEquals(200, indexSettingsService.getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue()); + IndexService indexService = getInstanceFromNode(IndicesService.class).indexService("test"); + assertEquals(200, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue()); client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_PRIORITY, 400).build()).get(); - assertEquals(400, indexSettingsService.getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue()); + assertEquals(400, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue()); } public void testRecoverIntoLeftover() throws IOException { @@ -720,6 +721,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { CyclicBarrier barrier = new CyclicBarrier(numThreads + 1); for (int i = 0; i < threads.length; i++) { threads[i] = new Thread() { + @Override public void run() { try { barrier.await(); @@ -747,7 +749,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertEquals(total + 1, shard.flushStats().getTotal()); } - public void testRecoverFromStore() { + public void testRecoverFromStore() throws IOException { createIndex("test"); ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -761,10 +763,11 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardRouting routing = new ShardRouting(shard.routingEntry()); test.removeShard(0, "b/c simon says so"); ShardRoutingHelper.reinit(routing); - IndexShard newShard = test.createShard(0, routing); + IndexShard newShard = test.createShard(routing); newShard.updateRoutingEntry(routing, false); DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); - assertTrue(newShard.recoverFromStore(routing, localNode)); + newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); + assertTrue(newShard.recoverFromStore(localNode)); routing = new ShardRouting(routing); ShardRoutingHelper.moveToStarted(routing); newShard.updateRoutingEntry(routing, true); @@ -792,10 +795,11 @@ public class IndexShardTests extends ESSingleNodeTestCase { Lucene.cleanLuceneIndex(store.directory()); store.decRef(); ShardRoutingHelper.reinit(routing); - IndexShard newShard = test.createShard(0, routing); + IndexShard newShard = test.createShard(routing); newShard.updateRoutingEntry(routing, false); + newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); try { - newShard.recoverFromStore(routing, localNode); + newShard.recoverFromStore(localNode); fail("index not there!"); } catch (IndexShardRecoveryException ex) { assertTrue(ex.getMessage().contains("failed to fetch index version after copying it over")); @@ -803,12 +807,18 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardRoutingHelper.moveToUnassigned(routing, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "because I say so")); ShardRoutingHelper.initialize(routing, origRouting.currentNodeId()); - - assertFalse("it's already recovering", newShard.recoverFromStore(routing, localNode)); + assertTrue("it's already recovering, we should ignore new ones", newShard.ignoreRecoveryAttempt()); + try { + newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); + fail("we are already recovering, can't mark again"); + } catch (IllegalIndexShardStateException e) { + // OK! + } test.removeShard(0, "I broken it"); - newShard = test.createShard(0, routing); + newShard = test.createShard(routing); newShard.updateRoutingEntry(routing, false); - assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore(routing, localNode)); + newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); + assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore(localNode)); routing = new ShardRouting(routing); ShardRoutingHelper.moveToStarted(routing); @@ -838,13 +848,14 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardRoutingHelper.reinit(routing); routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(new SnapshotId("foo", "bar"), Version.CURRENT, "test")); test_target.removeShard(0, "just do it man!"); - final IndexShard test_target_shard = test_target.createShard(0, routing); + final IndexShard test_target_shard = test_target.createShard(routing); Store sourceStore = test_shard.store(); Store targetStore = test_target_shard.store(); test_target_shard.updateRoutingEntry(routing, false); DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); - assertTrue(test_target_shard.restoreFromRepository(routing, new IndexShardRepository() { + test_target_shard.markAsRecovering("store", new RecoveryState(routing.shardId(), routing.primary(), RecoveryState.Type.SNAPSHOT, routing.restoreSource(), localNode)); + assertTrue(test_target_shard.restoreFromRepository(new IndexShardRepository() { @Override public void snapshot(SnapshotId snapshotId, ShardId shardId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) { } @@ -881,18 +892,6 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertSearchHits(client().prepareSearch("test_target").get(), "0"); } - public void testListenersAreRemoved() { - createIndex("test"); - ensureGreen(); - IndicesService indicesService = getInstanceFromNode(IndicesService.class); - IndexService indexService = indicesService.indexService("test"); - IndexShard shard = indexService.getShardOrNull(0); - IndexSettingsService settingsService = indexService.settingsService(); - assertTrue(settingsService.isRegistered(shard)); - indexService.removeShard(0, "simon says so"); - assertFalse(settingsService.isRegistered(shard)); - } - public void testSearcherWrapperIsUsed() throws IOException { createIndex("test"); ensureGreen(); @@ -919,7 +918,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; @@ -958,7 +957,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; @@ -969,13 +968,15 @@ public class IndexShardTests extends ESSingleNodeTestCase { MappedFieldType foo = newShard.mapperService().indexName("foo"); IndexFieldData.Global ifd = shard.indexFieldDataService().getForField(foo); FieldDataStats before = shard.fieldData().stats("foo"); + assertThat(before.getMemorySizeInBytes(), equalTo(0l)); FieldDataStats after = null; try (Engine.Searcher searcher = newShard.acquireSearcher("test")) { assumeTrue("we have to have more than one segment", searcher.getDirectoryReader().leaves().size() > 1); IndexFieldData indexFieldData = ifd.loadGlobal(searcher.getDirectoryReader()); after = shard.fieldData().stats("foo"); assertEquals(after.getEvictions(), before.getEvictions()); - assertTrue(indexFieldData.toString(), after.getMemorySizeInBytes() > before.getMemorySizeInBytes()); + // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached: + assertThat(after.getMemorySizeInBytes(), equalTo(0l)); } assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions()); assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes()); @@ -1001,8 +1002,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { throw new RuntimeException("boom"); } - @Override - public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { return searcher; } }; @@ -1022,42 +1022,17 @@ public class IndexShardTests extends ESSingleNodeTestCase { private final IndexShard reinitWithWrapper(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper) throws IOException { ShardRouting routing = new ShardRouting(shard.routingEntry()); shard.close("simon says", true); - IndexServicesProvider indexServices = indexService.getIndexServices(); - IndexServicesProvider newProvider = new IndexServicesProvider(indexServices.getIndicesLifecycle(), indexServices.getThreadPool(), indexServices.getMapperService(), indexServices.getQueryParserService(), indexServices.getIndexCache(), indexServices.getIndicesQueryCache(), indexServices.getCodecService(), indexServices.getTermVectorsService(), indexServices.getIndexFieldDataService(), indexServices.getWarmer(), indexServices.getSimilarityService(), indexServices.getFactory(), indexServices.getBigArrays(), wrapper, indexServices.getIndexingMemoryController()); - IndexShard newShard = new IndexShard(shard.shardId(), shard.indexSettings, shard.shardPath(), shard.store(), newProvider); + NodeServicesProvider indexServices = indexService.getIndexServices(); + IndexShard newShard = new IndexShard(shard.shardId(), indexService.getIndexSettings(), shard.shardPath(), shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(), indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper, indexServices); ShardRoutingHelper.reinit(routing); newShard.updateRoutingEntry(routing, false); DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); - assertTrue(newShard.recoverFromStore(routing, localNode)); + newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); + assertTrue(newShard.recoverFromStore(localNode)); routing = new ShardRouting(routing); ShardRoutingHelper.moveToStarted(routing); newShard.updateRoutingEntry(routing, true); return newShard; } - private static class FieldMaskingReader extends FilterDirectoryReader { - - private final String field; - public FieldMaskingReader(String field, DirectoryReader in) throws IOException { - super(in, new SubReaderWrapper() { - private final String filteredField = field; - @Override - public LeafReader wrap(LeafReader reader) { - return new FieldFilterLeafReader(reader, Collections.singleton(field), true); - } - }); - this.field = field; - - } - - @Override - protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return new FieldMaskingReader(field, in); - } - - @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); - } - } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/MergePolicySettingsTests.java b/core/src/test/java/org/elasticsearch/index/shard/MergePolicySettingsTests.java index 0f6b2bd8d9e..84fe7b1f96b 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/MergePolicySettingsTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/MergePolicySettingsTests.java @@ -24,9 +24,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; @@ -34,12 +32,9 @@ import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.hamcrest.Matchers.equalTo; public class MergePolicySettingsTests extends ESTestCase { - protected final ShardId shardId = new ShardId(new Index("index"), 1); - @Test public void testCompoundFileSettings() throws IOException { - assertThat(new MergePolicyConfig(logger, EMPTY_SETTINGS).getMergePolicy().getNoCFSRatio(), equalTo(0.1)); assertThat(new MergePolicyConfig(logger, build(true)).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); assertThat(new MergePolicyConfig(logger, build(0.5)).getMergePolicy().getNoCFSRatio(), equalTo(0.5)); @@ -58,21 +53,19 @@ public class MergePolicySettingsTests extends ESTestCase { assertTrue(mp.getMergePolicy() instanceof NoMergePolicy); } - @Test public void testUpdateSettings() throws IOException { { - IndexSettingsService service = new IndexSettingsService(new Index("test"), EMPTY_SETTINGS); MergePolicyConfig mp = new MergePolicyConfig(logger, EMPTY_SETTINGS); - assertThat(((TieredMergePolicy) mp.getMergePolicy()).getNoCFSRatio(), equalTo(0.1)); + assertThat((mp.getMergePolicy()).getNoCFSRatio(), equalTo(0.1)); mp.onRefreshSettings(build(1.0)); - assertThat(((TieredMergePolicy) mp.getMergePolicy()).getNoCFSRatio(), equalTo(1.0)); + assertThat((mp.getMergePolicy()).getNoCFSRatio(), equalTo(1.0)); mp.onRefreshSettings(build(0.1)); - assertThat(((TieredMergePolicy) mp.getMergePolicy()).getNoCFSRatio(), equalTo(0.1)); + assertThat((mp.getMergePolicy()).getNoCFSRatio(), equalTo(0.1)); mp.onRefreshSettings(build(0.0)); - assertThat(((TieredMergePolicy) mp.getMergePolicy()).getNoCFSRatio(), equalTo(0.0)); + assertThat((mp.getMergePolicy()).getNoCFSRatio(), equalTo(0.0)); } diff --git a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java index e2affa21f10..449fd1df96b 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java @@ -20,14 +20,16 @@ package org.elasticsearch.index.shard; import org.apache.lucene.mockfile.FilterFileSystemProvider; -import org.apache.lucene.util.Constants; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment.NodePath; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.env.NodeEnvironment.NodePath; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -38,15 +40,13 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttributeView; import java.nio.file.attribute.FileStoreAttributeView; import java.nio.file.spi.FileSystemProvider; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** Separate test class from ShardPathTests because we need static (BeforeClass) setup to install mock filesystems... */ public class NewPathForShardTests extends ESTestCase { + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings(new Index("index"), Settings.EMPTY); + // Sneakiness to install mock file stores so we can pretend how much free space we have on each path.data: private static MockFileStore aFileStore = new MockFileStore("mocka"); private static MockFileStore bFileStore = new MockFileStore("mockb"); @@ -179,7 +179,7 @@ public class NewPathForShardTests extends ESTestCase { bFileStore.usableSpace = 1000; ShardId shardId = new ShardId("index", 0); - ShardPath result = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, Collections.emptyMap()); + ShardPath result = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, Collections.emptyMap()); assertTrue(result.getDataPath().toString().contains(aPathPart)); // Test the reverse: b has lots of free space, but a has little, so new shard should go to b: @@ -187,7 +187,7 @@ public class NewPathForShardTests extends ESTestCase { bFileStore.usableSpace = 100000; shardId = new ShardId("index", 0); - result = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, Collections.emptyMap()); + result = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, Collections.emptyMap()); assertTrue(result.getDataPath().toString().contains(bPathPart)); // Now a and be have equal usable space; we allocate two shards to the node, and each should go to different paths: @@ -195,9 +195,9 @@ public class NewPathForShardTests extends ESTestCase { bFileStore.usableSpace = 100000; Map dataPathToShardCount = new HashMap<>(); - ShardPath result1 = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount); + ShardPath result1 = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, dataPathToShardCount); dataPathToShardCount.put(NodeEnvironment.shardStatePathToDataPath(result1.getDataPath()), 1); - ShardPath result2 = ShardPath.selectNewPathForShard(nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount); + ShardPath result2 = ShardPath.selectNewPathForShard(nodeEnv, shardId, INDEX_SETTINGS, 100, dataPathToShardCount); // #11122: this was the original failure: on a node with 2 disks that have nearly equal // free space, we would always allocate all N incoming shards to the one path that diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index 32d20190890..5a82a8942aa 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -18,32 +18,34 @@ */ package org.elasticsearch.index.shard; -import com.carrotsearch.randomizedtesting.annotations.Repeat; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Path; -import java.util.Set; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; /** */ public class ShardPathTests extends ESTestCase { - public void testLoadShardPath() throws IOException { try (final NodeEnvironment env = newNodeEnvironment(settingsBuilder().build())) { - Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF"); + Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF") + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); Settings settings = builder.build(); ShardId shardId = new ShardId("foo", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path); - ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, settings); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path); + ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), settings)); assertEquals(path, shardPath.getDataPath()); assertEquals("0xDEADBEEF", shardPath.getIndexUUID()); assertEquals("foo", shardPath.getShardId().getIndex()); @@ -52,38 +54,48 @@ public class ShardPathTests extends ESTestCase { } } - @Test(expected = IllegalStateException.class) public void testFailLoadShardPathOnMultiState() throws IOException { try (final NodeEnvironment env = newNodeEnvironment(settingsBuilder().build())) { - Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF"); + Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF") + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); Settings settings = builder.build(); ShardId shardId = new ShardId("foo", 0); Path[] paths = env.availableShardPaths(shardId); assumeTrue("This test tests multi data.path but we only got one", paths.length > 1); int id = randomIntBetween(1, 10); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF"), id, paths); - ShardPath.loadShardPath(logger, env, shardId, settings); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, paths); + ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), settings)); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), containsString("more than one shard state found")); } } - @Test(expected = IllegalStateException.class) public void testFailLoadShardPathIndexUUIDMissmatch() throws IOException { try (final NodeEnvironment env = newNodeEnvironment(settingsBuilder().build())) { - Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "foobar"); + Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "foobar") + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); Settings settings = builder.build(); ShardId shardId = new ShardId("foo", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); int id = randomIntBetween(1, 10); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF"), id, path); - ShardPath.loadShardPath(logger, env, shardId, settings); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, path); + ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), settings)); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), containsString("expected: foobar on shard path")); } } - @Test(expected = IllegalArgumentException.class) public void testIllegalCustomDataPath() { final Path path = createTempDir().resolve("foo").resolve("0"); - new ShardPath(true, path, path, "foo", new ShardId("foo", 0)); + try { + new ShardPath(true, path, path, "foo", new ShardId("foo", 0)); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("shard state path must be different to the data path when using custom data paths")); + } } public void testValidCtor() { @@ -98,7 +110,9 @@ public class ShardPathTests extends ESTestCase { boolean useCustomDataPath = randomBoolean(); final Settings indexSetttings; final Settings nodeSettings; - Settings.Builder indexSettingsBuilder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF"); + Settings.Builder indexSettingsBuilder = settingsBuilder() + .put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF") + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); final Path customPath; if (useCustomDataPath) { final Path path = createTempDir(); @@ -120,8 +134,8 @@ public class ShardPathTests extends ESTestCase { ShardId shardId = new ShardId("foo", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path); - ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, indexSetttings); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path); + ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), indexSetttings)); boolean found = false; for (Path p : env.nodeDataPaths()) { if (p.equals(shardPath.getRootStatePath())) { diff --git a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityModuleTests.java b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityModuleTests.java deleted file mode 100644 index a73d2a5dac4..00000000000 --- a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityModuleTests.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.similarity; - -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.CollectionStatistics; -import org.apache.lucene.search.TermStatistics; -import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.search.similarities.Similarity; -import org.elasticsearch.common.inject.ModuleTestCase; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; - -import java.io.IOException; - -public class SimilarityModuleTests extends ModuleTestCase { - - public void testAddSimilarity() { - Settings indexSettings = Settings.settingsBuilder() - .put("index.similarity.my_similarity.type", "test_similarity") - .put("index.similarity.my_similarity.key", "there is a key") - .build(); - SimilarityModule module = new SimilarityModule(new Index("foo"), indexSettings); - module.addSimilarity("test_similarity", (string, settings) -> new SimilarityProvider() { - @Override - public String name() { - return string; - } - - @Override - public Similarity get() { - return new TestSimilarity(settings.get("key")); - } - }); - assertInstanceBinding(module, SimilarityService.class, (inst) -> { - if (inst instanceof SimilarityService) { - assertNotNull(inst.getSimilarity("my_similarity")); - assertTrue(inst.getSimilarity("my_similarity").get() instanceof TestSimilarity); - assertEquals("my_similarity", inst.getSimilarity("my_similarity").name()); - assertEquals("there is a key" , ((TestSimilarity)inst.getSimilarity("my_similarity").get()).key); - return true; - } - return false; - }); - } - - public void testSetupUnknownSimilarity() { - Settings indexSettings = Settings.settingsBuilder() - .put("index.similarity.my_similarity.type", "test_similarity") - .build(); - SimilarityModule module = new SimilarityModule(new Index("foo"), indexSettings); - try { - assertInstanceBinding(module, SimilarityService.class, (inst) -> inst instanceof SimilarityService); - } catch (IllegalArgumentException ex) { - assertEquals("Unknown Similarity type [test_similarity] for [my_similarity]", ex.getMessage()); - } - } - - - public void testSetupWithoutType() { - Settings indexSettings = Settings.settingsBuilder() - .put("index.similarity.my_similarity.foo", "bar") - .build(); - SimilarityModule module = new SimilarityModule(new Index("foo"), indexSettings); - try { - assertInstanceBinding(module, SimilarityService.class, (inst) -> inst instanceof SimilarityService); - } catch (IllegalArgumentException ex) { - assertEquals("Similarity [my_similarity] must have an associated type", ex.getMessage()); - } - } - - - private static class TestSimilarity extends Similarity { - private final Similarity delegate = new BM25Similarity(); - private final String key; - - - public TestSimilarity(String key) { - if (key == null) { - throw new AssertionError("key is null"); - } - this.key = key; - } - - @Override - public long computeNorm(FieldInvertState state) { - return delegate.computeNorm(state); - } - - @Override - public SimWeight computeWeight(CollectionStatistics collectionStats, TermStatistics... termStats) { - return delegate.computeWeight(collectionStats, termStats); - } - - @Override - public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException { - return delegate.simScorer(weight, context); - } - } -} diff --git a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java index 28f5e5c62f6..965916284a3 100644 --- a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java +++ b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java @@ -19,13 +19,22 @@ package org.elasticsearch.index.similarity; -import org.apache.lucene.search.similarities.*; +import org.apache.lucene.search.similarities.AfterEffectL; +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.BasicModelG; +import org.apache.lucene.search.similarities.DFRSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DistributionSPL; +import org.apache.lucene.search.similarities.IBSimilarity; +import org.apache.lucene.search.similarities.LMDirichletSimilarity; +import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; +import org.apache.lucene.search.similarities.LambdaTTF; +import org.apache.lucene.search.similarities.NormalizationH2; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.io.IOException; @@ -33,15 +42,12 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; public class SimilarityTests extends ESSingleNodeTestCase { - - @Test public void testResolveDefaultSimilarities() { SimilarityService similarityService = createIndex("foo").similarityService(); assertThat(similarityService.getSimilarity("default").get(), instanceOf(DefaultSimilarity.class)); assertThat(similarityService.getSimilarity("BM25").get(), instanceOf(BM25Similarity.class)); } - @Test public void testResolveSimilaritiesFromMapping_default() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -61,7 +67,6 @@ public class SimilarityTests extends ESSingleNodeTestCase { assertThat(similarity.getDiscountOverlaps(), equalTo(false)); } - @Test public void testResolveSimilaritiesFromMapping_bm25() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -85,7 +90,6 @@ public class SimilarityTests extends ESSingleNodeTestCase { assertThat(similarity.getDiscountOverlaps(), equalTo(false)); } - @Test public void testResolveSimilaritiesFromMapping_DFR() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -111,7 +115,6 @@ public class SimilarityTests extends ESSingleNodeTestCase { assertThat(((NormalizationH2) similarity.getNormalization()).getC(), equalTo(3f)); } - @Test public void testResolveSimilaritiesFromMapping_IB() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -137,7 +140,6 @@ public class SimilarityTests extends ESSingleNodeTestCase { assertThat(((NormalizationH2) similarity.getNormalization()).getC(), equalTo(3f)); } - @Test public void testResolveSimilaritiesFromMapping_LMDirichlet() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") @@ -157,7 +159,6 @@ public class SimilarityTests extends ESSingleNodeTestCase { assertThat(similarity.getMu(), equalTo(3000f)); } - @Test public void testResolveSimilaritiesFromMapping_LMJelinekMercer() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java index cd1b05f1528..650388c5f09 100644 --- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java +++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java @@ -22,11 +22,14 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.Fields; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.Fields; -import org.junit.Test; import java.io.IOException; @@ -37,8 +40,6 @@ import static org.hamcrest.Matchers.is; /** */ public class FileInfoTests extends ESTestCase { - - @Test public void testToFromXContent() throws IOException { final int iters = scaledRandomIntBetween(1, 10); for (int iter = 0; iter < iters; iter++) { @@ -71,7 +72,6 @@ public class FileInfoTests extends ESTestCase { } } - @Test public void testInvalidFieldsInFromXContent() throws IOException { final int iters = scaledRandomIntBetween(1, 10); for (int iter = 0; iter < iters; iter++) { diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java index e9deadb9d04..38fc17c777e 100644 --- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java +++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java @@ -19,18 +19,20 @@ package org.elasticsearch.index.snapshots.blobstore; import com.carrotsearch.randomizedtesting.generators.RandomInts; -import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import java.io.*; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Random; import static org.hamcrest.Matchers.equalTo; public class SlicedInputStreamTests extends ESTestCase { - - @Test - public void readRandom() throws IOException { + public void testReadRandom() throws IOException { int parts = randomIntBetween(1, 20); ByteArrayOutputStream stream = new ByteArrayOutputStream(); int numWriteOps = scaledRandomIntBetween(1000, 10000); diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index f32b6b0b994..53d18eaef81 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -19,28 +19,21 @@ package org.elasticsearch.index.store; import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import java.nio.charset.StandardCharsets; - import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexFileNames; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.GroupShardsIterator; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.ShardIterator; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.elasticsearch.common.Nullable; @@ -50,46 +43,33 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.discovery.Discovery; import org.elasticsearch.gateway.PrimaryShardAllocator; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.IndexShardState; -import org.elasticsearch.index.shard.MergePolicyConfig; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndicesLifecycle; -import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.index.shard.*; import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.snapshots.SnapshotState; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.CorruptionUtils; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.store.MockFSDirectoryService; +import org.elasticsearch.test.MockIndexEventListener; +import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import org.junit.Test; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -98,20 +78,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.hamcrest.Matchers.*; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) public class CorruptedFileIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -127,13 +98,12 @@ public class CorruptedFileIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return pluginList(MockTransportService.TestPlugin.class); + return pluginList(MockTransportService.TestPlugin.class, MockIndexEventListener.TestPlugin.class); } /** * Tests that we can actually recover from a corruption on the primary given that we have replica shards around. */ - @Test public void testCorruptFileAndRecover() throws ExecutionException, InterruptedException, IOException { int numDocs = scaledRandomIntBetween(100, 1000); // have enough space for 3 copies @@ -148,7 +118,7 @@ public class CorruptedFileIT extends ESIntegTestCase { .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose .put(IndexShard.INDEX_TRANSLOG_DISABLE_FLUSH, true) // no translog based flush - it might change the .liv / segments.N files .put("indices.recovery.concurrent_streams", 10) )); @@ -162,7 +132,7 @@ public class CorruptedFileIT extends ESIntegTestCase { ensureGreen(); assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet()); // we have to flush at least once here since we don't corrupt the translog - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); final int numShards = numShards("test"); @@ -197,11 +167,11 @@ public class CorruptedFileIT extends ESIntegTestCase { */ final CountDownLatch latch = new CountDownLatch(numShards * 3); // primary + 2 replicas final CopyOnWriteArrayList exception = new CopyOnWriteArrayList<>(); - final IndicesLifecycle.Listener listener = new IndicesLifecycle.Listener() { + final IndexEventListener listener = new IndexEventListener() { @Override - public void afterIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard, @IndexSettings Settings indexSettings) { + public void afterIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard, Settings indexSettings) { if (indexShard != null) { - Store store = ((IndexShard) indexShard).store(); + Store store = indexShard.store(); store.incRef(); try { if (!Lucene.indexExists(store.directory()) && indexShard.state() == IndexShardState.STARTED) { @@ -228,16 +198,16 @@ public class CorruptedFileIT extends ESIntegTestCase { } }; - for (IndicesService service : internalCluster().getDataNodeInstances(IndicesService.class)) { - service.indicesLifecycle().addListener(listener); + for (MockIndexEventListener.TestEventListener eventListener : internalCluster().getDataNodeInstances(MockIndexEventListener.TestEventListener.class)) { + eventListener.setNewDelegate(listener); } try { client().admin().indices().prepareDelete("test").get(); latch.await(); assertThat(exception, empty()); } finally { - for (IndicesService service : internalCluster().getDataNodeInstances(IndicesService.class)) { - service.indicesLifecycle().removeListener(listener); + for (MockIndexEventListener.TestEventListener eventListener : internalCluster().getDataNodeInstances(MockIndexEventListener.TestEventListener.class)) { + eventListener.setNewDelegate(null); } } } @@ -246,7 +216,6 @@ public class CorruptedFileIT extends ESIntegTestCase { * Tests corruption that happens on a single shard when no replicas are present. We make sure that the primary stays unassigned * and all other replicas for the healthy shards happens */ - @Test public void testCorruptPrimaryNoReplica() throws ExecutionException, InterruptedException, IOException { int numDocs = scaledRandomIntBetween(100, 1000); internalCluster().ensureAtLeastNumDataNodes(2); @@ -254,7 +223,7 @@ public class CorruptedFileIT extends ESIntegTestCase { assertAcked(prepareCreate("test").setSettings(Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose .put(IndexShard.INDEX_TRANSLOG_DISABLE_FLUSH, true) // no translog based flush - it might change the .liv / segments.N files .put("indices.recovery.concurrent_streams", 10) )); @@ -267,7 +236,7 @@ public class CorruptedFileIT extends ESIntegTestCase { ensureGreen(); assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet()); // we have to flush at least once here since we don't corrupt the translog - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); ShardRouting shardRouting = corruptRandomPrimaryFile(); @@ -330,7 +299,7 @@ public class CorruptedFileIT extends ESIntegTestCase { } assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2)); - Collections.shuffle(dataNodeStats, getRandom()); + Collections.shuffle(dataNodeStats, random()); NodeStats primariesNode = dataNodeStats.get(0); NodeStats unluckyNode = dataNodeStats.get(1); assertAcked(prepareCreate("test").setSettings(Settings.builder() @@ -345,7 +314,7 @@ public class CorruptedFileIT extends ESIntegTestCase { final CountDownLatch hasCorrupted = new CountDownLatch(1); for (NodeStats dataNode : dataNodeStats) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().name())); - mockTransportService.addDelegate(internalCluster().getInstance(Discovery.class, unluckyNode.getNode().name()).localNode(), new MockTransportService.DelegateTransport(mockTransportService.original()) { + mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().name()), new MockTransportService.DelegateTransport(mockTransportService.original()) { @Override public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { @@ -375,7 +344,6 @@ public class CorruptedFileIT extends ESIntegTestCase { * Tests corruption that happens on the network layer and that the primary does not get affected by corruption that happens on the way * to the replica. The file on disk stays uncorrupted */ - @Test public void testCorruptionOnNetworkLayer() throws ExecutionException, InterruptedException { int numDocs = scaledRandomIntBetween(100, 1000); internalCluster().ensureAtLeastNumDataNodes(2); @@ -391,7 +359,7 @@ public class CorruptedFileIT extends ESIntegTestCase { } assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2)); - Collections.shuffle(dataNodeStats, getRandom()); + Collections.shuffle(dataNodeStats, random()); NodeStats primariesNode = dataNodeStats.get(0); NodeStats unluckyNode = dataNodeStats.get(1); @@ -400,7 +368,7 @@ public class CorruptedFileIT extends ESIntegTestCase { .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 4)) // don't go crazy here it must recovery fast // This does corrupt files on the replica, so we can't check: - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) .put("index.routing.allocation.include._name", primariesNode.getNode().name()) .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE, EnableAllocationDecider.Rebalance.NONE) )); @@ -413,12 +381,12 @@ public class CorruptedFileIT extends ESIntegTestCase { ensureGreen(); assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet()); // we have to flush at least once here since we don't corrupt the translog - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); final boolean truncate = randomBoolean(); for (NodeStats dataNode : dataNodeStats) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().name())); - mockTransportService.addDelegate(internalCluster().getInstance(Discovery.class, unluckyNode.getNode().name()).localNode(), new MockTransportService.DelegateTransport(mockTransportService.original()) { + mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().name()), new MockTransportService.DelegateTransport(mockTransportService.original()) { @Override public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { @@ -474,7 +442,6 @@ public class CorruptedFileIT extends ESIntegTestCase { * TODO once checksum verification on snapshotting is implemented this test needs to be fixed or split into several * parts... We should also corrupt files on the actual snapshot and check that we don't restore the corrupted shard. */ - @Test public void testCorruptFileThenSnapshotAndRestore() throws ExecutionException, InterruptedException, IOException { int numDocs = scaledRandomIntBetween(100, 1000); internalCluster().ensureAtLeastNumDataNodes(2); @@ -482,7 +449,7 @@ public class CorruptedFileIT extends ESIntegTestCase { assertAcked(prepareCreate("test").setSettings(Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") // no replicas for this test .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose .put(IndexShard.INDEX_TRANSLOG_DISABLE_FLUSH, true) // no translog based flush - it might change the .liv / segments.N files .put("indices.recovery.concurrent_streams", 10) )); @@ -495,7 +462,7 @@ public class CorruptedFileIT extends ESIntegTestCase { ensureGreen(); assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet()); // we have to flush at least once here since we don't corrupt the translog - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); ShardRouting shardRouting = corruptRandomPrimaryFile(false); @@ -529,7 +496,6 @@ public class CorruptedFileIT extends ESIntegTestCase { * nodes, so that replica won't be sneaky and allocated on a node that doesn't have a corrupted * replica. */ - @Test public void testReplicaCorruption() throws Exception { int numDocs = scaledRandomIntBetween(100, 1000); internalCluster().ensureAtLeastNumDataNodes(2); @@ -538,7 +504,7 @@ public class CorruptedFileIT extends ESIntegTestCase { .put(PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS, "one") .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1) .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose .put(IndexShard.INDEX_TRANSLOG_DISABLE_FLUSH, true) // no translog based flush - it might change the .liv / segments.N files .put("indices.recovery.concurrent_streams", 10) )); @@ -551,7 +517,7 @@ public class CorruptedFileIT extends ESIntegTestCase { ensureGreen(); assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet()); // we have to flush at least once here since we don't corrupt the translog - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); final Map> filesToCorrupt = findFilesToCorruptForReplica(); diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java index c5158b782e3..653a7d04e9e 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.store; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; @@ -37,7 +38,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.engine.MockEngineSupport; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; -import org.junit.Test; import java.io.IOException; import java.nio.ByteBuffer; @@ -62,7 +62,6 @@ import static org.hamcrest.Matchers.notNullValue; */ @ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE, numDataNodes = 0) public class CorruptedTranslogIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { // we really need local GW here since this also checks for corruption etc. @@ -70,7 +69,6 @@ public class CorruptedTranslogIT extends ESIntegTestCase { return pluginList(MockTransportService.TestPlugin.class); } - @Test @TestLogging("index.translog:TRACE,index.gateway:TRACE") public void testCorruptTranslogFiles() throws Exception { internalCluster().startNodesAsync(1, Settings.EMPTY).get(); @@ -153,12 +151,12 @@ public class CorruptedTranslogIT extends ESIntegTestCase { ByteBuffer bb = ByteBuffer.wrap(new byte[1]); raf.read(bb); bb.flip(); - + // corrupt byte oldValue = bb.get(0); byte newValue = (byte) (oldValue + 1); bb.put(0, newValue); - + // rewrite raf.position(filePointer); raf.write(bb); diff --git a/core/src/test/java/org/elasticsearch/index/store/DirectoryUtilsTests.java b/core/src/test/java/org/elasticsearch/index/store/DirectoryUtilsTests.java index 04ae4676fe4..57265872c4c 100644 --- a/core/src/test/java/org/elasticsearch/index/store/DirectoryUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/DirectoryUtilsTests.java @@ -18,20 +18,23 @@ */ package org.elasticsearch.index.store; -import org.apache.lucene.store.*; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FileSwitchDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.RAMDirectory; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.nio.file.Path; import java.util.Collections; import java.util.Set; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.CoreMatchers.sameInstance; public class DirectoryUtilsTests extends ESTestCase { - - @Test public void testGetLeave() throws IOException { Path file = createTempDir(); final int iters = scaledRandomIntBetween(10, 100); diff --git a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java index 91d17f45038..f5b7fc250aa 100644 --- a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java @@ -26,15 +26,17 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.TransportShardBulkAction; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.*; -import org.junit.Test; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.Collection; @@ -45,7 +47,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -60,7 +64,7 @@ public class ExceptionRetryIT extends ESIntegTestCase { @Override protected void beforeIndexDeletion() { // a write operation might still be in flight when the test has finished - // so we should not check the operation counter here + // so we should not check the operation counter here } /** @@ -68,7 +72,6 @@ public class ExceptionRetryIT extends ESIntegTestCase { * If auto generated ids are used this must not lead to duplicate ids * see https://github.com/elasticsearch/elasticsearch/issues/8788 */ - @Test public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException { final AtomicBoolean exceptionThrown = new AtomicBoolean(false); int numDocs = scaledRandomIntBetween(100, 1000); @@ -80,7 +83,7 @@ public class ExceptionRetryIT extends ESIntegTestCase { //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry. for (NodeStats dataNode : nodeStats.getNodes()) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().name())); - mockTransportService.addDelegate(internalCluster().getInstance(Discovery.class, unluckyNode.getNode().name()).localNode(), new MockTransportService.DelegateTransport(mockTransportService.original()) { + mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().name()), new MockTransportService.DelegateTransport(mockTransportService.original()) { @Override public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index e921f95394c..ee3ad6b8b29 100644 --- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -21,10 +21,16 @@ package org.elasticsearch.index.store; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.store.*; import org.apache.lucene.util.Constants; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Path; @@ -36,10 +42,12 @@ public class IndexStoreTests extends ESTestCase { public void testStoreDirectory() throws IOException { final Path tempDir = createTempDir().resolve("foo").resolve("0"); - final IndexStoreModule.Type[] values = IndexStoreModule.Type.values(); - final IndexStoreModule.Type type = RandomPicks.randomFrom(random(), values); - Settings settings = Settings.settingsBuilder().put(IndexStoreModule.STORE_TYPE, type.name().toLowerCase(Locale.ROOT)).build(); - FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); + final IndexModule.Type[] values = IndexModule.Type.values(); + final IndexModule.Type type = RandomPicks.randomFrom(random(), values); + Settings settings = Settings.settingsBuilder().put(IndexModule.STORE_TYPE, type.name().toLowerCase(Locale.ROOT)) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(new Index("foo"), settings); + FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { switch (type) { case NIOFS: @@ -71,13 +79,14 @@ public class IndexStoreTests extends ESTestCase { public void testStoreDirectoryDefault() throws IOException { final Path tempDir = createTempDir().resolve("foo").resolve("0"); - Settings settings = Settings.EMPTY; - FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); + FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings(new Index("foo"), Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { if (Constants.WINDOWS) { assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory); - } else { + } else if (Constants.JRE_IS_64BIT) { assertTrue(directory.toString(), directory instanceof FileSwitchDirectory); + } else { + assertTrue(directory.toString(), directory instanceof NIOFSDirectory); } } } diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java index 123e4e01bcc..1e1e9487668 100644 --- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -23,41 +23,16 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.lucene50.Lucene50SegmentInfoFormat; -import org.apache.lucene.codecs.lucene53.Lucene53Codec; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.IndexNotFoundException; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; -import org.apache.lucene.index.NoDeletionPolicy; -import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.index.SnapshotDeletionPolicy; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.BaseDirectoryWrapper; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.codecs.lucene54.Lucene54Codec; +import org.apache.lucene.document.*; +import org.apache.lucene.index.*; +import org.apache.lucene.store.*; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.Version; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; @@ -66,14 +41,15 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.ShardLock; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -81,38 +57,25 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.Adler32; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.test.VersionUtils.randomVersion; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class StoreTests extends ESTestCase { - @Test + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings(new Index("index"), Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT).build()); + public void testRefCount() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + IndexSettings indexSettings = INDEX_SETTINGS; + + Store store = new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId)); int incs = randomIntBetween(1, 100); for (int i = 0; i < incs; i++) { if (randomBoolean()) { @@ -129,7 +92,6 @@ public class StoreTests extends ESTestCase { } store.incRef(); - final AtomicBoolean called = new AtomicBoolean(false); store.close(); for (int i = 0; i < incs; i++) { if (randomBoolean()) { @@ -162,7 +124,6 @@ public class StoreTests extends ESTestCase { } } - @Test public void testVerifyingIndexOutput() throws IOException { Directory dir = newDirectory(); IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT); @@ -276,7 +237,6 @@ public class StoreTests extends ESTestCase { } } - @Test public void testVerifyingIndexOutputWithBogusInput() throws IOException { Directory dir = newDirectory(); int length = scaledRandomIntBetween(10, 1024); @@ -297,7 +257,7 @@ public class StoreTests extends ESTestCase { private static final class OldSIMockingCodec extends FilterCodec { protected OldSIMockingCodec() { - super(new Lucene53Codec().getName(), new Lucene53Codec()); + super(new Lucene54Codec().getName(), new Lucene54Codec()); } @Override @@ -359,11 +319,10 @@ public class StoreTests extends ESTestCase { // The test currently fails because the segment infos and the index don't // agree on the oldest version of a segment. We should fix this test by // switching to a static bw index - @Test public void testWriteLegacyChecksums() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); // set default codec - all segments need checksums final boolean usesOldCodec = randomBoolean(); IndexWriter writer = new IndexWriter(store.directory(), newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(usesOldCodec ? new OldSIMockingCodec() : TestUtil.getDefaultCodec())); @@ -444,11 +403,10 @@ public class StoreTests extends ESTestCase { } - @Test public void testNewChecksums() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); // set default codec - all segments need checksums IndexWriter writer = new IndexWriter(store.directory(), newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec())); int docs = 1 + random().nextInt(100); @@ -504,11 +462,10 @@ public class StoreTests extends ESTestCase { IOUtils.close(store); } - @Test public void testMixedChecksums() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); // this time random codec.... IndexWriter writer = new IndexWriter(store.directory(), newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec())); int docs = 1 + random().nextInt(100); @@ -596,11 +553,10 @@ public class StoreTests extends ESTestCase { IOUtils.close(store); } - @Test public void testRenameFile() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); { IndexOutput output = store.directory().createOutput("foo.bar", IOContext.DEFAULT); int iters = scaledRandomIntBetween(10, 100); @@ -713,7 +669,6 @@ public class StoreTests extends ESTestCase { } - @Test public void testVerifyingIndexInput() throws IOException { Directory dir = newDirectory(); IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT); @@ -808,7 +763,7 @@ public class StoreTests extends ESTestCase { } public LuceneManagedDirectoryService(Random random, boolean preventDoubleWrite) { - super(new ShardId("fake", 1), Settings.EMPTY); + super(new ShardId(INDEX_SETTINGS.getIndex(), 1), INDEX_SETTINGS); dir = StoreTests.newDirectory(random); if (dir instanceof MockDirectoryWrapper) { ((MockDirectoryWrapper) dir).setPreventDoubleWrite(preventDoubleWrite); @@ -843,7 +798,6 @@ public class StoreTests extends ESTestCase { * Legacy indices without lucene CRC32 did never write or calculate checksums for segments_N files * but for other files */ - @Test public void testRecoveryDiffWithLegacyCommit() { Map metaDataMap = new HashMap<>(); metaDataMap.put("segments_1", new StoreFileMetaData("segments_1", 50, null, null, new BytesRef(new byte[]{1}))); @@ -855,8 +809,6 @@ public class StoreTests extends ESTestCase { assertEquals(recoveryDiff.toString(), recoveryDiff.different.size(), 2); } - - @Test public void testRecoveryDiff() throws IOException, InterruptedException { int numDocs = 2 + random().nextInt(100); List docs = new ArrayList<>(); @@ -876,7 +828,7 @@ public class StoreTests extends ESTestCase { iwc.setUseCompoundFile(random.nextBoolean()); final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); IndexWriter writer = new IndexWriter(store.directory(), iwc); final boolean lotsOfSegments = rarely(random); for (Document d : docs) { @@ -906,7 +858,7 @@ public class StoreTests extends ESTestCase { iwc.setUseCompoundFile(random.nextBoolean()); final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random); - store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); IndexWriter writer = new IndexWriter(store.directory(), iwc); final boolean lotsOfSegments = rarely(random); for (Document d : docs) { @@ -1001,11 +953,10 @@ public class StoreTests extends ESTestCase { IOUtils.close(store); } - @Test public void testCleanupFromSnapshot() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); // this time random codec.... IndexWriterConfig indexWriterConfig = newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec()); // we keep all commits and that allows us clean based on multiple snapshots @@ -1111,7 +1062,6 @@ public class StoreTests extends ESTestCase { IOUtils.close(store); } - @Test public void testCleanUpWithLegacyChecksums() throws IOException { Map metaDataMap = new HashMap<>(); metaDataMap.put("segments_1", new StoreFileMetaData("segments_1", 50, null, null, new BytesRef(new byte[]{1}))); @@ -1120,7 +1070,7 @@ public class StoreTests extends ESTestCase { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); for (String file : metaDataMap.keySet()) { try (IndexOutput output = store.directory().createOutput(file, IOContext.DEFAULT)) { BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024)); @@ -1140,7 +1090,7 @@ public class StoreTests extends ESTestCase { final AtomicInteger count = new AtomicInteger(0); final ShardLock lock = new DummyShardLock(shardId); - Store store = new Store(shardId, Settings.EMPTY, directoryService, lock, new Store.OnClose() { + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, lock, new Store.OnClose() { @Override public void handle(ShardLock theLock) { assertEquals(shardId, theLock.getShardId()); @@ -1158,12 +1108,13 @@ public class StoreTests extends ESTestCase { assertEquals(count.get(), 1); } - @Test public void testStoreStats() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Settings settings = Settings.builder().put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL, TimeValue.timeValueMinutes(0)).build(); - Store store = new Store(shardId, settings, directoryService, new DummyShardLock(shardId)); + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT) + .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL, TimeValue.timeValueMinutes(0)).build(); + Store store = new Store(shardId, IndexSettingsModule.newIndexSettings(new Index("index"), settings), directoryService, new DummyShardLock(shardId)); long initialStoreSize = 0; for (String extraFiles : store.directory().listAll()) { assertTrue("expected extraFS file but got: " + extraFiles, extraFiles.startsWith("extra")); @@ -1217,9 +1168,7 @@ public class StoreTests extends ESTestCase { return numNonExtra; } - @Test public void testMetadataSnapshotStreaming() throws Exception { - Store.MetadataSnapshot outMetadataSnapshot = createMetaDataSnapshot(); org.elasticsearch.Version targetNodeVersion = randomVersion(random()); @@ -1253,11 +1202,10 @@ public class StoreTests extends ESTestCase { return new Store.MetadataSnapshot(unmodifiableMap(storeFileMetaDataMap), unmodifiableMap(commitUserData), 0); } - @Test public void testUserDataRead() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec()); SnapshotDeletionPolicy deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); config.setIndexDeletionPolicy(deletionPolicy); @@ -1288,7 +1236,6 @@ public class StoreTests extends ESTestCase { IOUtils.close(store); } - @Test public void testStreamStoreFilesMetaData() throws Exception { Store.MetadataSnapshot metadataSnapshot = createMetaDataSnapshot(); TransportNodesListShardStoreMetaData.StoreFilesMetaData outStoreFileMetaData = new TransportNodesListShardStoreMetaData.StoreFilesMetaData(randomBoolean(), new ShardId("test", 0),metadataSnapshot); @@ -1312,7 +1259,7 @@ public class StoreTests extends ESTestCase { IndexWriterConfig iwc = newIndexWriterConfig(); final ShardId shardId = new ShardId(new Index("index"), 1); DirectoryService directoryService = new LuceneManagedDirectoryService(random()); - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); IndexWriter writer = new IndexWriter(store.directory(), iwc); int numDocs = 1 + random().nextInt(10); @@ -1367,7 +1314,7 @@ public class StoreTests extends ESTestCase { assertTrue(Store.canOpenIndex(logger, tempDir)); final ShardId shardId = new ShardId(new Index("index"), 1); - DirectoryService directoryService = new DirectoryService(shardId, Settings.EMPTY) { + DirectoryService directoryService = new DirectoryService(shardId, INDEX_SETTINGS) { @Override public long throttleTimeInNanos() { return 0; @@ -1378,7 +1325,7 @@ public class StoreTests extends ESTestCase { return dir; } }; - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); store.markStoreCorrupted(new CorruptIndexException("foo", "bar")); assertFalse(Store.canOpenIndex(logger, tempDir)); store.close(); @@ -1387,7 +1334,7 @@ public class StoreTests extends ESTestCase { public void testDeserializeCorruptionException() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); final Directory dir = new RAMDirectory(); // I use ram dir to prevent that virusscanner being a PITA - DirectoryService directoryService = new DirectoryService(shardId, Settings.EMPTY) { + DirectoryService directoryService = new DirectoryService(shardId, INDEX_SETTINGS) { @Override public long throttleTimeInNanos() { return 0; @@ -1398,7 +1345,7 @@ public class StoreTests extends ESTestCase { return dir; } }; - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); CorruptIndexException ex = new CorruptIndexException("foo", "bar"); store.markStoreCorrupted(ex); try { @@ -1407,7 +1354,7 @@ public class StoreTests extends ESTestCase { } catch (CorruptIndexException e) { assertEquals(ex.getMessage(), e.getMessage()); assertEquals(ex.toString(), e.toString()); - assertEquals(ExceptionsHelper.stackTrace(ex), ExceptionsHelper.stackTrace(e)); + assertArrayEquals(ex.getStackTrace(), e.getStackTrace()); } store.removeCorruptionMarker(); @@ -1419,7 +1366,7 @@ public class StoreTests extends ESTestCase { fail("should be corrupted"); } catch (CorruptIndexException e) { assertEquals("foobar (resource=preexisting_corruption)", e.getMessage()); - assertEquals(ExceptionsHelper.stackTrace(ioe), ExceptionsHelper.stackTrace(e.getCause())); + assertArrayEquals(ioe.getStackTrace(), e.getCause().getStackTrace()); } store.close(); } @@ -1427,7 +1374,7 @@ public class StoreTests extends ESTestCase { public void testCanReadOldCorruptionMarker() throws IOException { final ShardId shardId = new ShardId(new Index("index"), 1); final Directory dir = new RAMDirectory(); // I use ram dir to prevent that virusscanner being a PITA - DirectoryService directoryService = new DirectoryService(shardId, Settings.EMPTY) { + DirectoryService directoryService = new DirectoryService(shardId, INDEX_SETTINGS) { @Override public long throttleTimeInNanos() { return 0; @@ -1438,7 +1385,7 @@ public class StoreTests extends ESTestCase { return dir; } }; - Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); CorruptIndexException exception = new CorruptIndexException("foo", "bar"); String uuid = Store.CORRUPTED + Strings.randomBase64UUID(); diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java index 02d5279abb3..68c19d56e56 100644 --- a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java +++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.HashSet; import java.util.Set; @@ -41,19 +40,20 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; /** */ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SuggestStatsIT extends ESIntegTestCase { - @Override protected int numberOfReplicas() { return 0; } - @Test public void testSimpleStats() throws Exception { // clear all stats first client().admin().indices().prepareStats().clear().execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/index/translog/BufferedTranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/BufferedTranslogTests.java index 2e5375efd51..a29cc6cf8d0 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/BufferedTranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/BufferedTranslogTests.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.translog; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Path; @@ -32,12 +34,12 @@ import java.nio.file.Path; public class BufferedTranslogTests extends TranslogTests { @Override - protected Translog create(Path path) throws IOException { + protected TranslogConfig getTranslogConfig(Path path) { Settings build = Settings.settingsBuilder() .put("index.translog.fs.type", TranslogWriter.Type.BUFFERED.name()) .put("index.translog.fs.buffer_size", 10 + randomInt(128 * 1024), ByteSizeUnit.BYTES) + .put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT) .build(); - TranslogConfig translogConfig = new TranslogConfig(shardId, path, build, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, null); - return new Translog(translogConfig); + return new TranslogConfig(shardId, path, IndexSettingsModule.newIndexSettings(shardId.index(), build), Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, null); } } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 0b3e12dd9cd..e35c04dcd6b 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -22,19 +22,19 @@ package org.elasticsearch.index.translog; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.Term; +import org.apache.lucene.mockfile.FilterFileChannel; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -42,26 +42,26 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.EOFException; import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; -import java.nio.file.*; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import static org.hamcrest.Matchers.*; @@ -71,8 +71,6 @@ import static org.hamcrest.Matchers.*; @LuceneTestCase.SuppressFileSystems("ExtrasFS") public class TranslogTests extends ESTestCase { - private static final Pattern PARSE_LEGACY_ID_PATTERN = Pattern.compile("^" + Translog.TRANSLOG_FILE_PREFIX + "(\\d+)((\\.recovering))?$"); - protected final ShardId shardId = new ShardId(new Index("index"), 1); protected Translog translog; @@ -114,15 +112,19 @@ public class TranslogTests extends ESTestCase { } } - protected Translog create(Path path) throws IOException { - Settings build = Settings.settingsBuilder() - .put(TranslogConfig.INDEX_TRANSLOG_FS_TYPE, TranslogWriter.Type.SIMPLE.name()) - .build(); - TranslogConfig translogConfig = new TranslogConfig(shardId, path, build, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, null); - return new Translog(translogConfig); + private Translog create(Path path) throws IOException { + return new Translog(getTranslogConfig(path)); } - protected void addToTranslogAndList(Translog translog, ArrayList list, Translog.Operation op) { + protected TranslogConfig getTranslogConfig(Path path) { + Settings build = Settings.settingsBuilder() + .put(TranslogConfig.INDEX_TRANSLOG_FS_TYPE, TranslogWriter.Type.SIMPLE.name()) + .put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT) + .build(); + return new TranslogConfig(shardId, path, IndexSettingsModule.newIndexSettings(shardId.index(), build), Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, null); + } + + protected void addToTranslogAndList(Translog translog, ArrayList list, Translog.Operation op) throws IOException { list.add(op); translog.add(op); } @@ -182,7 +184,6 @@ public class TranslogTests extends ESTestCase { return string; } - @Test public void testRead() throws IOException { Translog.Location loc1 = translog.add(new Translog.Index("test", "1", new byte[]{1})); Translog.Location loc2 = translog.add(new Translog.Index("test", "2", new byte[]{2})); @@ -209,7 +210,6 @@ public class TranslogTests extends ESTestCase { } } - @Test public void testSimpleOperations() throws IOException { ArrayList ops = new ArrayList<>(); Translog.Snapshot snapshot = translog.newSnapshot(); @@ -272,7 +272,6 @@ public class TranslogTests extends ESTestCase { return stats; } - @Test public void testStats() throws IOException { final long firstOperationPosition = translog.getFirstOperationPosition(); TranslogStats stats = stats(); @@ -336,8 +335,7 @@ public class TranslogTests extends ESTestCase { } } - @Test - public void testSnapshot() { + public void testSnapshot() throws IOException { ArrayList ops = new ArrayList<>(); Translog.Snapshot snapshot = translog.newSnapshot(); assertThat(snapshot, SnapshotMatchers.size(0)); @@ -363,7 +361,6 @@ public class TranslogTests extends ESTestCase { snapshot1.close(); } - @Test public void testSnapshotWithNewTranslog() throws IOException { ArrayList ops = new ArrayList<>(); Translog.Snapshot snapshot = translog.newSnapshot(); @@ -397,12 +394,11 @@ public class TranslogTests extends ESTestCase { Translog.Snapshot snapshot = translog.newSnapshot(); fail("translog is closed"); } catch (AlreadyClosedException ex) { - assertThat(ex.getMessage(), containsString("translog-1.tlog is already closed can't increment")); + assertEquals(ex.getMessage(), "translog is already closed"); } } - @Test - public void deleteOnSnapshotRelease() throws Exception { + public void testDeleteOnSnapshotRelease() throws Exception { ArrayList firstOps = new ArrayList<>(); addToTranslogAndList(translog, firstOps, new Translog.Index("test", "1", new byte[]{1})); @@ -457,7 +453,6 @@ public class TranslogTests extends ESTestCase { } - @Test public void testConcurrentWritesWithVaryingSize() throws Throwable { final int opsPerThread = randomIntBetween(10, 200); int threadCount = 2 + randomInt(5); @@ -470,36 +465,7 @@ public class TranslogTests extends ESTestCase { final CountDownLatch downLatch = new CountDownLatch(1); for (int i = 0; i < threadCount; i++) { final int threadId = i; - threads[i] = new Thread(new Runnable() { - @Override - public void run() { - try { - downLatch.await(); - for (int opCount = 0; opCount < opsPerThread; opCount++) { - Translog.Operation op; - switch (randomFrom(Translog.Operation.Type.values())) { - case CREATE: - case INDEX: - op = new Translog.Index("test", threadId + "_" + opCount, - randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8")); - break; - case DELETE: - op = new Translog.Delete(new Term("_uid", threadId + "_" + opCount), - 1 + randomInt(100000), - randomFrom(VersionType.values())); - break; - default: - throw new ElasticsearchException("not supported op type"); - } - - Translog.Location loc = translog.add(op); - writtenOperations.add(new LocationOperation(op, loc)); - } - } catch (Throwable t) { - threadExceptions[threadId] = t; - } - } - }); + threads[i] = new TranslogThread(translog, downLatch, opsPerThread, threadId, writtenOperations, threadExceptions); threads[i].setDaemon(true); threads[i].start(); } @@ -543,7 +509,6 @@ public class TranslogTests extends ESTestCase { } - @Test public void testTranslogChecksums() throws Exception { List locations = new ArrayList<>(); @@ -567,7 +532,6 @@ public class TranslogTests extends ESTestCase { assertThat("at least one corruption was caused and caught", corruptionsCaught.get(), greaterThanOrEqualTo(1)); } - @Test public void testTruncatedTranslogs() throws Exception { List locations = new ArrayList<>(); @@ -634,8 +598,6 @@ public class TranslogTests extends ESTestCase { return new Term("_uid", id); } - - @Test public void testVerifyTranslogIsNotDeleted() throws IOException { assertFileIsPresent(translog, 1); translog.add(new Translog.Index("test", "1", new byte[]{1})); @@ -655,7 +617,6 @@ public class TranslogTests extends ESTestCase { } /** Tests that concurrent readers and writes maintain view and snapshot semantics */ - @Test public void testConcurrentWriteViewsAndSnapshot() throws Throwable { final Thread[] writers = new Thread[randomIntBetween(1, 10)]; final Thread[] readers = new Thread[randomIntBetween(1, 10)]; @@ -678,7 +639,7 @@ public class TranslogTests extends ESTestCase { final String threadId = "writer_" + i; writers[i] = new Thread(new AbstractRunnable() { @Override - public void doRun() throws BrokenBarrierException, InterruptedException { + public void doRun() throws BrokenBarrierException, InterruptedException, IOException { barrier.await(); int counter = 0; while (run.get()) { @@ -1055,7 +1016,109 @@ public class TranslogTests extends ESTestCase { } } } + } + + public void testRecoveryUncommittedFileExists() throws IOException { + List locations = new ArrayList<>(); + int translogOperations = randomIntBetween(10, 100); + final int prepareOp = randomIntBetween(0, translogOperations-1); + Translog.TranslogGeneration translogGeneration = null; + final boolean sync = randomBoolean(); + for (int op = 0; op < translogOperations; op++) { + locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + if (op == prepareOp) { + translogGeneration = translog.getGeneration(); + translog.prepareCommit(); + assertEquals("expected this to be the first commit", 1l, translogGeneration.translogFileGeneration); + assertNotNull(translogGeneration.translogUUID); + } + } + if (sync) { + translog.sync(); + } + // we intentionally don't close the tlog that is in the prepareCommit stage since we try to recovery the uncommitted + // translog here as well. + TranslogConfig config = translog.getConfig(); + config.setTranslogGeneration(translogGeneration); + Path ckp = config.getTranslogPath().resolve(Translog.CHECKPOINT_FILE_NAME); + Checkpoint read = Checkpoint.read(ckp); + Files.copy(ckp, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation))); + + try (Translog translog = new Translog(config)) { + assertNotNull(translogGeneration); + assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); + assertFalse(translog.syncNeeded()); + try (Translog.Snapshot snapshot = translog.newSnapshot()) { + int upTo = sync ? translogOperations : prepareOp; + for (int i = 0; i < upTo; i++) { + Translog.Operation next = snapshot.next(); + assertNotNull("operation " + i + " must be non-null synced: " + sync, next); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + } + } + } + + if (randomBoolean()) { // recover twice + try (Translog translog = new Translog(config)) { + assertNotNull(translogGeneration); + assertEquals("lastCommitted must be 3 less than current - we never finished the commit and run recovery twice", translogGeneration.translogFileGeneration + 3, translog.currentFileGeneration()); + assertFalse(translog.syncNeeded()); + try (Translog.Snapshot snapshot = translog.newSnapshot()) { + int upTo = sync ? translogOperations : prepareOp; + for (int i = 0; i < upTo; i++) { + Translog.Operation next = snapshot.next(); + assertNotNull("operation " + i + " must be non-null synced: " + sync, next); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + } + } + } + } + } + + public void testRecoveryUncommittedCorryptedCheckpoint() throws IOException { + List locations = new ArrayList<>(); + int translogOperations = 100; + final int prepareOp = 44; + Translog.TranslogGeneration translogGeneration = null; + final boolean sync = randomBoolean(); + for (int op = 0; op < translogOperations; op++) { + locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + if (op == prepareOp) { + translogGeneration = translog.getGeneration(); + translog.prepareCommit(); + assertEquals("expected this to be the first commit", 1l, translogGeneration.translogFileGeneration); + assertNotNull(translogGeneration.translogUUID); + } + } + translog.sync(); + // we intentionally don't close the tlog that is in the prepareCommit stage since we try to recovery the uncommitted + // translog here as well. + TranslogConfig config = translog.getConfig(); + config.setTranslogGeneration(translogGeneration); + Path ckp = config.getTranslogPath().resolve(Translog.CHECKPOINT_FILE_NAME); + Checkpoint read = Checkpoint.read(ckp); + Checkpoint corrupted = new Checkpoint(0,0,0); + Checkpoint.write(config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), corrupted, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); + try (Translog translog = new Translog(config)) { + fail("corrupted"); + } catch (IllegalStateException ex) { + assertEquals(ex.getMessage(), "Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=2683, numOps=55, translogFileGeneration= 2} but got: Checkpoint{offset=0, numOps=0, translogFileGeneration= 0}"); + } + Checkpoint.write(config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + try (Translog translog = new Translog(config)) { + assertNotNull(translogGeneration); + assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); + assertFalse(translog.syncNeeded()); + try (Translog.Snapshot snapshot = translog.newSnapshot()) { + int upTo = sync ? translogOperations : prepareOp; + for (int i = 0; i < upTo; i++) { + Translog.Operation next = snapshot.next(); + assertNotNull("operation " + i + " must be non-null synced: " + sync, next); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + } + } + } } public void testSnapshotFromStreamInput() throws IOException { @@ -1133,4 +1196,210 @@ public class TranslogTests extends ESTestCase { assertNull(snapshot.next()); } } + + public void testFailOnClosedWrite() throws IOException { + translog.add(new Translog.Index("test", "1", Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + translog.close(); + try { + translog.add(new Translog.Index("test", "1", Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + fail("closed"); + } catch (AlreadyClosedException ex) { + // all is welll + } + } + + public void testCloseConcurrently() throws Throwable { + final int opsPerThread = randomIntBetween(10, 200); + int threadCount = 2 + randomInt(5); + + logger.info("testing with [{}] threads, each doing [{}] ops", threadCount, opsPerThread); + final BlockingQueue writtenOperations = new ArrayBlockingQueue<>(threadCount * opsPerThread); + + Thread[] threads = new Thread[threadCount]; + final Throwable[] threadExceptions = new Throwable[threadCount]; + final CountDownLatch downLatch = new CountDownLatch(1); + for (int i = 0; i < threadCount; i++) { + final int threadId = i; + threads[i] = new TranslogThread(translog, downLatch, opsPerThread, threadId, writtenOperations, threadExceptions); + threads[i].setDaemon(true); + threads[i].start(); + } + + downLatch.countDown(); + translog.close(); + + for (int i = 0; i < threadCount; i++) { + if (threadExceptions[i] != null) { + if ((threadExceptions[i] instanceof AlreadyClosedException) == false) { + throw threadExceptions[i]; + } + } + threads[i].join(60 * 1000); + } + } + + private static class TranslogThread extends Thread { + private final CountDownLatch downLatch; + private final int opsPerThread; + private final int threadId; + private final BlockingQueue writtenOperations; + private final Throwable[] threadExceptions; + private final Translog translog; + + public TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, BlockingQueue writtenOperations, Throwable[] threadExceptions) { + this.translog = translog; + this.downLatch = downLatch; + this.opsPerThread = opsPerThread; + this.threadId = threadId; + this.writtenOperations = writtenOperations; + this.threadExceptions = threadExceptions; + } + + @Override + public void run() { + try { + downLatch.await(); + for (int opCount = 0; opCount < opsPerThread; opCount++) { + Translog.Operation op; + switch (randomFrom(Translog.Operation.Type.values())) { + case CREATE: + case INDEX: + op = new Translog.Index("test", threadId + "_" + opCount, + randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8")); + break; + case DELETE: + op = new Translog.Delete(new Term("_uid", threadId + "_" + opCount), + 1 + randomInt(100000), + randomFrom(VersionType.values())); + break; + default: + throw new ElasticsearchException("not supported op type"); + } + + Translog.Location loc = translog.add(op); + writtenOperations.add(new LocationOperation(op, loc)); + } + } catch (Throwable t) { + threadExceptions[threadId] = t; + } + } + } + + public void testFailFlush() throws IOException { + Path tempDir = createTempDir(); + final AtomicBoolean simulateDiskFull = new AtomicBoolean(); + TranslogConfig config = getTranslogConfig(tempDir); + Translog translog = new Translog(config) { + @Override + TranslogWriter.ChannelFactory getChannelFactory() { + final TranslogWriter.ChannelFactory factory = super.getChannelFactory(); + + return new TranslogWriter.ChannelFactory() { + @Override + public FileChannel open(Path file) throws IOException { + FileChannel channel = factory.open(file); + return new FilterFileChannel(channel) { + + @Override + public int write(ByteBuffer src) throws IOException { + if (simulateDiskFull.get()) { + if (src.limit() > 1) { + final int pos = src.position(); + final int limit = src.limit(); + src.limit(limit / 2); + super.write(src); + src.position(pos); + src.limit(limit); + throw new IOException("__FAKE__ no space left on device"); + } + } + return super.write(src); + } + }; + } + }; + } + }; + + List locations = new ArrayList<>(); + int opsSynced = 0; + int opsAdded = 0; + boolean failed = false; + while(failed == false) { + try { + locations.add(translog.add(new Translog.Index("test", "" + opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); + opsAdded++; + translog.sync(); + opsSynced++; + } catch (IOException ex) { + failed = true; + assertFalse(translog.isOpen()); + assertEquals("__FAKE__ no space left on device", ex.getMessage()); + } + simulateDiskFull.set(randomBoolean()); + } + simulateDiskFull.set(false); + if (randomBoolean()) { + try { + locations.add(translog.add(new Translog.Index("test", "" + opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); + fail("we are already closed"); + } catch (AlreadyClosedException ex) { + assertNotNull(ex.getCause()); + assertEquals(ex.getCause().getMessage(), "__FAKE__ no space left on device"); + } + + } + Translog.TranslogGeneration translogGeneration = translog.getGeneration(); + try { + translog.newSnapshot(); + fail("already closed"); + } catch (AlreadyClosedException ex) { + // all is well + assertNotNull(ex.getCause()); + assertSame(translog.getTragicException(), ex.getCause()); + } + + try { + translog.commit(); + fail("already closed"); + } catch (AlreadyClosedException ex) { + assertNotNull(ex.getCause()); + assertSame(translog.getTragicException(), ex.getCause()); + } + + assertFalse(translog.isOpen()); + translog.close(); // we are closed + config.setTranslogGeneration(translogGeneration); + try (Translog tlog = new Translog(config)){ + assertEquals("lastCommitted must be 1 less than current", translogGeneration.translogFileGeneration + 1, tlog.currentFileGeneration()); + assertFalse(tlog.syncNeeded()); + + try (Translog.Snapshot snapshot = tlog.newSnapshot()) { + assertEquals(opsSynced, snapshot.estimatedTotalOperations()); + for (int i = 0; i < opsSynced; i++) { + assertEquals("expected operation" + i + " to be in the previous translog but wasn't", tlog.currentFileGeneration() - 1, locations.get(i).generation); + Translog.Operation next = snapshot.next(); + assertNotNull("operation " + i + " must be non-null", next); + assertEquals(i, Integer.parseInt(next.getSource().source.toUtf8())); + } + } + } + } + + public void testTranslogOpsCountIsCorrect() throws IOException { + List locations = new ArrayList<>(); + int numOps = randomIntBetween(100, 200); + LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer boarders regularly + for (int opsAdded = 0; opsAdded < numOps; opsAdded++) { + locations.add(translog.add(new Translog.Index("test", "" + opsAdded, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8"))))); + try (Translog.Snapshot snapshot = translog.newSnapshot()) { + assertEquals(opsAdded+1, snapshot.estimatedTotalOperations()); + for (int i = 0; i < opsAdded; i++) { + assertEquals("expected operation" + i + " to be in the current translog but wasn't", translog.currentFileGeneration(), locations.get(i).generation); + Translog.Operation next = snapshot.next(); + assertNotNull("operation " + i + " must be non-null", next); + } + } + } + } } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java index 283124d09ed..68f26c504fb 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.translog; import org.apache.lucene.util.IOUtils; import org.elasticsearch.index.VersionType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.nio.channels.FileChannel; @@ -36,8 +35,6 @@ import static org.hamcrest.Matchers.equalTo; * Tests for reading old and new translog files */ public class TranslogVersionTests extends ESTestCase { - - @Test public void testV0LegacyTranslogVersion() throws Exception { Path translogFile = getDataPath("/org/elasticsearch/index/translog/translog-v0.binary"); assertThat("test file should exist", Files.exists(translogFile), equalTo(true)); @@ -62,7 +59,6 @@ public class TranslogVersionTests extends ESTestCase { } } - @Test public void testV1ChecksummedTranslogVersion() throws Exception { Path translogFile = getDataPath("/org/elasticsearch/index/translog/translog-v1.binary"); assertThat("test file should exist", Files.exists(translogFile), equalTo(true)); @@ -95,7 +91,6 @@ public class TranslogVersionTests extends ESTestCase { } } - @Test public void testCorruptedTranslogs() throws Exception { try { Path translogFile = getDataPath("/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary"); @@ -135,7 +130,6 @@ public class TranslogVersionTests extends ESTestCase { } - @Test public void testTruncatedTranslog() throws Exception { try { Path translogFile = getDataPath("/org/elasticsearch/index/translog/translog-v1-truncated.binary"); diff --git a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java index 1c914c10c03..f4a70a2afa4 100644 --- a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java +++ b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.index.VersionType; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -44,12 +43,10 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; * */ public class IndexActionIT extends ESIntegTestCase { - /** * This test tries to simulate load while creating an index and indexing documents * while the index is being created. */ - @Test public void testAutoGenerateIdNoDuplicates() throws Exception { int numberOfIterations = scaledRandomIntBetween(10, 50); for (int i = 0; i < numberOfIterations; i++) { @@ -92,7 +89,6 @@ public class IndexActionIT extends ESIntegTestCase { } } - @Test public void testCreatedFlag() throws Exception { createIndex("test"); ensureGreen(); @@ -110,7 +106,6 @@ public class IndexActionIT extends ESIntegTestCase { } - @Test public void testCreatedFlagWithFlush() throws Exception { createIndex("test"); ensureGreen(); @@ -126,7 +121,6 @@ public class IndexActionIT extends ESIntegTestCase { assertTrue(indexResponse.isCreated()); } - @Test public void testCreatedFlagParallelExecution() throws Exception { createIndex("test"); ensureGreen(); @@ -159,7 +153,6 @@ public class IndexActionIT extends ESIntegTestCase { terminate(threadPool); } - @Test public void testCreatedFlagWithExternalVersioning() throws Exception { createIndex("test"); ensureGreen(); @@ -169,7 +162,6 @@ public class IndexActionIT extends ESIntegTestCase { assertTrue(indexResponse.isCreated()); } - @Test public void testCreateFlagWithBulk() { createIndex("test"); ensureGreen(); @@ -181,7 +173,6 @@ public class IndexActionIT extends ESIntegTestCase { assertTrue(indexResponse.isCreated()); } - @Test public void testCreateIndexWithLongName() { int min = MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES + 1; int max = MetaDataCreateIndexService.MAX_INDEX_NAME_BYTES * 2; diff --git a/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java b/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java index 8fe2bf01ca3..818937c511e 100644 --- a/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java +++ b/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java @@ -20,19 +20,20 @@ package org.elasticsearch.indexlifecycle; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; -import org.junit.Test; import java.util.Set; import java.util.stream.Collectors; @@ -42,16 +43,9 @@ import static org.elasticsearch.client.Requests.clusterHealthRequest; import static org.elasticsearch.client.Requests.createIndexRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; /** @@ -59,12 +53,11 @@ import static org.hamcrest.Matchers.nullValue; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class IndexLifecycleActionIT extends ESIntegTestCase { - - @Test public void testIndexLifecycleActionsWith11Shards1Backup() throws Exception { Settings settings = settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 11) .put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "0s") .build(); // start one server diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java index bf3b84e4d46..8de3af25827 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java @@ -30,16 +30,18 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.MockIndexEventListener; import org.hamcrest.Matchers; -import org.junit.Test; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -65,7 +67,11 @@ import static org.hamcrest.Matchers.hasSize; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class IndicesLifecycleListenerIT extends ESIntegTestCase { - @Test + @Override + protected Collection> nodePlugins() { + return pluginList(MockIndexEventListener.TestPlugin.class); + } + public void testBeforeIndexAddedToCluster() throws Exception { String node1 = internalCluster().startNode(); String node2 = internalCluster().startNode(); @@ -74,9 +80,9 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { final AtomicInteger beforeAddedCount = new AtomicInteger(0); final AtomicInteger allCreatedCount = new AtomicInteger(0); - IndicesLifecycle.Listener listener = new IndicesLifecycle.Listener() { + IndexEventListener listener = new IndexEventListener() { @Override - public void beforeIndexAddedToCluster(Index index, @IndexSettings Settings indexSettings) { + public void beforeIndexAddedToCluster(Index index, Settings indexSettings) { beforeAddedCount.incrementAndGet(); if (indexSettings.getAsBoolean("index.fail", false)) { throw new ElasticsearchException("failing on purpose"); @@ -84,14 +90,14 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { } @Override - public void beforeIndexCreated(Index index, @IndexSettings Settings indexSettings) { + public void beforeIndexCreated(Index index, Settings indexSettings) { allCreatedCount.incrementAndGet(); } }; - internalCluster().getInstance(IndicesLifecycle.class, node1).addListener(listener); - internalCluster().getInstance(IndicesLifecycle.class, node2).addListener(listener); - internalCluster().getInstance(IndicesLifecycle.class, node3).addListener(listener); + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node1).setNewDelegate(listener); + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node2).setNewDelegate(listener); + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node3).setNewDelegate(listener); client().admin().indices().prepareCreate("test") .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).get(); @@ -113,15 +119,14 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { * Tests that if an *index* structure creation fails on relocation to a new node, the shard * is not stuck but properly failed. */ - @Test public void testIndexShardFailedOnRelocation() throws Throwable { String node1 = internalCluster().startNode(); client().admin().indices().prepareCreate("index1").setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0).get(); ensureGreen("index1"); String node2 = internalCluster().startNode(); - internalCluster().getInstance(IndicesLifecycle.class, node2).addListener(new IndexShardStateChangeListener() { + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node2).setNewDelegate(new IndexShardStateChangeListener() { @Override - public void beforeIndexCreated(Index index, @IndexSettings Settings indexSettings) { + public void beforeIndexCreated(Index index, Settings indexSettings) { throw new RuntimeException("FAIL"); } }); @@ -133,14 +138,12 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { assertThat(state.nodes().resolveNode(shard.get(0).currentNodeId()).getName(), Matchers.equalTo(node1)); } - @Test public void testIndexStateShardChanged() throws Throwable { - //start with a single node String node1 = internalCluster().startNode(); IndexShardStateChangeListener stateChangeListenerNode1 = new IndexShardStateChangeListener(); //add a listener that keeps track of the shard state changes - internalCluster().getInstance(IndicesLifecycle.class, node1).addListener(stateChangeListenerNode1); + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node1).setNewDelegate(stateChangeListenerNode1); //create an index that should fail try { @@ -171,7 +174,7 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { String node2 = internalCluster().startNode(); IndexShardStateChangeListener stateChangeListenerNode2 = new IndexShardStateChangeListener(); //add a listener that keeps track of the shard state changes - internalCluster().getInstance(IndicesLifecycle.class, node2).addListener(stateChangeListenerNode2); + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node2).setNewDelegate(stateChangeListenerNode2); //re-enable allocation assertAcked(client().admin().cluster().prepareUpdateSettings() .setPersistentSettings(builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "all"))); @@ -232,7 +235,7 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { stateChangeListener.shardStates.clear(); } - private static class IndexShardStateChangeListener extends IndicesLifecycle.Listener { + private static class IndexShardStateChangeListener implements IndexEventListener { //we keep track of all the states (ordered) a shard goes through final ConcurrentMap> shardStates = new ConcurrentHashMap<>(); Settings creationSettings = Settings.EMPTY; @@ -248,7 +251,7 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { } @Override - public void beforeIndexCreated(Index index, @IndexSettings Settings indexSettings) { + public void beforeIndexCreated(Index index, Settings indexSettings) { this.creationSettings = indexSettings; if (indexSettings.getAsBoolean("index.fail", false)) { throw new ElasticsearchException("failing on purpose"); @@ -256,7 +259,7 @@ public class IndicesLifecycleListenerIT extends ESIntegTestCase { } @Override - public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, @IndexSettings Settings indexSettings) { + public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { this.afterCloseSettings = indexSettings; } diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index 9f9ed46f695..e34e1d6bd6b 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -18,14 +18,24 @@ */ package org.elasticsearch.indices; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingHelper; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.index.shard.IndexEventListener; +import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; @@ -34,21 +44,17 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCase { - @Override - protected boolean resetNodeAfterTest() { - return true; - } - - @Test public void testCloseDeleteCallback() throws Throwable { - - final AtomicInteger counter = new AtomicInteger(1); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); assertAcked(client().admin().indices().prepareCreate("test") .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0)); ensureGreen(); - getInstanceFromNode(IndicesLifecycle.class).addListener(new IndicesLifecycle.Listener() { + IndexMetaData metaData = indicesService.indexService("test").getMetaData(); + ShardRouting shardRouting = indicesService.indexService("test").getShard(0).routingEntry(); + final AtomicInteger counter = new AtomicInteger(1); + IndexEventListener countingListener = new IndexEventListener() { @Override - public void afterIndexClosed(Index index, @IndexSettings Settings indexSettings) { + public void afterIndexClosed(Index index, Settings indexSettings) { assertEquals(counter.get(), 5); counter.incrementAndGet(); } @@ -60,13 +66,13 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas } @Override - public void afterIndexDeleted(Index index, @IndexSettings Settings indexSettings) { + public void afterIndexDeleted(Index index, Settings indexSettings) { assertEquals(counter.get(), 6); counter.incrementAndGet(); } @Override - public void beforeIndexDeleted(IndexService indexService) { + public void beforeIndexDeleted(IndexService indexService) { assertEquals(counter.get(), 2); counter.incrementAndGet(); } @@ -82,9 +88,27 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas assertEquals(counter.get(), 4); counter.incrementAndGet(); } - }); - assertAcked(client().admin().indices().prepareDelete("test").get()); + }; + indicesService.deleteIndex("test", "simon says"); + try { + NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class); + IndexService index = indicesService.createIndex(nodeServicesProvider, metaData, Arrays.asList(countingListener)); + ShardRouting newRouting = new ShardRouting(shardRouting); + String nodeId = newRouting.currentNodeId(); + ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom")); + ShardRoutingHelper.initialize(newRouting, nodeId); + IndexShard shard = index.createShard(newRouting); + shard.updateRoutingEntry(newRouting, true); + final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); + shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode)); + shard.recoverFromStore(localNode); + newRouting = new ShardRouting(newRouting); + ShardRoutingHelper.moveToStarted(newRouting); + shard.updateRoutingEntry(newRouting, true); + } finally { + indicesService.deleteIndex("test", "simon says"); + } assertEquals(7, counter.get()); } - + } diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index c7a2624d380..968266f00c3 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -48,13 +48,13 @@ public class IndicesModuleTests extends ModuleTestCase { } public void testRegisterQueryParser() { - IndicesModule module = new IndicesModule(Settings.EMPTY); + IndicesModule module = new IndicesModule(); module.registerQueryParser(FakeQueryParser.class); assertSetMultiBinding(module, QueryParser.class, FakeQueryParser.class); } public void testRegisterQueryParserDuplicate() { - IndicesModule module = new IndicesModule(Settings.EMPTY); + IndicesModule module = new IndicesModule(); try { module.registerQueryParser(TermQueryParser.class); } catch (IllegalArgumentException e) { @@ -62,17 +62,8 @@ public class IndicesModuleTests extends ModuleTestCase { } } - public void testRegisterHunspellDictionary() throws Exception { - IndicesModule module = new IndicesModule(Settings.EMPTY); - InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff"); - InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic"); - Dictionary dictionary = new Dictionary(aff, dic); - module.registerHunspellDictionary("foo", dictionary); - assertMapInstanceBinding(module, String.class, Dictionary.class, Collections.singletonMap("foo", dictionary)); - } - public void testRegisterHunspellDictionaryDuplicate() { - IndicesModule module = new IndicesModule(Settings.EMPTY); + IndicesModule module = new IndicesModule(); try { module.registerQueryParser(TermQueryParser.class); } catch (IllegalArgumentException e) { diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 9cf2034e05a..66cb5e7ea7d 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -27,9 +27,9 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder; @@ -37,7 +37,6 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersRequestBuilder; -import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder; import org.elasticsearch.action.percolate.PercolateRequestBuilder; import org.elasticsearch.action.percolate.PercolateSourceBuilder; @@ -48,37 +47,35 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.suggest.SuggestRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.SuggestBuilders; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class IndicesOptionsIntegrationIT extends ESIntegTestCase { - - @Test - public void testSpecifiedIndexUnavailable_multipleIndices() throws Exception { + public void testSpecifiedIndexUnavailableMultipleIndices() throws Exception { createIndex("test1"); ensureYellow(); // Verify defaults verify(search("test1", "test2"), true); verify(msearch(null, "test1", "test2"), true); - verify(count("test1", "test2"), true); verify(clearCache("test1", "test2"), true); verify(_flush("test1", "test2"),true); verify(segments("test1", "test2"), true); verify(stats("test1", "test2"), true); - verify(optimize("test1", "test2"), true); + verify(forceMerge("test1", "test2"), true); verify(refresh("test1", "test2"), true); verify(validateQuery("test1", "test2"), true); verify(aliasExists("test1", "test2"), true); @@ -95,12 +92,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { IndicesOptions options = IndicesOptions.strictExpandOpen(); verify(search("test1", "test2").setIndicesOptions(options), true); verify(msearch(options, "test1", "test2"), true); - verify(count("test1", "test2").setIndicesOptions(options), true); verify(clearCache("test1", "test2").setIndicesOptions(options), true); verify(_flush("test1", "test2").setIndicesOptions(options),true); verify(segments("test1", "test2").setIndicesOptions(options), true); verify(stats("test1", "test2").setIndicesOptions(options), true); - verify(optimize("test1", "test2").setIndicesOptions(options), true); + verify(forceMerge("test1", "test2").setIndicesOptions(options), true); verify(refresh("test1", "test2").setIndicesOptions(options), true); verify(validateQuery("test1", "test2").setIndicesOptions(options), true); verify(aliasExists("test1", "test2").setIndicesOptions(options), true); @@ -117,12 +113,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { options = IndicesOptions.lenientExpandOpen(); verify(search("test1", "test2").setIndicesOptions(options), false); verify(msearch(options, "test1", "test2").setIndicesOptions(options), false); - verify(count("test1", "test2").setIndicesOptions(options), false); verify(clearCache("test1", "test2").setIndicesOptions(options), false); verify(_flush("test1", "test2").setIndicesOptions(options), false); verify(segments("test1", "test2").setIndicesOptions(options), false); verify(stats("test1", "test2").setIndicesOptions(options), false); - verify(optimize("test1", "test2").setIndicesOptions(options), false); + verify(forceMerge("test1", "test2").setIndicesOptions(options), false); verify(refresh("test1", "test2").setIndicesOptions(options), false); verify(validateQuery("test1", "test2").setIndicesOptions(options), false); verify(aliasExists("test1", "test2").setIndicesOptions(options), false); @@ -141,12 +136,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { ensureYellow(); verify(search("test1", "test2").setIndicesOptions(options), false); verify(msearch(options, "test1", "test2").setIndicesOptions(options), false); - verify(count("test1", "test2").setIndicesOptions(options), false); verify(clearCache("test1", "test2").setIndicesOptions(options), false); verify(_flush("test1", "test2").setIndicesOptions(options),false); verify(segments("test1", "test2").setIndicesOptions(options), false); verify(stats("test1", "test2").setIndicesOptions(options), false); - verify(optimize("test1", "test2").setIndicesOptions(options), false); + verify(forceMerge("test1", "test2").setIndicesOptions(options), false); verify(refresh("test1", "test2").setIndicesOptions(options), false); verify(validateQuery("test1", "test2").setIndicesOptions(options), false); verify(aliasExists("test1", "test2").setIndicesOptions(options), false); @@ -161,8 +155,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(getSettings("test1", "test2").setIndicesOptions(options), false); } - @Test - public void testSpecifiedIndexUnavailable_singleIndexThatIsClosed() throws Exception { + public void testSpecifiedIndexUnavailableSingleIndexThatIsClosed() throws Exception { assertAcked(prepareCreate("test1")); // we need to wait until all shards are allocated since recovery from // gateway will fail unless the majority of the replicas was allocated @@ -174,12 +167,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { IndicesOptions options = IndicesOptions.strictExpandOpenAndForbidClosed(); verify(search("test1").setIndicesOptions(options), true); verify(msearch(options, "test1"), true); - verify(count("test1").setIndicesOptions(options), true); verify(clearCache("test1").setIndicesOptions(options), true); verify(_flush("test1").setIndicesOptions(options),true); verify(segments("test1").setIndicesOptions(options), true); verify(stats("test1").setIndicesOptions(options), true); - verify(optimize("test1").setIndicesOptions(options), true); + verify(forceMerge("test1").setIndicesOptions(options), true); verify(refresh("test1").setIndicesOptions(options), true); verify(validateQuery("test1").setIndicesOptions(options), true); verify(aliasExists("test1").setIndicesOptions(options), true); @@ -196,12 +188,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { options = IndicesOptions.fromOptions(true, options.allowNoIndices(), options.expandWildcardsOpen(), options.expandWildcardsClosed(), options); verify(search("test1").setIndicesOptions(options), false); verify(msearch(options, "test1"), false); - verify(count("test1").setIndicesOptions(options), false); verify(clearCache("test1").setIndicesOptions(options), false); verify(_flush("test1").setIndicesOptions(options),false); verify(segments("test1").setIndicesOptions(options), false); verify(stats("test1").setIndicesOptions(options), false); - verify(optimize("test1").setIndicesOptions(options), false); + verify(forceMerge("test1").setIndicesOptions(options), false); verify(refresh("test1").setIndicesOptions(options), false); verify(validateQuery("test1").setIndicesOptions(options), false); verify(aliasExists("test1").setIndicesOptions(options), false); @@ -221,12 +212,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { options = IndicesOptions.strictExpandOpenAndForbidClosed(); verify(search("test1").setIndicesOptions(options), false); verify(msearch(options, "test1"), false); - verify(count("test1").setIndicesOptions(options), false); verify(clearCache("test1").setIndicesOptions(options), false); verify(_flush("test1").setIndicesOptions(options),false); verify(segments("test1").setIndicesOptions(options), false); verify(stats("test1").setIndicesOptions(options), false); - verify(optimize("test1").setIndicesOptions(options), false); + verify(forceMerge("test1").setIndicesOptions(options), false); verify(refresh("test1").setIndicesOptions(options), false); verify(validateQuery("test1").setIndicesOptions(options), false); verify(aliasExists("test1").setIndicesOptions(options), false); @@ -241,17 +231,15 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(getSettings("test1").setIndicesOptions(options), false); } - @Test - public void testSpecifiedIndexUnavailable_singleIndex() throws Exception { + public void testSpecifiedIndexUnavailableSingleIndex() throws Exception { IndicesOptions options = IndicesOptions.strictExpandOpenAndForbidClosed(); verify(search("test1").setIndicesOptions(options), true); verify(msearch(options, "test1"), true); - verify(count("test1").setIndicesOptions(options), true); verify(clearCache("test1").setIndicesOptions(options), true); verify(_flush("test1").setIndicesOptions(options),true); verify(segments("test1").setIndicesOptions(options), true); verify(stats("test1").setIndicesOptions(options), true); - verify(optimize("test1").setIndicesOptions(options), true); + verify(forceMerge("test1").setIndicesOptions(options), true); verify(refresh("test1").setIndicesOptions(options), true); verify(validateQuery("test1").setIndicesOptions(options), true); verify(aliasExists("test1").setIndicesOptions(options), true); @@ -267,12 +255,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { options = IndicesOptions.fromOptions(true, options.allowNoIndices(), options.expandWildcardsOpen(), options.expandWildcardsClosed(), options); verify(search("test1").setIndicesOptions(options), false); verify(msearch(options, "test1"), false); - verify(count("test1").setIndicesOptions(options), false); verify(clearCache("test1").setIndicesOptions(options), false); verify(_flush("test1").setIndicesOptions(options),false); verify(segments("test1").setIndicesOptions(options), false); verify(stats("test1").setIndicesOptions(options), false); - verify(optimize("test1").setIndicesOptions(options), false); + verify(forceMerge("test1").setIndicesOptions(options), false); verify(refresh("test1").setIndicesOptions(options), false); verify(validateQuery("test1").setIndicesOptions(options), false); verify(aliasExists("test1").setIndicesOptions(options), false); @@ -291,12 +278,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { options = IndicesOptions.strictExpandOpenAndForbidClosed(); verify(search("test1").setIndicesOptions(options), false); verify(msearch(options, "test1"), false); - verify(count("test1").setIndicesOptions(options), false); verify(clearCache("test1").setIndicesOptions(options), false); verify(_flush("test1").setIndicesOptions(options),false); verify(segments("test1").setIndicesOptions(options), false); verify(stats("test1").setIndicesOptions(options), false); - verify(optimize("test1").setIndicesOptions(options), false); + verify(forceMerge("test1").setIndicesOptions(options), false); verify(refresh("test1").setIndicesOptions(options), false); verify(validateQuery("test1").setIndicesOptions(options), false); verify(aliasExists("test1").setIndicesOptions(options), false); @@ -310,8 +296,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(getSettings("test1").setIndicesOptions(options), false); } - @Test - public void testSpecifiedIndexUnavailable_snapshotRestore() throws Exception { + public void testSpecifiedIndexUnavailableSnapshotRestore() throws Exception { createIndex("test1"); ensureGreen("test1"); waitForRelocation(); @@ -341,18 +326,16 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(restore("snap3", "test1", "test2").setIndicesOptions(options), false); } - @Test public void testWildcardBehaviour() throws Exception { // Verify defaults for wildcards, when specifying no indices (*, _all, /) String[] indices = Strings.EMPTY_ARRAY; verify(search(indices), false); verify(msearch(null, indices), false); - verify(count(indices), false); verify(clearCache(indices), false); verify(_flush(indices),false); verify(segments(indices), false); verify(stats(indices), false); - verify(optimize(indices), false); + verify(forceMerge(indices), false); verify(refresh(indices), false); verify(validateQuery(indices), true); verify(aliasExists(indices), false); @@ -370,12 +353,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { IndicesOptions options = IndicesOptions.fromOptions(false, true, true, false); verify(search(indices).setIndicesOptions(options), false); verify(msearch(options, indices).setIndicesOptions(options), false); - verify(count(indices).setIndicesOptions(options), false); verify(clearCache(indices).setIndicesOptions(options), false); verify(_flush(indices).setIndicesOptions(options),false); verify(segments(indices).setIndicesOptions(options), false); verify(stats(indices).setIndicesOptions(options), false); - verify(optimize(indices).setIndicesOptions(options), false); + verify(forceMerge(indices).setIndicesOptions(options), false); verify(refresh(indices).setIndicesOptions(options), false); verify(validateQuery(indices).setIndicesOptions(options), false); verify(aliasExists(indices).setIndicesOptions(options), false); @@ -396,12 +378,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { indices = new String[]{"foo*"}; verify(search(indices), false, 1); verify(msearch(null, indices), false, 1); - verify(count(indices), false, 1); verify(clearCache(indices), false); verify(_flush(indices),false); verify(segments(indices), false); verify(stats(indices), false); - verify(optimize(indices), false); + verify(forceMerge(indices), false); verify(refresh(indices), false); verify(validateQuery(indices), false); verify(aliasExists(indices), false); @@ -419,12 +400,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { indices = new String[]{"foo*", "bar*"}; verify(search(indices), false, 1); verify(msearch(null, indices), false, 1); - verify(count(indices), false, 1); verify(clearCache(indices), false); verify(_flush(indices),false); verify(segments(indices), false); verify(stats(indices), false); - verify(optimize(indices), false); + verify(forceMerge(indices), false); verify(refresh(indices), false); verify(validateQuery(indices), true); verify(aliasExists(indices), false); @@ -442,12 +422,11 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { options = IndicesOptions.fromOptions(false, true, true, false); verify(search(indices).setIndicesOptions(options), false, 1); verify(msearch(options, indices).setIndicesOptions(options), false, 1); - verify(count(indices).setIndicesOptions(options), false, 1); verify(clearCache(indices).setIndicesOptions(options), false); verify(_flush(indices).setIndicesOptions(options),false); verify(segments(indices).setIndicesOptions(options), false); verify(stats(indices).setIndicesOptions(options), false); - verify(optimize(indices).setIndicesOptions(options), false); + verify(forceMerge(indices).setIndicesOptions(options), false); verify(refresh(indices).setIndicesOptions(options), false); verify(validateQuery(indices).setIndicesOptions(options), false); verify(aliasExists(indices).setIndicesOptions(options), false); @@ -462,8 +441,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(getSettings(indices).setIndicesOptions(options), false); } - @Test - public void testWildcardBehaviour_snapshotRestore() throws Exception { + public void testWildcardBehaviourSnapshotRestore() throws Exception { createIndex("foobar"); ensureGreen("foobar"); waitForRelocation(); @@ -494,8 +472,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(restore("snap3", "foo*", "baz*").setIndicesOptions(options), true); } - @Test - public void testAllMissing_lenient() throws Exception { + public void testAllMissingLenient() throws Exception { createIndex("test1"); client().prepareIndex("test1", "type", "1").setSource("k", "v").setRefresh(true).execute().actionGet(); SearchResponse response = client().prepareSearch("test2") @@ -508,7 +485,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .execute().actionGet(); assertHitCount(response, 0l); - + //you should still be able to run empty searches without things blowing up response = client().prepareSearch() .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -517,8 +494,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertHitCount(response, 1l); } - @Test - public void testAllMissing_strict() throws Exception { + public void testAllMissingStrict() throws Exception { createIndex("test1"); ensureYellow(); try { @@ -541,31 +517,24 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { client().prepareSearch().setQuery(matchAllQuery()).execute().actionGet(); } - @Test // For now don't handle closed indices - public void testCloseApi_specifiedIndices() throws Exception { + public void testCloseApiSpecifiedIndices() throws Exception { createIndex("test1", "test2"); ensureGreen(); verify(search("test1", "test2"), false); - verify(count("test1", "test2"), false); assertAcked(client().admin().indices().prepareClose("test2").get()); verify(search("test1", "test2"), true); - verify(count("test1", "test2"), true); IndicesOptions options = IndicesOptions.fromOptions(true, true, true, false, IndicesOptions.strictExpandOpenAndForbidClosed()); verify(search("test1", "test2").setIndicesOptions(options), false); - verify(count("test1", "test2").setIndicesOptions(options), false); verify(search(), false); - verify(count(), false); verify(search("t*"), false); - verify(count("t*"), false); } - @Test - public void testCloseApi_wildcards() throws Exception { + public void testCloseApiWildcards() throws Exception { createIndex("foo", "foobar", "bar", "barbaz"); ensureGreen(); @@ -581,7 +550,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(client().admin().indices().prepareOpen("_all"), true); } - @Test public void testDeleteIndex() throws Exception { createIndex("foobar"); ensureYellow(); @@ -592,8 +560,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareExists("foobar").get().isExists(), equalTo(false)); } - @Test - public void testDeleteIndex_wildcard() throws Exception { + public void testDeleteIndexWildcard() throws Exception { verify(client().admin().indices().prepareDelete("_all"), false); createIndex("foo", "foobar", "bar", "barbaz"); @@ -613,8 +580,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareExists("bar").get().isExists(), equalTo(false)); assertThat(client().admin().indices().prepareExists("barbaz").get().isExists(), equalTo(false)); } - - @Test + public void testPutWarmer() throws Exception { createIndex("foobar"); ensureYellow(); @@ -622,29 +588,27 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer1").get().getWarmers().size(), equalTo(1)); } - - @Test - public void testPutWarmer_wildcard() throws Exception { + + public void testPutWarmerWildcard() throws Exception { createIndex("foo", "foobar", "bar", "barbaz"); ensureYellow(); verify(client().admin().indices().preparePutWarmer("warmer1").setSearchRequest(client().prepareSearch().setIndices("foo*").setQuery(QueryBuilders.matchAllQuery())), false); - + assertThat(client().admin().indices().prepareGetWarmers("foo").setWarmers("warmer1").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer1").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("bar").setWarmers("warmer1").get().getWarmers().size(), equalTo(0)); assertThat(client().admin().indices().prepareGetWarmers("barbaz").setWarmers("warmer1").get().getWarmers().size(), equalTo(0)); verify(client().admin().indices().preparePutWarmer("warmer2").setSearchRequest(client().prepareSearch().setIndices().setQuery(QueryBuilders.matchAllQuery())), false); - + assertThat(client().admin().indices().prepareGetWarmers("foo").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("bar").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("barbaz").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); - + } - @Test public void testPutAlias() throws Exception { createIndex("foobar"); ensureYellow(); @@ -652,9 +616,8 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true)); } - - @Test - public void testPutAlias_wildcard() throws Exception { + + public void testPutAliasWildcard() throws Exception { createIndex("foo", "foobar", "bar", "barbaz"); ensureYellow(); @@ -669,14 +632,13 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true)); assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("bar").get().exists(), equalTo(true)); assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("barbaz").get().exists(), equalTo(true)); - + } - - @Test + public void testDeleteWarmer() throws Exception { - IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry( - "test1", new String[]{"typ1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}") - ); + SearchSourceBuilder source = new SearchSourceBuilder(); + source.query(QueryBuilders.matchAllQuery()); + IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "typ1" }, false, new IndexWarmersMetaData.SearchSource(source)); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); ensureYellow(); @@ -686,13 +648,12 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("test1").get().getWarmers().size(), equalTo(0)); } - @Test - public void testDeleteWarmer_wildcard() throws Exception { + public void testDeleteWarmerWildcard() throws Exception { verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true); - IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry( - "test1", new String[]{"type1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}") - ); + SearchSourceBuilder source = new SearchSourceBuilder(); + source.query(QueryBuilders.matchAllQuery()); + IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "type1" }, false, new IndexWarmersMetaData.SearchSource(source)); assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("bar").addCustom(new IndexWarmersMetaData(entry))); @@ -714,7 +675,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareGetWarmers("barbaz").setWarmers("test1").get().getWarmers().size(), equalTo(0)); } - @Test public void testPutMapping() throws Exception { verify(client().admin().indices().preparePutMapping("foo").setType("type1").setSource("field", "type=string"), true); verify(client().admin().indices().preparePutMapping("_all").setType("type1").setSource("field", "type=string"), true); @@ -737,7 +697,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareGetMappings("foobar").get().mappings().get("foobar").get("type3"), notNullValue()); assertThat(client().admin().indices().prepareGetMappings("bar").get().mappings().get("bar").get("type3"), notNullValue()); assertThat(client().admin().indices().prepareGetMappings("barbaz").get().mappings().get("barbaz").get("type3"), notNullValue()); - + verify(client().admin().indices().preparePutMapping("c*").setType("type1").setSource("field", "type=string"), true); @@ -746,7 +706,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareGetMappings("barbaz").get().mappings().get("barbaz").get("type4"), notNullValue()); } - @Test public void testUpdateSettings() throws Exception { verify(client().admin().indices().prepareUpdateSettings("foo").setSettings(Settings.builder().put("a", "b")), true); verify(client().admin().indices().prepareUpdateSettings("_all").setSettings(Settings.builder().put("a", "b")), true); @@ -791,10 +750,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { return multiSearchRequestBuilder.add(client().prepareSearch(indices).setQuery(matchAllQuery())); } - private static CountRequestBuilder count(String... indices) { - return client().prepareCount(indices).setQuery(matchAllQuery()); - } - private static ClearIndicesCacheRequestBuilder clearCache(String... indices) { return client().admin().indices().prepareClearCache(indices); } @@ -811,8 +766,8 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { return client().admin().indices().prepareStats(indices); } - private static OptimizeRequestBuilder optimize(String... indices) { - return client().admin().indices().prepareOptimize(indices); + private static ForceMergeRequestBuilder forceMerge(String... indices) { + return client().admin().indices().prepareForceMerge(indices); } private static RefreshRequestBuilder refresh(String... indices) { @@ -883,7 +838,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { private static void verify(ActionRequestBuilder requestBuilder, boolean fail) { verify(requestBuilder, fail, 0); } - + private static void verify(ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) { if (fail) { if (requestBuilder instanceof MultiSearchRequestBuilder) { @@ -900,9 +855,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { if (requestBuilder instanceof SearchRequestBuilder) { SearchRequestBuilder searchRequestBuilder = (SearchRequestBuilder) requestBuilder; assertHitCount(searchRequestBuilder.get(), expectedCount); - } else if (requestBuilder instanceof CountRequestBuilder) { - CountRequestBuilder countRequestBuilder = (CountRequestBuilder) requestBuilder; - assertHitCount(countRequestBuilder.get(), expectedCount); } else if (requestBuilder instanceof MultiSearchRequestBuilder) { MultiSearchResponse multiSearchResponse = ((MultiSearchRequestBuilder) requestBuilder).get(); assertThat(multiSearchResponse.getResponses().length, equalTo(1)); diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 995deaca10c..522ebfb0f3b 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -29,11 +29,12 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.GatewayMetaState; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.IndexSettingsModule; -import java.io.IOException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -44,6 +45,7 @@ public class IndicesServiceTests extends ESSingleNodeTestCase { public IndicesService getIndicesService() { return getInstanceFromNode(IndicesService.class); } + public NodeEnvironment getNodeEnvironment() { return getInstanceFromNode(NodeEnvironment.class); } @@ -56,12 +58,12 @@ public class IndicesServiceTests extends ESSingleNodeTestCase { public void testCanDeleteIndexContent() { IndicesService indicesService = getIndicesService(); - Settings idxSettings = settings(Version.CURRENT) + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", Settings.builder() .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true) .put(IndexMetaData.SETTING_DATA_PATH, "/foo/bar") .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 4)) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 3)) - .build(); + .build()); assertFalse("shard on shared filesystem", indicesService.canDeleteIndexContents(new Index("test"), idxSettings, false)); assertTrue("shard on shared filesystem and closed", indicesService.canDeleteIndexContents(new Index("test"), idxSettings, true)); } @@ -70,12 +72,13 @@ public class IndicesServiceTests extends ESSingleNodeTestCase { IndicesService indicesService = getIndicesService(); IndexMetaData meta = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas( 1).build(); - assertFalse("no shard location", indicesService.canDeleteShardContent(new ShardId("test", 0), meta)); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", meta.getSettings()); + assertFalse("no shard location", indicesService.canDeleteShardContent(new ShardId("test", 0), indexSettings)); IndexService test = createIndex("test"); assertTrue(test.hasShard(0)); - assertFalse("shard is allocated", indicesService.canDeleteShardContent(new ShardId("test", 0), meta)); + assertFalse("shard is allocated", indicesService.canDeleteShardContent(new ShardId("test", 0), indexSettings)); test.removeShard(0, "boom"); - assertTrue("shard is removed", indicesService.canDeleteShardContent(new ShardId("test", 0), meta)); + assertTrue("shard is removed", indicesService.canDeleteShardContent(new ShardId("test", 0), indexSettings)); } public void testDeleteIndexStore() throws Exception { @@ -132,7 +135,7 @@ public class IndicesServiceTests extends ESSingleNodeTestCase { ensureGreen("test"); } - public void testPendingTasks() throws IOException { + public void testPendingTasks() throws Exception { IndicesService indicesService = getIndicesService(); IndexService test = createIndex("test"); @@ -175,7 +178,7 @@ public class IndicesServiceTests extends ESSingleNodeTestCase { indicesService.addPendingDelete(new ShardId("bogus", 1), test.getIndexSettings()); assertEquals(indicesService.numPendingDeletes(test.index()), 2); // shard lock released... we can now delete - indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS)); + indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS)); assertEquals(indicesService.numPendingDeletes(test.index()), 0); } assertAcked(client().admin().indices().prepareOpen("test")); diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisBinderProcessor.java b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisBinderProcessor.java deleted file mode 100644 index fdb5ab05bde..00000000000 --- a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisBinderProcessor.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.analysis; - -import org.elasticsearch.index.analysis.AnalysisModule; - -/** - */ -public class DummyAnalysisBinderProcessor extends AnalysisModule.AnalysisBinderProcessor { - - @Override - public void processAnalyzers(AnalyzersBindings analyzersBindings) { - analyzersBindings.processAnalyzer("dummy", DummyAnalyzerProvider.class); - } - - @Override - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - tokenFiltersBindings.processTokenFilter("dummy_token_filter", DummyTokenFilterFactory.class); - } - - @Override - public void processTokenizers(TokenizersBindings tokenizersBindings) { - tokenizersBindings.processTokenizer("dummy_tokenizer", DummyTokenizerFactory.class); - } - - @Override - public void processCharFilters(CharFiltersBindings charFiltersBindings) { - charFiltersBindings.processCharFilter("dummy_char_filter", DummyCharFilterFactory.class); - } -} diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java index 003771f8cc7..8e15a6f02f4 100644 --- a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java +++ b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java @@ -19,13 +19,8 @@ package org.elasticsearch.indices.analysis; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.index.analysis.AnalysisModule; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - public class DummyAnalysisPlugin extends Plugin { /** * The name of the plugin. @@ -43,13 +38,12 @@ public class DummyAnalysisPlugin extends Plugin { return "Analysis Dummy Plugin"; } - @Override - public Collection nodeModules() { - return Collections.singletonList(new DummyIndicesAnalysisModule()); - } public void onModule(AnalysisModule module) { - module.addProcessor(new DummyAnalysisBinderProcessor()); + module.registerAnalyzer("dummy", (a, b, c, d) -> new DummyAnalyzerProvider()); + module.registerTokenFilter("dummy_token_filter", (a, b, c, d) -> new DummyTokenFilterFactory()); + module.registerTokenizer("dummy_tokenizer", (a, b, c, d) -> new DummyTokenizerFactory()); + module.registerCharFilter("dummy_char_filter", (a, b, c, d) -> new DummyCharFilterFactory()); } } diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/DummyIndicesAnalysis.java b/core/src/test/java/org/elasticsearch/indices/analysis/DummyIndicesAnalysis.java deleted file mode 100644 index 9642b610f69..00000000000 --- a/core/src/test/java/org/elasticsearch/indices/analysis/DummyIndicesAnalysis.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.analysis; - -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.analysis.*; - -public class DummyIndicesAnalysis extends AbstractComponent { - - @Inject - public DummyIndicesAnalysis(Settings settings, IndicesAnalysisService indicesAnalysisService) { - super(settings); - indicesAnalysisService.analyzerProviderFactories().put("dummy", - new PreBuiltAnalyzerProviderFactory("dummy", AnalyzerScope.INDICES, - new DummyAnalyzer())); - indicesAnalysisService.tokenFilterFactories().put("dummy_token_filter", - new PreBuiltTokenFilterFactoryFactory(new DummyTokenFilterFactory())); - indicesAnalysisService.charFilterFactories().put("dummy_char_filter", - new PreBuiltCharFilterFactoryFactory(new DummyCharFilterFactory())); - indicesAnalysisService.tokenizerFactories().put("dummy_tokenizer", - new PreBuiltTokenizerFactoryFactory(new DummyTokenizerFactory())); - } -} diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java index 208922b885b..4130cf5ad8a 100644 --- a/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESBackcompatTestCase; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -45,13 +44,11 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) @ESBackcompatTestCase.CompatibilityVersion(version = Version.V_1_2_0_ID) // we throw an exception if we create an index with _field_names that is 1.3 public class PreBuiltAnalyzerIntegrationIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(DummyAnalysisPlugin.class); } - @Test public void testThatPreBuiltAnalyzersAreNotClosedOnIndexClose() throws Exception { Map> loadedAnalyzers = new HashMap<>(); List indexNames = new ArrayList<>(); @@ -121,7 +118,6 @@ public class PreBuiltAnalyzerIntegrationIT extends ESIntegTestCase { * Test case for #5030: Upgrading analysis plugins fails * See https://github.com/elasticsearch/elasticsearch/issues/5030 */ - @Test public void testThatPluginAnalyzersCanBeUpdated() throws Exception { final XContentBuilder mapping = jsonBuilder().startObject() .startObject("type") diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java index a9de21b22f1..80993229bec 100644 --- a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java +++ b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java @@ -22,26 +22,32 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.rest.action.admin.indices.analyze.RestAnalyzeAction; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.hamcrest.core.IsNull; import java.io.IOException; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; + /** * */ public class AnalyzeActionIT extends ESIntegTestCase { - - @Test - public void simpleAnalyzerTests() throws Exception { + public void testSimpleAnalyzerTests() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); @@ -70,9 +76,8 @@ public class AnalyzeActionIT extends ESIntegTestCase { assertThat(token.getPosition(), equalTo(3)); } } - - @Test - public void analyzeNumericField() throws IOException { + + public void testAnalyzeNumericField() throws IOException { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "long", "type=long", "double", "type=double")); ensureGreen("test"); @@ -90,9 +95,7 @@ public class AnalyzeActionIT extends ESIntegTestCase { } } - @Test - public void analyzeWithNoIndex() throws Exception { - + public void testAnalyzeWithNoIndex() throws Exception { AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); @@ -120,9 +123,7 @@ public class AnalyzeActionIT extends ESIntegTestCase { } - @Test - public void analyzeWithCharFilters() throws Exception { - + public void testAnalyzeWithCharFilters() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(settingsBuilder().put(indexSettings()) .put("index.analysis.char_filter.custom_mapping.type", "mapping") @@ -152,8 +153,7 @@ public class AnalyzeActionIT extends ESIntegTestCase { assertThat(token.getTerm(), equalTo("fish")); } - @Test - public void analyzerWithFieldOrTypeTests() throws Exception { + public void testAnalyzerWithFieldOrTypeTests() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); @@ -173,7 +173,7 @@ public class AnalyzeActionIT extends ESIntegTestCase { } } - @Test // issue #5974 + // issue #5974 public void testThatStandardAndDefaultAnalyzersAreSame() throws Exception { AnalyzeResponse response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("standard").get(); assertTokens(response, "this", "is", "a", "test"); @@ -196,7 +196,6 @@ public class AnalyzeActionIT extends ESIntegTestCase { return randomBoolean() ? "test" : "alias"; } - @Test public void testParseXContentForAnalyzeReuqest() throws Exception { BytesReference content = XContentFactory.jsonBuilder() .startObject() @@ -207,7 +206,7 @@ public class AnalyzeActionIT extends ESIntegTestCase { AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test"); - RestAnalyzeAction.buildFromContent(content, analyzeRequest); + RestAnalyzeAction.buildFromContent(content, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)); assertThat(analyzeRequest.text().length, equalTo(1)); assertThat(analyzeRequest.text(), equalTo(new String[]{"THIS IS A TEST"})); @@ -215,12 +214,11 @@ public class AnalyzeActionIT extends ESIntegTestCase { assertThat(analyzeRequest.tokenFilters(), equalTo(new String[]{"lowercase"})); } - @Test public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception { AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test"); try { - RestAnalyzeAction.buildFromContent(new BytesArray("{invalid_json}"), analyzeRequest); + RestAnalyzeAction.buildFromContent(new BytesArray("{invalid_json}"), analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)); fail("shouldn't get here"); } catch (Exception e) { assertThat(e, instanceOf(IllegalArgumentException.class)); @@ -228,7 +226,6 @@ public class AnalyzeActionIT extends ESIntegTestCase { } } - @Test public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() throws Exception { AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test"); BytesReference invalidContent =XContentFactory.jsonBuilder() @@ -238,7 +235,7 @@ public class AnalyzeActionIT extends ESIntegTestCase { .endObject().bytes(); try { - RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest); + RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)); fail("shouldn't get here"); } catch (Exception e) { assertThat(e, instanceOf(IllegalArgumentException.class)); @@ -246,9 +243,7 @@ public class AnalyzeActionIT extends ESIntegTestCase { } } - @Test - public void analyzerWithMultiValues() throws Exception { - + public void testAnalyzerWithMultiValues() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); @@ -277,4 +272,235 @@ public class AnalyzeActionIT extends ESIntegTestCase { } + public void testDetailAnalyze() throws Exception { + assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .setSettings( + settingsBuilder() + .put("index.analysis.char_filter.my_mapping.type", "mapping") + .putArray("index.analysis.char_filter.my_mapping.mappings", "PH=>F") + .put("index.analysis.analyzer.test_analyzer.type", "custom") + .put("index.analysis.analyzer.test_analyzer.position_increment_gap", "100") + .put("index.analysis.analyzer.test_analyzer.tokenizer", "standard") + .putArray("index.analysis.analyzer.test_analyzer.char_filter", "my_mapping") + .putArray("index.analysis.analyzer.test_analyzer.filter", "snowball"))); + ensureGreen(); + + for (int i = 0; i < 10; i++) { + AnalyzeResponse analyzeResponse = admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText("THIS IS A PHISH") + .setExplain(true).setCharFilters("my_mapping").setTokenizer("keyword").setTokenFilters("lowercase").get(); + + assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); + //charfilters + // global charfilter is not change text. + assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); + assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("my_mapping")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("THIS IS A FISH")); + //tokenizer + assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); + assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("THIS IS A FISH")); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(15)); + //tokenfilters + assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("this is a fish")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), equalTo(15)); + } + } + + public void testDetailAnalyzeWithNoIndex() throws Exception { + //analyzer only + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setExplain(true).setAnalyzer("simple").get(); + + assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue()); + assertThat(analyzeResponse.detail().tokenfilters(), IsNull.nullValue()); + assertThat(analyzeResponse.detail().charfilters(), IsNull.nullValue()); + assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple")); + assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(4)); + } + + public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception { + //analyzer only + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setExplain(true).setAnalyzer("simple").get(); + + assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue()); + assertThat(analyzeResponse.detail().tokenfilters(), IsNull.nullValue()); + assertThat(analyzeResponse.detail().charfilters(), IsNull.nullValue()); + assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple")); + assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(4)); + + //custom analyzer + analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setExplain(true).setCharFilters("html_strip").setTokenizer("keyword").setTokenFilters("lowercase").get(); + assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); + //charfilters + // global charfilter is not change text. + assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); + assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("html_strip")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("\nTHIS IS A TEST\n")); + //tokenizer + assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); + assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("\nTHIS IS A TEST\n")); + //tokenfilters + assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("\nthis is a test\n")); + + + //check other attributes + analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled") + .setExplain(true).setTokenizer("standard").setTokenFilters("snowball").get(); + + assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl")); + String[] expectedAttributesKey = { + "bytes", + "positionLength", + "keyword"}; + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length)); + Object extendedAttribute; + + for (String key : expectedAttributesKey) { + extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key); + assertThat(extendedAttribute, notNullValue()); + } + } + + public void testDetailAnalyzeSpecifyAttributes() throws Exception { + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled") + .setExplain(true).setTokenizer("standard").setTokenFilters("snowball").setAttributes("keyword").get(); + + assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl")); + String[] expectedAttributesKey = { + "keyword"}; + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length)); + Object extendedAttribute; + + for (String key : expectedAttributesKey) { + extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key); + assertThat(extendedAttribute, notNullValue()); + } + } + + public void testDetailAnalyzeWithMultiValues() throws Exception { + assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); + ensureGreen(); + client().admin().indices().preparePutMapping("test") + .setType("document").setSource("simple", "type=string,analyzer=simple,position_increment_gap=100").get(); + + String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"}; + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts) + .setExplain(true).setField("simple").setText(texts).execute().get(); + + assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple")); + assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(7)); + AnalyzeResponse.AnalyzeToken token = analyzeResponse.detail().analyzer().getTokens()[3]; + + assertThat(token.getTerm(), equalTo("test")); + assertThat(token.getPosition(), equalTo(3)); + assertThat(token.getStartOffset(), equalTo(10)); + assertThat(token.getEndOffset(), equalTo(14)); + + token = analyzeResponse.detail().analyzer().getTokens()[5]; + assertThat(token.getTerm(), equalTo("second")); + assertThat(token.getPosition(), equalTo(105)); + assertThat(token.getStartOffset(), equalTo(19)); + assertThat(token.getEndOffset(), equalTo(25)); + } + + public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exception { + assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .setSettings( + settingsBuilder() + .put("index.analysis.char_filter.my_mapping.type", "mapping") + .putArray("index.analysis.char_filter.my_mapping.mappings", "PH=>F") + .put("index.analysis.analyzer.test_analyzer.type", "custom") + .put("index.analysis.analyzer.test_analyzer.position_increment_gap", "100") + .put("index.analysis.analyzer.test_analyzer.tokenizer", "standard") + .putArray("index.analysis.analyzer.test_analyzer.char_filter", "my_mapping") + .putArray("index.analysis.analyzer.test_analyzer.filter", "snowball", "lowercase"))); + ensureGreen(); + + client().admin().indices().preparePutMapping("test") + .setType("document").setSource("simple", "type=string,analyzer=simple,position_increment_gap=100").get(); + + //only analyzer = + String[] texts = new String[]{"this is a PHISH", "the troubled text"}; + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts) + .setExplain(true).setAnalyzer("test_analyzer").setText(texts).execute().get(); + + // charfilter + assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); + assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("my_mapping")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(2)); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("this is a FISH")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[1], equalTo("the troubled text")); + + // tokenizer + assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("standard")); + assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(7)); + AnalyzeResponse.AnalyzeToken token = analyzeResponse.detail().tokenizer().getTokens()[3]; + + assertThat(token.getTerm(), equalTo("FISH")); + assertThat(token.getPosition(), equalTo(3)); + assertThat(token.getStartOffset(), equalTo(10)); + assertThat(token.getEndOffset(), equalTo(15)); + + token = analyzeResponse.detail().tokenizer().getTokens()[5]; + assertThat(token.getTerm(), equalTo("troubled")); + assertThat(token.getPosition(), equalTo(105)); + assertThat(token.getStartOffset(), equalTo(20)); + assertThat(token.getEndOffset(), equalTo(28)); + + // tokenfilter(snowball) + assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(2)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(7)); + token = analyzeResponse.detail().tokenfilters()[0].getTokens()[3]; + + assertThat(token.getTerm(), equalTo("FISH")); + assertThat(token.getPosition(), equalTo(3)); + assertThat(token.getStartOffset(), equalTo(10)); + assertThat(token.getEndOffset(), equalTo(15)); + + token = analyzeResponse.detail().tokenfilters()[0].getTokens()[5]; + assertThat(token.getTerm(), equalTo("troubl")); + assertThat(token.getPosition(), equalTo(105)); + assertThat(token.getStartOffset(), equalTo(20)); + assertThat(token.getEndOffset(), equalTo(28)); + + // tokenfilter(lowercase) + assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("lowercase")); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens().length, equalTo(7)); + token = analyzeResponse.detail().tokenfilters()[1].getTokens()[3]; + + assertThat(token.getTerm(), equalTo("fish")); + assertThat(token.getPosition(), equalTo(3)); + assertThat(token.getStartOffset(), equalTo(10)); + assertThat(token.getEndOffset(), equalTo(15)); + + token = analyzeResponse.detail().tokenfilters()[0].getTokens()[5]; + assertThat(token.getTerm(), equalTo("troubl")); + assertThat(token.getPosition(), equalTo(105)); + assertThat(token.getStartOffset(), equalTo(20)); + assertThat(token.getEndOffset(), equalTo(28)); + + + } + } diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java index 96fc85ad85a..722a4ebde8a 100644 --- a/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java @@ -27,9 +27,9 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.hamcrest.Matchers; -import org.junit.Test; -import static org.elasticsearch.indices.analysis.HunspellService.*; +import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_IGNORE_CASE; +import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_LAZY_LOAD; import static org.hamcrest.Matchers.notNullValue; /** @@ -37,8 +37,6 @@ import static org.hamcrest.Matchers.notNullValue; */ @ClusterScope(scope= Scope.TEST, numDataNodes=0) public class HunspellServiceIT extends ESIntegTestCase { - - @Test public void testLocaleDirectoryWithNodeLevelConfig() throws Exception { Settings settings = Settings.settingsBuilder() .put("path.conf", getDataPath("/indices/analyze/conf_dir")) @@ -52,7 +50,6 @@ public class HunspellServiceIT extends ESIntegTestCase { assertIgnoreCase(true, dictionary); } - @Test public void testLocaleDirectoryWithLocaleSpecificConfig() throws Exception { Settings settings = Settings.settingsBuilder() .put("path.conf", getDataPath("/indices/analyze/conf_dir")) @@ -75,7 +72,6 @@ public class HunspellServiceIT extends ESIntegTestCase { assertIgnoreCase(true, dictionary); } - @Test public void testDicWithNoAff() throws Exception { Settings settings = Settings.settingsBuilder() .put("path.conf", getDataPath("/indices/analyze/no_aff_conf_dir")) @@ -94,7 +90,6 @@ public class HunspellServiceIT extends ESIntegTestCase { } } - @Test public void testDicWithTwoAffs() throws Exception { Settings settings = Settings.settingsBuilder() .put("path.conf", getDataPath("/indices/analyze/two_aff_conf_dir")) diff --git a/core/src/test/java/org/elasticsearch/indices/cache/query/IndicesQueryCacheTests.java b/core/src/test/java/org/elasticsearch/indices/cache/query/IndicesQueryCacheTests.java new file mode 100644 index 00000000000..c25b20699aa --- /dev/null +++ b/core/src/test/java/org/elasticsearch/indices/cache/query/IndicesQueryCacheTests.java @@ -0,0 +1,327 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.cache.query; + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.cache.query.QueryCacheStats; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; + +public class IndicesQueryCacheTests extends ESTestCase { + + private static class DummyQuery extends Query { + + private final int id; + + DummyQuery(int id) { + this.id = id; + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && id == ((DummyQuery) obj).id; + } + + @Override + public int hashCode() { + return 31 * super.hashCode() + id; + } + + @Override + public String toString(String field) { + return "dummy"; + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores) + throws IOException { + return new ConstantScoreWeight(this) { + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); + } + }; + } + + } + + public void testBasics() throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); + w.addDocument(new Document()); + DirectoryReader r = DirectoryReader.open(w, false); + w.close(); + ShardId shard = new ShardId(new Index("index"), 0); + r = ElasticsearchDirectoryReader.wrap(r, shard); + IndexSearcher s = new IndexSearcher(r); + s.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + + Settings settings = Settings.builder() + .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT, 10) + .build(); + IndicesQueryCache cache = new IndicesQueryCache(settings); + s.setQueryCache(cache); + + QueryCacheStats stats = cache.getStats(shard); + assertEquals(0L, stats.getCacheSize()); + assertEquals(0L, stats.getCacheCount()); + assertEquals(0L, stats.getHitCount()); + assertEquals(0L, stats.getMissCount()); + + assertEquals(1, s.count(new DummyQuery(0))); + + stats = cache.getStats(shard); + assertEquals(1L, stats.getCacheSize()); + assertEquals(1L, stats.getCacheCount()); + assertEquals(0L, stats.getHitCount()); + assertEquals(1L, stats.getMissCount()); + + for (int i = 1; i < 20; ++i) { + assertEquals(1, s.count(new DummyQuery(i))); + } + + stats = cache.getStats(shard); + assertEquals(10L, stats.getCacheSize()); + assertEquals(20L, stats.getCacheCount()); + assertEquals(0L, stats.getHitCount()); + assertEquals(20L, stats.getMissCount()); + + s.count(new DummyQuery(10)); + + stats = cache.getStats(shard); + assertEquals(10L, stats.getCacheSize()); + assertEquals(20L, stats.getCacheCount()); + assertEquals(1L, stats.getHitCount()); + assertEquals(20L, stats.getMissCount()); + + IOUtils.close(r, dir); + + // got emptied, but no changes to other metrics + stats = cache.getStats(shard); + assertEquals(0L, stats.getCacheSize()); + assertEquals(20L, stats.getCacheCount()); + assertEquals(1L, stats.getHitCount()); + assertEquals(20L, stats.getMissCount()); + + cache.onClose(shard); + + // forgot everything + stats = cache.getStats(shard); + assertEquals(0L, stats.getCacheSize()); + assertEquals(0L, stats.getCacheCount()); + assertEquals(0L, stats.getHitCount()); + assertEquals(0L, stats.getMissCount()); + + cache.close(); // this triggers some assertions + } + + public void testTwoShards() throws IOException { + Directory dir1 = newDirectory(); + IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig()); + w1.addDocument(new Document()); + DirectoryReader r1 = DirectoryReader.open(w1, false); + w1.close(); + ShardId shard1 = new ShardId(new Index("index"), 0); + r1 = ElasticsearchDirectoryReader.wrap(r1, shard1); + IndexSearcher s1 = new IndexSearcher(r1); + s1.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + + Directory dir2 = newDirectory(); + IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig()); + w2.addDocument(new Document()); + DirectoryReader r2 = DirectoryReader.open(w2, false); + w2.close(); + ShardId shard2 = new ShardId(new Index("index"), 1); + r2 = ElasticsearchDirectoryReader.wrap(r2, shard2); + IndexSearcher s2 = new IndexSearcher(r2); + s2.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + + Settings settings = Settings.builder() + .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT, 10) + .build(); + IndicesQueryCache cache = new IndicesQueryCache(settings); + s1.setQueryCache(cache); + s2.setQueryCache(cache); + + assertEquals(1, s1.count(new DummyQuery(0))); + + QueryCacheStats stats1 = cache.getStats(shard1); + assertEquals(1L, stats1.getCacheSize()); + assertEquals(1L, stats1.getCacheCount()); + assertEquals(0L, stats1.getHitCount()); + assertEquals(1L, stats1.getMissCount()); + + QueryCacheStats stats2 = cache.getStats(shard2); + assertEquals(0L, stats2.getCacheSize()); + assertEquals(0L, stats2.getCacheCount()); + assertEquals(0L, stats2.getHitCount()); + assertEquals(0L, stats2.getMissCount()); + + assertEquals(1, s2.count(new DummyQuery(0))); + + stats1 = cache.getStats(shard1); + assertEquals(1L, stats1.getCacheSize()); + assertEquals(1L, stats1.getCacheCount()); + assertEquals(0L, stats1.getHitCount()); + assertEquals(1L, stats1.getMissCount()); + + stats2 = cache.getStats(shard2); + assertEquals(1L, stats2.getCacheSize()); + assertEquals(1L, stats2.getCacheCount()); + assertEquals(0L, stats2.getHitCount()); + assertEquals(1L, stats2.getMissCount()); + + for (int i = 0; i < 20; ++i) { + assertEquals(1, s2.count(new DummyQuery(i))); + } + + stats1 = cache.getStats(shard1); + assertEquals(0L, stats1.getCacheSize()); // evicted + assertEquals(1L, stats1.getCacheCount()); + assertEquals(0L, stats1.getHitCount()); + assertEquals(1L, stats1.getMissCount()); + + stats2 = cache.getStats(shard2); + assertEquals(10L, stats2.getCacheSize()); + assertEquals(20L, stats2.getCacheCount()); + assertEquals(1L, stats2.getHitCount()); + assertEquals(20L, stats2.getMissCount()); + + IOUtils.close(r1, dir1); + + // no changes + stats1 = cache.getStats(shard1); + assertEquals(0L, stats1.getCacheSize()); + assertEquals(1L, stats1.getCacheCount()); + assertEquals(0L, stats1.getHitCount()); + assertEquals(1L, stats1.getMissCount()); + + stats2 = cache.getStats(shard2); + assertEquals(10L, stats2.getCacheSize()); + assertEquals(20L, stats2.getCacheCount()); + assertEquals(1L, stats2.getHitCount()); + assertEquals(20L, stats2.getMissCount()); + + cache.onClose(shard1); + + // forgot everything about shard1 + stats1 = cache.getStats(shard1); + assertEquals(0L, stats1.getCacheSize()); + assertEquals(0L, stats1.getCacheCount()); + assertEquals(0L, stats1.getHitCount()); + assertEquals(0L, stats1.getMissCount()); + + stats2 = cache.getStats(shard2); + assertEquals(10L, stats2.getCacheSize()); + assertEquals(20L, stats2.getCacheCount()); + assertEquals(1L, stats2.getHitCount()); + assertEquals(20L, stats2.getMissCount()); + + IOUtils.close(r2, dir2); + cache.onClose(shard2); + + // forgot everything about shard2 + stats1 = cache.getStats(shard1); + assertEquals(0L, stats1.getCacheSize()); + assertEquals(0L, stats1.getCacheCount()); + assertEquals(0L, stats1.getHitCount()); + assertEquals(0L, stats1.getMissCount()); + + stats2 = cache.getStats(shard2); + assertEquals(0L, stats2.getCacheSize()); + assertEquals(0L, stats2.getCacheCount()); + assertEquals(0L, stats2.getHitCount()); + assertEquals(0L, stats2.getMissCount()); + + cache.close(); // this triggers some assertions + } + + // Make sure the cache behaves correctly when a segment that is associated + // with an empty cache gets closed. In that particular case, the eviction + // callback is called with a number of evicted entries equal to 0 + // see https://github.com/elastic/elasticsearch/issues/15043 + public void testStatsOnEviction() throws IOException { + Directory dir1 = newDirectory(); + IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig()); + w1.addDocument(new Document()); + DirectoryReader r1 = DirectoryReader.open(w1, false); + w1.close(); + ShardId shard1 = new ShardId(new Index("index"), 0); + r1 = ElasticsearchDirectoryReader.wrap(r1, shard1); + IndexSearcher s1 = new IndexSearcher(r1); + s1.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + + Directory dir2 = newDirectory(); + IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig()); + w2.addDocument(new Document()); + DirectoryReader r2 = DirectoryReader.open(w2, false); + w2.close(); + ShardId shard2 = new ShardId(new Index("index"), 1); + r2 = ElasticsearchDirectoryReader.wrap(r2, shard2); + IndexSearcher s2 = new IndexSearcher(r2); + s2.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + + Settings settings = Settings.builder() + .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT, 10) + .build(); + IndicesQueryCache cache = new IndicesQueryCache(settings); + s1.setQueryCache(cache); + s2.setQueryCache(cache); + + assertEquals(1, s1.count(new DummyQuery(0))); + + for (int i = 1; i <= 20; ++i) { + assertEquals(1, s2.count(new DummyQuery(i))); + } + + QueryCacheStats stats1 = cache.getStats(shard1); + assertEquals(0L, stats1.getCacheSize()); + assertEquals(1L, stats1.getCacheCount()); + + // this used to fail because we were evicting an empty cache on + // the segment from r1 + IOUtils.close(r1, dir1); + cache.onClose(shard1); + + IOUtils.close(r2, dir2); + cache.onClose(shard2); + + cache.close(); // this triggers some assertions + } + +} diff --git a/core/src/test/java/org/elasticsearch/indices/cache/query/terms/TermsLookupTests.java b/core/src/test/java/org/elasticsearch/indices/cache/query/terms/TermsLookupTests.java index 6474547cf29..bf0394988b4 100644 --- a/core/src/test/java/org/elasticsearch/indices/cache/query/terms/TermsLookupTests.java +++ b/core/src/test/java/org/elasticsearch/indices/cache/query/terms/TermsLookupTests.java @@ -22,13 +22,12 @@ package org.elasticsearch.indices.cache.query.terms; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; -public class TermsLookupTests extends ESTestCase { +import static org.hamcrest.Matchers.containsString; - @Test +public class TermsLookupTests extends ESTestCase { public void testTermsLookup() { String index = randomAsciiOfLengthBetween(1, 10); String type = randomAsciiOfLengthBetween(1, 10); @@ -44,23 +43,28 @@ public class TermsLookupTests extends ESTestCase { assertEquals(routing, termsLookup.routing()); } - @Test(expected=IllegalArgumentException.class) public void testIllegalArguments() { String type = randomAsciiOfLength(5); String id = randomAsciiOfLength(5); String path = randomAsciiOfLength(5); switch (randomIntBetween(0, 2)) { case 0: - type = null; break; + type = null; + break; case 1: - id = null; break; + id = null; + break; case 2: - path = null; break; + path = null; + break; + } + try { + new TermsLookup(null, type, id, path); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("[terms] query lookup element requires specifying")); } - new TermsLookup(null, type, id, path); } - @Test public void testSerialization() throws IOException { TermsLookup termsLookup = randomTermsLookup(); try (BytesStreamOutput output = new BytesStreamOutput()) { diff --git a/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java b/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java index 199f4b6f77c..38bea16f83c 100644 --- a/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java @@ -23,16 +23,15 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.hamcrest.Matchers.equalTo; public class IndicesExistsIT extends ESIntegTestCase { - - @Test // Indices exists never throws IndexMissingException, the indices options control its behaviour (return true or false) public void testIndicesExists() throws Exception { assertThat(client().admin().indices().prepareExists("foo").get().isExists(), equalTo(false)); @@ -51,7 +50,6 @@ public class IndicesExistsIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareExists("_all").get().isExists(), equalTo(true)); } - @Test public void testIndicesExistsWithBlocks() { createIndex("ro"); ensureYellow(); diff --git a/core/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java b/core/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java index ffb2e2e186a..407ee6fbc43 100644 --- a/core/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java @@ -24,20 +24,19 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; public class TypesExistsIT extends ESIntegTestCase { - - @Test public void testSimple() throws Exception { Client client = client(); client.admin().indices().prepareCreate("test1") @@ -76,7 +75,6 @@ public class TypesExistsIT extends ESIntegTestCase { assertThat(response.isExists(), equalTo(false)); } - @Test public void testTypesExistsWithBlocks() throws IOException { assertAcked(prepareCreate("ro").addMapping("type1", jsonBuilder().startObject().startObject("type1").endObject().endObject())); ensureGreen("ro"); diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index 96b145a9bef..aa8c9f18c01 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -31,7 +31,6 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; @@ -47,8 +46,6 @@ import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; public class FlushIT extends ESIntegTestCase { - - @Test public void testWaitIfOngoing() throws InterruptedException { createIndex("test"); ensureGreen("test"); @@ -181,12 +178,12 @@ public class FlushIT extends ESIntegTestCase { indexStats = client().admin().indices().prepareStats("test").get().getIndex("test"); assertFlushResponseEqualsShardStats(indexStats.getShards(), syncedFlushResult.getShardsResultPerIndex().get("test")); refresh(); - assertThat(client().prepareCount().get().getCount(), equalTo((long) numDocs.get())); - logger.info("indexed {} docs", client().prepareCount().get().getCount()); + assertThat(client().prepareSearch().setSize(0).get().getHits().totalHits(), equalTo((long) numDocs.get())); + logger.info("indexed {} docs", client().prepareSearch().setSize(0).get().getHits().totalHits()); logClusterState(); internalCluster().fullRestart(); ensureGreen(); - assertThat(client().prepareCount().get().getCount(), equalTo((long) numDocs.get())); + assertThat(client().prepareSearch().setSize(0).get().getHits().totalHits(), equalTo((long) numDocs.get())); } private void assertFlushResponseEqualsShardStats(ShardStats[] shardsStats, List syncedFlushResults) { @@ -210,7 +207,6 @@ public class FlushIT extends ESIntegTestCase { } } - @Test public void testUnallocatedShardsDoesNotHang() throws InterruptedException { // create an index but disallow allocation prepareCreate("test").setSettings(Settings.builder().put("index.routing.allocation.include._name", "nonexistent")).get(); @@ -222,5 +218,4 @@ public class FlushIT extends ESIntegTestCase { assertThat(shardsResult.size(), equalTo(numShards)); assertThat(shardsResult.get(0).failureReason(), equalTo("no active shards")); } - } diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java index f9d597cdc62..0946d51a45c 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.List; @@ -32,17 +31,15 @@ import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.emptyIterable; @ESIntegTestCase.ClusterScope(randomDynamicTemplates = false) // this test takes a long time to delete the idx if all fields are eager loading public class ConcurrentDynamicTemplateIT extends ESIntegTestCase { - private final String mappingType = "test-mapping"; - @Test // see #3544 + // see #3544 public void testConcurrentDynamicMapping() throws Exception { final String fieldName = "field"; final String mapping = "{ \"" + mappingType + "\": {" + diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java index 504d9a5840a..2f4970ef4f8 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java @@ -25,12 +25,15 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_METADATA_BLOCK; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; @@ -41,25 +44,20 @@ import static org.hamcrest.Matchers.notNullValue; */ @ClusterScope(randomDynamicTemplates = false) public class SimpleGetMappingsIT extends ESIntegTestCase { - - @Test - public void getMappingsWhereThereAreNone() { + public void testGetMappingsWhereThereAreNone() { createIndex("index"); GetMappingsResponse response = client().admin().indices().prepareGetMappings().execute().actionGet(); assertThat(response.mappings().containsKey("index"), equalTo(true)); assertThat(response.mappings().get("index").size(), equalTo(0)); } - private XContentBuilder getMappingForType(String type) throws IOException { return jsonBuilder().startObject().startObject(type).startObject("properties") .startObject("field1").field("type", "string").endObject() .endObject().endObject().endObject(); } - - @Test - public void simpleGetMappings() throws Exception { + public void testSimpleGetMappings() throws Exception { client().admin().indices().prepareCreate("indexa") .addMapping("typeA", getMappingForType("typeA")) .addMapping("typeB", getMappingForType("typeB")) @@ -146,7 +144,6 @@ public class SimpleGetMappingsIT extends ESIntegTestCase { assertThat(response.mappings().get("indexb").get("Btype"), notNullValue()); } - @Test public void testGetMappingsWithBlocks() throws IOException { client().admin().indices().prepareCreate("test") .addMapping("typeA", getMappingForType("typeA")) diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index a7ad9285aee..ed4b95c03d8 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -22,8 +22,8 @@ package org.elasticsearch.indices.mapping; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Priority; @@ -31,11 +31,9 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -52,8 +50,7 @@ import static org.hamcrest.Matchers.*; @ClusterScope(randomDynamicTemplates = false) public class UpdateMappingIntegrationIT extends ESIntegTestCase { - @Test - public void dynamicUpdates() throws Exception { + public void testDynamicUpdates() throws Exception { client().admin().indices().prepareCreate("test") .setSettings( settingsBuilder() @@ -75,8 +72,8 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { logger.info("checking all the documents are there"); RefreshResponse refreshResponse = client().admin().indices().prepareRefresh().execute().actionGet(); assertThat(refreshResponse.getFailedShards(), equalTo(0)); - CountResponse response = client().prepareCount("test").execute().actionGet(); - assertThat(response.getCount(), equalTo((long) recCount)); + SearchResponse response = client().prepareSearch("test").setSize(0).execute().actionGet(); + assertThat(response.getHits().totalHits(), equalTo((long) recCount)); logger.info("checking all the fields are in the mappings"); @@ -87,8 +84,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { } } - @Test - public void updateMappingWithoutType() throws Exception { + public void testUpdateMappingWithoutType() throws Exception { client().admin().indices().prepareCreate("test") .setSettings( settingsBuilder() @@ -109,8 +105,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { equalTo("{\"doc\":{\"properties\":{\"body\":{\"type\":\"string\"},\"date\":{\"type\":\"integer\"}}}}")); } - @Test - public void updateMappingWithoutTypeMultiObjects() throws Exception { + public void testUpdateMappingWithoutTypeMultiObjects() throws Exception { client().admin().indices().prepareCreate("test") .setSettings( settingsBuilder() @@ -130,9 +125,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { equalTo("{\"doc\":{\"properties\":{\"date\":{\"type\":\"integer\"}}}}")); } - @Test(expected = MergeMappingException.class) - public void updateMappingWithConflicts() throws Exception { - + public void testUpdateMappingWithConflicts() throws Exception { client().admin().indices().prepareCreate("test") .setSettings( settingsBuilder() @@ -142,29 +135,33 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { .execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); - PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type") - .setSource("{\"type\":{\"properties\":{\"body\":{\"type\":\"integer\"}}}}") - .execute().actionGet(); - - assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); + try { + client().admin().indices().preparePutMapping("test").setType("type") + .setSource("{\"type\":{\"properties\":{\"body\":{\"type\":\"integer\"}}}}").execute().actionGet(); + fail("Expected MergeMappingException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [body] cannot be changed from type [string] to [int]")); + } } - @Test(expected = MergeMappingException.class) - public void updateMappingWithNormsConflicts() throws Exception { + public void testUpdateMappingWithNormsConflicts() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("type", "{\"type\":{\"properties\":{\"body\":{\"type\":\"string\", \"norms\": { \"enabled\": false }}}}}") .execute().actionGet(); - PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type") - .setSource("{\"type\":{\"properties\":{\"body\":{\"type\":\"string\", \"norms\": { \"enabled\": true }}}}}") - .execute().actionGet(); + try { + client().admin().indices().preparePutMapping("test").setType("type") + .setSource("{\"type\":{\"properties\":{\"body\":{\"type\":\"string\", \"norms\": { \"enabled\": true }}}}}").execute() + .actionGet(); + fail("Expected MergeMappingException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [body] has different [omit_norms]")); + } } /* Second regression test for https://github.com/elasticsearch/elasticsearch/issues/3381 */ - @Test - public void updateMappingNoChanges() throws Exception { - + public void testUpdateMappingNoChanges() throws Exception { client().admin().indices().prepareCreate("test") .setSettings( settingsBuilder() @@ -183,9 +180,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { } @SuppressWarnings("unchecked") - @Test - public void updateDefaultMappingSettings() throws Exception { - + public void testUpdateDefaultMappingSettings() throws Exception { logger.info("Creating index with _default_ mappings"); client().admin().indices().prepareCreate("test").addMapping(MapperService.DEFAULT_MAPPING, JsonXContent.contentBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING) @@ -245,8 +240,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { } - @Test - public void updateMappingConcurrently() throws Throwable { + public void testUpdateMappingConcurrently() throws Throwable { createIndex("test1", "test2"); // This is important. The test assumes all nodes are aware of all indices. Due to initializing shard throttling @@ -312,7 +306,6 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { } - @Test public void testPutMappingsWithBlocks() throws Exception { createIndex("test"); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java index 2d54fb0a1fd..8230274de5b 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java @@ -22,26 +22,22 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.engine.EngineConfig; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.translog.TranslogConfig; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESSingleNodeTestCase; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; -public class IndexingMemoryControllerTests extends ESTestCase { +public class IndexingMemoryControllerTests extends ESSingleNodeTestCase { static class MockController extends IndexingMemoryController { - final Map indexBufferRAMBytesUsed = new HashMap<>(); + final Map indexBufferRAMBytesUsed = new HashMap<>(); public MockController(Settings settings) { super(Settings.builder() @@ -51,23 +47,23 @@ public class IndexingMemoryControllerTests extends ESTestCase { null, null, 100 * 1024 * 1024); // fix jvm mem size to 100mb } - public void deleteShard(ShardId shardId) { - indexBufferRAMBytesUsed.remove(shardId); + public void deleteShard(IndexShard shard) { + indexBufferRAMBytesUsed.remove(shard); } @Override - protected List availableShards() { + protected List availableShards() { return new ArrayList<>(indexBufferRAMBytesUsed.keySet()); } @Override - protected boolean shardAvailable(ShardId shardId) { - return indexBufferRAMBytesUsed.containsKey(shardId); + protected boolean shardAvailable(IndexShard shard) { + return indexBufferRAMBytesUsed.containsKey(shard); } @Override - protected long getIndexBufferRAMBytesUsed(ShardId shardId) { - Long used = indexBufferRAMBytesUsed.get(shardId); + protected long getIndexBufferRAMBytesUsed(IndexShard shard) { + Long used = indexBufferRAMBytesUsed.get(shard); if (used == null) { return 0; } else { @@ -76,105 +72,106 @@ public class IndexingMemoryControllerTests extends ESTestCase { } @Override - protected void checkIdle(ShardId shardId, long inactiveTimeNS) { + protected void checkIdle(IndexShard shard, long inactiveTimeNS) { } @Override - public void refreshShardAsync(ShardId shardId) { - indexBufferRAMBytesUsed.put(shardId, 0L); + public void refreshShardAsync(IndexShard shard) { + indexBufferRAMBytesUsed.put(shard, 0L); } - public void assertBuffer(ShardId shardId, ByteSizeValue expected) { - Long actual = indexBufferRAMBytesUsed.get(shardId); + public void assertBuffer(IndexShard shard, ByteSizeValue expected) { + Long actual = indexBufferRAMBytesUsed.get(shard); assertEquals(expected.bytes(), actual.longValue()); } - public void simulateIndexing(ShardId shardId) { - Long bytes = indexBufferRAMBytesUsed.get(shardId); + public void simulateIndexing(IndexShard shard) { + Long bytes = indexBufferRAMBytesUsed.get(shard); if (bytes == null) { bytes = 0L; } // Each doc we index takes up a megabyte! bytes += 1024*1024; - indexBufferRAMBytesUsed.put(shardId, bytes); + indexBufferRAMBytesUsed.put(shard, bytes); forceCheck(); } } public void testShardAdditionAndRemoval() { + createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 3).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); + MockController controller = new MockController(Settings.builder() .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "4mb").build()); - final ShardId shard1 = new ShardId("test", 1); - controller.simulateIndexing(shard1); - controller.assertBuffer(shard1, new ByteSizeValue(1, ByteSizeUnit.MB)); + IndexShard shard0 = test.getShard(0); + controller.simulateIndexing(shard0); + controller.assertBuffer(shard0, new ByteSizeValue(1, ByteSizeUnit.MB)); // add another shard - final ShardId shard2 = new ShardId("test", 2); - controller.simulateIndexing(shard2); + IndexShard shard1 = test.getShard(1); + controller.simulateIndexing(shard1); + controller.assertBuffer(shard0, new ByteSizeValue(1, ByteSizeUnit.MB)); controller.assertBuffer(shard1, new ByteSizeValue(1, ByteSizeUnit.MB)); - controller.assertBuffer(shard2, new ByteSizeValue(1, ByteSizeUnit.MB)); // remove first shard - controller.deleteShard(shard1); + controller.deleteShard(shard0); controller.forceCheck(); - controller.assertBuffer(shard2, new ByteSizeValue(1, ByteSizeUnit.MB)); + controller.assertBuffer(shard1, new ByteSizeValue(1, ByteSizeUnit.MB)); // remove second shard - controller.deleteShard(shard2); + controller.deleteShard(shard1); controller.forceCheck(); // add a new one - final ShardId shard3 = new ShardId("test", 3); - controller.simulateIndexing(shard3); - controller.assertBuffer(shard3, new ByteSizeValue(1, ByteSizeUnit.MB)); + IndexShard shard2 = test.getShard(2); + controller.simulateIndexing(shard2); + controller.assertBuffer(shard2, new ByteSizeValue(1, ByteSizeUnit.MB)); } public void testActiveInactive() { + + createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); + MockController controller = new MockController(Settings.builder() .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "5mb") .build()); - final ShardId shard1 = new ShardId("test", 1); + IndexShard shard0 = test.getShard(0); + controller.simulateIndexing(shard0); + IndexShard shard1 = test.getShard(1); controller.simulateIndexing(shard1); - final ShardId shard2 = new ShardId("test", 2); - controller.simulateIndexing(shard2); + + controller.assertBuffer(shard0, new ByteSizeValue(1, ByteSizeUnit.MB)); controller.assertBuffer(shard1, new ByteSizeValue(1, ByteSizeUnit.MB)); - controller.assertBuffer(shard2, new ByteSizeValue(1, ByteSizeUnit.MB)); + controller.simulateIndexing(shard0); controller.simulateIndexing(shard1); - controller.simulateIndexing(shard2); + controller.assertBuffer(shard0, new ByteSizeValue(2, ByteSizeUnit.MB)); controller.assertBuffer(shard1, new ByteSizeValue(2, ByteSizeUnit.MB)); - controller.assertBuffer(shard2, new ByteSizeValue(2, ByteSizeUnit.MB)); // index into one shard only, crosses the 5mb limit, so shard1 is refreshed + controller.simulateIndexing(shard0); + controller.simulateIndexing(shard0); + controller.assertBuffer(shard0, new ByteSizeValue(0, ByteSizeUnit.MB)); + controller.assertBuffer(shard1, new ByteSizeValue(2, ByteSizeUnit.MB)); + controller.simulateIndexing(shard1); controller.simulateIndexing(shard1); + controller.assertBuffer(shard1, new ByteSizeValue(4, ByteSizeUnit.MB)); + controller.simulateIndexing(shard1); + controller.simulateIndexing(shard1); + // shard1 crossed 5 mb and is now cleared: controller.assertBuffer(shard1, new ByteSizeValue(0, ByteSizeUnit.MB)); - controller.assertBuffer(shard2, new ByteSizeValue(2, ByteSizeUnit.MB)); - - controller.simulateIndexing(shard2); - controller.simulateIndexing(shard2); - controller.assertBuffer(shard2, new ByteSizeValue(4, ByteSizeUnit.MB)); - controller.simulateIndexing(shard2); - controller.simulateIndexing(shard2); - // shard2 crossed 5 mb and is now cleared: - controller.assertBuffer(shard2, new ByteSizeValue(0, ByteSizeUnit.MB)); } - public void testRelativeBufferSizes() { - MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "50%") - .build()); - - assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(50, ByteSizeUnit.MB))); - } - - public void testMinBufferSizes() { MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "0.001%") - .put(IndexingMemoryController.MIN_INDEX_BUFFER_SIZE_SETTING, "6mb").build()); + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "0.001%") + .put(IndexingMemoryController.MIN_INDEX_BUFFER_SIZE_SETTING, "6mb").build()); assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(6, ByteSizeUnit.MB))); } diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java index 4992b9f72dc..3398839b905 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -39,7 +38,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke /** Tests for the noop breakers, which are non-dynamic settings */ @ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE, numDataNodes=0) public class CircuitBreakerNoopIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -52,7 +50,6 @@ public class CircuitBreakerNoopIT extends ESIntegTestCase { .build(); } - @Test public void testNoopRequestBreaker() throws Exception { assertAcked(prepareCreate("cb-test", 1, settingsBuilder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)))); Client client = client(); @@ -70,7 +67,6 @@ public class CircuitBreakerNoopIT extends ESIntegTestCase { // no exception because the breaker is a noop } - @Test public void testNoopFielddataBreaker() throws Exception { assertAcked(prepareCreate("cb-test", 1, settingsBuilder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)))); Client client = client(); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index 4ffe6361cb5..fcd94d99585 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -38,7 +38,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -60,7 +59,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; */ @ClusterScope(scope = TEST, randomDynamicTemplates = false) public class CircuitBreakerServiceIT extends ESIntegTestCase { - /** Reset all breaker settings back to their defaults */ private void reset() { logger.info("--> resetting breaker settings"); @@ -100,7 +98,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase { return false; } - @Test public void testMemoryBreaker() throws Exception { if (noopBreakerUsed()) { logger.info("--> noop breakers used, skipping test"); @@ -142,7 +139,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase { assertThat(breaks, greaterThanOrEqualTo(1)); } - @Test public void testRamAccountingTermsEnum() throws Exception { if (noopBreakerUsed()) { logger.info("--> noop breakers used, skipping test"); @@ -196,7 +192,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase { * Test that a breaker correctly redistributes to a different breaker, in * this case, the fielddata breaker borrows space from the request breaker */ - @Test public void testParentChecking() throws Exception { if (noopBreakerUsed()) { logger.info("--> noop breakers used, skipping test"); @@ -256,7 +251,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase { } } - @Test public void testRequestBreaker() throws Exception { if (noopBreakerUsed()) { logger.info("--> noop breakers used, skipping test"); @@ -307,7 +301,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase { }, 30, TimeUnit.SECONDS); } - @Test public void testCustomCircuitBreakerRegistration() throws Exception { Iterable serviceIter = internalCluster().getInstances(CircuitBreakerService.class); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java index 013bc718ec9..741ea305254 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerUnitTests.java @@ -26,20 +26,20 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; /** * Unit tests for the circuit breaker */ public class CircuitBreakerUnitTests extends ESTestCase { - public static long pctBytes(String percentString) { return Settings.EMPTY.getAsMemory("", percentString).bytes(); } - @Test public void testBreakerSettingsValidationWithValidSettings() { // parent: {:limit 70}, fd: {:limit 50}, request: {:limit 20} BreakerSettings fd = new BreakerSettings(CircuitBreaker.FIELDDATA, pctBytes("50%"), 1.0); @@ -52,7 +52,6 @@ public class CircuitBreakerUnitTests extends ESTestCase { HierarchyCircuitBreakerService.validateSettings(new BreakerSettings[]{fd, request}); } - @Test public void testBreakerSettingsValidationNegativeOverhead() { // parent: {:limit 70}, fd: {:limit 50}, request: {:limit 20} BreakerSettings fd = new BreakerSettings(CircuitBreaker.FIELDDATA, pctBytes("50%"), -0.1); @@ -66,7 +65,6 @@ public class CircuitBreakerUnitTests extends ESTestCase { } } - @Test public void testRegisterCustomBreaker() throws Exception { CircuitBreakerService service = new HierarchyCircuitBreakerService(Settings.EMPTY, new NodeSettingsService(Settings.EMPTY)); String customName = "custom"; @@ -78,5 +76,4 @@ public class CircuitBreakerUnitTests extends ESTestCase { assertThat(breaker, instanceOf(CircuitBreaker.class)); assertThat(breaker.getName(), is(customName)); } - } diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java index 4f0bd600be7..3422959771c 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java @@ -33,15 +33,14 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.engine.MockEngineSupport; -import org.elasticsearch.test.engine.MockEngineSupportModule; import org.elasticsearch.test.engine.ThrowingLeafReaderWrapper; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; @@ -57,13 +56,11 @@ import static org.hamcrest.Matchers.equalTo; * Tests for the circuit breaker while random exceptions are happening */ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(RandomExceptionDirectoryReaderWrapper.TestPlugin.class); } - @Test public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException { for (NodeStats node : client().admin().cluster().prepareNodesStats() .clear().setBreaker(true).execute().actionGet().getNodes()) { @@ -83,7 +80,6 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase { // I don't use randomNumericType() here because I don't want "byte", and I want "float" and "double" .field("type", randomFrom(Arrays.asList("float", "long", "double", "short", "integer"))) .startObject("fielddata") - .field("format", randomNumericFieldDataFormat()) .endObject() // fielddata .endObject() // test-num .endObject() // properties @@ -216,8 +212,9 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase { public String description() { return "a mock reader wrapper that throws random exceptions for testing"; } - public void onModule(MockEngineSupportModule module) { - module.wrapperImpl = RandomExceptionDirectoryReaderWrapper.class; + + public void onModule(MockEngineFactoryPlugin.MockEngineReaderModule module) { + module.setReaderClass(RandomExceptionDirectoryReaderWrapper.class); } } diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 2eedceffcec..4cf60289a18 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -49,6 +49,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.store.MockFSDirectoryService; @@ -59,7 +60,6 @@ import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -70,7 +70,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.arrayWithSize; @@ -139,27 +138,30 @@ public class IndexRecoveryIT extends ESIntegTestCase { } private void slowDownRecovery(ByteSizeValue shardSize) { - long chunkSize = shardSize.bytes() / 10; + long chunkSize = Math.max(1, shardSize.bytes() / 10); + for(RecoverySettings settings : internalCluster().getInstances(RecoverySettings.class)) { + setChunkSize(settings, new ByteSizeValue(chunkSize, ByteSizeUnit.BYTES)); + } assertTrue(client().admin().cluster().prepareUpdateSettings() .setTransientSettings(Settings.builder() // one chunk per sec.. .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, chunkSize, ByteSizeUnit.BYTES) - .put(RecoverySettings.INDICES_RECOVERY_FILE_CHUNK_SIZE, chunkSize, ByteSizeUnit.BYTES) ) .get().isAcknowledged()); } private void restoreRecoverySpeed() { + for(RecoverySettings settings : internalCluster().getInstances(RecoverySettings.class)) { + setChunkSize(settings, RecoverySettings.DEFAULT_CHUNK_SIZE); + } assertTrue(client().admin().cluster().prepareUpdateSettings() .setTransientSettings(Settings.builder() .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, "20mb") - .put(RecoverySettings.INDICES_RECOVERY_FILE_CHUNK_SIZE, "512kb") ) .get().isAcknowledged()); } - @Test - public void gatewayRecoveryTest() throws Exception { + public void testGatewayRecovery() throws Exception { logger.info("--> start nodes"); String node = internalCluster().startNode(); @@ -184,8 +186,7 @@ public class IndexRecoveryIT extends ESIntegTestCase { validateIndexRecoveryState(recoveryState.getIndex()); } - @Test - public void gatewayRecoveryTestActiveOnly() throws Exception { + public void testGatewayRecoveryTestActiveOnly() throws Exception { logger.info("--> start nodes"); internalCluster().startNode(); @@ -202,8 +203,7 @@ public class IndexRecoveryIT extends ESIntegTestCase { assertThat(recoveryStates.size(), equalTo(0)); // Should not expect any responses back } - @Test - public void replicaRecoveryTest() throws Exception { + public void testReplicaRecovery() throws Exception { logger.info("--> start node A"); String nodeA = internalCluster().startNode(); @@ -243,9 +243,8 @@ public class IndexRecoveryIT extends ESIntegTestCase { validateIndexRecoveryState(nodeBRecoveryState.getIndex()); } - @Test @TestLogging("indices.recovery:TRACE") - public void rerouteRecoveryTest() throws Exception { + public void testRerouteRecovery() throws Exception { logger.info("--> start node A"); final String nodeA = internalCluster().startNode(); @@ -433,8 +432,7 @@ public class IndexRecoveryIT extends ESIntegTestCase { validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex()); } - @Test - public void snapshotRecoveryTest() throws Exception { + public void testSnapshotRecovery() throws Exception { logger.info("--> start node A"); String nodeA = internalCluster().startNode(); @@ -516,7 +514,7 @@ public class IndexRecoveryIT extends ESIntegTestCase { indexRandom(true, docs); flush(); - assertThat(client().prepareCount(INDEX_NAME).get().getCount(), equalTo((long) numDocs)); + assertThat(client().prepareSearch(INDEX_NAME).setSize(0).get().getHits().totalHits(), equalTo((long) numDocs)); return client().admin().indices().prepareStats(INDEX_NAME).execute().actionGet(); } @@ -528,8 +526,7 @@ public class IndexRecoveryIT extends ESIntegTestCase { assertThat(indexState.recoveredBytesPercent(), lessThanOrEqualTo(100.0f)); } - @Test - public void disconnectsWhileRecoveringTest() throws Exception { + public void testDisconnectsWhileRecovering() throws Exception { final String indexName = "test"; final Settings nodeSettings = Settings.builder() .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK, "100ms") @@ -587,12 +584,12 @@ public class IndexRecoveryIT extends ESIntegTestCase { MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName); MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName); - DiscoveryNode redDiscoNode = internalCluster().getInstance(ClusterService.class, redNodeName).localNode(); - DiscoveryNode blueDiscoNode = internalCluster().getInstance(ClusterService.class, blueNodeName).localNode(); + TransportService redTransportService = internalCluster().getInstance(TransportService.class, redNodeName); + TransportService blueTransportService = internalCluster().getInstance(TransportService.class, blueNodeName); final CountDownLatch requestBlocked = new CountDownLatch(1); - blueMockTransportService.addDelegate(redDiscoNode, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, blueMockTransportService.original(), requestBlocked)); - redMockTransportService.addDelegate(blueDiscoNode, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, redMockTransportService.original(), requestBlocked)); + blueMockTransportService.addDelegate(redTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, blueMockTransportService.original(), requestBlocked)); + redMockTransportService.addDelegate(blueTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, redMockTransportService.original(), requestBlocked)); logger.info("--> starting recovery from blue to red"); client().admin().indices().prepareUpdateSettings(indexName).setSettings( @@ -638,4 +635,8 @@ public class IndexRecoveryIT extends ESIntegTestCase { transport.sendRequest(node, requestId, action, request, options); } } + + public static void setChunkSize(RecoverySettings recoverySettings, ByteSizeValue chunksSize) { + recoverySettings.setChunkSize(chunksSize); + } } diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index d658b07de4a..8346003287c 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -22,39 +22,44 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.*; -import org.apache.lucene.store.*; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.node.settings.NodeSettingsService; +import org.elasticsearch.test.CorruptionUtils; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.CorruptionUtils; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -import static org.hamcrest.Matchers.is; - public class RecoverySourceHandlerTests extends ESTestCase { - - private final ShardId shardId = new ShardId(new Index("index"), 1); + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings(new Index("index"), Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT).build()); + private final ShardId shardId = new ShardId(INDEX_SETTINGS.getIndex(), 1); private final NodeSettingsService service = new NodeSettingsService(Settings.EMPTY); public void testSendFiles() throws Throwable { @@ -233,7 +238,7 @@ public class RecoverySourceHandlerTests extends ESTestCase { return newStore(path, true); } private Store newStore(Path path, boolean checkIndex) throws IOException { - DirectoryService directoryService = new DirectoryService(shardId, Settings.EMPTY) { + DirectoryService directoryService = new DirectoryService(shardId, INDEX_SETTINGS) { @Override public long throttleTimeInNanos() { return 0; @@ -248,7 +253,7 @@ public class RecoverySourceHandlerTests extends ESTestCase { return baseDirectoryWrapper; } }; - return new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId)); + return new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId)); } diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStateTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStateTests.java index 3a571a6d917..8b23354ebb8 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStateTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStateTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.recovery.RecoveryState.*; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -41,9 +40,7 @@ import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.Matchers.*; public class RecoveryStateTests extends ESTestCase { - abstract class Streamer extends Thread { - private T lastRead; final private AtomicBoolean shouldStop; final private T source; @@ -191,7 +188,7 @@ public class RecoveryStateTests extends ESTestCase { } } - Collections.shuffle(Arrays.asList(files)); + Collections.shuffle(Arrays.asList(files), random()); final RecoveryState.Index index = new RecoveryState.Index(); if (randomBoolean()) { @@ -485,7 +482,6 @@ public class RecoveryStateTests extends ESTestCase { } } - @Test public void testConcurrentModificationIndexFileDetailsMap() throws InterruptedException { final Index index = new Index(); final AtomicBoolean stop = new AtomicBoolean(false); @@ -496,6 +492,7 @@ public class RecoveryStateTests extends ESTestCase { } }; Thread modifyThread = new Thread() { + @Override public void run() { for (int i = 0; i < 1000; i++) { index.addFileDetail(randomAsciiOfLength(10), 100, true); @@ -510,7 +507,6 @@ public class RecoveryStateTests extends ESTestCase { assertThat(readWriteIndex.error.get(), equalTo(null)); } - @Test public void testFileHashCodeAndEquals() { File f = new File("foo", randomIntBetween(0, 100), randomBoolean()); File anotherFile = new File(f.name(), f.length(), f.reused()); @@ -526,6 +522,5 @@ public class RecoveryStateTests extends ESTestCase { assertFalse(f.equals(anotherFile)); } } - } } diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java index 4df825704d7..4ad298e39a3 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java @@ -27,20 +27,16 @@ import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; /** */ public class StartRecoveryRequestTests extends ESTestCase { - - @Test public void testSerialization() throws Exception { Version targetNodeVersion = randomVersion(random()); StartRecoveryRequest outRequest = new StartRecoveryRequest( diff --git a/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java b/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java index 357e87f0692..a2a7c5fb493 100644 --- a/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java @@ -22,19 +22,19 @@ package org.elasticsearch.indices.settings; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class GetSettingsBlocksIT extends ESIntegTestCase { - - @Test public void testGetSettingsWithBlocks() throws Exception { assertAcked(prepareCreate("test") .setSettings(Settings.settingsBuilder() diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java index 9d3a6773c3d..efb94aff31e 100644 --- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java +++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java @@ -20,13 +20,12 @@ package org.elasticsearch.indices.settings; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; @@ -39,14 +38,12 @@ import static org.hamcrest.Matchers.equalTo; @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class UpdateNumberOfReplicasIT extends ESIntegTestCase { - @Override protected int maximumNumberOfReplicas() { return 1; } - @Test - public void simpleUpdateNumberOfReplicasIT() throws Exception { + public void testSimpleUpdateNumberOfReplicas() throws Exception { logger.info("Creating index test"); assertAcked(prepareCreate("test", 2)); logger.info("Running Cluster Health"); @@ -70,7 +67,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase { refresh(); for (int i = 0; i < 10; i++) { - CountResponse countResponse = client().prepareCount().setQuery(matchAllQuery()).get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(); assertHitCount(countResponse, 10l); } @@ -100,7 +97,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase { assertThat(clusterHealth.getIndices().get("test").getActiveShards(), equalTo(numShards.numPrimaries * 3)); for (int i = 0; i < 10; i++) { - CountResponse countResponse = client().prepareCount().setQuery(matchAllQuery()).get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(); assertHitCount(countResponse, 10l); } @@ -122,7 +119,6 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase { } } - @Test public void testAutoExpandNumberOfReplicas0ToData() throws IOException { internalCluster().ensureAtMostNumDataNodes(2); logger.info("--> creating index test with auto expand replicas"); @@ -178,7 +174,6 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase { assertThat(clusterHealth.getIndices().get("test").getActiveShards(), equalTo(numShards.numPrimaries)); } - @Test public void testAutoExpandNumberReplicas1ToData() throws IOException { logger.info("--> creating index test with auto expand replicas"); internalCluster().ensureAtMostNumDataNodes(2); @@ -234,7 +229,6 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase { assertThat(clusterHealth.getIndices().get("test").getActiveShards(), equalTo(numShards.numPrimaries)); } - @Test public void testAutoExpandNumberReplicas2() { logger.info("--> creating index test with auto expand replicas set to 0-2"); assertAcked(prepareCreate("test", 3, settingsBuilder().put("auto_expand_replicas", "0-2"))); @@ -267,7 +261,6 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase { assertThat(clusterHealth.getIndices().get("test").getActiveShards(), equalTo(numShards.numPrimaries * 4)); } - @Test public void testUpdateWithInvalidNumberOfReplicas() { createIndex("test"); try { diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java index f26a5b25862..0ef83337bfd 100644 --- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java @@ -23,7 +23,6 @@ import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; -import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -33,21 +32,24 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.shard.MergePolicyConfig; +import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.Store; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public class UpdateSettingsIT extends ESIntegTestCase { - - @Test public void testOpenCloseUpdateSettings() throws Exception { createIndex("test"); try { @@ -63,8 +65,8 @@ public class UpdateSettingsIT extends ESIntegTestCase { } IndexMetaData indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test"); - assertThat(indexMetaData.settings().get("index.refresh_interval"), nullValue()); - assertThat(indexMetaData.settings().get("index.cache.filter.type"), nullValue()); + assertThat(indexMetaData.getSettings().get("index.refresh_interval"), nullValue()); + assertThat(indexMetaData.getSettings().get("index.cache.filter.type"), nullValue()); // Now verify via dedicated get settings api: GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get(); @@ -78,7 +80,7 @@ public class UpdateSettingsIT extends ESIntegTestCase { .execute().actionGet(); indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test"); - assertThat(indexMetaData.settings().get("index.refresh_interval"), equalTo("-1")); + assertThat(indexMetaData.getSettings().get("index.refresh_interval"), equalTo("-1")); // Now verify via dedicated get settings api: getSettingsResponse = client().admin().indices().prepareGetSettings("test").get(); assertThat(getSettingsResponse.getSetting("test", "index.refresh_interval"), equalTo("-1")); @@ -110,8 +112,8 @@ public class UpdateSettingsIT extends ESIntegTestCase { .execute().actionGet(); indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test"); - assertThat(indexMetaData.settings().get("index.refresh_interval"), equalTo("1s")); - assertThat(indexMetaData.settings().get("index.cache.filter.type"), equalTo("none")); + assertThat(indexMetaData.getSettings().get("index.refresh_interval"), equalTo("1s")); + assertThat(indexMetaData.getSettings().get("index.cache.filter.type"), equalTo("none")); // Now verify via dedicated get settings api: getSettingsResponse = client().admin().indices().prepareGetSettings("test").get(); @@ -119,7 +121,6 @@ public class UpdateSettingsIT extends ESIntegTestCase { assertThat(getSettingsResponse.getSetting("test", "index.cache.filter.type"), equalTo("none")); } - @Test public void testEngineGCDeletesSetting() throws InterruptedException { createIndex("test"); client().prepareIndex("test", "type", "1").setSource("f", 1).get(); // set version to 1 @@ -137,9 +138,7 @@ public class UpdateSettingsIT extends ESIntegTestCase { } // #6626: make sure we can update throttle settings and the changes take effect - @Test public void testUpdateThrottleSettings() { - // No throttling at first, only 1 non-replicated shard, force lots of merging: assertAcked(prepareCreate("test") .setSettings(Settings.builder() @@ -214,7 +213,7 @@ public class UpdateSettingsIT extends ESIntegTestCase { } logger.info("test: disable merge throttling"); - + // Now updates settings to disable merge throttling client() .admin() @@ -226,7 +225,7 @@ public class UpdateSettingsIT extends ESIntegTestCase { // Optimize does a waitForMerges, which we must do to make sure all in-flight (throttled) merges finish: logger.info("test: optimize"); - client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get(); + client().admin().indices().prepareForceMerge("test").setMaxNumSegments(1).get(); logger.info("test: optimize done"); // Record current throttling so far @@ -264,16 +263,13 @@ public class UpdateSettingsIT extends ESIntegTestCase { // when ESIntegTestCase.after tries to remove indices created by the test: // Wait for merges to finish - client().admin().indices().prepareOptimize("test").get(); + client().admin().indices().prepareForceMerge("test").get(); flush(); logger.info("test: test done"); } private static class MockAppender extends AppenderSkeleton { - public boolean sawIndexWriterMessage; - public boolean sawFlushDeletes; - public boolean sawMergeThreadPaused; public boolean sawUpdateMaxThreadCount; public boolean sawUpdateAutoThrottle; @@ -282,8 +278,6 @@ public class UpdateSettingsIT extends ESIntegTestCase { String message = event.getMessage().toString(); if (event.getLevel() == Level.TRACE && event.getLoggerName().endsWith("lucene.iw")) { - sawFlushDeletes |= message.contains("IW: apply all deletes during flush"); - sawMergeThreadPaused |= message.contains("CMS: pause thread"); } if (event.getLevel() == Level.INFO && message.contains("updating [index.merge.scheduler.max_thread_count] from [10000] to [1]")) { sawUpdateMaxThreadCount = true; @@ -303,9 +297,7 @@ public class UpdateSettingsIT extends ESIntegTestCase { } } - @Test public void testUpdateAutoThrottleSettings() { - MockAppender mockAppender = new MockAppender(); Logger rootLogger = Logger.getRootLogger(); Level savedLevel = rootLogger.getLevel(); @@ -347,9 +339,7 @@ public class UpdateSettingsIT extends ESIntegTestCase { } // #6882: make sure we can change index.merge.scheduler.max_thread_count live - @Test public void testUpdateMergeMaxThreadCount() { - MockAppender mockAppender = new MockAppender(); Logger rootLogger = Logger.getRootLogger(); Level savedLevel = rootLogger.getLevel(); @@ -379,7 +369,7 @@ public class UpdateSettingsIT extends ESIntegTestCase { .put(MergeSchedulerConfig.MAX_THREAD_COUNT, "1") ) .get(); - + // Make sure we log the change: assertTrue(mockAppender.sawUpdateMaxThreadCount); @@ -393,7 +383,6 @@ public class UpdateSettingsIT extends ESIntegTestCase { } } - @Test public void testUpdateSettingsWithBlocks() { createIndex("test"); ensureGreen("test"); diff --git a/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java b/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java index daebe71ac56..bde40aa928f 100644 --- a/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java +++ b/core/src/test/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java @@ -21,22 +21,21 @@ package org.elasticsearch.indices.state; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; +import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; import org.elasticsearch.action.support.DestructiveOperations; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.elasticsearch.test.ESIntegTestCase.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -@ClusterScope(scope= Scope.TEST, numDataNodes =2) +@ClusterScope(scope=Scope.TEST, numDataNodes=2) public class CloseIndexDisableCloseAllIT extends ESIntegTestCase { - - @Test // Combined multiple tests into one, because cluster scope is test. // The cluster scope is test b/c we can't clear cluster settings. public void testCloseAllRequiresName() { @@ -86,6 +85,20 @@ public class CloseIndexDisableCloseAllIT extends ESIntegTestCase { CloseIndexResponse closeIndexResponse = client().admin().indices().prepareClose("test3", "test2").execute().actionGet(); assertThat(closeIndexResponse.isAcknowledged(), equalTo(true)); assertIndexIsClosed("test2", "test3"); + + // disable closing + Client client = client(); + createIndex("test_no_close"); + healthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + assertThat(healthResponse.isTimedOut(), equalTo(false)); + client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(TransportCloseIndexAction.SETTING_CLUSTER_INDICES_CLOSE_ENABLE, false)).get(); + + try { + client.admin().indices().prepareClose("test_no_close").execute().actionGet(); + fail("exception expected"); + } catch (IllegalStateException ex) { + assertEquals(ex.getMessage(), "closing indices is disabled - set [cluster.indices.close.enable: true] to enable it. NOTE: closed indices still consume a significant amount of diskspace"); + } } private void assertIndexIsClosed(String... indices) { diff --git a/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java index 5556b674aa0..e17b2a5c7b4 100644 --- a/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java +++ b/core/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java @@ -34,20 +34,25 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; import java.util.concurrent.ExecutionException; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; public class OpenCloseIndexIT extends ESIntegTestCase { - - @Test public void testSimpleCloseOpen() { Client client = client(); createIndex("test1"); @@ -63,28 +68,39 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1"); } - @Test(expected = IndexNotFoundException.class) public void testSimpleCloseMissingIndex() { Client client = client(); - client.admin().indices().prepareClose("test1").execute().actionGet(); + try { + client.admin().indices().prepareClose("test1").execute().actionGet(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test(expected = IndexNotFoundException.class) public void testSimpleOpenMissingIndex() { Client client = client(); - client.admin().indices().prepareOpen("test1").execute().actionGet(); + try { + client.admin().indices().prepareOpen("test1").execute().actionGet(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test(expected = IndexNotFoundException.class) public void testCloseOneMissingIndex() { Client client = client(); createIndex("test1"); ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); assertThat(healthResponse.isTimedOut(), equalTo(false)); - client.admin().indices().prepareClose("test1", "test2").execute().actionGet(); + try { + client.admin().indices().prepareClose("test1", "test2").execute().actionGet(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test public void testCloseOneMissingIndexIgnoreMissing() { Client client = client(); createIndex("test1"); @@ -96,16 +112,19 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsClosed("test1"); } - @Test(expected = IndexNotFoundException.class) public void testOpenOneMissingIndex() { Client client = client(); createIndex("test1"); ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); assertThat(healthResponse.isTimedOut(), equalTo(false)); - client.admin().indices().prepareOpen("test1", "test2").execute().actionGet(); + try { + client.admin().indices().prepareOpen("test1", "test2").execute().actionGet(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test public void testOpenOneMissingIndexIgnoreMissing() { Client client = client(); createIndex("test1"); @@ -117,7 +136,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1"); } - @Test public void testCloseOpenMultipleIndices() { Client client = client(); createIndex("test1", "test2", "test3"); @@ -138,7 +156,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1", "test2", "test3"); } - @Test public void testCloseOpenWildcard() { Client client = client(); createIndex("test1", "test2", "a"); @@ -155,7 +172,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1", "test2", "a"); } - @Test public void testCloseOpenAll() { Client client = client(); createIndex("test1", "test2", "test3"); @@ -171,7 +187,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1", "test2", "test3"); } - @Test public void testCloseOpenAllWildcard() { Client client = client(); createIndex("test1", "test2", "test3"); @@ -187,31 +202,46 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1", "test2", "test3"); } - @Test(expected = ActionRequestValidationException.class) public void testCloseNoIndex() { Client client = client(); - client.admin().indices().prepareClose().execute().actionGet(); + try { + client.admin().indices().prepareClose().execute().actionGet(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("index is missing")); + } } - @Test(expected = ActionRequestValidationException.class) public void testCloseNullIndex() { Client client = client(); - client.admin().indices().prepareClose((String[])null).execute().actionGet(); + try { + client.admin().indices().prepareClose((String[])null).execute().actionGet(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("index is missing")); + } } - @Test(expected = ActionRequestValidationException.class) public void testOpenNoIndex() { Client client = client(); - client.admin().indices().prepareOpen().execute().actionGet(); + try { + client.admin().indices().prepareOpen().execute().actionGet(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("index is missing")); + } } - @Test(expected = ActionRequestValidationException.class) public void testOpenNullIndex() { Client client = client(); - client.admin().indices().prepareOpen((String[])null).execute().actionGet(); + try { + client.admin().indices().prepareOpen((String[])null).execute().actionGet(); + fail("Expected ActionRequestValidationException"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("index is missing")); + } } - @Test public void testOpenAlreadyOpenedIndex() { Client client = client(); createIndex("test1"); @@ -224,7 +254,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1"); } - @Test public void testCloseAlreadyClosedIndex() { Client client = client(); createIndex("test1"); @@ -242,7 +271,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsClosed("test1"); } - @Test public void testSimpleCloseOpenAlias() { Client client = client(); createIndex("test1"); @@ -261,7 +289,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertIndexIsOpened("test1"); } - @Test public void testCloseOpenAliasMultipleIndices() { Client client = client(); createIndex("test1", "test2"); @@ -299,7 +326,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { } } - @Test public void testOpenCloseWithDocs() throws IOException, ExecutionException, InterruptedException { String mapping = XContentFactory.jsonBuilder(). startObject(). @@ -335,7 +361,6 @@ public class OpenCloseIndexIT extends ESIntegTestCase { assertHitCount(searchResponse, docs); } - @Test public void testOpenCloseIndexWithBlocks() { createIndex("test"); ensureGreen("test"); diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java index dbdfc2be84c..96611aeca8a 100644 --- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java @@ -54,29 +54,20 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.disruption.BlockClusterStateProcessing; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.junit.Test; import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.*; /** */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0) @ESIntegTestCase.SuppressLocalMode public class RareClusterStateIT extends ESIntegTestCase { - @Override protected int numberOfShards() { return 1; @@ -87,7 +78,6 @@ public class RareClusterStateIT extends ESIntegTestCase { return 0; } - @Test public void testUnassignedShardAndEmptyNodesInRoutingTable() throws Exception { internalCluster().startNode(); createIndex("a"); @@ -102,11 +92,10 @@ public class RareClusterStateIT extends ESIntegTestCase { .nodes(DiscoveryNodes.EMPTY_NODES) .build(), false ); - RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current.nodes(), ClusterInfo.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current.nodes(), ClusterInfo.EMPTY, System.nanoTime()); allocator.allocateUnassigned(routingAllocation); } - @Test @TestLogging("gateway:TRACE") public void testAssignmentWithJustAddedNodes() throws Exception { internalCluster().startNode(); @@ -139,7 +128,7 @@ public class RareClusterStateIT extends ESIntegTestCase { routingTable.addAsRecovery(updatedState.metaData().index(index)); updatedState = ClusterState.builder(updatedState).routingTable(routingTable.build()).build(); - RoutingAllocation.Result result = allocationService.reroute(updatedState); + RoutingAllocation.Result result = allocationService.reroute(updatedState, "reroute"); return ClusterState.builder(updatedState).routingResult(result).build(); } @@ -159,7 +148,7 @@ public class RareClusterStateIT extends ESIntegTestCase { builder.nodes(DiscoveryNodes.builder(currentState.nodes()).remove("_non_existent")); currentState = builder.build(); - RoutingAllocation.Result result = allocationService.reroute(currentState); + RoutingAllocation.Result result = allocationService.reroute(currentState, "reroute"); return ClusterState.builder(currentState).routingResult(result).build(); } @@ -171,9 +160,8 @@ public class RareClusterStateIT extends ESIntegTestCase { }); } - - @Test - @TestLogging(value = "cluster.service:TRACE") + @TestLogging("cluster.service:TRACE") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/14932") public void testDeleteCreateInOneBulk() throws Exception { internalCluster().startNodesAsync(2, Settings.builder() .put(DiscoveryModule.DISCOVERY_TYPE_KEY, "zen") diff --git a/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java index 3001c038486..a5cfa816455 100644 --- a/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java @@ -20,7 +20,7 @@ package org.elasticsearch.indices.state; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -35,7 +35,6 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.IndexPrimaryShardNotAllocatedException; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; @@ -46,10 +45,8 @@ import static org.hamcrest.Matchers.nullValue; */ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SimpleIndexStateIT extends ESIntegTestCase { - private final ESLogger logger = Loggers.getLogger(SimpleIndexStateIT.class); - @Test public void testSimpleOpenClose() { logger.info("--> creating test index"); createIndex("test"); @@ -60,7 +57,7 @@ public class SimpleIndexStateIT extends ESIntegTestCase { NumShards numShards = getNumShards("test"); ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(numShards.numPrimaries)); assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(numShards.totalNumShards)); @@ -72,7 +69,7 @@ public class SimpleIndexStateIT extends ESIntegTestCase { assertThat(closeIndexResponse.isAcknowledged(), equalTo(true)); stateResponse = client().admin().cluster().prepareState().get(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); logger.info("--> trying to index into a closed index ..."); @@ -91,7 +88,7 @@ public class SimpleIndexStateIT extends ESIntegTestCase { ensureGreen(); stateResponse = client().admin().cluster().prepareState().get(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(numShards.numPrimaries)); assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(numShards.totalNumShards)); @@ -100,7 +97,6 @@ public class SimpleIndexStateIT extends ESIntegTestCase { client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get(); } - @Test public void testFastCloseAfterCreateDoesNotClose() { logger.info("--> creating test index that cannot be allocated"); client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder() @@ -127,7 +123,7 @@ public class SimpleIndexStateIT extends ESIntegTestCase { NumShards numShards = getNumShards("test"); ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get(); - assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.OPEN)); + assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(numShards.numPrimaries)); assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(numShards.totalNumShards)); @@ -135,9 +131,7 @@ public class SimpleIndexStateIT extends ESIntegTestCase { client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get(); } - @Test public void testConsistencyAfterIndexCreationFailure() { - logger.info("--> deleting test index...."); try { client().admin().indices().prepareDelete("test").get(); @@ -157,5 +151,4 @@ public class SimpleIndexStateIT extends ESIntegTestCase { CreateIndexResponse response = client().admin().indices().prepareCreate("test").setSettings(settingsBuilder().put("number_of_shards", 1)).get(); assertThat(response.isAcknowledged(), equalTo(true)); } - } diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index d40ebf57a48..a87da6fc046 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -19,10 +19,6 @@ package org.elasticsearch.indices.stats; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.index.cache.IndexCacheModule; -import org.elasticsearch.index.engine.VersionConflictEngineException; -import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.Version; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -39,19 +35,22 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.VersionType; import org.elasticsearch.index.cache.query.QueryCacheStats; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.MergePolicyConfig; +import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.indices.cache.request.IndicesRequestCache; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.io.IOException; import java.util.EnumSet; @@ -73,18 +72,16 @@ import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, randomDynamicTemplates = false) @SuppressCodecs("*") // requires custom completion format public class IndexStatsIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { //Filter/Query cache is cleaned periodically, default is 60s, so make sure it runs often. Thread.sleep for 60s is bad return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)) .put(IndicesRequestCache.INDICES_CACHE_REQUEST_CLEAN_INTERVAL, "1ms") - .put(IndexCacheModule.QUERY_CACHE_EVERYTHING, true) - .put(IndexCacheModule.QUERY_CACHE_TYPE, IndexCacheModule.INDEX_QUERY_CACHE) + .put(IndexModule.QUERY_CACHE_EVERYTHING, true) + .put(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE) .build(); } - @Test public void testFieldDataStats() { client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.number_of_shards", 2)).execute().actionGet(); ensureGreen(); @@ -129,7 +126,6 @@ public class IndexStatsIT extends ESIntegTestCase { } - @Test public void testClearAllCaches() throws Exception { client().admin().indices().prepareCreate("test") .setSettings(Settings.settingsBuilder().put("index.number_of_replicas", 0).put("index.number_of_shards", 2)) @@ -186,7 +182,6 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(indicesStats.getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0l)); } - @Test public void testQueryCache() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED, true).get()); ensureGreen(); @@ -277,9 +272,7 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0l)); } - - @Test - public void nonThrottleStats() throws Exception { + public void testNonThrottleStats() throws Exception { assertAcked(prepareCreate("test") .setSettings(Settings.builder() .put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") @@ -311,8 +304,7 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0l)); } - @Test - public void throttleStats() throws Exception { + public void testThrottleStats() throws Exception { assertAcked(prepareCreate("test") .setSettings(Settings.builder() .put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") @@ -356,13 +348,12 @@ public class IndexStatsIT extends ESIntegTestCase { // Optimize & flush and wait; else we sometimes get a "Delete Index failed - not acked" // when ESIntegTestCase.after tries to remove indices created by the test: logger.info("test: now optimize"); - client().admin().indices().prepareOptimize("test").get(); + client().admin().indices().prepareForceMerge("test").get(); flush(); logger.info("test: test done"); } - @Test - public void simpleStats() throws Exception { + public void testSimpleStats() throws Exception { createIndex("test1", "test2"); ensureGreen(); @@ -491,7 +482,6 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(stats.getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(3L)); } - @Test public void testMergeStats() { createIndex("test1"); @@ -519,7 +509,7 @@ public class IndexStatsIT extends ESIntegTestCase { client().prepareIndex("test1", "type2", Integer.toString(i)).setSource("field", "value").execute().actionGet(); client().admin().indices().prepareFlush().execute().actionGet(); } - client().admin().indices().prepareOptimize().setMaxNumSegments(1).execute().actionGet(); + client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet(); stats = client().admin().indices().prepareStats() .setMerge(true) .execute().actionGet(); @@ -528,7 +518,6 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0l)); } - @Test public void testSegmentsStats() { assertAcked(prepareCreate("test1", 2, settingsBuilder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)))); ensureGreen(); @@ -546,7 +535,7 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(stats.getTotal().getSegments().getVersionMapMemoryInBytes(), greaterThan(0l)); client().admin().indices().prepareFlush().get(); - client().admin().indices().prepareOptimize().setMaxNumSegments(1).execute().actionGet(); + client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet(); stats = client().admin().indices().prepareStats().setSegments(true).get(); assertThat(stats.getTotal().getSegments(), notNullValue()); @@ -555,7 +544,6 @@ public class IndexStatsIT extends ESIntegTestCase { assertThat(stats.getTotal().getSegments().getMemoryInBytes(), greaterThan(0l)); } - @Test public void testAllFlags() throws Exception { // rely on 1 replica for this tests createIndex("test1"); @@ -617,7 +605,6 @@ public class IndexStatsIT extends ESIntegTestCase { } - @Test public void testEncodeDecodeCommonStats() throws IOException { CommonStatsFlags flags = new CommonStatsFlags(); Flag[] values = CommonStatsFlags.Flag.values(); @@ -661,7 +648,6 @@ public class IndexStatsIT extends ESIntegTestCase { } } - @Test public void testFlagOrdinalOrder() { Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh, Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Percolate, Flag.Completion, Flag.Segments, @@ -673,9 +659,7 @@ public class IndexStatsIT extends ESIntegTestCase { } } - @Test public void testMultiIndex() throws Exception { - createIndex("test1"); createIndex("test2"); @@ -714,9 +698,7 @@ public class IndexStatsIT extends ESIntegTestCase { } - @Test public void testFieldDataFieldsParam() throws Exception { - createIndex("test1"); ensureGreen(); @@ -761,9 +743,7 @@ public class IndexStatsIT extends ESIntegTestCase { } - @Test public void testCompletionFieldsParam() throws Exception { - assertAcked(prepareCreate("test1") .addMapping( "bar", @@ -808,9 +788,7 @@ public class IndexStatsIT extends ESIntegTestCase { } - @Test public void testGroupsParam() throws Exception { - createIndex("test1"); ensureGreen(); @@ -844,9 +822,7 @@ public class IndexStatsIT extends ESIntegTestCase { } - @Test public void testTypesParam() throws Exception { - createIndex("test1"); createIndex("test2"); diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java index 8562b0991c2..fc4dd4f6487 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java @@ -20,7 +20,7 @@ package org.elasticsearch.indices.store; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -44,13 +44,13 @@ import org.elasticsearch.indices.recovery.RecoverySource; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.BlockClusterStateProcessing; import org.elasticsearch.test.disruption.SingleNodeDisruption; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.*; -import org.junit.Test; import java.io.IOException; import java.nio.file.Files; @@ -63,7 +63,6 @@ import java.util.concurrent.TimeUnit; import static java.lang.Thread.sleep; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -72,7 +71,6 @@ import static org.hamcrest.Matchers.equalTo; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class IndicesStoreIntegrationIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { // simplify this and only use a single data path return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put("path.data", "") @@ -94,8 +92,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { // so we cannot check state consistency of this cluster } - @Test - public void indexCleanup() throws Exception { + public void testIndexCleanup() throws Exception { final String masterNode = internalCluster().startNode(Settings.builder().put("node.data", false)); final String node_1 = internalCluster().startNode(Settings.builder().put("node.master", false)); final String node_2 = internalCluster().startNode(Settings.builder().put("node.master", false)); @@ -164,9 +161,8 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { } - @Test /* Test that shard is deleted in case ShardActiveRequest after relocation and next incoming cluster state is an index delete. */ - public void shardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted() throws Exception { + public void testShardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted() throws Exception { final String node_1 = internalCluster().startNode(); logger.info("--> creating index [test] with one shard and on replica"); assertAcked(prepareCreate("test").setSettings( @@ -189,10 +185,9 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { // add a transport delegate that will prevent the shard active request to succeed the first time after relocation has finished. // node_1 will then wait for the next cluster state change before it tries a next attempt to delet the shard. MockTransportService transportServiceNode_1 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_1); - String node_2_id = internalCluster().getInstance(DiscoveryService.class, node_2).localNode().id(); - DiscoveryNode node_2_disco = internalCluster().clusterService().state().getNodes().dataNodes().get(node_2_id); + TransportService transportServiceNode_2 = internalCluster().getInstance(TransportService.class, node_2); final CountDownLatch shardActiveRequestSent = new CountDownLatch(1); - transportServiceNode_1.addDelegate(node_2_disco, new MockTransportService.DelegateTransport(transportServiceNode_1.original()) { + transportServiceNode_1.addDelegate(transportServiceNode_2, new MockTransportService.DelegateTransport(transportServiceNode_1.original()) { @Override public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { if (action.equals("internal:index/shard/exists") && shardActiveRequestSent.getCount() > 0) { @@ -226,8 +221,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(false)); } - @Test - public void shardsCleanup() throws Exception { + public void testShardsCleanup() throws Exception { final String node_1 = internalCluster().startNode(); final String node_2 = internalCluster().startNode(); logger.info("--> creating index [test] with one shard and on replica"); @@ -286,8 +280,6 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { assertThat(waitForShardDeletion(node_4, "test", 0), equalTo(false)); } - - @Test @TestLogging("cluster.service:TRACE") public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception { InternalTestCluster.Async masterFuture = internalCluster().startNodeAsync( @@ -367,7 +359,6 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { } - @Test public void testShardActiveElseWhere() throws Exception { List nodes = internalCluster().startNodesAsync(2).get(); @@ -406,7 +397,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { // disable relocations when we do this, to make sure the shards are not relocated from node2 // due to rebalancing, and delete its content client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, EnableAllocationDecider.Rebalance.NONE)).get(); - internalCluster().getInstance(ClusterService.class, nonMasterNode).submitStateUpdateTask("test", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + internalCluster().getInstance(ClusterService.class, nonMasterNode).submitStateUpdateTask("test", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public ClusterState execute(ClusterState currentState) throws Exception { IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder("test"); diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java index 890b94d1583..ec6a3b38491 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java @@ -26,12 +26,14 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import org.junit.Test; import java.util.Arrays; import java.util.HashSet; @@ -39,12 +41,10 @@ import java.util.Set; import static org.elasticsearch.Version.CURRENT; import static org.elasticsearch.test.VersionUtils.randomVersion; -import static org.hamcrest.Matchers.is; /** */ public class IndicesStoreTests extends ESTestCase { - private final static ShardRoutingState[] NOT_STARTED_STATES; static { @@ -63,8 +63,7 @@ public class IndicesStoreTests extends ESTestCase { indicesStore = new IndicesStore(); } - @Test - public void testShardCanBeDeleted_noShardRouting() throws Exception { + public void testShardCanBeDeletedNoShardRouting() throws Exception { int numShards = randomIntBetween(1, 7); int numReplicas = randomInt(2); @@ -75,8 +74,7 @@ public class IndicesStoreTests extends ESTestCase { assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); } - @Test - public void testShardCanBeDeleted_noShardStarted() throws Exception { + public void testShardCanBeDeletedNoShardStarted() throws Exception { int numShards = randomIntBetween(1, 7); int numReplicas = randomInt(2); @@ -103,8 +101,7 @@ public class IndicesStoreTests extends ESTestCase { assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); } - @Test - public void testShardCanBeDeleted_shardExistsLocally() throws Exception { + public void testShardCanBeDeletedShardExistsLocally() throws Exception { int numShards = randomIntBetween(1, 7); int numReplicas = randomInt(2); @@ -126,8 +123,7 @@ public class IndicesStoreTests extends ESTestCase { assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); } - @Test - public void testShardCanBeDeleted_nodeNotInList() throws Exception { + public void testShardCanBeDeletedNodeNotInList() throws Exception { int numShards = randomIntBetween(1, 7); int numReplicas = randomInt(2); @@ -147,8 +143,7 @@ public class IndicesStoreTests extends ESTestCase { assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); } - @Test - public void testShardCanBeDeleted_nodeVersion() throws Exception { + public void testShardCanBeDeletedNodeVersion() throws Exception { int numShards = randomIntBetween(1, 7); int numReplicas = randomInt(2); @@ -169,8 +164,7 @@ public class IndicesStoreTests extends ESTestCase { assertTrue(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); } - @Test - public void testShardCanBeDeleted_relocatingNode() throws Exception { + public void testShardCanBeDeletedRelocatingNode() throws Exception { int numShards = randomIntBetween(1, 7); int numReplicas = randomInt(2); @@ -194,5 +188,4 @@ public class IndicesStoreTests extends ESTestCase { // shard exist on other node (abc and def) assertTrue(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); } - } diff --git a/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java b/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java index c1c41cf6c37..c46c0385292 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResp import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.io.IOException; @@ -33,8 +32,6 @@ import static org.hamcrest.Matchers.hasSize; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndexTemplateBlocksIT extends ESIntegTestCase { - - @Test public void testIndexTemplatesWithBlocks() throws IOException { // creates a simple index template client().admin().indices().preparePutTemplate("template_blocks") diff --git a/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java b/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java index f67e12095f3..ee0f8748083 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java @@ -26,12 +26,10 @@ import org.elasticsearch.cluster.metadata.IndexTemplateFilter; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.util.Collection; @@ -41,13 +39,11 @@ import static org.hamcrest.core.IsNull.notNullValue; @ClusterScope(scope = Scope.SUITE) public class IndexTemplateFilteringIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(TestPlugin.class); } - @Test public void testTemplateFiltering() throws Exception { client().admin().indices().preparePutTemplate("template1") .setTemplate("test*") @@ -70,7 +66,6 @@ public class IndexTemplateFilteringIT extends ESIntegTestCase { assertThat(metadata.get("type2"), notNullValue()); } - public static class TestFilter implements IndexTemplateFilter { @Override public boolean apply(CreateIndexClusterStateUpdateRequest request, IndexTemplateMetaData template) { diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 1084d4d27d7..b32cfef76b6 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -29,15 +29,14 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.indices.IndexTemplateAlreadyExistsException; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -64,9 +63,7 @@ import static org.hamcrest.Matchers.nullValue; * */ public class SimpleIndexTemplateIT extends ESIntegTestCase { - - @Test - public void simpleIndexTemplateTests() throws Exception { + public void testSimpleIndexTemplateTests() throws Exception { // clean all templates setup by the framework. client().admin().indices().prepareDeleteTemplate("*").get(); @@ -121,7 +118,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertHitCount(searchResponse, 1); assertThat(searchResponse.getHits().getAt(0).field("field1").value().toString(), equalTo("value1")); - assertThat(searchResponse.getHits().getAt(0).field("field2").value().toString(), equalTo("value 2")); // this will still be loaded because of the source feature + // field2 is not stored. + assertThat(searchResponse.getHits().getAt(0).field("field2"), nullValue()); client().prepareIndex("text_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefresh(true).execute().actionGet(); @@ -139,7 +137,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).field("field2").value().toString(), equalTo("value 2")); } - @Test public void testDeleteIndexTemplate() throws Exception { final int existingTemplates = admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().size(); logger.info("--> put template_1 and template_2"); @@ -186,7 +183,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(admin().cluster().prepareState().execute().actionGet().getState().metaData().templates().size(), equalTo(0)); } - @Test public void testThatGetIndexTemplatesWorks() throws Exception { logger.info("--> put template_1"); client().admin().indices().preparePutTemplate("template_1") @@ -210,7 +206,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(getTemplate2Response.getIndexTemplates(), hasSize(0)); } - @Test public void testThatGetIndexTemplatesWithSimpleRegexWorks() throws Exception { logger.info("--> put template_1"); client().admin().indices().preparePutTemplate("template_1") @@ -271,7 +266,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(templateNames, containsInAnyOrder("template_1", "template_2")); } - @Test public void testThatInvalidGetIndexTemplatesFails() throws Exception { logger.info("--> get template null"); testExpectActionRequestValidationException((String[])null); @@ -292,7 +286,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { "get template with " + Arrays.toString(names)); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/8802") public void testBrokenMapping() throws Exception { // clean all templates setup by the framework. @@ -320,7 +313,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { } } - @Test public void testInvalidSettings() throws Exception { // clean all templates setup by the framework. client().admin().indices().prepareDeleteTemplate("*").get(); @@ -398,7 +390,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(types, containsInAnyOrder("typeX", "typeY", "typeZ")); } - @Test public void testIndexTemplateWithAliasesInSource() { client().admin().indices().preparePutTemplate("template_1") .setSource("{\n" + @@ -434,7 +425,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).type(), equalTo("type2")); } - @Test public void testIndexTemplateWithAliasesSource() { client().admin().indices().preparePutTemplate("template_1") .setTemplate("te*") @@ -473,7 +463,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).type(), equalTo("type2")); } - @Test public void testDuplicateAlias() throws Exception { client().admin().indices().preparePutTemplate("template_1") .setTemplate("te*") @@ -487,9 +476,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(response.getIndexTemplates().get(0).getAliases().get("my_alias").filter().string(), containsString("\"value1\"")); } - @Test public void testAliasInvalidFilterValidJson() throws Exception { - //invalid filter but valid json: put index template works fine, fails during index creation client().admin().indices().preparePutTemplate("template_1") .setTemplate("te*") @@ -510,9 +497,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { } } - @Test public void testAliasInvalidFilterInvalidJson() throws Exception { - //invalid json: put index template fails PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1") .setTemplate("te*") @@ -528,9 +513,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { assertThat(response.getIndexTemplates().size(), equalTo(0)); } - @Test public void testAliasNameExistingIndex() throws Exception { - createIndex("index"); client().admin().indices().preparePutTemplate("template_1") @@ -545,7 +528,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { } } - @Test public void testAliasEmptyName() throws Exception { PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1") .setTemplate("te*") @@ -559,7 +541,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { } } - @Test public void testAliasWithMultipleIndexRoutings() throws Exception { PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1") .setTemplate("te*") @@ -573,14 +554,13 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { } } - @Test public void testMultipleAliasesPrecedence() throws Exception { client().admin().indices().preparePutTemplate("template1") .setTemplate("*") .setOrder(0) .addAlias(new Alias("alias1")) .addAlias(new Alias("{index}-alias")) - .addAlias(new Alias("alias3").filter(QueryBuilders.missingQuery("test"))) + .addAlias(new Alias("alias3").filter(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("test")))) .addAlias(new Alias("alias4")).get(); client().admin().indices().preparePutTemplate("template2") @@ -611,7 +591,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { } } - @Test public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exception { // Indexing into a should succeed, because the field mapping for field 'field' is defined in the test mapping. client().admin().indices().preparePutTemplate("template1") diff --git a/core/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerIT.java b/core/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerIT.java index 1c1b1939868..7c5a154ebcb 100644 --- a/core/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerIT.java +++ b/core/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerIT.java @@ -28,11 +28,10 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; import org.hamcrest.Matchers; -import org.junit.Test; -import static org.elasticsearch.test.ESIntegTestCase.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -40,12 +39,9 @@ import static org.hamcrest.Matchers.equalTo; */ @ClusterScope(numDataNodes =0, scope= Scope.TEST) public class GatewayIndicesWarmerIT extends ESIntegTestCase { - private final ESLogger logger = Loggers.getLogger(GatewayIndicesWarmerIT.class); - @Test public void testStatePersistence() throws Exception { - logger.info("--> starting 1 nodes"); internalCluster().startNode(); diff --git a/core/src/test/java/org/elasticsearch/indices/warmer/IndicesWarmerBlocksIT.java b/core/src/test/java/org/elasticsearch/indices/warmer/IndicesWarmerBlocksIT.java index 0ee4ab6329f..62bac50b0a1 100644 --- a/core/src/test/java/org/elasticsearch/indices/warmer/IndicesWarmerBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/indices/warmer/IndicesWarmerBlocksIT.java @@ -21,17 +21,23 @@ package org.elasticsearch.indices.warmer; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersResponse; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Arrays; import java.util.List; -import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_METADATA_BLOCK; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_READ_BLOCK; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_READ_ONLY_BLOCK; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.cluster.metadata.MetaData.CLUSTER_READ_ONLY_BLOCK; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; @@ -39,8 +45,6 @@ import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndicesWarmerBlocksIT extends ESIntegTestCase { - - @Test public void testPutWarmerWithBlocks() { createIndex("test-blocks"); ensureGreen("test-blocks"); @@ -91,7 +95,6 @@ public class IndicesWarmerBlocksIT extends ESIntegTestCase { } } - @Test public void testGetWarmerWithBlocks() { createIndex("test-blocks"); ensureGreen("test-blocks"); @@ -124,7 +127,6 @@ public class IndicesWarmerBlocksIT extends ESIntegTestCase { } } - @Test public void testDeleteWarmerWithBlocks() { createIndex("test-blocks"); ensureGreen("test-blocks"); diff --git a/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java b/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java index 82e08588b58..8470020f823 100644 --- a/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java +++ b/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java @@ -33,7 +33,6 @@ import org.elasticsearch.search.warmer.IndexWarmerMissingException; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.List; @@ -44,9 +43,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; public class SimpleIndicesWarmerIT extends ESIntegTestCase { - - @Test - public void simpleWarmerTests() { + public void testSimpleWarmers() { createIndex("test"); ensureGreen(); @@ -99,8 +96,7 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { assertThat(getWarmersResponse.getWarmers().size(), equalTo(0)); } - @Test - public void templateWarmer() { + public void testTtemplateWarmer() { client().admin().indices().preparePutTemplate("template_1") .setSource("{\n" + " \"template\" : \"*\",\n" + @@ -129,8 +125,7 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { client().prepareIndex("test", "type1", "2").setSource("field", "value2").setRefresh(true).execute().actionGet(); } - @Test - public void createIndexWarmer() { + public void testCreateIndexWarmer() { assertAcked(prepareCreate("test") .setSource("{\n" + " \"warmers\" : {\n" + @@ -154,8 +149,7 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { client().prepareIndex("test", "type1", "2").setSource("field", "value2").setRefresh(true).execute().actionGet(); } - @Test - public void deleteNonExistentIndexWarmerTest() { + public void testDeleteNonExistentIndexWarmer() { createIndex("test"); try { client().admin().indices().prepareDeleteWarmer().setIndices("test").setNames("foo").execute().actionGet(); @@ -165,8 +159,8 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { } } - @Test // issue 8991 - public void deleteAllIndexWarmerDoesNotThrowWhenNoWarmers() { + // issue 8991 + public void testDeleteAllIndexWarmerDoesNotThrowWhenNoWarmers() { createIndex("test"); DeleteWarmerResponse deleteWarmerResponse = client().admin().indices().prepareDeleteWarmer() .setIndices("test").setNames("_all").execute().actionGet(); @@ -177,8 +171,7 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { assertThat(deleteWarmerResponse.isAcknowledged(), equalTo(true)); } - @Test - public void deleteIndexWarmerTest() { + public void testDeleteIndexWarmerTest() { createIndex("test"); ensureGreen(); @@ -201,8 +194,8 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { assertThat(getWarmersResponse.warmers().size(), equalTo(0)); } - @Test // issue 3246 - public void ensureThatIndexWarmersCanBeChangedOnRuntime() throws Exception { + // issue 3246 + public void testEnsureThatIndexWarmersCanBeChangedOnRuntime() throws Exception { createIndex("test"); ensureGreen(); @@ -224,8 +217,7 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { assertThat(getWarmerRuns(), equalTo(warmerRunsAfterDisabling)); } - @Test - public void gettingAllWarmersUsingAllAndWildcardsShouldWork() throws Exception { + public void testGettingAllWarmersUsingAllAndWildcardsShouldWork() throws Exception { createIndex("test"); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java index 7716bf7300b..b661e785329 100644 --- a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java +++ b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java @@ -29,18 +29,18 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; public class SimpleMgetIT extends ESIntegTestCase { - - @Test public void testThatMgetShouldWorkWithOneIndexMissing() throws IOException { createIndex("test"); ensureYellow(); @@ -74,7 +74,6 @@ public class SimpleMgetIT extends ESIntegTestCase { } - @Test public void testThatParentPerDocumentIsSupported() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .addMapping("test", jsonBuilder() @@ -106,7 +105,6 @@ public class SimpleMgetIT extends ESIntegTestCase { } @SuppressWarnings("unchecked") - @Test public void testThatSourceFilteringIsSupported() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureYellow(); @@ -146,7 +144,6 @@ public class SimpleMgetIT extends ESIntegTestCase { } } - @Test public void testThatRoutingPerDocumentIsSupported() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.builder() diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java index 383bd0c891b..26a56529e05 100644 --- a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java @@ -22,15 +22,14 @@ package org.elasticsearch.monitor.fs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.not; public class FsProbeTests extends ESTestCase { - - @Test public void testFsInfo() throws IOException { try (NodeEnvironment env = newNodeEnvironment()) { FsProbe probe = new FsProbe(Settings.EMPTY, env); diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java index 011edcb5b73..d0b1d54171f 100644 --- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java @@ -22,17 +22,17 @@ package org.elasticsearch.monitor.jvm; import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; import java.util.List; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class JvmStatsTests extends ESTestCase { - - @Test public void testJvmStats() throws IOException { JvmStats stats = JvmStats.jvmStats(); assertNotNull(stats); diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java index 6b8818a4931..0d59341f1c9 100644 --- a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java @@ -21,15 +21,12 @@ package org.elasticsearch.monitor.os; import org.apache.lucene.util.Constants; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.*; public class OsProbeTests extends ESTestCase { - OsProbe probe = OsProbe.getInstance(); - @Test public void testOsInfo() { OsInfo info = probe.osInfo(); assertNotNull(info); @@ -40,17 +37,17 @@ public class OsProbeTests extends ESTestCase { assertThat(info.getAvailableProcessors(), equalTo(Runtime.getRuntime().availableProcessors())); } - @Test public void testOsStats() { OsStats stats = probe.osStats(); assertNotNull(stats); assertThat(stats.getTimestamp(), greaterThan(0L)); + assertThat(stats.getCpu().getPercent(), anyOf(equalTo((short) -1), is(both(greaterThanOrEqualTo((short) 0)).and(lessThanOrEqualTo((short) 100))))); if (Constants.WINDOWS) { // Load average is always -1 on Windows platforms - assertThat(stats.getLoadAverage(), equalTo((double) -1)); + assertThat(stats.getCpu().getLoadAverage(), equalTo((double) -1)); } else { // Load average can be negative if not available or not computed yet, otherwise it should be >= 0 - assertThat(stats.getLoadAverage(), anyOf(lessThan((double) 0), greaterThanOrEqualTo((double) 0))); + assertThat(stats.getCpu().getLoadAverage(), anyOf(lessThan((double) 0), greaterThanOrEqualTo((double) 0))); } assertNotNull(stats.getMem()); diff --git a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java index 18b5f7a7e19..181932513df 100644 --- a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java @@ -22,16 +22,19 @@ package org.elasticsearch.monitor.process; import org.apache.lucene.util.Constants; import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.elasticsearch.monitor.jvm.JvmInfo.jvmInfo; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class ProcessProbeTests extends ESTestCase { - ProcessProbe probe = ProcessProbe.getInstance(); - @Test public void testProcessInfo() { ProcessInfo info = probe.processInfo(); assertNotNull(info); @@ -40,7 +43,6 @@ public class ProcessProbeTests extends ESTestCase { assertThat(info.isMlockall(), equalTo(BootstrapInfo.isMemoryLocked())); } - @Test public void testProcessStats() { ProcessStats stats = probe.processStats(); assertNotNull(stats); diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java index 4cd61ebdd14..9236a16dcbd 100644 --- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java +++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.node.internal; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.cli.CliToolTestCase; import org.elasticsearch.common.cli.Terminal; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.env.Environment; @@ -35,9 +34,7 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -45,38 +42,8 @@ import static org.hamcrest.Matchers.*; public class InternalSettingsPreparerTests extends ESTestCase { - Map savedProperties = new HashMap<>(); Settings baseEnvSettings; - @Before - public void saveSettingsSystemProperties() { - // clear out any properties the settings preparer may look for - savedProperties.clear(); - for (Object propObj : System.getProperties().keySet()) { - String property = (String)propObj; - // NOTE: these prefixes are prefixes of the defaults, so both are handled here - for (String prefix : InternalSettingsPreparer.PROPERTY_PREFIXES) { - if (property.startsWith(prefix)) { - savedProperties.put(property, System.getProperty(property)); - } - } - } - String name = System.getProperty("name"); - if (name != null) { - savedProperties.put("name", name); - } - for (String property : savedProperties.keySet()) { - System.clearProperty(property); - } - } - - @After - public void restoreSettingsSystemProperties() { - for (Map.Entry property : savedProperties.entrySet()) { - System.setProperty(property.getKey(), property.getValue()); - } - } - @Before public void createBaseEnvSettings() { baseEnvSettings = settingsBuilder() @@ -93,13 +60,13 @@ public class InternalSettingsPreparerTests extends ESTestCase { Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY); assertNotNull(settings.get("name")); // a name was set assertNotNull(settings.get(ClusterName.SETTING)); // a cluster name was set - assertEquals(settings.toString(), 2, settings.names().size()); + int size = settings.names().size(); Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null); settings = env.settings(); assertNotNull(settings.get("name")); // a name was set assertNotNull(settings.get(ClusterName.SETTING)); // a cluster name was set - assertEquals(settings.toString(), 3 /* path.home is in the base settings */, settings.names().size()); + assertEquals(settings.toString(), size + 1 /* path.home is in the base settings */, settings.names().size()); String home = baseEnvSettings.get("path.home"); String configDir = env.configFile().toString(); assertTrue(configDir, configDir.startsWith(home)); @@ -112,30 +79,6 @@ public class InternalSettingsPreparerTests extends ESTestCase { assertEquals(ClusterName.DEFAULT.value(), settings.get(ClusterName.SETTING)); } - public void testIgnoreSystemProperties() { - try { - System.setProperty("es.node.zone", "foo"); - Settings settings = settingsBuilder() - .put("node.zone", "bar") - .put(baseEnvSettings) - .build(); - Environment env = InternalSettingsPreparer.prepareEnvironment(settings, null); - // Should use setting from the system property - assertThat(env.settings().get("node.zone"), equalTo("foo")); - - settings = settingsBuilder() - .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) - .put("node.zone", "bar") - .put(baseEnvSettings) - .build(); - env = InternalSettingsPreparer.prepareEnvironment(settings, null); - // Should use setting from the system property - assertThat(env.settings().get("node.zone"), equalTo("bar")); - } finally { - System.clearProperty("es.node.zone"); - } - } - public void testReplacePromptPlaceholders() { final List replacedSecretProperties = new ArrayList<>(); final List replacedTextProperties = new ArrayList<>(); @@ -205,74 +148,6 @@ public class InternalSettingsPreparerTests extends ESTestCase { } } - public void testNameSettingsPreference() { - try { - System.setProperty("name", "sys-prop-name"); - // Test system property overrides node.name - Settings settings = settingsBuilder() - .put("node.name", "node-name") - .put(baseEnvSettings) - .build(); - Environment env = InternalSettingsPreparer.prepareEnvironment(settings, null); - assertThat(env.settings().get("name"), equalTo("sys-prop-name")); - - // test name in settings overrides sys prop and node.name - settings = settingsBuilder() - .put("name", "name-in-settings") - .put("node.name", "node-name") - .put(baseEnvSettings) - .build(); - env = InternalSettingsPreparer.prepareEnvironment(settings, null); - assertThat(env.settings().get("name"), equalTo("name-in-settings")); - - // test only node.name in settings - System.clearProperty("name"); - settings = settingsBuilder() - .put("node.name", "node-name") - .put(baseEnvSettings) - .build(); - env = InternalSettingsPreparer.prepareEnvironment(settings, null); - assertThat(env.settings().get("name"), equalTo("node-name")); - - // test no name at all results in name being set - env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null); - assertThat(env.settings().get("name"), not("name-in-settings")); - assertThat(env.settings().get("name"), not("sys-prop-name")); - assertThat(env.settings().get("name"), not("node-name")); - assertThat(env.settings().get("name"), notNullValue()); - } finally { - System.clearProperty("name"); - } - } - - public void testPromptForNodeNameOnlyPromptsOnce() { - final AtomicInteger counter = new AtomicInteger(); - final Terminal terminal = new CliToolTestCase.MockTerminal() { - @Override - public char[] readSecret(String message, Object... args) { - fail("readSecret should never be called by this test"); - return null; - } - - @Override - public String readText(String message, Object... args) { - int count = counter.getAndIncrement(); - return "prompted name " + count; - } - }; - - System.clearProperty("name"); - Settings settings = Settings.builder() - .put(baseEnvSettings) - .put("node.name", InternalSettingsPreparer.TEXT_PROMPT_VALUE) - .build(); - Environment env = InternalSettingsPreparer.prepareEnvironment(settings, terminal); - settings = env.settings(); - assertThat(counter.intValue(), is(1)); - assertThat(settings.get("name"), is("prompted name 0")); - assertThat(settings.get("node.name"), is("prompted name 0")); - } - public void testGarbageIsNotSwallowed() throws IOException { try { InputStream garbage = getClass().getResourceAsStream("/config/garbage/garbage.yml"); diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java index 406cf68a98e..6b23bb09f24 100644 --- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java +++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java @@ -22,14 +22,15 @@ package org.elasticsearch.nodesinfo; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.util.List; import static org.elasticsearch.client.Requests.nodesInfoRequest; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.Matchers.*; /** @@ -37,15 +38,12 @@ import static org.hamcrest.Matchers.*; */ @ClusterScope(scope= Scope.TEST, numDataNodes =0) public class SimpleNodesInfoIT extends ESIntegTestCase { - static final class Fields { static final String SITE_PLUGIN = "dummy"; static final String SITE_PLUGIN_DESCRIPTION = "This is a description for a dummy test site plugin."; static final String SITE_PLUGIN_VERSION = "0.0.7-BOND-SITE"; } - - @Test public void testNodesInfos() throws Exception { List nodesIds = internalCluster().startNodesAsync(2).get(); final String node_1 = nodesIds.get(0); @@ -84,4 +82,36 @@ public class SimpleNodesInfoIT extends ESIntegTestCase { assertThat(response.getNodes().length, is(1)); assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); } + + public void testAllocatedProcessors() throws Exception { + List nodesIds = internalCluster(). + startNodesAsync( + Settings.builder().put(EsExecutors.PROCESSORS, 3).build(), + Settings.builder().put(EsExecutors.PROCESSORS, 6).build() + ).get(); + + final String node_1 = nodesIds.get(0); + final String node_2 = nodesIds.get(1); + + ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").get(); + logger.info("--> done cluster_health, status " + clusterHealth.getStatus()); + + String server1NodeId = internalCluster().getInstance(ClusterService.class, node_1).state().nodes().localNodeId(); + String server2NodeId = internalCluster().getInstance(ClusterService.class, node_2).state().nodes().localNodeId(); + logger.info("--> started nodes: " + server1NodeId + " and " + server2NodeId); + + NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); + + assertThat(response.getNodes().length, is(2)); + assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); + assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); + + assertThat(response.getNodesMap().get(server1NodeId).getOs().getAvailableProcessors(), + equalTo(Runtime.getRuntime().availableProcessors())); + assertThat(response.getNodesMap().get(server2NodeId).getOs().getAvailableProcessors(), + equalTo(Runtime.getRuntime().availableProcessors())); + + assertThat(response.getNodesMap().get(server1NodeId).getOs().getAllocatedProcessors(), equalTo(3)); + assertThat(response.getNodesMap().get(server2NodeId).getOs().getAllocatedProcessors(), equalTo(6)); + } } diff --git a/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIntegrationIT.java b/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIntegrationIT.java index 3e2d0e70d9f..49d22b87bf8 100644 --- a/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIntegrationIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.operateAllIndices; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -31,8 +30,6 @@ import static org.hamcrest.Matchers.equalTo; */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class DestructiveOperationsIntegrationIT extends ESIntegTestCase { - - @Test // One test for test performance, since cluster scope is test // The cluster scope is test b/c we can't clear cluster settings. public void testDestructiveOperations() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java index e6282d803b3..a0751dffac5 100644 --- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java +++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.rest.client.http.HttpDeleteWithEntity; import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.Test; import static org.hamcrest.Matchers.is; @@ -39,7 +38,6 @@ import static org.hamcrest.Matchers.is; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 1) public class DetailedErrorsDisabledIT extends ESIntegTestCase { - // Build our cluster settings @Override protected Settings nodeSettings(int nodeOrdinal) { @@ -50,7 +48,6 @@ public class DetailedErrorsDisabledIT extends ESIntegTestCase { .build(); } - @Test public void testThatErrorTraceParamReturns400() throws Exception { // Make the HTTP request HttpResponse response = new HttpRequestBuilder(HttpClients.createDefault()) diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java index 050d88c2f39..935b4e21ad2 100644 --- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java +++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.rest.client.http.HttpDeleteWithEntity; import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.Test; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; @@ -39,8 +38,6 @@ import static org.hamcrest.Matchers.not; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 1) public class DetailedErrorsEnabledIT extends ESIntegTestCase { - - // Build our cluster settings @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.settingsBuilder() @@ -49,7 +46,6 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase { .build(); } - @Test public void testThatErrorTraceWorksByDefault() throws Exception { // Make the HTTP request HttpResponse response = new HttpRequestBuilder(HttpClients.createDefault()) diff --git a/core/src/test/java/org/elasticsearch/percolator/ConcurrentPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/ConcurrentPercolatorIT.java index 4a6b835c834..b11f24377ad 100644 --- a/core/src/test/java/org/elasticsearch/percolator/ConcurrentPercolatorIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/ConcurrentPercolatorIT.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Random; import java.util.Set; @@ -41,15 +40,19 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.emptyArray; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; /** * */ public class ConcurrentPercolatorIT extends ESIntegTestCase { - - @Test public void testSimpleConcurrentPercolator() throws Exception { // We need to index a document / define mapping, otherwise field1 doesn't get reconized as number field. // If we don't do this, then 'test2' percolate query gets parsed as a TermQuery and not a RangeQuery. @@ -144,7 +147,6 @@ public class ConcurrentPercolatorIT extends ESIntegTestCase { assertThat(assertionError + " should be null", assertionError, nullValue()); } - @Test public void testConcurrentAddingAndPercolating() throws Exception { assertAcked(prepareCreate("index").addMapping("type", "field1", "type=string", "field2", "type=string")); ensureGreen(); @@ -291,7 +293,6 @@ public class ConcurrentPercolatorIT extends ESIntegTestCase { assertThat(exceptionsHolder.isEmpty(), equalTo(true)); } - @Test public void testConcurrentAddingAndRemovingWhilePercolating() throws Exception { assertAcked(prepareCreate("index").addMapping("type", "field1", "type=string")); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java index 2eb763959f8..7674ef83b5c 100644 --- a/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java @@ -26,27 +26,33 @@ import org.elasticsearch.action.percolate.PercolateSourceBuilder; import org.elasticsearch.client.Requests; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.*; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** */ public class MultiPercolatorIT extends ESIntegTestCase { - - @Test public void testBasics() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "field1", "type=string")); ensureGreen(); @@ -116,7 +122,6 @@ public class MultiPercolatorIT extends ESIntegTestCase { assertThat(item.getErrorMessage(), containsString("document missing")); } - @Test public void testWithRouting() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "field1", "type=string")); ensureGreen(); @@ -195,7 +200,6 @@ public class MultiPercolatorIT extends ESIntegTestCase { assertThat(item.getErrorMessage(), containsString("document missing")); } - @Test public void testExistingDocsOnly() throws Exception { createIndex("test"); @@ -265,7 +269,6 @@ public class MultiPercolatorIT extends ESIntegTestCase { assertThat(response.items()[numPercolateRequest].getResponse().getMatches().length, equalTo(numQueries)); } - @Test public void testWithDocsOnly() throws Exception { createIndex("test"); ensureGreen(); @@ -338,8 +341,6 @@ public class MultiPercolatorIT extends ESIntegTestCase { assertThat(response.items()[numPercolateRequest].getResponse().getMatches().length, equalTo(numQueries)); } - - @Test public void testNestedMultiPercolation() throws IOException { initNestedIndexAndPercolation(); MultiPercolateRequestBuilder mpercolate= client().prepareMultiPercolate(); diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java index f250e9231fa..8254932c304 100644 --- a/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java @@ -23,9 +23,8 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.percolate.PercolateSourceBuilder; import org.elasticsearch.index.percolator.PercolatorException; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.index.query.QueryShardException; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -36,8 +35,6 @@ import static org.hamcrest.Matchers.instanceOf; /** */ public class PercolatorBackwardsCompatibilityIT extends ESIntegTestCase { - - @Test public void testPercolatorUpgrading() throws Exception { // Simulates an index created on an node before 1.4.0 where the field resolution isn't strict. assertAcked(prepareCreate("test") @@ -53,7 +50,7 @@ public class PercolatorBackwardsCompatibilityIT extends ESIntegTestCase { PercolateResponse response = client().preparePercolate().setIndices("test").setDocumentType("type") .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("field1", "value")) .get(); - assertMatchCount(response, (long) numDocs); + assertMatchCount(response, numDocs); // After upgrade indices, indices created before the upgrade allow that queries refer to fields not available in mapping client().prepareIndex("test", PercolatorService.TYPE_NAME) diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorFacetsAndAggregationsIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorFacetsAndAggregationsIT.java index c1326845b49..85783e3d456 100644 --- a/core/src/test/java/org/elasticsearch/percolator/PercolatorFacetsAndAggregationsIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorFacetsAndAggregationsIT.java @@ -31,7 +31,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -51,8 +50,6 @@ import static org.hamcrest.Matchers.notNullValue; * */ public class PercolatorFacetsAndAggregationsIT extends ESIntegTestCase { - - @Test // Just test the integration with facets and aggregations, not the facet and aggregation functionality! public void testFacetsAndAggregations() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "field1", "type=string", "field2", "type=string")); @@ -115,7 +112,6 @@ public class PercolatorFacetsAndAggregationsIT extends ESIntegTestCase { } } - @Test // Just test the integration with facets and aggregations, not the facet and aggregation functionality! public void testAggregationsAndPipelineAggregations() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "field1", "type=string", "field2", "type=string")); @@ -188,7 +184,6 @@ public class PercolatorFacetsAndAggregationsIT extends ESIntegTestCase { } } - @Test public void testSignificantAggs() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -199,7 +194,6 @@ public class PercolatorFacetsAndAggregationsIT extends ESIntegTestCase { assertNoFailures(response); } - @Test public void testSingleShardAggregations() throws Exception { assertAcked(prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put("SETTING_NUMBER_OF_SHARDS", 1)) .addMapping("type", "field1", "type=string", "field2", "type=string")); diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java index 4cde86a6677..f627e0217e6 100644 --- a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java @@ -26,7 +26,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.percolate.PercolateSourceBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -50,27 +50,59 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeSet; import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.*; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyArray; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * */ public class PercolatorIT extends ESIntegTestCase { - - @Test public void testSimple1() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -149,7 +181,6 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test public void testSimple2() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", "field1", "type=long,doc_values=true")); ensureGreen(); @@ -201,7 +232,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContaining("test1")); } - @Test public void testPercolateQueriesWithRouting() throws Exception { client().admin().indices().prepareCreate("test") .setSettings(settingsBuilder().put("index.number_of_shards", 2)) @@ -243,8 +273,7 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(response.getMatches(), arrayWithSize(50)); } - @Test - public void storePeroclateQueriesOnRecreatedIndex() throws Exception { + public void testStorePeroclateQueriesOnRecreatedIndex() throws Exception { createIndex("test"); ensureGreen(); @@ -273,9 +302,8 @@ public class PercolatorIT extends ESIntegTestCase { .execute().actionGet(); } - @Test // see #2814 - public void percolateCustomAnalyzer() throws Exception { + public void testPercolateCustomAnalyzer() throws Exception { Builder builder = builder(); builder.put("index.analysis.analyzer.lwhitespacecomma.tokenizer", "whitespacecomma"); builder.putArray("index.analysis.analyzer.lwhitespacecomma.filter", "lowercase"); @@ -312,8 +340,7 @@ public class PercolatorIT extends ESIntegTestCase { } - @Test - public void createIndexAndThenRegisterPercolator() throws Exception { + public void testCreateIndexAndThenRegisterPercolator() throws Exception { prepareCreate("test") .addMapping("type1", "field1", "type=string") .get(); @@ -328,10 +355,10 @@ public class PercolatorIT extends ESIntegTestCase { .execute().actionGet(); refresh(); - CountResponse countResponse = client().prepareCount() + SearchResponse countResponse = client().prepareSearch().setSize(0) .setQuery(matchAllQuery()).setTypes(PercolatorService.TYPE_NAME) .execute().actionGet(); - assertThat(countResponse.getCount(), equalTo(1l)); + assertThat(countResponse.getHits().totalHits(), equalTo(1l)); for (int i = 0; i < 10; i++) { @@ -357,14 +384,13 @@ public class PercolatorIT extends ESIntegTestCase { logger.info("--> delete the index"); client().admin().indices().prepareDelete("test").execute().actionGet(); logger.info("--> make sure percolated queries for it have been deleted as well"); - countResponse = client().prepareCount() + countResponse = client().prepareSearch().setSize(0) .setQuery(matchAllQuery()).setTypes(PercolatorService.TYPE_NAME) .execute().actionGet(); assertHitCount(countResponse, 0l); } - @Test - public void multiplePercolators() throws Exception { + public void testMultiplePercolators() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", "field1", "type=string")); ensureGreen(); @@ -404,8 +430,7 @@ public class PercolatorIT extends ESIntegTestCase { } - @Test - public void dynamicAddingRemovingQueries() throws Exception { + public void testDynamicAddingRemovingQueries() throws Exception { assertAcked( prepareCreate("test") .addMapping("type1", "field1", "type=string") @@ -479,7 +504,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(percolate.getMatches(), emptyArray()); } - @Test public void testPercolateStatistics() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -564,7 +588,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(percolateSumTime, greaterThan(0l)); } - @Test public void testPercolatingExistingDocs() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -638,7 +661,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(3).type(), equalTo("type")); } - @Test public void testPercolatingExistingDocs_routing() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -704,7 +726,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContaining("4")); } - @Test public void testPercolatingExistingDocs_versionCheck() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -765,7 +786,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContainingInAnyOrder("2", "4")); } - @Test public void testPercolateMultipleIndicesAndAliases() throws Exception { createIndex("test1", "test2"); ensureGreen(); @@ -843,7 +863,6 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test public void testPercolateWithAliasFilter() throws Exception { assertAcked(prepareCreate("my-index") .addMapping(PercolatorService.TYPE_NAME, "a", "type=string,index=not_analyzed") @@ -921,7 +940,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(response.getCount(), equalTo(0l)); } - @Test public void testCountPercolation() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -990,7 +1008,6 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test public void testCountPercolatingExistingDocs() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -1052,7 +1069,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(response.getMatches(), nullValue()); } - @Test public void testPercolateSizingWithQueryAndFilter() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -1141,7 +1157,6 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test public void testPercolateScoreAndSorting() throws Exception { createIndex("my-index"); ensureGreen(); @@ -1231,7 +1246,6 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test public void testPercolateSortingWithNoSize() throws Exception { createIndex("my-index"); ensureGreen(); @@ -1269,8 +1283,7 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test - public void testPercolateSorting_unsupportedField() throws Exception { + public void testPercolateSortingUnsupportedField() throws Exception { client().admin().indices().prepareCreate("my-index") .addMapping("my-type", "field", "type=string") .addMapping(PercolatorService.TYPE_NAME, "level", "type=integer", "query", "type=object,enabled=false") @@ -1297,7 +1310,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(response.getShardFailures()[0].reason(), containsString("Only _score desc is supported")); } - @Test public void testPercolateOnEmptyIndex() throws Exception { client().admin().indices().prepareCreate("my-index").execute().actionGet(); ensureGreen(); @@ -1311,7 +1323,6 @@ public class PercolatorIT extends ESIntegTestCase { assertMatchCount(response, 0l); } - @Test public void testPercolateNotEmptyIndexButNoRefresh() throws Exception { client().admin().indices().prepareCreate("my-index") .setSettings(settingsBuilder().put("index.refresh_interval", -1)) @@ -1331,7 +1342,6 @@ public class PercolatorIT extends ESIntegTestCase { assertMatchCount(response, 0l); } - @Test public void testPercolatorWithHighlighting() throws Exception { StringBuilder fieldMapping = new StringBuilder("type=string") .append(",store=").append(randomBoolean()); @@ -1547,8 +1557,7 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy dog")); } - @Test - public void percolateNonMatchingConstantScoreQuery() throws Exception { + public void testPercolateNonMatchingConstantScoreQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("doc", "message", "type=string")); ensureGreen(); @@ -1572,7 +1581,6 @@ public class PercolatorIT extends ESIntegTestCase { assertMatchCount(percolate, 0l); } - @Test public void testNestedPercolation() throws IOException { initNestedIndexAndPercolation(); PercolateResponse response = client().preparePercolate().setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getNotMatchingNestedDoc())).setIndices("nestedindex").setDocumentType("company").get(); @@ -1582,8 +1590,7 @@ public class PercolatorIT extends ESIntegTestCase { assertEquals(response.getMatches()[0].getId().string(), "Q"); } - @Test - public void makeSureNonNestedDocumentDoesNotTriggerAssertion() throws IOException { + public void testNonNestedDocumentDoesNotTriggerAssertion() throws IOException { initNestedIndexAndPercolation(); XContentBuilder doc = jsonBuilder(); doc.startObject(); @@ -1592,7 +1599,6 @@ public class PercolatorIT extends ESIntegTestCase { assertNoFailures(response); } - @Test public void testNestedPercolationOnExistingDoc() throws IOException { initNestedIndexAndPercolation(); client().prepareIndex("nestedindex", "company", "notmatching").setSource(getNotMatchingNestedDoc()).get(); @@ -1605,7 +1611,6 @@ public class PercolatorIT extends ESIntegTestCase { assertEquals(response.getMatches()[0].getId().string(), "Q"); } - @Test public void testPercolationWithDynamicTemplates() throws Exception { assertAcked(prepareCreate("idx").addMapping("type", jsonBuilder().startObject().startObject("type") .field("dynamic", false) @@ -1662,7 +1667,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("2")); } - @Test public void testUpdateMappingDynamicallyWhilePercolating() throws Exception { createIndex("test"); ensureSearchable(); @@ -1691,7 +1695,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(((Map) properties.get("field2")).get("type"), equalTo("string")); } - @Test public void testDontReportDeletedPercolatorDocs() throws Exception { client().admin().indices().prepareCreate("test").execute().actionGet(); ensureGreen(); @@ -1714,7 +1717,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContainingInAnyOrder("1")); } - @Test public void testAddQueryWithNoMapping() throws Exception { client().admin().indices().prepareCreate("test").get(); ensureGreen(); @@ -1738,7 +1740,6 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test public void testPercolatorQueryWithNowRange() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("my-type", "timestamp", "type=date,format=epoch_millis") @@ -1798,7 +1799,6 @@ public class PercolatorIT extends ESIntegTestCase { } // issue - @Test public void testNestedDocFilter() throws IOException { String mapping = "{\n" + " \"doc\": {\n" + @@ -1935,7 +1935,6 @@ public class PercolatorIT extends ESIntegTestCase { assertMatchCount(response, 3l); } - @Test public void testMapUnmappedFieldAsString() throws IOException{ // If index.percolator.map_unmapped_fields_as_string is set to true, unmapped field is mapped as an analyzed string. Settings.Builder settings = Settings.settingsBuilder() @@ -1954,7 +1953,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(response1.getMatches(), arrayWithSize(1)); } - @Test public void testFailNicelyWithInnerHits() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("mapping") @@ -1982,7 +1980,6 @@ public class PercolatorIT extends ESIntegTestCase { } } - @Test public void testParentChild() throws Exception { // We don't fail p/c queries, but those queries are unusable because only a single document can be provided in // the percolate api @@ -1993,7 +1990,6 @@ public class PercolatorIT extends ESIntegTestCase { .execute().actionGet(); } - @Test public void testPercolateDocumentWithParentField() throws Exception { assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent").addMapping("parent")); client().prepareIndex("index", PercolatorService.TYPE_NAME, "1") @@ -2009,7 +2005,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(response.getMatches()[0].getId().string(), equalTo("1")); } - @Test public void testFilterByNow() throws Exception { client().prepareIndex("index", PercolatorService.TYPE_NAME, "1") .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("created", "2015-07-10T14:41:54+0000").endObject()) @@ -2024,6 +2019,5 @@ public class PercolatorIT extends ESIntegTestCase { .get(); assertMatchCount(response, 1); } - } diff --git a/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorIT.java index eb694ff5ed6..db660695843 100644 --- a/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorIT.java @@ -21,7 +21,7 @@ package org.elasticsearch.percolator; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder; import org.elasticsearch.action.percolate.MultiPercolateResponse; import org.elasticsearch.action.percolate.PercolateRequestBuilder; @@ -33,7 +33,8 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -47,8 +48,6 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount; @@ -61,13 +60,11 @@ import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0) public class RecoveryPercolatorIT extends ESIntegTestCase { - @Override protected int numberOfShards() { return 1; } - @Test public void testRestartNodePercolator1() throws Exception { internalCluster().startNode(); assertAcked(prepareCreate("test").addMapping("type1", "field1", "type=string").addMapping(PercolatorService.TYPE_NAME, "color", "type=string")); @@ -104,7 +101,6 @@ public class RecoveryPercolatorIT extends ESIntegTestCase { assertThat(percolate.getMatches(), arrayWithSize(1)); } - @Test public void testRestartNodePercolator2() throws Exception { internalCluster().startNode(); assertAcked(prepareCreate("test").addMapping("type1", "field1", "type=string").addMapping(PercolatorService.TYPE_NAME, "color", "type=string")); @@ -118,7 +114,7 @@ public class RecoveryPercolatorIT extends ESIntegTestCase { .setRefresh(true) .get(); - assertThat(client().prepareCount().setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getHits().totalHits(), equalTo(1l)); PercolateResponse percolate = client().preparePercolate() .setIndices("test").setDocumentType("type1") @@ -135,7 +131,7 @@ public class RecoveryPercolatorIT extends ESIntegTestCase { ClusterHealthResponse clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)).actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); - CountResponse countResponse = client().prepareCount().setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get(); assertHitCount(countResponse, 1l); DeleteIndexResponse actionGet = client().admin().indices().prepareDelete("test").get(); @@ -144,7 +140,7 @@ public class RecoveryPercolatorIT extends ESIntegTestCase { clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)).actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); - assertThat(client().prepareCount().setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getCount(), equalTo(0l)); + assertThat(client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getHits().totalHits(), equalTo(0l)); percolate = client().preparePercolate() .setIndices("test").setDocumentType("type1") @@ -164,7 +160,7 @@ public class RecoveryPercolatorIT extends ESIntegTestCase { .setRefresh(true) .get(); - assertThat(client().prepareCount().setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getHits().totalHits(), equalTo(1l)); percolate = client().preparePercolate() .setIndices("test").setDocumentType("type1") @@ -176,7 +172,6 @@ public class RecoveryPercolatorIT extends ESIntegTestCase { assertThat(percolate.getMatches(), arrayWithSize(1)); } - @Test public void testLoadingPercolateQueriesDuringCloseAndOpen() throws Exception { internalCluster().startNode(); internalCluster().startNode(); @@ -223,13 +218,11 @@ public class RecoveryPercolatorIT extends ESIntegTestCase { assertThat(response.getMatches()[0].getId().string(), equalTo("100")); } - @Test - public void testSinglePercolator_recovery() throws Exception { + public void testSinglePercolatorRecovery() throws Exception { percolatorRecovery(false); } - @Test - public void testMultiPercolator_recovery() throws Exception { + public void testMultiPercolatorRecovery() throws Exception { percolatorRecovery(true); } diff --git a/core/src/test/java/org/elasticsearch/percolator/TTLPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/TTLPercolatorIT.java index 7fbedca0440..4b4d4a84237 100644 --- a/core/src/test/java/org/elasticsearch/percolator/TTLPercolatorIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/TTLPercolatorIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.concurrent.TimeUnit; @@ -48,7 +47,6 @@ import static org.hamcrest.Matchers.equalTo; */ @ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class TTLPercolatorIT extends ESIntegTestCase { - private static final long PURGE_INTERVAL = 200; @Override @@ -63,7 +61,6 @@ public class TTLPercolatorIT extends ESIntegTestCase { .build(); } - @Test public void testPercolatingWithTimeToLive() throws Exception { final Client client = client(); ensureGreen(); @@ -155,8 +152,6 @@ public class TTLPercolatorIT extends ESIntegTestCase { assertThat(percolateResponse.getMatches(), emptyArray()); } - - @Test public void testEnsureTTLDoesNotCreateIndex() throws IOException, InterruptedException { ensureGreen(); client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder() @@ -207,8 +202,5 @@ public class TTLPercolatorIT extends ESIntegTestCase { client().admin().indices().prepareCreate("test") .addMapping("type1", typeMapping) .execute().actionGet(); - - } - } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluggableTransportModuleIT.java b/core/src/test/java/org/elasticsearch/plugins/PluggableTransportModuleIT.java index a6ac6bf1c69..7831b7ca994 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluggableTransportModuleIT.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluggableTransportModuleIT.java @@ -25,26 +25,30 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.transport.AssertingLocalTransport; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.*; -import org.junit.Test; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportModule; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; import java.io.IOException; import java.util.Collection; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; /** * */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 2) public class PluggableTransportModuleIT extends ESIntegTestCase { - public static final AtomicInteger SENT_REQUEST_COUNTER = new AtomicInteger(0); @Override @@ -65,7 +69,6 @@ public class PluggableTransportModuleIT extends ESIntegTestCase { return pluginList(CountingSentRequestsPlugin.class); } - @Test public void testThatPluginFunctionalityIsLoadedWithoutConfiguration() throws Exception { for (Transport transport : internalCluster().getInstances(Transport.class)) { assertThat(transport, instanceOf(CountingAssertingLocalTransport.class)); diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java index 27379dfa838..000365f6a20 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.plugins; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -35,22 +35,9 @@ import static org.hamcrest.Matchers.contains; public class PluginInfoTests extends ESTestCase { - static void writeProperties(Path pluginDir, String... stringProps) throws IOException { - assert stringProps.length % 2 == 0; - Files.createDirectories(pluginDir); - Path propertiesFile = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES); - Properties properties = new Properties(); - for (int i = 0; i < stringProps.length; i += 2) { - properties.put(stringProps[i], stringProps[i + 1]); - } - try (OutputStream out = Files.newOutputStream(propertiesFile)) { - properties.store(out, ""); - } - } - public void testReadFromProperties() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -71,7 +58,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesNameMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir); + PluginTestUtil.writeProperties(pluginDir); try { PluginInfo.readFromProperties(pluginDir); fail("expected missing name exception"); @@ -79,7 +66,7 @@ public class PluginInfoTests extends ESTestCase { assertTrue(e.getMessage().contains("Property [name] is missing in")); } - writeProperties(pluginDir, "name", ""); + PluginTestUtil.writeProperties(pluginDir, "name", ""); try { PluginInfo.readFromProperties(pluginDir); fail("expected missing name exception"); @@ -90,7 +77,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesDescriptionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, "name", "fake-plugin"); + PluginTestUtil.writeProperties(pluginDir, "name", "fake-plugin"); try { PluginInfo.readFromProperties(pluginDir); fail("expected missing description exception"); @@ -101,7 +88,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesVersionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, "description", "fake desc", "name", "fake-plugin"); + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "fake-plugin"); try { PluginInfo.readFromProperties(pluginDir); fail("expected missing version exception"); @@ -112,7 +99,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesJvmAndSiteMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "version", "1.0", "name", "my_plugin"); @@ -126,7 +113,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesElasticsearchVersionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -141,7 +128,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesJavaVersionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "elasticsearch.version", Version.CURRENT.toString(), @@ -158,7 +145,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesJavaVersionIncompatible() throws Exception { String pluginName = "fake-plugin"; Path pluginDir = createTempDir().resolve(pluginName); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", pluginName, "elasticsearch.version", Version.CURRENT.toString(), @@ -177,7 +164,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesBadJavaVersionFormat() throws Exception { String pluginName = "fake-plugin"; Path pluginDir = createTempDir().resolve(pluginName); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", pluginName, "elasticsearch.version", Version.CURRENT.toString(), @@ -195,7 +182,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "version", "1.0", "jvm", "true", @@ -205,13 +192,13 @@ public class PluginInfoTests extends ESTestCase { PluginInfo.readFromProperties(pluginDir); fail("expected bogus elasticsearch version exception"); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("version needs to contain major, minor and revision")); + assertTrue(e.getMessage().contains("version needs to contain major, minor, and revision")); } } public void testReadFromPropertiesOldElasticsearchVersion() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -227,7 +214,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesJvmMissingClassname() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -245,7 +232,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesSitePlugin() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); Files.createDirectories(pluginDir.resolve("_site")); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -258,7 +245,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromPropertiesSitePluginWithoutSite() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - writeProperties(pluginDir, + PluginTestUtil.writeProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -272,14 +259,14 @@ public class PluginInfoTests extends ESTestCase { } public void testPluginListSorted() { - PluginsInfo pluginsInfo = new PluginsInfo(5); - pluginsInfo.add(new PluginInfo("c", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("b", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("e", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("a", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("d", "foo", true, "dummy", true, "dummyclass", true)); + PluginsAndModules pluginsInfo = new PluginsAndModules(); + pluginsInfo.addPlugin(new PluginInfo("c", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("b", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("e", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("a", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("d", "foo", true, "dummy", true, "dummyclass", true)); - final List infos = pluginsInfo.getInfos(); + final List infos = pluginsInfo.getPluginInfos(); List names = infos.stream().map((input) -> input.getName()).collect(Collectors.toList()); assertThat(names, contains("a", "b", "c", "d", "e")); } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerCliTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginManagerCliTests.java index f21609a5d55..f16f9981d93 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerCliTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginManagerCliTests.java @@ -21,19 +21,18 @@ package org.elasticsearch.plugins; import org.elasticsearch.common.cli.CliTool; import org.elasticsearch.common.cli.CliToolTestCase; -import org.junit.Test; import java.io.IOException; import java.net.MalformedURLException; import java.nio.file.Path; -import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK_AND_EXIT; import static org.elasticsearch.common.cli.CliTool.ExitStatus.IO_ERROR; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK_AND_EXIT; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.is; public class PluginManagerCliTests extends CliToolTestCase { - - @Test public void testHelpWorks() throws IOException { CliToolTestCase.CaptureOutputTerminal terminal = new CliToolTestCase.CaptureOutputTerminal(); assertThat(new PluginManagerCliParser(terminal).execute(args("--help")), is(OK_AND_EXIT)); diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index d4553bd0c46..660f1015c3d 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -19,11 +19,15 @@ package org.elasticsearch.plugins; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.store.IndexStoreModule; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.test.ESTestCase; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; public class PluginsServiceTests extends ESTestCase { @@ -38,7 +42,7 @@ public class PluginsServiceTests extends ESTestCase { } @Override public Settings additionalSettings() { - return Settings.builder().put("foo.bar", "1").put(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.MMAPFS.getSettingsKey()).build(); + return Settings.builder().put("foo.bar", "1").put(IndexModule.STORE_TYPE, IndexModule.Type.MMAPFS.getSettingsKey()).build(); } } public static class AdditionalSettingsPlugin2 extends Plugin { @@ -56,20 +60,42 @@ public class PluginsServiceTests extends ESTestCase { } } + public static class FailOnModule extends Plugin { + @Override + public String name() { + return "fail-on-module"; + } + @Override + public String description() { + return "fails in onModule"; + } + + public void onModule(BrokenModule brokenModule) { + throw new IllegalStateException("boom"); + } + } + + public static class BrokenModule extends AbstractModule { + + @Override + protected void configure() { + } + } + static PluginsService newPluginsService(Settings settings, Class... classpathPlugins) { - return new PluginsService(settings, new Environment(settings).pluginsFile(), Arrays.asList(classpathPlugins)); + return new PluginsService(settings, null, new Environment(settings).pluginsFile(), Arrays.asList(classpathPlugins)); } public void testAdditionalSettings() { Settings settings = Settings.builder() .put("path.home", createTempDir()) .put("my.setting", "test") - .put(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.SIMPLEFS.getSettingsKey()).build(); + .put(IndexModule.STORE_TYPE, IndexModule.Type.SIMPLEFS.getSettingsKey()).build(); PluginsService service = newPluginsService(settings, AdditionalSettingsPlugin1.class); Settings newSettings = service.updatedSettings(); assertEquals("test", newSettings.get("my.setting")); // previous settings still exist assertEquals("1", newSettings.get("foo.bar")); // added setting exists - assertEquals(IndexStoreModule.Type.SIMPLEFS.getSettingsKey(), newSettings.get(IndexStoreModule.STORE_TYPE)); // does not override pre existing settings + assertEquals(IndexModule.Type.SIMPLEFS.getSettingsKey(), newSettings.get(IndexModule.STORE_TYPE)); // does not override pre existing settings } public void testAdditionalSettingsClash() { @@ -86,4 +112,28 @@ public class PluginsServiceTests extends ESTestCase { assertTrue(msg, msg.contains("plugin [additional-settings2]")); } } + + public void testOnModuleExceptionsArePropagated() { + Settings settings = Settings.builder() + .put("path.home", createTempDir()).build(); + PluginsService service = newPluginsService(settings, FailOnModule.class); + try { + service.processModule(new BrokenModule()); + fail("boom"); + } catch (ElasticsearchException ex) { + assertEquals("failed to invoke onModule", ex.getMessage()); + assertEquals("boom", ex.getCause().getCause().getMessage()); + } + } + + public void testExistingPluginMissingDescriptor() throws Exception { + Path pluginsDir = createTempDir(); + Files.createDirectory(pluginsDir.resolve("plugin-missing-descriptor")); + try { + PluginsService.getPluginBundles(pluginsDir); + fail(); + } catch (IllegalStateException e) { + assertTrue(e.getMessage(), e.getMessage().contains("Could not load plugin descriptor for existing plugin")); + } + } } diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java index d9580854c14..5d7b8068fe4 100644 --- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java +++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java @@ -22,14 +22,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.responseheader.TestResponseHeaderPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.Test; import java.util.Collection; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasStatus; import static org.hamcrest.Matchers.equalTo; @@ -38,7 +37,6 @@ import static org.hamcrest.Matchers.equalTo; */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class ResponseHeaderPluginIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.settingsBuilder() @@ -52,7 +50,6 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase { return pluginList(TestResponseHeaderPlugin.class); } - @Test public void testThatSettingHeadersWorks() throws Exception { ensureGreen(); HttpResponse response = httpClient().method("GET").path("/_protected").execute(); @@ -63,5 +60,4 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase { assertThat(authResponse, hasStatus(OK)); assertThat(authResponse.getHeaders().get("Secret"), equalTo("granted")); } - } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/plugins/SitePluginIT.java b/core/src/test/java/org/elasticsearch/plugins/SitePluginIT.java index 6e62fd9e675..e2df2518f1c 100644 --- a/core/src/test/java/org/elasticsearch/plugins/SitePluginIT.java +++ b/core/src/test/java/org/elasticsearch/plugins/SitePluginIT.java @@ -25,9 +25,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.Test; import java.nio.file.Path; import java.util.ArrayList; @@ -35,8 +35,10 @@ import java.util.List; import java.util.Locale; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.rest.RestStatus.*; -import static org.elasticsearch.test.ESIntegTestCase.Scope; +import static org.elasticsearch.rest.RestStatus.FORBIDDEN; +import static org.elasticsearch.rest.RestStatus.MOVED_PERMANENTLY; +import static org.elasticsearch.rest.RestStatus.NOT_FOUND; +import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasStatus; import static org.hamcrest.Matchers.containsString; @@ -45,8 +47,6 @@ import static org.hamcrest.Matchers.containsString; */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class SitePluginIT extends ESIntegTestCase { - - @Override protected Settings nodeSettings(int nodeOrdinal) { Path pluginDir = getDataPath("/org/elasticsearch/test_plugins"); @@ -57,13 +57,13 @@ public class SitePluginIT extends ESIntegTestCase { .build(); } + @Override public HttpRequestBuilder httpClient() { RequestConfig.Builder builder = RequestConfig.custom().setRedirectsEnabled(false); CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(builder.build()).build(); return new HttpRequestBuilder(httpClient).httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class)); } - @Test public void testRedirectSitePlugin() throws Exception { // We use an HTTP Client to test redirection HttpResponse response = httpClient().method("GET").path("/_plugin/dummy").execute(); @@ -79,7 +79,6 @@ public class SitePluginIT extends ESIntegTestCase { /** * Test direct access to an existing file (index.html) */ - @Test public void testAnyPage() throws Exception { HttpResponse response = httpClient().path("/_plugin/dummy/index.html").execute(); assertThat(response, hasStatus(OK)); @@ -89,7 +88,6 @@ public class SitePluginIT extends ESIntegTestCase { /** * Test normalizing of path */ - @Test public void testThatPathsAreNormalized() throws Exception { // more info: https://www.owasp.org/index.php/Path_Traversal List notFoundUris = new ArrayList<>(); @@ -100,7 +98,7 @@ public class SitePluginIT extends ESIntegTestCase { notFoundUris.add("/_plugin/dummy/%2e%2e/%2e%2e/%2e%2e/%2e%2e/index.html"); notFoundUris.add("/_plugin/dummy/%2e%2e%2f%2e%2e%2f%2e%2e%2f%2e%2e%2findex.html"); notFoundUris.add("/_plugin/dummy/%2E%2E/%2E%2E/%2E%2E/%2E%2E/index.html"); - notFoundUris.add("/_plugin/dummy/..\\..\\..\\..\\..\\log4j.properties"); + notFoundUris.add("/_plugin/dummy/..%5C..%5C..%5C..%5C..%5Clog4j.properties"); for (String uri : notFoundUris) { HttpResponse response = httpClient().path(uri).execute(); @@ -118,7 +116,6 @@ public class SitePluginIT extends ESIntegTestCase { * Test case for #4845: https://github.com/elasticsearch/elasticsearch/issues/4845 * Serving _site plugins do not pick up on index.html for sub directories */ - @Test public void testWelcomePageInSubDirs() throws Exception { HttpResponse response = httpClient().path("/_plugin/subdir/dir/").execute(); assertThat(response, hasStatus(OK)); diff --git a/core/src/test/java/org/elasticsearch/plugins/SitePluginRelativePathConfigIT.java b/core/src/test/java/org/elasticsearch/plugins/SitePluginRelativePathConfigIT.java index ed3062620bc..1cde90d6984 100644 --- a/core/src/test/java/org/elasticsearch/plugins/SitePluginRelativePathConfigIT.java +++ b/core/src/test/java/org/elasticsearch/plugins/SitePluginRelativePathConfigIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.Test; import java.nio.file.Path; @@ -39,7 +38,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasStatus; @ClusterScope(scope = SUITE, numDataNodes = 1) public class SitePluginRelativePathConfigIT extends ESIntegTestCase { - private final Path root = PathUtils.get(".").toAbsolutePath().getRoot(); @Override @@ -60,7 +58,6 @@ public class SitePluginRelativePathConfigIT extends ESIntegTestCase { .build(); } - @Test public void testThatRelativePathsDontAffectPlugins() throws Exception { HttpResponse response = httpClient().method("GET").path("/_plugin/dummy/").execute(); assertThat(response, hasStatus(OK)); @@ -83,6 +80,7 @@ public class SitePluginRelativePathConfigIT extends ESIntegTestCase { return sb.toString(); } + @Override public HttpRequestBuilder httpClient() { CloseableHttpClient httpClient = HttpClients.createDefault(); return new HttpRequestBuilder(httpClient).httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class)); diff --git a/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java b/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java index c0d64533725..8d33758ef26 100644 --- a/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/FullRollingRestartIT.java @@ -21,16 +21,21 @@ package org.elasticsearch.recovery; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.zen.ZenDiscovery; +import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; /** @@ -38,7 +43,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0.0) public class FullRollingRestartIT extends ESIntegTestCase { - protected void assertTimeout(ClusterHealthRequestBuilder requestBuilder) { ClusterHealthResponse clusterHealth = requestBuilder.get(); if (clusterHealth.isTimedOut()) { @@ -52,7 +56,6 @@ public class FullRollingRestartIT extends ESIntegTestCase { return 1; } - @Test public void testFullRollingRestart() throws Exception { Settings settings = Settings.builder().put(ZenDiscovery.SETTING_JOIN_TIMEOUT, "30s").build(); internalCluster().startNode(settings); @@ -88,7 +91,7 @@ public class FullRollingRestartIT extends ESIntegTestCase { logger.info("--> refreshing and checking data"); refresh(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 2000l); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2000l); } // now start shutting nodes down @@ -106,7 +109,7 @@ public class FullRollingRestartIT extends ESIntegTestCase { logger.info("--> stopped two nodes, verifying data"); refresh(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 2000l); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2000l); } // closing the 3rd node @@ -124,7 +127,39 @@ public class FullRollingRestartIT extends ESIntegTestCase { logger.info("--> one node left, verifying data"); refresh(); for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 2000l); + assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2000l); + } + } + + public void testNoRebalanceOnRollingRestart() throws Exception { + // see https://github.com/elastic/elasticsearch/issues/14387 + internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startDataOnlyNodesAsync(3).get(); + /** + * We start 3 nodes and a dedicated master. Restart on of the data-nodes and ensure that we got no relocations. + * Yet we have 6 shards 0 replica so that means if the restarting node comes back both other nodes are subject + * to relocating to the restarting node since all had 2 shards and now one node has nothing allocated. + * We have a fix for this to wait until we have allocated unallocated shards now so this shouldn't happen. + */ + prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "6").put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0").put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueMinutes(1))).get(); + + for (int i = 0; i < 100; i++) { + client().prepareIndex("test", "type1", Long.toString(i)) + .setSource(MapBuilder.newMapBuilder().put("test", "value" + i).map()).execute().actionGet(); + } + ensureGreen(); + ClusterState state = client().admin().cluster().prepareState().get().getState(); + RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get(); + for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) { + assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "\n" + state.prettyPrint(), recoveryState.getType() != RecoveryState.Type.RELOCATION); + } + internalCluster().restartRandomDataNode(); + ensureGreen(); + ClusterState afterState = client().admin().cluster().prepareState().get().getState(); + + recoveryResponse = client().admin().indices().prepareRecoveries("test").get(); + for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) { + assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode()+ "-- \nbefore: \n" + state.prettyPrint() + "\nafter: \n" + afterState.prettyPrint(), recoveryState.getType() != RecoveryState.Type.RELOCATION); } } } diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java index 5a5f3163a4c..4cad0b2bf05 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.indices.recovery.RecoveryStatus; import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.junit.Test; import java.util.ArrayList; import java.util.concurrent.CountDownLatch; @@ -46,7 +45,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; public class RecoveriesCollectionTests extends ESSingleNodeTestCase { - final static RecoveryTarget.RecoveryListener listener = new RecoveryTarget.RecoveryListener() { @Override public void onRecoveryDone(RecoveryState state) { @@ -59,7 +57,6 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase { } }; - @Test public void testLastAccessTimeUpdate() throws Exception { createIndex(); final RecoveriesCollection collection = new RecoveriesCollection(logger, getInstanceFromNode(ThreadPool.class)); @@ -79,7 +76,6 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase { } } - @Test public void testRecoveryTimeout() throws InterruptedException { createIndex(); final RecoveriesCollection collection = new RecoveriesCollection(logger, getInstanceFromNode(ThreadPool.class)); @@ -106,7 +102,6 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase { } - @Test public void testRecoveryCancellationNoPredicate() throws Exception { createIndex(); final RecoveriesCollection collection = new RecoveriesCollection(logger, getInstanceFromNode(ThreadPool.class)); @@ -122,7 +117,6 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase { } } - @Test public void testRecoveryCancellationPredicate() throws Exception { createIndex(); final RecoveriesCollection collection = new RecoveriesCollection(logger, getInstanceFromNode(ThreadPool.class)); diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoverySettingsTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoverySettingsTests.java index a722e3ab026..4bcbb8c8ee7 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoverySettingsTests.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoverySettingsTests.java @@ -22,37 +22,16 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.util.concurrent.TimeUnit; public class RecoverySettingsTests extends ESSingleNodeTestCase { - @Override protected boolean resetNodeAfterTest() { return true; } - @Test public void testAllSettingsAreDynamicallyUpdatable() { - innerTestSettings(RecoverySettings.INDICES_RECOVERY_FILE_CHUNK_SIZE, randomIntBetween(1, 200), ByteSizeUnit.BYTES, new Validator() { - @Override - public void validate(RecoverySettings recoverySettings, int expectedValue) { - assertEquals(expectedValue, recoverySettings.fileChunkSize().bytesAsInt()); - } - }); - innerTestSettings(RecoverySettings.INDICES_RECOVERY_TRANSLOG_OPS, randomIntBetween(1, 200), new Validator() { - @Override - public void validate(RecoverySettings recoverySettings, int expectedValue) { - assertEquals(expectedValue, recoverySettings.translogOps()); - } - }); - innerTestSettings(RecoverySettings.INDICES_RECOVERY_TRANSLOG_SIZE, randomIntBetween(1, 200), ByteSizeUnit.BYTES, new Validator() { - @Override - public void validate(RecoverySettings recoverySettings, int expectedValue) { - assertEquals(expectedValue, recoverySettings.translogSize().bytesAsInt()); - } - }); innerTestSettings(RecoverySettings.INDICES_RECOVERY_CONCURRENT_STREAMS, randomIntBetween(1, 200), new Validator() { @Override public void validate(RecoverySettings recoverySettings, int expectedValue) { @@ -101,13 +80,6 @@ public class RecoverySettingsTests extends ESSingleNodeTestCase { assertEquals(expectedValue, recoverySettings.internalActionLongTimeout().millis()); } }); - - innerTestSettings(RecoverySettings.INDICES_RECOVERY_COMPRESS, false, new Validator() { - @Override - public void validate(RecoverySettings recoverySettings, boolean expectedValue) { - assertEquals(expectedValue, recoverySettings.compress()); - } - }); } private static class Validator { diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java index 29eb8266d10..7095639eafc 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java @@ -23,16 +23,19 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Priority; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.math.MathUtils; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.concurrent.TimeUnit; @@ -41,17 +44,12 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { - private final ESLogger logger = Loggers.getLogger(RecoveryWhileUnderLoadIT.class); - @Test - public void recoverWhileUnderLoadAllocateReplicasTest() throws Exception { + public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception { logger.info("--> creating test index ..."); int numberOfShards = numberOfShards(); assertAcked(prepareCreate("test", 1, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(TranslogConfig.INDEX_TRANSLOG_DURABILITY, Translog.Durabilty.ASYNC))); @@ -104,8 +102,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { } } - @Test - public void recoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() throws Exception { + public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() throws Exception { logger.info("--> creating test index ..."); int numberOfShards = numberOfShards(); assertAcked(prepareCreate("test", 1, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(TranslogConfig.INDEX_TRANSLOG_DURABILITY, Translog.Durabilty.ASYNC))); @@ -156,8 +153,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { } } - @Test - public void recoverWhileUnderLoadWithReducedAllowedNodes() throws Exception { + public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception { logger.info("--> creating test index ..."); int numberOfShards = numberOfShards(); assertAcked(prepareCreate("test", 2, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(TranslogConfig.INDEX_TRANSLOG_DURABILITY, Translog.Durabilty.ASYNC))); @@ -226,15 +222,14 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { } } - @Test - public void recoverWhileRelocating() throws Exception { - final int numShards = between(2, 10); + public void testRecoverWhileRelocating() throws Exception { + final int numShards = between(2, 5); final int numReplicas = 0; logger.info("--> creating test index ..."); int allowNodes = 2; assertAcked(prepareCreate("test", 3, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(SETTING_NUMBER_OF_REPLICAS, numReplicas).put(TranslogConfig.INDEX_TRANSLOG_DURABILITY, Translog.Durabilty.ASYNC))); - final int numDocs = scaledRandomIntBetween(200, 20000); + final int numDocs = scaledRandomIntBetween(200, 9999); try (BackgroundIndexer indexer = new BackgroundIndexer("test", "type", client(), numDocs)) { @@ -268,12 +263,14 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { private void iterateAssertCount(final int numberOfShards, final long numberOfDocs, final int iterations) throws Exception { SearchResponse[] iterationResults = new SearchResponse[iterations]; boolean error = false; + SearchResponse lastErroneousResponse = null; for (int i = 0; i < iterations; i++) { - SearchResponse searchResponse = client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(); + SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).addSort("id", SortOrder.ASC).get(); logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse); iterationResults[i] = searchResponse; if (searchResponse.getHits().totalHits() != numberOfDocs) { error = true; + lastErroneousResponse = searchResponse; } } @@ -285,6 +282,15 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary()); } + + for (int doc = 1, hit = 0; hit < lastErroneousResponse.getHits().getHits().length; hit++, doc++) { + SearchHit searchHit = lastErroneousResponse.getHits().getAt(hit); + while (doc < Integer.parseInt(searchHit.id())) { + logger.info("missing doc [{}], indexed to shard [{}]", doc, MathUtils.mod(Murmur3HashFunction.hash(Integer.toString(doc)), numberOfShards)); + doc++; + } + } + //if there was an error we try to wait and see if at some point it'll get fixed logger.info("--> trying to wait"); assertTrue(awaitBusy(() -> { diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 5637b2d8127..57b5e888ea9 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.recovery; import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.hppc.procedures.IntProcedure; + import org.apache.lucene.index.IndexFileNames; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -39,10 +40,10 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest; import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.plugins.Plugin; @@ -51,10 +52,15 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.MockIndexEventListener; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.*; -import org.junit.Test; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -71,10 +77,11 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; /** */ @@ -83,12 +90,13 @@ import static org.hamcrest.Matchers.*; public class RelocationIT extends ESIntegTestCase { private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES); + + @Override protected Collection> nodePlugins() { - return pluginList(MockTransportService.TestPlugin.class); + return pluginList(MockTransportService.TestPlugin.class, MockIndexEventListener.TestPlugin.class); } - @Test public void testSimpleRelocationNoIndexing() { logger.info("--> starting [node1] ..."); final String node_1 = internalCluster().startNode(); @@ -114,7 +122,7 @@ public class RelocationIT extends ESIntegTestCase { logger.info("--> verifying count"); client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().prepareCount("test").execute().actionGet().getCount(), equalTo(20l)); + assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().totalHits(), equalTo(20l)); logger.info("--> start another node"); final String node_2 = internalCluster().startNode(); @@ -133,10 +141,9 @@ public class RelocationIT extends ESIntegTestCase { logger.info("--> verifying count again..."); client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().prepareCount("test").execute().actionGet().getCount(), equalTo(20l)); + assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().totalHits(), equalTo(20l)); } - @Test public void testRelocationWhileIndexingRandom() throws Exception { int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4); int numberOfReplicas = randomBoolean() ? 0 : 1; @@ -245,7 +252,6 @@ public class RelocationIT extends ESIntegTestCase { } } - @Test public void testRelocationWhileRefreshing() throws Exception { int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4); int numberOfReplicas = randomBoolean() ? 0 : 1; @@ -279,16 +285,16 @@ public class RelocationIT extends ESIntegTestCase { } final Semaphore postRecoveryShards = new Semaphore(0); - - for (IndicesLifecycle indicesLifecycle : internalCluster().getInstances(IndicesLifecycle.class)) { - indicesLifecycle.addListener(new IndicesLifecycle.Listener() { - @Override - public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) { - if (currentState == IndexShardState.POST_RECOVERY) { - postRecoveryShards.release(); - } + final IndexEventListener listener = new IndexEventListener() { + @Override + public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) { + if (currentState == IndexShardState.POST_RECOVERY) { + postRecoveryShards.release(); } - }); + } + }; + for (MockIndexEventListener.TestEventListener eventListener : internalCluster().getInstances(MockIndexEventListener.TestEventListener.class)) { + eventListener.setNewDelegate(listener); } @@ -345,7 +351,6 @@ public class RelocationIT extends ESIntegTestCase { } } - @Test public void testCancellationCleansTempFiles() throws Exception { final String indexName = "test"; @@ -372,7 +377,7 @@ public class RelocationIT extends ESIntegTestCase { MockTransportService mockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, p_node); for (DiscoveryNode node : clusterService.state().nodes()) { if (!node.equals(clusterService.localNode())) { - mockTransportService.addDelegate(node, new RecoveryCorruption(mockTransportService.original(), corruptionCount)); + mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, node.getName()), new RecoveryCorruption(mockTransportService.original(), corruptionCount)); } } diff --git a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java index d19117e1d23..26291b7ac9c 100644 --- a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java @@ -24,15 +24,16 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import static org.elasticsearch.client.Requests.*; +import static org.elasticsearch.client.Requests.flushRequest; +import static org.elasticsearch.client.Requests.getRequest; +import static org.elasticsearch.client.Requests.indexRequest; +import static org.elasticsearch.client.Requests.refreshRequest; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; public class SimpleRecoveryIT extends ESIntegTestCase { - @Override public Settings indexSettings() { return settingsBuilder().put(super.indexSettings()).put(recoverySettings()).build(); @@ -47,7 +48,6 @@ public class SimpleRecoveryIT extends ESIntegTestCase { return 1; } - @Test public void testSimpleRecovery() throws Exception { assertAcked(prepareCreate("test", 1).execute().actionGet()); diff --git a/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java index 25347fa1fab..60a14abac7c 100644 --- a/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java @@ -29,16 +29,18 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.discovery.Discovery; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.indices.recovery.IndexRecoveryIT; import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.*; -import org.junit.Test; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.ArrayList; @@ -57,14 +59,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; @SuppressCodecs("*") // test relies on exact file extensions public class TruncatedRecoveryIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings.Builder builder = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(RecoverySettings.INDICES_RECOVERY_FILE_CHUNK_SIZE, new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES)); - return builder.build(); - } - @Override protected Collection> nodePlugins() { return pluginList(MockTransportService.TestPlugin.class); @@ -76,8 +70,11 @@ public class TruncatedRecoveryIT extends ESIntegTestCase { * we just throw an exception to make sure the recovery fails and we leave some half baked files on the target. * Later we allow full recovery to ensure we can still recover and don't run into corruptions. */ - @Test public void testCancelRecoveryAndResume() throws Exception { + for(RecoverySettings settings : internalCluster().getInstances(RecoverySettings.class)) { + IndexRecoveryIT.setChunkSize(settings, new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES)); + } + NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get(); List dataNodeStats = new ArrayList<>(); for (NodeStats stat : nodeStats.getNodes()) { @@ -86,7 +83,7 @@ public class TruncatedRecoveryIT extends ESIntegTestCase { } } assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2)); - Collections.shuffle(dataNodeStats, getRandom()); + Collections.shuffle(dataNodeStats, random()); // we use 2 nodes a lucky and unlucky one // the lucky one holds the primary // the unlucky one gets the replica and the truncated leftovers @@ -115,13 +112,13 @@ public class TruncatedRecoveryIT extends ESIntegTestCase { ensureGreen(); // ensure we have flushed segments and make them a big one via optimize client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).get(); - client().admin().indices().prepareOptimize().setMaxNumSegments(1).setFlush(true).get(); + client().admin().indices().prepareForceMerge().setMaxNumSegments(1).setFlush(true).get(); final CountDownLatch latch = new CountDownLatch(1); final AtomicBoolean truncate = new AtomicBoolean(true); for (NodeStats dataNode : dataNodeStats) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().name())); - mockTransportService.addDelegate(internalCluster().getInstance(Discovery.class, unluckyNode.getNode().name()).localNode(), new MockTransportService.DelegateTransport(mockTransportService.original()) { + mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().name()), new MockTransportService.DelegateTransport(mockTransportService.original()) { @Override public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { @@ -155,4 +152,4 @@ public class TruncatedRecoveryIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setQuery(QueryBuilders.termQuery("the_id", id)).get(), 1); } } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index 76d08eadc8d..0a40da3403d 100644 --- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -28,19 +28,20 @@ import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.transport.RemoteTransportException; -import org.junit.Test; import java.io.FileNotFoundException; import java.io.IOException; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; /** * */ public class BytesRestResponseTests extends ESTestCase { - @Test public void testWithHeaders() throws Exception { RestRequest request = new FakeRestRequest(); RestChannel channel = randomBoolean() ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request); @@ -52,7 +53,6 @@ public class BytesRestResponseTests extends ESTestCase { assertThat(response.getHeaders().get("n2"), contains("v21", "v22")); } - @Test public void testSimpleExceptionMessage() throws Exception { RestRequest request = new FakeRestRequest(); RestChannel channel = new SimpleExceptionRestChannel(request); @@ -66,7 +66,6 @@ public class BytesRestResponseTests extends ESTestCase { assertThat(text, not(containsString("error_trace"))); } - @Test public void testDetailedExceptionMessage() throws Exception { RestRequest request = new FakeRestRequest(); RestChannel channel = new DetailedExceptionRestChannel(request); @@ -78,7 +77,6 @@ public class BytesRestResponseTests extends ESTestCase { assertThat(text, containsString("{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}")); } - @Test public void testNonElasticsearchExceptionIsNotShownAsSimpleMessage() throws Exception { RestRequest request = new FakeRestRequest(); RestChannel channel = new SimpleExceptionRestChannel(request); @@ -92,7 +90,6 @@ public class BytesRestResponseTests extends ESTestCase { assertThat(text, containsString("\"error\":\"No ElasticsearchException found\"")); } - @Test public void testErrorTrace() throws Exception { RestRequest request = new FakeRestRequest(); request.params().put("error_trace", "true"); @@ -123,7 +120,6 @@ public class BytesRestResponseTests extends ESTestCase { } } - @Test public void testNullThrowable() throws Exception { RestRequest request = new FakeRestRequest(); RestChannel channel = new SimpleExceptionRestChannel(request); @@ -134,7 +130,6 @@ public class BytesRestResponseTests extends ESTestCase { assertThat(text, not(containsString("error_trace"))); } - @Test public void testConvert() throws IOException { RestRequest request = new FakeRestRequest(); RestChannel channel = new DetailedExceptionRestChannel(request); diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexDefaultIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexDefaultIT.java index fa7eccd6658..2b7533cae19 100644 --- a/core/src/test/java/org/elasticsearch/rest/CorsRegexDefaultIT.java +++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexDefaultIT.java @@ -22,9 +22,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; /** * @@ -38,7 +39,6 @@ public class CorsRegexDefaultIT extends ESIntegTestCase { .put(super.nodeSettings(nodeOrdinal)).build(); } - @Test public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception { String corsValue = "http://localhost:9200"; HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", corsValue).execute(); @@ -48,7 +48,6 @@ public class CorsRegexDefaultIT extends ESIntegTestCase { assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Credentials"))); } - @Test public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception { HttpResponse response = httpClient().method("GET").path("/").execute(); diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java index d0e0282f2ca..3828ae0ad74 100644 --- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java +++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java @@ -23,16 +23,16 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.Test; -import static org.elasticsearch.http.netty.NettyHttpServerTransport.SETTING_CORS_ALLOW_ORIGIN; import static org.elasticsearch.http.netty.NettyHttpServerTransport.SETTING_CORS_ALLOW_CREDENTIALS; +import static org.elasticsearch.http.netty.NettyHttpServerTransport.SETTING_CORS_ALLOW_ORIGIN; import static org.elasticsearch.http.netty.NettyHttpServerTransport.SETTING_CORS_ENABLED; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; /** * @@ -53,7 +53,6 @@ public class CorsRegexIT extends ESIntegTestCase { .build(); } - @Test public void testThatRegularExpressionWorksOnMatch() throws Exception { String corsValue = "http://localhost:9200"; HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", corsValue).execute(); @@ -66,34 +65,29 @@ public class CorsRegexIT extends ESIntegTestCase { assertThat(response.getHeaders().get("Access-Control-Allow-Credentials"), is("true")); } - @Test public void testThatRegularExpressionReturnsNullOnNonMatch() throws Exception { HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", "http://evil-host:9200").execute(); assertResponseWithOriginheader(response, "null"); } - @Test public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws Exception { HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").execute(); assertThat(response.getStatusCode(), is(200)); assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin"))); } - @Test public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception { HttpResponse response = httpClient().method("GET").path("/").execute(); assertThat(response.getStatusCode(), is(200)); assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin"))); } - @Test public void testThatPreFlightRequestWorksOnMatch() throws Exception { String corsValue = "http://localhost:9200"; HttpResponse response = httpClient().method("OPTIONS").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", corsValue).execute(); assertResponseWithOriginheader(response, corsValue); } - @Test public void testThatPreFlightRequestReturnsNullOnNonMatch() throws Exception { HttpResponse response = httpClient().method("OPTIONS").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", "http://evil-host:9200").execute(); assertResponseWithOriginheader(response, "null"); diff --git a/core/src/test/java/org/elasticsearch/rest/HeadersAndContextCopyClientTests.java b/core/src/test/java/org/elasticsearch/rest/HeadersAndContextCopyClientTests.java index b099f97ddff..2a8299226c0 100644 --- a/core/src/test/java/org/elasticsearch/rest/HeadersAndContextCopyClientTests.java +++ b/core/src/test/java/org/elasticsearch/rest/HeadersAndContextCopyClientTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; -import org.junit.Test; import java.util.Arrays; import java.util.HashMap; @@ -53,7 +52,6 @@ import static org.hamcrest.Matchers.is; public class HeadersAndContextCopyClientTests extends ESTestCase { - @Test public void testRegisterRelevantHeaders() throws InterruptedException { final RestController restController = new RestController(Settings.EMPTY); @@ -91,7 +89,6 @@ public class HeadersAndContextCopyClientTests extends ESTestCase { assertThat(relevantHeaders, equalTo(headersArray)); } - @Test public void testCopyHeadersRequest() { Map transportHeaders = randomHeaders(randomIntBetween(0, 10)); Map restHeaders = randomHeaders(randomIntBetween(0, 10)); @@ -137,7 +134,6 @@ public class HeadersAndContextCopyClientTests extends ESTestCase { } } - @Test public void testCopyHeadersClusterAdminRequest() { Map transportHeaders = randomHeaders(randomIntBetween(0, 10)); Map restHeaders = randomHeaders(randomIntBetween(0, 10)); @@ -183,7 +179,6 @@ public class HeadersAndContextCopyClientTests extends ESTestCase { } } - @Test public void testCopyHeadersIndicesAdminRequest() { Map transportHeaders = randomHeaders(randomIntBetween(0, 10)); Map restHeaders = randomHeaders(randomIntBetween(0, 10)); @@ -229,7 +224,6 @@ public class HeadersAndContextCopyClientTests extends ESTestCase { } } - @Test public void testCopyHeadersRequestBuilder() { Map transportHeaders = randomHeaders(randomIntBetween(0, 10)); Map restHeaders = randomHeaders(randomIntBetween(0, 10)); @@ -270,7 +264,6 @@ public class HeadersAndContextCopyClientTests extends ESTestCase { } } - @Test public void testCopyHeadersClusterAdminRequestBuilder() { Map transportHeaders = randomHeaders(randomIntBetween(0, 10)); Map restHeaders = randomHeaders(randomIntBetween(0, 10)); @@ -310,7 +303,6 @@ public class HeadersAndContextCopyClientTests extends ESTestCase { } } - @Test public void testCopyHeadersIndicesAdminRequestBuilder() { Map transportHeaders = randomHeaders(randomIntBetween(0, 10)); Map restHeaders = randomHeaders(randomIntBetween(0, 10)); diff --git a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java index 5760c284161..b66d00cd6a8 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -42,8 +41,6 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.CoreMatchers.equalTo; public class RestFilterChainTests extends ESTestCase { - - @Test public void testRestFilters() throws InterruptedException { RestController restController = new RestController(Settings.EMPTY); @@ -120,7 +117,6 @@ public class RestFilterChainTests extends ESTestCase { } } - @Test public void testTooManyContinueProcessing() throws InterruptedException { final int additionalContinueCount = randomInt(10); diff --git a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java index 12bbef4735b..8e60b28f376 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java @@ -22,18 +22,16 @@ package org.elasticsearch.rest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; /** * */ public class RestRequestTests extends ESTestCase { - - @Test public void testContext() throws Exception { int count = randomInt(10); Request request = new Request(); diff --git a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java index 237c62d93c7..161668dba04 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java @@ -23,13 +23,14 @@ import org.elasticsearch.common.Table; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.junit.Before; -import org.junit.Test; import java.util.ArrayList; import java.util.List; import static org.elasticsearch.rest.action.support.RestTable.buildDisplayHeaders; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; public class RestTableTests extends ESTestCase { @@ -51,7 +52,6 @@ public class RestTableTests extends ESTestCase { table.endHeaders(); } - @Test public void testThatDisplayHeadersSupportWildcards() throws Exception { restRequest.params().put("h", "bulk*"); List headers = buildDisplayHeaders(table, restRequest); @@ -61,7 +61,6 @@ public class RestTableTests extends ESTestCase { assertThat(headerNames, not(hasItem("unmatched"))); } - @Test public void testThatDisplayHeadersAreNotAddedTwice() throws Exception { restRequest.params().put("h", "nonexistent,bulk*,bul*"); List headers = buildDisplayHeaders(table, restRequest); diff --git a/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java b/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java index f2022511324..e60a120ff18 100644 --- a/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java @@ -19,27 +19,21 @@ package org.elasticsearch.rest.util; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.support.RestUtils; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.regex.Pattern; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; /** * */ public class RestUtilsTests extends ESTestCase { - @Test public void testDecodeQueryString() { Map params = new HashMap<>(); @@ -66,7 +60,6 @@ public class RestUtilsTests extends ESTestCase { assertThat(params.size(), equalTo(0)); } - @Test public void testDecodeQueryStringEdgeCases() { Map params = new HashMap<>(); @@ -127,7 +120,6 @@ public class RestUtilsTests extends ESTestCase { assertThat(params.get("p1"), equalTo("v1")); } - @Test public void testCorsSettingIsARegex() { assertCorsSettingRegex("/foo/", Pattern.compile("foo")); assertCorsSettingRegex("/.*/", Pattern.compile(".*")); @@ -139,7 +131,6 @@ public class RestUtilsTests extends ESTestCase { assertCorsSettingRegexIsNull("/foo"); assertCorsSettingRegexIsNull("foo"); assertCorsSettingRegexIsNull(""); - assertThat(RestUtils.getCorsSettingRegex(Settings.EMPTY), is(nullValue())); } public void testCrazyURL() { @@ -153,15 +144,15 @@ public class RestUtilsTests extends ESTestCase { } private void assertCorsSettingRegexIsNull(String settingsValue) { - assertThat(RestUtils.getCorsSettingRegex(settingsBuilder().put("http.cors.allow-origin", settingsValue).build()), is(nullValue())); + assertThat(RestUtils.checkCorsSettingForRegex(settingsValue), is(nullValue())); } private void assertCorsSettingRegex(String settingsValue, Pattern pattern) { - assertThat(RestUtils.getCorsSettingRegex(settingsBuilder().put("http.cors.allow-origin", settingsValue).build()).toString(), is(pattern.toString())); + assertThat(RestUtils.checkCorsSettingForRegex(settingsValue).toString(), is(pattern.toString())); } private void assertCorsSettingRegexMatches(String settingsValue, boolean expectMatch, String ... candidates) { - Pattern pattern = RestUtils.getCorsSettingRegex(settingsBuilder().put("http.cors.allow-origin", settingsValue).build()); + Pattern pattern = RestUtils.checkCorsSettingForRegex(settingsValue); for (String candidate : candidates) { assertThat(String.format(Locale.ROOT, "Expected pattern %s to match against %s: %s", settingsValue, candidate, expectMatch), pattern.matcher(candidate).matches(), is(expectMatch)); diff --git a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java index 2d823292e55..2740dd73246 100644 --- a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.Priority; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -38,8 +37,6 @@ import static org.hamcrest.Matchers.nullValue; * */ public class AliasResolveRoutingIT extends ESIntegTestCase { - - @Test public void testResolveIndexRouting() throws Exception { createIndex("test1"); createIndex("test2"); @@ -78,8 +75,6 @@ public class AliasResolveRoutingIT extends ESIntegTestCase { } } - - @Test public void testResolveSearchRouting() throws Exception { createIndex("test1"); createIndex("test2"); diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java index c418f68bd26..9fc6bcfb739 100644 --- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.cluster.metadata.AliasAction.newAddAliasAction; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -40,7 +39,6 @@ public class AliasRoutingIT extends ESIntegTestCase { return 2; } - @Test public void testAliasCrudRouting() throws Exception { createIndex("test"); ensureGreen(); @@ -102,7 +100,6 @@ public class AliasRoutingIT extends ESIntegTestCase { } } - @Test public void testAliasSearchRouting() throws Exception { createIndex("test"); ensureGreen(); @@ -131,17 +128,17 @@ public class AliasRoutingIT extends ESIntegTestCase { logger.info("--> search with wrong routing, should not find"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); - assertThat(client().prepareCount().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(0l)); + assertThat(client().prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); assertThat(client().prepareSearch("alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); - assertThat(client().prepareCount("alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(0l)); + assertThat(client().prepareSearch("alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); } logger.info("--> search with correct routing, should find"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); assertThat(client().prepareSearch("alias0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount("alias0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch("alias0").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); } logger.info("--> indexing with id [2], and routing [1] using alias"); @@ -150,54 +147,53 @@ public class AliasRoutingIT extends ESIntegTestCase { logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } logger.info("--> search with 0 routing, should find one"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); assertThat(client().prepareSearch("alias0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount("alias0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch("alias0").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); } logger.info("--> search with 1 routing, should find one"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); assertThat(client().prepareSearch("alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount("alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch("alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); } logger.info("--> search with 0,1 routings , should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch().setSize(0).setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); assertThat(client().prepareSearch("alias01").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount("alias01").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch("alias01").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } logger.info("--> search with two routing aliases , should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch("alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount("alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch("alias0", "alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } logger.info("--> search with alias0, alias1 and alias01, should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch("alias0", "alias1", "alias01").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount("alias0", "alias1", "alias01").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch("alias0", "alias1", "alias01").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } logger.info("--> search with test, alias0 and alias1, should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch("test", "alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount("test", "alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch("test", "alias0", "alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } } - @Test public void testAliasSearchRoutingWithTwoIndices() throws Exception { createIndex("test-a"); createIndex("test-b"); @@ -236,19 +232,19 @@ public class AliasRoutingIT extends ESIntegTestCase { logger.info("--> search with alias-a1,alias-b0, should not find"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch("alias-a1", "alias-b0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); - assertThat(client().prepareCount("alias-a1", "alias-b0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(0l)); + assertThat(client().prepareSearch("alias-a1", "alias-b0").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); } logger.info("--> search with alias-ab, should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch("alias-ab").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount("alias-ab").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch("alias-ab").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } logger.info("--> search with alias-a0,alias-b1 should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch("alias-a0", "alias-b1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount("alias-a0", "alias-b1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch("alias-a0", "alias-b1").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } } @@ -258,7 +254,6 @@ public class AliasRoutingIT extends ESIntegTestCase { to the other indices (without routing) were not taken into account in PlainOperationRouting#searchShards. That affected the number of shards that we executed the search on, thus some documents were missing in the search results. */ - @Test public void testAliasSearchRoutingWithConcreteAndAliasedIndices_issue2682() throws Exception { createIndex("index", "index_2"); ensureGreen(); @@ -284,7 +279,6 @@ public class AliasRoutingIT extends ESIntegTestCase { That could cause returning 1, which led to forcing the QUERY_AND_FETCH mode. As a result, (size * number of hit shards) results were returned and no reduce phase was taking place. */ - @Test public void testAliasSearchRoutingWithConcreteAndAliasedIndices_issue3268() throws Exception { createIndex("index", "index_2"); ensureGreen(); @@ -305,7 +299,6 @@ public class AliasRoutingIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getHits().length, equalTo(1)); } - @Test public void testIndexingAliasesOverTime() throws Exception { createIndex("test"); ensureGreen(); @@ -321,7 +314,7 @@ public class AliasRoutingIT extends ESIntegTestCase { for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "0").setRouting("3").execute().actionGet().isExists(), equalTo(true)); assertThat(client().prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount("alias").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); } logger.info("--> creating alias with routing [4]"); @@ -331,7 +324,7 @@ public class AliasRoutingIT extends ESIntegTestCase { logger.info("--> verifying search with wrong routing should not find"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); - assertThat(client().prepareCount("alias").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(0l)); + assertThat(client().prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); } logger.info("--> creating alias with search routing [3,4] and index routing 4"); @@ -347,7 +340,7 @@ public class AliasRoutingIT extends ESIntegTestCase { assertThat(client().prepareGet("test", "type1", "0").setRouting("3").execute().actionGet().isExists(), equalTo(true)); assertThat(client().prepareGet("test", "type1", "1").setRouting("4").execute().actionGet().isExists(), equalTo(true)); assertThat(client().prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount("alias").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } } diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java index 1a45d0fd151..a5b7da7796f 100644 --- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -111,13 +111,13 @@ public class SimpleRoutingIT extends ESIntegTestCase { logger.info("--> search with wrong routing, should not find"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); - assertThat(client().prepareCount().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(0l)); + assertThat(client().prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(0l)); } logger.info("--> search with correct routing, should find"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); } logger.info("--> indexing with id [2], and routing [1]"); @@ -126,31 +126,31 @@ public class SimpleRoutingIT extends ESIntegTestCase { logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } logger.info("--> search with 0 routing, should find one"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); } logger.info("--> search with 1 routing, should find one"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); - assertThat(client().prepareCount().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(1l)); + assertThat(client().prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(1l)); } logger.info("--> search with 0,1 routings , should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch().setSize(0).setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } logger.info("--> search with 0,1,0 routings , should find two"); for (int i = 0; i < 5; i++) { assertThat(client().prepareSearch().setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); - assertThat(client().prepareCount().setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getCount(), equalTo(2l)); + assertThat(client().prepareSearch().setSize(0).setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(2l)); } } diff --git a/core/src/test/java/org/elasticsearch/script/ClassPermissionTests.java b/core/src/test/java/org/elasticsearch/script/ClassPermissionTests.java new file mode 100644 index 00000000000..05a65363ff5 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/script/ClassPermissionTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.test.ESTestCase; + +import java.security.AllPermission; +import java.security.PermissionCollection; + +/** Very simple sanity checks for {@link ClassPermission} */ +public class ClassPermissionTests extends ESTestCase { + + public void testEquals() { + assertEquals(new ClassPermission("pkg.MyClass"), new ClassPermission("pkg.MyClass")); + assertFalse(new ClassPermission("pkg.MyClass").equals(new AllPermission())); + } + + public void testImplies() { + assertTrue(new ClassPermission("pkg.MyClass").implies(new ClassPermission("pkg.MyClass"))); + assertFalse(new ClassPermission("pkg.MyClass").implies(new ClassPermission("pkg.MyOtherClass"))); + assertFalse(new ClassPermission("pkg.MyClass").implies(null)); + assertFalse(new ClassPermission("pkg.MyClass").implies(new AllPermission())); + } + + public void testStandard() { + assertTrue(new ClassPermission("<>").implies(new ClassPermission("java.lang.Math"))); + assertFalse(new ClassPermission("<>").implies(new ClassPermission("pkg.MyClass"))); + } + + public void testPermissionCollection() { + ClassPermission math = new ClassPermission("java.lang.Math"); + PermissionCollection collection = math.newPermissionCollection(); + collection.add(math); + assertTrue(collection.implies(new ClassPermission("java.lang.Math"))); + assertFalse(collection.implies(new ClassPermission("pkg.MyClass"))); + } + + public void testPermissionCollectionStandard() { + ClassPermission standard = new ClassPermission("<>"); + PermissionCollection collection = standard.newPermissionCollection(); + collection.add(standard); + assertTrue(collection.implies(new ClassPermission("java.lang.Math"))); + assertFalse(collection.implies(new ClassPermission("pkg.MyClass"))); + } + + /** not recommended but we test anyway */ + public void testWildcards() { + assertTrue(new ClassPermission("*").implies(new ClassPermission("pkg.MyClass"))); + assertTrue(new ClassPermission("pkg.*").implies(new ClassPermission("pkg.MyClass"))); + assertTrue(new ClassPermission("pkg.*").implies(new ClassPermission("pkg.sub.MyClass"))); + assertFalse(new ClassPermission("pkg.My*").implies(new ClassPermission("pkg.MyClass"))); + assertFalse(new ClassPermission("pkg*").implies(new ClassPermission("pkg.MyClass"))); + } + + public void testPermissionCollectionWildcards() { + ClassPermission lang = new ClassPermission("java.lang.*"); + PermissionCollection collection = lang.newPermissionCollection(); + collection.add(lang); + assertTrue(collection.implies(new ClassPermission("java.lang.Math"))); + assertFalse(collection.implies(new ClassPermission("pkg.MyClass"))); + } +} diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java index daefc205933..fc888c79a8c 100644 --- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java @@ -19,12 +19,9 @@ package org.elasticsearch.script; import org.elasticsearch.common.ContextAndHeaderHolder; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.nio.file.Files; import java.nio.file.Path; @@ -32,9 +29,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.containsString; - // TODO: these really should just be part of ScriptService tests, there is nothing special about them public class FileScriptTests extends ESTestCase { diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java index 99982347700..02fad319846 100644 --- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; @@ -32,7 +33,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPoolModule; import org.elasticsearch.watcher.ResourceWatcherService; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -45,8 +45,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; public class NativeScriptTests extends ESTestCase { - - @Test public void testNativeScript() throws InterruptedException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); Settings settings = Settings.settingsBuilder() @@ -58,7 +56,7 @@ public class NativeScriptTests extends ESTestCase { Injector injector = new ModulesBuilder().add( new EnvironmentModule(new Environment(settings)), new ThreadPoolModule(new ThreadPool(settings)), - new SettingsModule(settings), + new SettingsModule(settings, new SettingsFilter(settings)), scriptModule).createInjector(); ScriptService scriptService = injector.getInstance(ScriptService.class); @@ -69,7 +67,6 @@ public class NativeScriptTests extends ESTestCase { terminate(injector.getInstance(ThreadPool.class)); } - @Test public void testFineGrainedSettingsDontAffectNativeScripts() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); Settings.Builder builder = Settings.settingsBuilder(); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextRegistryTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextRegistryTests.java index c7d3a52bf7e..a43589fe211 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptContextRegistryTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptContextRegistryTests.java @@ -21,15 +21,14 @@ package org.elasticsearch.script; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.Arrays; import java.util.Collections; -public class ScriptContextRegistryTests extends ESTestCase { +import static org.hamcrest.Matchers.containsString; - @Test +public class ScriptContextRegistryTests extends ESTestCase { public void testValidateCustomScriptContextsOperation() throws IOException { for (final String rejectedContext : ScriptContextRegistry.RESERVED_SCRIPT_CONTEXTS) { try { @@ -42,7 +41,6 @@ public class ScriptContextRegistryTests extends ESTestCase { } } - @Test public void testValidateCustomScriptContextsPluginName() throws IOException { for (final String rejectedContext : ScriptContextRegistry.RESERVED_SCRIPT_CONTEXTS) { try { @@ -55,28 +53,34 @@ public class ScriptContextRegistryTests extends ESTestCase { } } - @Test(expected = IllegalArgumentException.class) public void testValidateCustomScriptContextsEmptyPluginName() throws IOException { - new ScriptContext.Plugin(randomBoolean() ? null : "", "test"); + try { + new ScriptContext.Plugin(randomBoolean() ? null : "", "test"); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("plugin name cannot be empty")); + } } - @Test(expected = IllegalArgumentException.class) public void testValidateCustomScriptContextsEmptyOperation() throws IOException { - new ScriptContext.Plugin("test", randomBoolean() ? null : ""); + try { + new ScriptContext.Plugin("test", randomBoolean() ? null : ""); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("operation name cannot be empty")); + } } - @Test public void testDuplicatedPluginScriptContexts() throws IOException { try { //try to register a prohibited script context new ScriptContextRegistry(Arrays.asList(new ScriptContext.Plugin("testplugin", "test"), new ScriptContext.Plugin("testplugin", "test"))); fail("ScriptContextRegistry initialization should have failed"); } catch(IllegalArgumentException e) { - assertThat(e.getMessage(), Matchers.containsString("script context [testplugin_test] cannot be registered twice")); + assertThat(e.getMessage(), containsString("script context [testplugin_test] cannot be registered twice")); } } - @Test public void testNonDuplicatedPluginScriptContexts() throws IOException { new ScriptContextRegistry(Arrays.asList(new ScriptContext.Plugin("testplugin1", "test"), new ScriptContext.Plugin("testplugin2", "test"))); } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java index e3fba013fc8..0edaedbb28e 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java @@ -30,6 +30,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; + public class ScriptContextTests extends ESTestCase { private static final String PLUGIN_NAME = "testplugin"; @@ -59,7 +61,7 @@ public class ScriptContextTests extends ESTestCase { scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"), contextAndHeaders); fail("script compilation should have been rejected"); } catch (ScriptException e) { - assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled")); + assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled")); } } } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java index 4fdfbb09e5e..71a41750c9c 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java @@ -21,7 +21,6 @@ package org.elasticsearch.script; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService.ScriptType; @@ -34,7 +33,6 @@ import java.util.Collection; import java.util.Map; import java.util.concurrent.ExecutionException; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = Scope.SUITE, numDataNodes = 3) diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java index e38c9930d5c..3e476d2bebb 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java @@ -22,12 +22,10 @@ package org.elasticsearch.script; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.Collections; import java.util.HashMap; @@ -41,11 +39,12 @@ import static java.util.Collections.unmodifiableSet; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.containsString; // TODO: this needs to be a base test class, and all scripting engines extend it public class ScriptModesTests extends ESTestCase { private static final Set ALL_LANGS = unmodifiableSet( - newHashSet(MustacheScriptEngineService.NAME, "custom", "test")); + newHashSet("custom", "test")); static final String[] ENABLE_VALUES = new String[]{"on", "true", "yes", "1"}; static final String[] DISABLE_VALUES = new String[]{"off", "false", "no", "0"}; @@ -73,7 +72,6 @@ public class ScriptModesTests extends ESTestCase { scriptContextRegistry = new ScriptContextRegistry(contexts.values()); scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]); scriptEngines = buildScriptEnginesByLangMap(newHashSet( - new MustacheScriptEngineService(Settings.EMPTY), //add the native engine just to make sure it gets filtered out new NativeScriptEngineService(Settings.EMPTY, Collections.emptyMap()), new CustomScriptEngineService())); @@ -93,8 +91,8 @@ public class ScriptModesTests extends ESTestCase { public void assertAllSettingsWereChecked() { if (assertScriptModesNonNull) { assertThat(scriptModes, notNullValue()); - //3 is the number of engines (native excluded), custom is counted twice though as it's associated with two different names - int numberOfSettings = 3 * ScriptType.values().length * scriptContextRegistry.scriptContexts().size(); + //2 is the number of engines (native excluded), custom is counted twice though as it's associated with two different names + int numberOfSettings = 2 * ScriptType.values().length * scriptContextRegistry.scriptContexts().size(); assertThat(scriptModes.scriptModes.size(), equalTo(numberOfSettings)); if (assertAllSettingsWereChecked) { assertThat(checkedSettings.size(), equalTo(numberOfSettings)); @@ -102,21 +100,23 @@ public class ScriptModesTests extends ESTestCase { } } - @Test public void testDefaultSettings() { this.scriptModes = new ScriptModes(scriptEngines, scriptContextRegistry, Settings.EMPTY); assertScriptModesAllOps(ScriptMode.ON, ALL_LANGS, ScriptType.FILE); assertScriptModesAllOps(ScriptMode.SANDBOX, ALL_LANGS, ScriptType.INDEXED, ScriptType.INLINE); } - @Test(expected = IllegalArgumentException.class) public void testMissingSetting() { assertAllSettingsWereChecked = false; this.scriptModes = new ScriptModes(scriptEngines, scriptContextRegistry, Settings.EMPTY); - scriptModes.getScriptMode("non_existing", randomFrom(ScriptType.values()), randomFrom(scriptContexts)); + try { + scriptModes.getScriptMode("non_existing", randomFrom(ScriptType.values()), randomFrom(scriptContexts)); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("not found for lang [non_existing]")); + } } - @Test public void testScriptTypeGenericSettings() { int randomInt = randomIntBetween(1, ScriptType.values().length - 1); Set randomScriptTypesSet = new HashSet<>(); @@ -149,7 +149,6 @@ public class ScriptModesTests extends ESTestCase { } } - @Test public void testScriptContextGenericSettings() { int randomInt = randomIntBetween(1, scriptContexts.length - 1); Set randomScriptContextsSet = new HashSet<>(); @@ -177,7 +176,6 @@ public class ScriptModesTests extends ESTestCase { assertScriptModes(ScriptMode.SANDBOX, ALL_LANGS, new ScriptType[]{ScriptType.INDEXED, ScriptType.INLINE}, complementOf); } - @Test public void testConflictingScriptTypeAndOpGenericSettings() { ScriptContext scriptContext = randomFrom(scriptContexts); Settings.Builder builder = Settings.builder().put(ScriptModes.SCRIPT_SETTINGS_PREFIX + scriptContext.getKey(), randomFrom(DISABLE_VALUES)) @@ -190,22 +188,6 @@ public class ScriptModesTests extends ESTestCase { assertScriptModes(ScriptMode.SANDBOX, ALL_LANGS, new ScriptType[]{ScriptType.INLINE}, complementOf); } - @Test - public void testInteractionBetweenGenericAndEngineSpecificSettings() { - Settings.Builder builder = Settings.builder().put("script.inline", randomFrom(DISABLE_VALUES)) - .put(specificEngineOpSettings(MustacheScriptEngineService.NAME, ScriptType.INLINE, ScriptContext.Standard.AGGS), randomFrom(ENABLE_VALUES)) - .put(specificEngineOpSettings(MustacheScriptEngineService.NAME, ScriptType.INLINE, ScriptContext.Standard.SEARCH), randomFrom(ENABLE_VALUES)); - Set mustacheLangSet = singleton(MustacheScriptEngineService.NAME); - Set allButMustacheLangSet = new HashSet<>(ALL_LANGS); - allButMustacheLangSet.remove(MustacheScriptEngineService.NAME); - this.scriptModes = new ScriptModes(scriptEngines, scriptContextRegistry, builder.build()); - assertScriptModes(ScriptMode.ON, mustacheLangSet, new ScriptType[]{ScriptType.INLINE}, ScriptContext.Standard.AGGS, ScriptContext.Standard.SEARCH); - assertScriptModes(ScriptMode.OFF, mustacheLangSet, new ScriptType[]{ScriptType.INLINE}, complementOf(ScriptContext.Standard.AGGS, ScriptContext.Standard.SEARCH)); - assertScriptModesAllOps(ScriptMode.OFF, allButMustacheLangSet, ScriptType.INLINE); - assertScriptModesAllOps(ScriptMode.SANDBOX, ALL_LANGS, ScriptType.INDEXED); - assertScriptModesAllOps(ScriptMode.ON, ALL_LANGS, ScriptType.FILE); - } - private void assertScriptModesAllOps(ScriptMode expectedScriptMode, Set langs, ScriptType... scriptTypes) { assertScriptModes(expectedScriptMode, langs, scriptTypes, scriptContexts); } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptParameterParserTests.java b/core/src/test/java/org/elasticsearch/script/ScriptParameterParserTests.java index 85dc650c6a3..c3c80c50850 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptParameterParserTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptParameterParserTests.java @@ -30,17 +30,22 @@ import org.elasticsearch.script.Script.ScriptParseException; import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import static java.util.Collections.singleton; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; public class ScriptParameterParserTests extends ESTestCase { - - @Test public void testTokenDefaultInline() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"script\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -61,7 +66,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenDefaultFile() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"script_file\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -84,7 +88,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenDefaultIndexed() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"script_id\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -107,7 +110,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenDefaultNotFound() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"bar\" }")); Token token = parser.nextToken(); @@ -121,7 +123,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenSingleParameter() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -136,7 +137,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenSingleParameterFile() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo_file\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -151,7 +151,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenSingleParameterIndexed() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo_id\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -166,7 +165,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test(expected = ScriptParseException.class) public void testTokenSingleParameterDelcaredTwiceInlineFile() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"scriptValue\", \"foo_file\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -183,10 +181,14 @@ public class ScriptParameterParserTests extends ESTestCase { while (token != Token.VALUE_STRING) { token = parser.nextToken(); } - paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + try { + paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testTokenSingleParameterDelcaredTwiceInlineIndexed() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"scriptValue\", \"foo_id\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -203,10 +205,14 @@ public class ScriptParameterParserTests extends ESTestCase { while (token != Token.VALUE_STRING) { token = parser.nextToken(); } - paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + try { + paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testTokenSingleParameterDelcaredTwiceFileInline() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo_file\" : \"scriptValue\", \"foo\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -223,10 +229,14 @@ public class ScriptParameterParserTests extends ESTestCase { while (token != Token.VALUE_STRING) { token = parser.nextToken(); } - paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + try { + paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testTokenSingleParameterDelcaredTwiceFileIndexed() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo_file\" : \"scriptValue\", \"foo_id\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -243,10 +253,14 @@ public class ScriptParameterParserTests extends ESTestCase { while (token != Token.VALUE_STRING) { token = parser.nextToken(); } - paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + try { + paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testTokenSingleParameterDelcaredTwiceIndexedInline() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo_id\" : \"scriptValue\", \"foo\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -263,10 +277,14 @@ public class ScriptParameterParserTests extends ESTestCase { while (token != Token.VALUE_STRING) { token = parser.nextToken(); } - paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + try { + paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testTokenSingleParameterDelcaredTwiceIndexedFile() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo_id\" : \"scriptValue\", \"foo_file\" : \"scriptValue\" }")); Token token = parser.nextToken(); @@ -283,10 +301,14 @@ public class ScriptParameterParserTests extends ESTestCase { while (token != Token.VALUE_STRING) { token = parser.nextToken(); } - paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + try { + paramParser.token(parser.currentName(), parser.currentToken(), parser, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test public void testTokenMultipleParameters() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"fooScriptValue\", \"bar_file\" : \"barScriptValue\", \"baz_id\" : \"bazScriptValue\" }")); Set parameters = new HashSet<>(); @@ -335,7 +357,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenMultipleParametersWithLang() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"fooScriptValue\", \"bar_file\" : \"barScriptValue\", \"lang\" : \"myLang\", \"baz_id\" : \"bazScriptValue\" }")); Set parameters = new HashSet<>(); @@ -395,7 +416,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), equalTo("myLang")); } - @Test public void testTokenMultipleParametersNotFound() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"other\" : \"scriptValue\" }")); Set parameters = new HashSet<>(); @@ -423,7 +443,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenMultipleParametersSomeNotFound() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"fooScriptValue\", \"other_file\" : \"barScriptValue\", \"baz_id\" : \"bazScriptValue\" }")); Set parameters = new HashSet<>(); @@ -480,7 +499,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testTokenMultipleParametersWrongType() throws IOException { XContentParser parser = XContentHelper.createParser(new BytesArray("{ \"foo\" : \"fooScriptValue\", \"bar_file\" : \"barScriptValue\", \"baz_id\" : \"bazScriptValue\" }")); Set parameters = new HashSet<>(); @@ -503,13 +521,15 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test(expected=IllegalArgumentException.class) public void testReservedParameters() { - Set parameterNames = Collections.singleton("lang"); - new ScriptParameterParser(parameterNames ); + try { + new ScriptParameterParser(singleton("lang")); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("lang is reserved")); + } } - @Test public void testConfigDefaultInline() throws IOException { Map config = new HashMap<>(); config.put("script", "scriptValue"); @@ -534,7 +554,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test public void testConfigDefaultFile() throws IOException { Map config = new HashMap<>(); config.put("script_file", "scriptValue"); @@ -553,7 +572,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test public void testConfigDefaultIndexed() throws IOException { Map config = new HashMap<>(); config.put("script_id", "scriptValue"); @@ -572,7 +590,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test public void testConfigDefaultIndexedNoRemove() throws IOException { Map config = new HashMap<>(); config.put("script_id", "scriptValue"); @@ -593,7 +610,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat((String) config.get("scriptId"), equalTo("scriptValue")); } - @Test public void testConfigDefaultNotFound() throws IOException { Map config = new HashMap<>(); config.put("foo", "bar"); @@ -606,7 +622,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat((String) config.get("foo"), equalTo("bar")); } - @Test public void testConfigSingleParameter() throws IOException { Map config = new HashMap<>(); config.put("foo", "scriptValue"); @@ -619,7 +634,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test public void testConfigSingleParameterFile() throws IOException { Map config = new HashMap<>(); config.put("foo_file", "scriptValue"); @@ -632,7 +646,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test public void testConfigSingleParameterIndexed() throws IOException { Map config = new HashMap<>(); config.put("foo_id", "scriptValue"); @@ -645,7 +658,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test(expected = ScriptParseException.class) public void testConfigSingleParameterDelcaredTwiceInlineFile() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo", "scriptValue"); @@ -653,10 +665,14 @@ public class ScriptParameterParserTests extends ESTestCase { Set parameters = Collections.singleton("foo"); ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testConfigSingleParameterDelcaredTwiceInlineIndexed() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo", "scriptValue"); @@ -664,10 +680,14 @@ public class ScriptParameterParserTests extends ESTestCase { Set parameters = Collections.singleton("foo"); ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testConfigSingleParameterDelcaredTwiceFileInline() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_file", "scriptValue"); @@ -675,10 +695,14 @@ public class ScriptParameterParserTests extends ESTestCase { Set parameters = Collections.singleton("foo"); ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testConfigSingleParameterDelcaredTwiceFileIndexed() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_file", "scriptValue"); @@ -686,10 +710,14 @@ public class ScriptParameterParserTests extends ESTestCase { Set parameters = Collections.singleton("foo"); ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testConfigSingleParameterDelcaredTwiceIndexedInline() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_id", "scriptValue"); @@ -697,10 +725,14 @@ public class ScriptParameterParserTests extends ESTestCase { Set parameters = Collections.singleton("foo"); ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testConfigSingleParameterDelcaredTwiceIndexedFile() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_id", "scriptValue"); @@ -708,10 +740,14 @@ public class ScriptParameterParserTests extends ESTestCase { Set parameters = Collections.singleton("foo"); ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test public void testConfigMultipleParameters() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -738,7 +774,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test public void testConfigMultipleParametersWithLang() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -766,7 +801,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(config.isEmpty(), equalTo(true)); } - @Test public void testConfigMultipleParametersWithLangNoRemove() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -798,7 +832,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat((String) config.get("lang"), equalTo("myLang")); } - @Test public void testConfigMultipleParametersNotFound() throws IOException { Map config = new HashMap<>(); config.put("other", "scriptValue"); @@ -825,7 +858,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat((String) config.get("other"), equalTo("scriptValue")); } - @Test public void testConfigMultipleParametersSomeNotFound() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -857,7 +889,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat((String) config.get("other_file"), equalTo("barScriptValue")); } - @Test(expected = ScriptParseException.class) public void testConfigMultipleParametersInlineWrongType() throws IOException { Map config = new HashMap<>(); config.put("foo", 1l); @@ -875,10 +906,14 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.getScriptParameterValue("bar_file"), nullValue()); assertThat(paramParser.getScriptParameterValue("baz_id"), nullValue()); assertThat(paramParser.lang(), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Value must be of type String: [foo]")); + } } - @Test(expected = ScriptParseException.class) public void testConfigMultipleParametersFileWrongType() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -896,10 +931,15 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.getScriptParameterValue("bar_file"), nullValue()); assertThat(paramParser.getScriptParameterValue("baz_id"), nullValue()); assertThat(paramParser.lang(), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Value must be of type String: [bar_file]")); + } + } - @Test(expected = ScriptParseException.class) public void testConfigMultipleParametersIndexedWrongType() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -917,10 +957,14 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.getScriptParameterValue("bar_file"), nullValue()); assertThat(paramParser.getScriptParameterValue("baz_id"), nullValue()); assertThat(paramParser.lang(), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Value must be of type String: [baz_id]")); + } } - @Test(expected = ScriptParseException.class) public void testConfigMultipleParametersLangWrongType() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -938,10 +982,14 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.getScriptParameterValue("bar_file"), nullValue()); assertThat(paramParser.getScriptParameterValue("baz_id"), nullValue()); assertThat(paramParser.lang(), nullValue()); - paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + try { + paramParser.parseConfig(config, true, ParseFieldMatcher.STRICT); + fail("Expected ScriptParseException"); + } catch (ScriptParseException e) { + assertThat(e.getMessage(), is("Value must be of type String: [lang]")); + } } - @Test public void testParamsDefaultInline() throws IOException { Map config = new HashMap<>(); config.put("script", "scriptValue"); @@ -950,7 +998,7 @@ public class ScriptParameterParserTests extends ESTestCase { paramParser.parseParams(params); assertDefaultParameterValue(paramParser, "scriptValue", ScriptType.INLINE); assertThat(paramParser.lang(), nullValue()); - + paramParser = new ScriptParameterParser(null); paramParser.parseParams(params); assertDefaultParameterValue(paramParser, "scriptValue", ScriptType.INLINE); @@ -962,7 +1010,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsDefaultFile() throws IOException { Map config = new HashMap<>(); config.put("script_file", "scriptValue"); @@ -973,7 +1020,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsDefaultIndexed() throws IOException { Map config = new HashMap<>(); config.put("script_id", "scriptValue"); @@ -984,7 +1030,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsDefaultNotFound() throws IOException { Map config = new HashMap<>(); config.put("foo", "bar"); @@ -996,7 +1041,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsSingleParameter() throws IOException { Map config = new HashMap<>(); config.put("foo", "scriptValue"); @@ -1009,7 +1053,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsSingleParameterFile() throws IOException { Map config = new HashMap<>(); config.put("foo_file", "scriptValue"); @@ -1022,7 +1065,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsSingleParameterIndexed() throws IOException { Map config = new HashMap<>(); config.put("foo_id", "scriptValue"); @@ -1035,7 +1077,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test(expected = ScriptParseException.class) public void testParamsSingleParameterDelcaredTwiceInlineFile() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo", "scriptValue"); @@ -1044,10 +1085,14 @@ public class ScriptParameterParserTests extends ESTestCase { ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); MapParams params = new MapParams(config); - paramParser.parseParams(params); + try { + paramParser.parseParams(params); + fail("Expected ScriptParseException"); + } catch(ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testParamsSingleParameterDelcaredTwiceInlineIndexed() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo", "scriptValue"); @@ -1056,10 +1101,14 @@ public class ScriptParameterParserTests extends ESTestCase { ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); MapParams params = new MapParams(config); - paramParser.parseParams(params); + try { + paramParser.parseParams(params); + fail("Expected ScriptParseException"); + } catch(ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testParamsSingleParameterDelcaredTwiceFileInline() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_file", "scriptValue"); @@ -1068,10 +1117,14 @@ public class ScriptParameterParserTests extends ESTestCase { ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); MapParams params = new MapParams(config); - paramParser.parseParams(params); + try { + paramParser.parseParams(params); + fail("Expected ScriptParseException"); + } catch(ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testParamsSingleParameterDelcaredTwiceFileIndexed() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_file", "scriptValue"); @@ -1080,10 +1133,14 @@ public class ScriptParameterParserTests extends ESTestCase { ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); MapParams params = new MapParams(config); - paramParser.parseParams(params); + try { + paramParser.parseParams(params); + fail("Expected ScriptParseException"); + } catch(ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testParamsSingleParameterDelcaredTwiceIndexedInline() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_id", "scriptValue"); @@ -1092,10 +1149,14 @@ public class ScriptParameterParserTests extends ESTestCase { ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); MapParams params = new MapParams(config); - paramParser.parseParams(params); + try { + paramParser.parseParams(params); + fail("Expected ScriptParseException"); + } catch(ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test(expected = ScriptParseException.class) public void testParamsSingleParameterDelcaredTwiceIndexedFile() throws IOException { Map config = new LinkedHashMap<>(); config.put("foo_id", "scriptValue"); @@ -1104,10 +1165,14 @@ public class ScriptParameterParserTests extends ESTestCase { ScriptParameterParser paramParser = new ScriptParameterParser(parameters); assertThat(paramParser.getScriptParameterValue("foo"), nullValue()); MapParams params = new MapParams(config); - paramParser.parseParams(params); + try { + paramParser.parseParams(params); + fail("Expected ScriptParseException"); + } catch(ScriptParseException e) { + assertThat(e.getMessage(), is("Only one of [foo, foo_file, foo_id] is allowed.")); + } } - @Test public void testParamsMultipleParameters() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -1134,7 +1199,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsMultipleParametersWithLang() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -1162,7 +1226,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), equalTo("myLang")); } - @Test public void testParamsMultipleParametersWithLangNoRemove() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); @@ -1190,7 +1253,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), equalTo("myLang")); } - @Test public void testParamsMultipleParametersNotFound() throws IOException { Map config = new HashMap<>(); config.put("other", "scriptValue"); @@ -1216,7 +1278,6 @@ public class ScriptParameterParserTests extends ESTestCase { assertThat(paramParser.lang(), nullValue()); } - @Test public void testParamsMultipleParametersSomeNotFound() throws IOException { Map config = new HashMap<>(); config.put("foo", "fooScriptValue"); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 26ba5807b2a..23cada02c6c 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -25,12 +25,10 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.watcher.ResourceWatcherService; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.nio.file.Files; @@ -74,8 +72,7 @@ public class ScriptServiceTests extends ESTestCase { .put("path.conf", genericConfigFolder) .build(); resourceWatcherService = new ResourceWatcherService(baseSettings, null); - scriptEngineServices = newHashSet(new TestEngineService(), - new MustacheScriptEngineService(baseSettings)); + scriptEngineServices = newHashSet(new TestEngineService()); scriptEnginesByLangMap = ScriptModesTests.buildScriptEnginesByLangMap(scriptEngineServices); //randomly register custom script contexts int randomInt = randomIntBetween(0, 3); @@ -112,7 +109,6 @@ public class ScriptServiceTests extends ESTestCase { }; } - @Test public void testNotSupportedDisableDynamicSetting() throws IOException { try { buildScriptService(Settings.builder().put(ScriptService.DISABLE_DYNAMIC_SCRIPTING_SETTING, randomUnicodeOfLength(randomIntBetween(1, 10))).build()); @@ -122,7 +118,6 @@ public class ScriptServiceTests extends ESTestCase { } } - @Test public void testScriptsWithoutExtensions() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); @@ -154,7 +149,6 @@ public class ScriptServiceTests extends ESTestCase { } } - @Test public void testInlineScriptCompiledOnceCache() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -165,7 +159,6 @@ public class ScriptServiceTests extends ESTestCase { assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled())); } - @Test public void testInlineScriptCompiledOnceMultipleLangAcronyms() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -176,7 +169,6 @@ public class ScriptServiceTests extends ESTestCase { assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled())); } - @Test public void testFileScriptCompiledOnceMultipleLangAcronyms() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -188,7 +180,6 @@ public class ScriptServiceTests extends ESTestCase { assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled())); } - @Test public void testDefaultBehaviourFineGrainedSettings() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); Settings.Builder builder = Settings.builder(); @@ -206,10 +197,6 @@ public class ScriptServiceTests extends ESTestCase { createFileScripts("groovy", "mustache", "test"); for (ScriptContext scriptContext : scriptContexts) { - //mustache engine is sandboxed, all scripts are enabled by default - assertCompileAccepted(MustacheScriptEngineService.NAME, "script", ScriptType.INLINE, scriptContext, contextAndHeaders); - assertCompileAccepted(MustacheScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext, contextAndHeaders); - assertCompileAccepted(MustacheScriptEngineService.NAME, "file_script", ScriptType.FILE, scriptContext, contextAndHeaders); //custom engine is sandboxed, all scripts are enabled by default assertCompileAccepted("test", "script", ScriptType.INLINE, scriptContext, contextAndHeaders); assertCompileAccepted("test", "script", ScriptType.INDEXED, scriptContext, contextAndHeaders); @@ -217,7 +204,6 @@ public class ScriptServiceTests extends ESTestCase { } } - @Test public void testFineGrainedSettings() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); //collect the fine-grained settings to set for this run @@ -342,7 +328,6 @@ public class ScriptServiceTests extends ESTestCase { } } - @Test public void testCompileNonRegisteredContext() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -366,7 +351,6 @@ public class ScriptServiceTests extends ESTestCase { } } - @Test public void testCompileCountedInCompilationStats() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -374,7 +358,6 @@ public class ScriptServiceTests extends ESTestCase { assertEquals(1L, scriptService.stats().getCompilations()); } - @Test public void testExecutableCountedInCompilationStats() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -382,14 +365,12 @@ public class ScriptServiceTests extends ESTestCase { assertEquals(1L, scriptService.stats().getCompilations()); } - @Test public void testSearchCountedInCompilationStats() throws IOException { buildScriptService(Settings.EMPTY); scriptService.search(null, new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts)); assertEquals(1L, scriptService.stats().getCompilations()); } - @Test public void testMultipleCompilationsCountedInCompilationStats() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -401,7 +382,6 @@ public class ScriptServiceTests extends ESTestCase { assertEquals(numberOfCompilations, scriptService.stats().getCompilations()); } - @Test public void testCompilationStatsOnCacheHit() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); Settings.Builder builder = Settings.builder(); @@ -412,7 +392,6 @@ public class ScriptServiceTests extends ESTestCase { assertEquals(1L, scriptService.stats().getCompilations()); } - @Test public void testFileScriptCountedInCompilationStats() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -421,7 +400,6 @@ public class ScriptServiceTests extends ESTestCase { assertEquals(1L, scriptService.stats().getCompilations()); } - @Test public void testIndexedScriptCountedInCompilationStats() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); buildScriptService(Settings.EMPTY); @@ -429,7 +407,6 @@ public class ScriptServiceTests extends ESTestCase { assertEquals(1L, scriptService.stats().getCompilations()); } - @Test public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException { ContextAndHeaderHolder contextAndHeaders = new ContextAndHeaderHolder(); Settings.Builder builder = Settings.builder(); diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index a84bc09b94d..fb3de6b0faa 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -20,8 +20,9 @@ package org.elasticsearch.search; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.index.*; -import org.apache.lucene.util.BitDocIdSet; +import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -170,7 +171,7 @@ public class MultiValueModeTests extends ESTestCase { private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { - final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitDocIdSet(innerDocs), maxDoc); + final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final long actual = selected.get(root); @@ -320,7 +321,7 @@ public class MultiValueModeTests extends ESTestCase { private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { - final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitDocIdSet(innerDocs), maxDoc); + final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final double actual = selected.get(root); @@ -460,7 +461,7 @@ public class MultiValueModeTests extends ESTestCase { private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { - final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitDocIdSet(innerDocs), maxDoc); + final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final BytesRef actual = selected.get(root); @@ -600,7 +601,7 @@ public class MultiValueModeTests extends ESTestCase { private void verify(RandomAccessOrds values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { - final SortedDocValues selected = mode.select(values, rootDocs, new BitDocIdSet(innerDocs)); + final SortedDocValues selected = mode.select(values, rootDocs, new BitSetIterator(innerDocs, 0L)); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final int actual = selected.getOrd(root); @@ -649,7 +650,6 @@ public class MultiValueModeTests extends ESTestCase { }; final SortedNumericDoubleValues singletonValues = FieldData.singleton(singleValues, docsWithValue); final MultiValueMode.UnsortedNumericDoubleValues multiValues = new MultiValueMode.UnsortedNumericDoubleValues() { - int doc; @Override public int count() { diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java index efdcf0062c3..376e8578e2e 100644 --- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -31,7 +31,7 @@ import org.elasticsearch.search.suggest.phrase.PhraseSuggester; public class SearchModuleTests extends ModuleTestCase { public void testDoubleRegister() { - SearchModule module = new SearchModule(Settings.EMPTY); + SearchModule module = new SearchModule(); try { module.registerHighlighter("fvh", PlainHighlighter.class); } catch (IllegalArgumentException e) { @@ -46,7 +46,7 @@ public class SearchModuleTests extends ModuleTestCase { } public void testRegisterSuggester() { - SearchModule module = new SearchModule(Settings.EMPTY); + SearchModule module = new SearchModule(); module.registerSuggester("custom", CustomSuggester.class); try { module.registerSuggester("custom", CustomSuggester.class); @@ -57,7 +57,7 @@ public class SearchModuleTests extends ModuleTestCase { } public void testRegisterHighlighter() { - SearchModule module = new SearchModule(Settings.EMPTY); + SearchModule module = new SearchModule(); module.registerHighlighter("custom", CustomHighlighter.class); try { module.registerHighlighter("custom", CustomHighlighter.class); diff --git a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java new file mode 100644 index 00000000000..a819317b62d --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) +public class SearchTimeoutIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(ScriptedTimeoutPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)).build(); + } + + public void testSimpleTimeout() throws Exception { + client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet(); + + SearchResponse searchResponse = client().prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.MILLISECONDS)) + .setQuery(scriptQuery(new Script(NativeTestScriptedTimeout.TEST_NATIVE_SCRIPT_TIMEOUT, ScriptType.INLINE, "native", null))) + .execute().actionGet(); + assertThat(searchResponse.isTimedOut(), equalTo(true)); + } + + public static class ScriptedTimeoutPlugin extends Plugin { + @Override + public String name() { + return "test-scripted-search-timeout"; + } + + @Override + public String description() { + return "Test for scripted timeouts on searches"; + } + + public void onModule(ScriptModule module) { + module.registerScript(NativeTestScriptedTimeout.TEST_NATIVE_SCRIPT_TIMEOUT, NativeTestScriptedTimeout.Factory.class); + } + } + + public static class NativeTestScriptedTimeout extends AbstractSearchScript { + + public static final String TEST_NATIVE_SCRIPT_TIMEOUT = "native_test_search_timeout_script"; + + public static class Factory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(Map params) { + return new NativeTestScriptedTimeout(); + } + + @Override + public boolean needsScores() { + return false; + } + } + + @Override + public Object run() { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + } + } + +} diff --git a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java index b12843b1ea6..d3e3de5fda1 100644 --- a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java +++ b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -38,13 +37,11 @@ public class SearchWithRejectionsIT extends ESIntegTestCase { @Override public Settings nodeSettings(int nodeOrdinal) { return settingsBuilder().put(super.nodeSettings(nodeOrdinal)) - .put("threadpool.search.type", "fixed") .put("threadpool.search.size", 1) .put("threadpool.search.queue_size", 1) .build(); } - @Test public void testOpenContextsAfterRejections() throws InterruptedException { createIndex("test"); ensureGreen("test"); diff --git a/core/src/test/java/org/elasticsearch/search/StressSearchServiceReaperIT.java b/core/src/test/java/org/elasticsearch/search/StressSearchServiceReaperIT.java index c97e29a0f9c..addfe14c488 100644 --- a/core/src/test/java/org/elasticsearch/search/StressSearchServiceReaperIT.java +++ b/core/src/test/java/org/elasticsearch/search/StressSearchServiceReaperIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.concurrent.ExecutionException; @@ -44,7 +43,7 @@ public class StressSearchServiceReaperIT extends ESIntegTestCase { .put(SearchService.KEEPALIVE_INTERVAL_KEY, TimeValue.timeValueMillis(1)).build(); } - @Test // see issue #5165 - this test fails each time without the fix in pull #5170 + // see issue #5165 - this test fails each time without the fix in pull #5170 public void testStressReaper() throws ExecutionException, InterruptedException { int num = randomIntBetween(100, 150); IndexRequestBuilder[] builders = new IndexRequestBuilder[num]; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsBinaryIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsBinaryIT.java index 631f705bf5b..80227ff7f66 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsBinaryIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationsBinaryIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; @@ -30,7 +31,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -41,6 +41,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.IsNull.notNullValue; @ESIntegTestCase.SuiteScopeTestCase +@AwaitsFix(bugUrl = "needs fixing after the search request refactor. Do we need agg binary?") +// NO RELEASE public class AggregationsBinaryIT extends ESIntegTestCase { private static final String STRING_FIELD_NAME = "s_value"; @@ -58,7 +60,6 @@ public class AggregationsBinaryIT extends ESIntegTestCase { ensureSearchable(); } - @Test public void testAggregationsBinary() throws Exception { TermsBuilder termsBuilder = AggregationBuilders.terms("terms").field(STRING_FIELD_NAME); TermsBuilder subTerm = AggregationBuilders.terms("subterms").field(INT_FIELD_NAME); @@ -98,7 +99,6 @@ public class AggregationsBinaryIT extends ESIntegTestCase { } } - @Test public void testAggregationsBinarySameContentType() throws Exception { TermsBuilder termsBuilder = AggregationBuilders.terms("terms").field(STRING_FIELD_NAME); TermsBuilder subTerm = AggregationBuilders.terms("subterms").field(INT_FIELD_NAME); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java index 8282a538bec..5154dcc39e1 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/CombiIT.java @@ -19,8 +19,8 @@ package org.elasticsearch.search.aggregations; -import com.carrotsearch.hppc.IntIntMap; import com.carrotsearch.hppc.IntIntHashMap; +import com.carrotsearch.hppc.IntIntMap; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.Collection; @@ -55,8 +54,7 @@ public class CombiIT extends ESIntegTestCase { * it as "numeric", it didn't work. Now we cache the Value Sources by a custom key (field name + ValueSource type) * so there's no conflict there. */ - @Test - public void multipleAggs_OnSameField_WithDifferentRequiredValueSourceType() throws Exception { + public void testMultipleAggsOnSameField_WithDifferentRequiredValueSourceType() throws Exception { createIndex("idx"); IndexRequestBuilder[] builders = new IndexRequestBuilder[randomInt(30)]; @@ -115,8 +113,7 @@ public class CombiIT extends ESIntegTestCase { * when the sub aggregator is then created, it will take this estimation into account. This used to cause * and an ArrayIndexOutOfBoundsException... */ - @Test - public void subAggregationForTopAggregationOnUnmappedField() throws Exception { + public void testSubAggregationForTopAggregationOnUnmappedField() throws Exception { prepareCreate("idx").addMapping("type", jsonBuilder() .startObject() diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/MetaDataIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/MetaDataIT.java index c83ad5777bc..9d83428038d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/MetaDataIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/MetaDataIT.java @@ -19,13 +19,13 @@ package org.elasticsearch.search.aggregations; -import com.carrotsearch.hppc.IntIntMap; import com.carrotsearch.hppc.IntIntHashMap; +import com.carrotsearch.hppc.IntIntMap; + import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -47,8 +47,7 @@ public class MetaDataIT extends ESIntegTestCase { * it as "numeric", it didn't work. Now we cache the Value Sources by a custom key (field name + ValueSource type) * so there's no conflict there. */ - @Test - public void meta_data_set_on_aggregation_result() throws Exception { + public void testMetaDataSetOnAggregationResult() throws Exception { createIndex("idx"); IndexRequestBuilder[] builders = new IndexRequestBuilder[randomInt(30)]; @@ -112,6 +111,4 @@ public class MetaDataIT extends ESIntegTestCase { Map nestedMap = (Map)nestedObject; assertEquals("value", nestedMap.get("nested")); } - - } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java index f76e1ad9e56..f2a78295664 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/MissingValueIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBounds; +import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroid; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.test.ESIntegTestCase; @@ -33,12 +34,14 @@ import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.geoBounds; +import static org.elasticsearch.search.aggregations.AggregationBuilders.geoCentroid; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles; import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.closeTo; @ESIntegTestCase.SuiteScopeTestCase public class MissingValueIT extends ESIntegTestCase { @@ -180,16 +183,28 @@ public class MissingValueIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("idx").addAggregation(geoBounds("bounds").field("non_existing_field").missing("2,1")).get(); assertSearchResponse(response); GeoBounds bounds = response.getAggregations().get("bounds"); - assertEquals(new GeoPoint(2,1), bounds.bottomRight()); - assertEquals(new GeoPoint(2,1), bounds.topLeft()); + assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); } public void testGeoBounds() { SearchResponse response = client().prepareSearch("idx").addAggregation(geoBounds("bounds").field("location").missing("2,1")).get(); assertSearchResponse(response); GeoBounds bounds = response.getAggregations().get("bounds"); - assertEquals(new GeoPoint(1,2), bounds.bottomRight()); - assertEquals(new GeoPoint(2,1), bounds.topLeft()); + assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); } + public void testGeoCentroid() { + SearchResponse response = client().prepareSearch("idx").addAggregation(geoCentroid("centroid").field("location").missing("2,1")).get(); + assertSearchResponse(response); + GeoCentroid centroid = response.getAggregations().get("centroid"); + GeoPoint point = new GeoPoint(1.5, 1.5); + assertThat(point.lat(), closeTo(centroid.centroid().lat(), 1E-5)); + assertThat(point.lon(), closeTo(centroid.centroid().lon(), 1E-5)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/ParsingIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/ParsingIT.java index 64f80d66e18..1ac06e61c70 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/ParsingIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/ParsingIT.java @@ -19,162 +19,150 @@ package org.elasticsearch.search.aggregations; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +// NORELEASE move these tests to unit tests when aggs refactoring is done +// @Test(expected=SearchPhaseExecutionException.class) +// public void testTwoTypes() throws Exception { +// createIndex("idx"); +// ensureGreen(); +// client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() +// .startObject() +// .startObject("in_stock") +// .startObject("filter") +// .startObject("range") +// .startObject("stock") +// .field("gt", 0) +// .endObject() +// .endObject() +// .endObject() +// .startObject("terms") +// .field("field", "stock") +// .endObject() +// .endObject() +// .endObject()).execute().actionGet(); +// } +// +// @Test(expected=SearchPhaseExecutionException.class) +// public void testTwoAggs() throws Exception { +// createIndex("idx"); +// ensureGreen(); +// client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() +// .startObject() +// .startObject("by_date") +// .startObject("date_histogram") +// .field("field", "timestamp") +// .field("interval", "month") +// .endObject() +// .startObject("aggs") +// .startObject("tag_count") +// .startObject("cardinality") +// .field("field", "tag") +// .endObject() +// .endObject() +// .endObject() +// .startObject("aggs") // 2nd "aggs": illegal +// .startObject("tag_count2") +// .startObject("cardinality") +// .field("field", "tag") +// .endObject() +// .endObject() +// .endObject() +// .endObject()).execute().actionGet(); +// } +// +// @Test(expected=SearchPhaseExecutionException.class) +// public void testInvalidAggregationName() throws Exception { +// +// Matcher matcher = Pattern.compile("[^\\[\\]>]+").matcher(""); +// String name; +// SecureRandom rand = new SecureRandom(); +// int len = randomIntBetween(1, 5); +// char[] word = new char[len]; +// while(true) { +// for (int i = 0; i < word.length; i++) { +// word[i] = (char) rand.nextInt(127); +// } +// name = String.valueOf(word); +// if (!matcher.reset(name).matches()) { +// break; +// } +// } +// +// createIndex("idx"); +// ensureGreen(); +// client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() +// .startObject() +// .startObject(name) +// .startObject("filter") +// .startObject("range") +// .startObject("stock") +// .field("gt", 0) +// .endObject() +// .endObject() +// .endObject() +// .endObject()).execute().actionGet(); +// } +// +// @Test(expected=SearchPhaseExecutionException.class) +// public void testSameAggregationName() throws Exception { +// createIndex("idx"); +// ensureGreen(); +// final String name = RandomStrings.randomAsciiOfLength(getRandom(), 10); +// client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() +// .startObject() +// .startObject(name) +// .startObject("terms") +// .field("field", "a") +// .endObject() +// .endObject() +// .startObject(name) +// .startObject("terms") +// .field("field", "b") +// .endObject() +// .endObject() +// .endObject()).execute().actionGet(); +// } +// +// @Test(expected=SearchPhaseExecutionException.class) +// public void testMissingName() throws Exception { +// createIndex("idx"); +// ensureGreen(); +// client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() +// .startObject() +// .startObject("by_date") +// .startObject("date_histogram") +// .field("field", "timestamp") +// .field("interval", "month") +// .endObject() +// .startObject("aggs") +// // the aggregation name is missing +// //.startObject("tag_count") +// .startObject("cardinality") +// .field("field", "tag") +// .endObject() +// //.endObject() +// .endObject() +// .endObject()).execute().actionGet(); +// } +// +// @Test(expected=SearchPhaseExecutionException.class) +// public void testMissingType() throws Exception { +// createIndex("idx"); +// ensureGreen(); +// client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() +// .startObject() +// .startObject("by_date") +// .startObject("date_histogram") +// .field("field", "timestamp") +// .field("interval", "month") +// .endObject() +// .startObject("aggs") +// .startObject("tag_count") +// // the aggregation type is missing +// //.startObject("cardinality") +// .field("field", "tag") +// //.endObject() +// .endObject() +// .endObject() +// .endObject()).execute().actionGet(); +// } -import java.security.SecureRandom; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class ParsingIT extends ESIntegTestCase { - - @Test(expected=SearchPhaseExecutionException.class) - public void testTwoTypes() throws Exception { - createIndex("idx"); - ensureGreen(); - client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() - .startObject() - .startObject("in_stock") - .startObject("filter") - .startObject("range") - .startObject("stock") - .field("gt", 0) - .endObject() - .endObject() - .endObject() - .startObject("terms") - .field("field", "stock") - .endObject() - .endObject() - .endObject()).execute().actionGet(); - } - - @Test(expected=SearchPhaseExecutionException.class) - public void testTwoAggs() throws Exception { - createIndex("idx"); - ensureGreen(); - client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() - .startObject() - .startObject("by_date") - .startObject("date_histogram") - .field("field", "timestamp") - .field("interval", "month") - .endObject() - .startObject("aggs") - .startObject("tag_count") - .startObject("cardinality") - .field("field", "tag") - .endObject() - .endObject() - .endObject() - .startObject("aggs") // 2nd "aggs": illegal - .startObject("tag_count2") - .startObject("cardinality") - .field("field", "tag") - .endObject() - .endObject() - .endObject() - .endObject()).execute().actionGet(); - } - - @Test(expected=SearchPhaseExecutionException.class) - public void testInvalidAggregationName() throws Exception { - - Matcher matcher = Pattern.compile("[^\\[\\]>]+").matcher(""); - String name; - SecureRandom rand = new SecureRandom(); - int len = randomIntBetween(1, 5); - char[] word = new char[len]; - while(true) { - for (int i = 0; i < word.length; i++) { - word[i] = (char) rand.nextInt(127); - } - name = String.valueOf(word); - if (!matcher.reset(name).matches()) { - break; - } - } - - createIndex("idx"); - ensureGreen(); - client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() - .startObject() - .startObject(name) - .startObject("filter") - .startObject("range") - .startObject("stock") - .field("gt", 0) - .endObject() - .endObject() - .endObject() - .endObject()).execute().actionGet(); - } - - @Test(expected=SearchPhaseExecutionException.class) - public void testSameAggregationName() throws Exception { - createIndex("idx"); - ensureGreen(); - final String name = RandomStrings.randomAsciiOfLength(getRandom(), 10); - client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() - .startObject() - .startObject(name) - .startObject("terms") - .field("field", "a") - .endObject() - .endObject() - .startObject(name) - .startObject("terms") - .field("field", "b") - .endObject() - .endObject() - .endObject()).execute().actionGet(); - } - - @Test(expected=SearchPhaseExecutionException.class) - public void testMissingName() throws Exception { - createIndex("idx"); - ensureGreen(); - client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() - .startObject() - .startObject("by_date") - .startObject("date_histogram") - .field("field", "timestamp") - .field("interval", "month") - .endObject() - .startObject("aggs") - // the aggregation name is missing - //.startObject("tag_count") - .startObject("cardinality") - .field("field", "tag") - .endObject() - //.endObject() - .endObject() - .endObject()).execute().actionGet(); - } - - @Test(expected=SearchPhaseExecutionException.class) - public void testMissingType() throws Exception { - createIndex("idx"); - ensureGreen(); - client().prepareSearch("idx").setAggregations(JsonXContent.contentBuilder() - .startObject() - .startObject("by_date") - .startObject("date_histogram") - .field("field", "timestamp") - .field("interval", "month") - .endObject() - .startObject("aggs") - .startObject("tag_count") - // the aggregation type is missing - //.startObject("cardinality") - .field("field", "tag") - //.endObject() - .endObject() - .endObject() - .endObject()).execute().actionGet(); - } - -} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java index b4c28ac3521..0a660b85374 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; @@ -85,8 +84,7 @@ public class BooleanTermsIT extends ESIntegTestCase { indexRandom(true, builders); } - @Test - public void singleValueField() throws Exception { + public void testSingleValueField() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -118,8 +116,7 @@ public class BooleanTermsIT extends ESIntegTestCase { } } - @Test - public void multiValueField() throws Exception { + public void testMultiValueField() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -151,8 +148,7 @@ public class BooleanTermsIT extends ESIntegTestCase { } } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java index c729c2f2d04..b6611a956af 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.metrics.tophits.TopHits; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.HashMap; @@ -42,9 +41,18 @@ import java.util.Set; import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.children; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; /** */ @@ -120,7 +128,6 @@ public class ChildrenIT extends ESIntegTestCase { ensureSearchable("test"); } - @Test public void testChildrenAggs() throws Exception { SearchResponse searchResponse = client().prepareSearch("test") .setQuery(matchQuery("randomized", true)) @@ -162,7 +169,6 @@ public class ChildrenIT extends ESIntegTestCase { } } - @Test public void testParentWithMultipleBuckets() throws Exception { SearchResponse searchResponse = client().prepareSearch("test") .setQuery(matchQuery("randomized", false)) @@ -225,7 +231,6 @@ public class ChildrenIT extends ESIntegTestCase { assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment")); } - @Test public void testWithDeletes() throws Exception { String indexName = "xyz"; assertAcked( @@ -270,7 +275,6 @@ public class ChildrenIT extends ESIntegTestCase { } } - @Test public void testNonExistingChildType() throws Exception { SearchResponse searchResponse = client().prepareSearch("test") .addAggregation( @@ -283,7 +287,6 @@ public class ChildrenIT extends ESIntegTestCase { assertThat(children.getDocCount(), equalTo(0l)); } - @Test public void testPostCollection() throws Exception { String indexName = "prodcatalog"; String masterType = "masterprod"; @@ -343,7 +346,6 @@ public class ChildrenIT extends ESIntegTestCase { assertThat(termsAgg.getBucketByKey("44").getDocCount(), equalTo(1l)); } - @Test public void testHierarchicalChildrenAggs() { String indexName = "geo"; String grandParentType = "continent"; diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java similarity index 83% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateHistogramTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index cea4a32ab9a..9a1d498ad6b 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateHistogramTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -16,8 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.bucket; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; @@ -27,49 +29,61 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.junit.After; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.max; +import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; /** * */ @ESIntegTestCase.SuiteScopeTestCase -public class DateHistogramTests extends ESIntegTestCase { +public class DateHistogramIT extends ESIntegTestCase { - @Override - protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - private DateTime date(int month, int day) { return new DateTime(2012, month, day, 0, 0, DateTimeZone.UTC); } @@ -124,6 +138,13 @@ public class DateHistogramTests extends ESIntegTestCase { ensureSearchable(); } + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ExtractFieldScriptPlugin.class, + FieldValueScriptPlugin.class); + } + @After public void afterEachTest() throws IOException { internalCluster().wipeIndices("idx2"); @@ -137,8 +158,7 @@ public class DateHistogramTests extends ESIntegTestCase { return Joda.forPattern(DateFieldMapper.Defaults.DATE_TIME_FORMATTER.format()).printer().withZone(tz).print(key); } - @Test - public void singleValuedField() throws Exception { + public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo").field("date").interval(DateHistogramInterval.MONTH)) .execute().actionGet(); @@ -173,8 +193,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(3l)); } - @Test - public void singleValuedField_WithTimeZone() throws Exception { + public void testSingleValuedFieldWithTimeZone() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo").field("date").interval(DateHistogramInterval.DAY).minDocCount(1).timeZone("+01:00")).execute() .actionGet(); @@ -230,8 +249,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(1l)); } - @Test - public void singleValuedField_OrderedByKeyAsc() throws Exception { + public void testSingleValuedFieldOrderedByKeyAsc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -254,8 +272,7 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByKeyDesc() throws Exception { + public void testSingleValuedFieldOrderedByKeyDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -277,8 +294,7 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByCountAsc() throws Exception { + public void testSingleValuedFieldOrderedByCountAsc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -300,8 +316,7 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByCountDesc() throws Exception { + public void testSingleValuedFieldOrderedByCountDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -323,8 +338,7 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo").field("date").interval(DateHistogramInterval.MONTH) .subAggregation(sum("sum").field("value"))) @@ -381,8 +395,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat((double) propertiesCounts[2], equalTo(15.0)); } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregationInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo").field("date").interval(DateHistogramInterval.MONTH) .subAggregation(max("max"))) @@ -427,8 +440,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(max.getValue(), equalTo((double) new DateTime(2012, 3, 23, 0, 0, DateTimeZone.UTC).getMillis())); } - @Test - public void singleValuedField_OrderedBySubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationAsc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -451,8 +463,7 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedBySubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -475,8 +486,7 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregationAsc_Inherited() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationAscInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -499,8 +509,7 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") @@ -523,12 +532,11 @@ public class DateHistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("date") - .script(new Script("new DateTime(_value).plusMonths(1).getMillis()")) + .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null)) .interval(DateHistogramInterval.MONTH)).execute().actionGet(); assertSearchResponse(response); @@ -562,8 +570,6 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(3l)); } - - /* [ Jan 2, Feb 3] [ Feb 2, Mar 3] @@ -573,8 +579,7 @@ public class DateHistogramTests extends ESIntegTestCase { [ Mar 23, Apr 24] */ - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo").field("dates").interval(DateHistogramInterval.MONTH)) .execute().actionGet(); @@ -616,8 +621,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(3l)); } - @Test - public void multiValuedField_OrderedByKeyDesc() throws Exception { + public void testMultiValuedFieldOrderedByKeyDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("dates") @@ -662,12 +666,11 @@ public class DateHistogramTests extends ESIntegTestCase { * doc 5: [ Apr 15, May 16] * doc 6: [ Apr 23, May 24] */ - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("dates") - .script(new Script("new DateTime(_value, DateTimeZone.UTC).plusMonths(1).getMillis()")) + .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null)) .interval(DateHistogramInterval.MONTH)).execute().actionGet(); assertSearchResponse(response); @@ -717,12 +720,11 @@ public class DateHistogramTests extends ESIntegTestCase { * doc 5: [ Apr 15, May 16] * doc 6: [ Apr 23, May 24] */ - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") .field("dates") - .script(new Script("new DateTime((long)_value, DateTimeZone.UTC).plusMonths(1).getMillis()")) + .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null)) .interval(DateHistogramInterval.MONTH).subAggregation(max("max"))).execute().actionGet(); assertSearchResponse(response); @@ -782,10 +784,9 @@ public class DateHistogramTests extends ESIntegTestCase { * Mar 15 * Mar 23 */ - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client().prepareSearch("idx") - .addAggregation(dateHistogram("histo").script(new Script("doc['date'].value")).interval(DateHistogramInterval.MONTH)) + .addAggregation(dateHistogram("histo").script(new Script("date", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)).interval(DateHistogramInterval.MONTH)) .execute().actionGet(); assertSearchResponse(response); @@ -818,11 +819,10 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(3l)); } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") - .script(new Script("doc['date'].value")).interval(DateHistogramInterval.MONTH) + .script(new Script("date", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)).interval(DateHistogramInterval.MONTH) .subAggregation(max("max"))).execute().actionGet(); assertSearchResponse(response); @@ -864,10 +864,9 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(max.getValue(), equalTo((double) new DateTime(2012, 3, 23, 0, 0, DateTimeZone.UTC).getMillis())); } - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client().prepareSearch("idx") - .addAggregation(dateHistogram("histo").script(new Script("doc['dates'].values")).interval(DateHistogramInterval.MONTH)) + .addAggregation(dateHistogram("histo").script(new Script("dates", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)).interval(DateHistogramInterval.MONTH)) .execute().actionGet(); assertSearchResponse(response); @@ -918,11 +917,10 @@ public class DateHistogramTests extends ESIntegTestCase { [ Mar 23, Apr 24] */ - @Test - public void script_MultiValued_WithAggregatorInherited() throws Exception { + public void testScriptMultiValuedWithAggregatorInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateHistogram("histo") - .script(new Script("doc['dates'].values")).interval(DateHistogramInterval.MONTH) + .script(new Script("dates", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null)).interval(DateHistogramInterval.MONTH) .subAggregation(max("max"))).execute().actionGet(); assertSearchResponse(response); @@ -974,8 +972,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat((long) max.getValue(), equalTo(new DateTime(2012, 4, 24, 0, 0, DateTimeZone.UTC).getMillis())); } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped") .addAggregation(dateHistogram("histo").field("date").interval(DateHistogramInterval.MONTH)) .execute().actionGet(); @@ -988,8 +985,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") .addAggregation(dateHistogram("histo").field("date").interval(DateHistogramInterval.MONTH)) .execute().actionGet(); @@ -1024,8 +1020,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(3l)); } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(dateHistogram("date_histo").interval(1))) @@ -1048,8 +1043,7 @@ public class DateHistogramTests extends ESIntegTestCase { } - @Test - public void singleValue_WithTimeZone() throws Exception { + public void testSingleValueWithTimeZone() throws Exception { prepareCreate("idx2").addMapping("type", "date", "type=date").execute().actionGet(); IndexRequestBuilder[] reqs = new IndexRequestBuilder[5]; DateTime date = date("2014-03-11T00:00:00+00:00"); @@ -1085,9 +1079,7 @@ public class DateHistogramTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(3l)); } - @Test - public void singleValueField_WithExtendedBounds() throws Exception { - + public void testSingleValueFieldWithExtendedBounds() throws Exception { String pattern = "yyyy-MM-dd"; // we're testing on days, so the base must be rounded to a day int interval = randomIntBetween(1, 2); // in days @@ -1202,9 +1194,7 @@ public class DateHistogramTests extends ESIntegTestCase { * Test date histogram aggregation with hour interval, timezone shift and * extended bounds (see https://github.com/elastic/elasticsearch/issues/12278) */ - @Test - public void singleValueField_WithExtendedBoundsTimezone() throws Exception { - + public void testSingleValueFieldWithExtendedBoundsTimezone() throws Exception { String index = "test12278"; prepareCreate(index) .setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) @@ -1263,9 +1253,7 @@ public class DateHistogramTests extends ESIntegTestCase { internalCluster().wipeIndices("test12278"); } - @Test - public void singleValue_WithMultipleDateFormatsFromMapping() throws Exception { - + public void testSingleValueWithMultipleDateFormatsFromMapping() throws Exception { String mappingJson = jsonBuilder().startObject().startObject("type").startObject("properties").startObject("date").field("type", "date").field("format", "dateOptionalTime||dd-MM-yyyy").endObject().endObject().endObject().endObject().string(); prepareCreate("idx2").addMapping("type", mappingJson).execute().actionGet(); IndexRequestBuilder[] reqs = new IndexRequestBuilder[5]; @@ -1392,4 +1380,258 @@ public class DateHistogramTests extends ESIntegTestCase { Histogram histo = response.getAggregations().get("histo"); assertThat(histo.getBuckets().size(), greaterThan(0)); } + + /** + * Mock plugin for the {@link ExtractFieldScriptEngine} + */ + public static class ExtractFieldScriptPlugin extends Plugin { + + @Override + public String name() { + return ExtractFieldScriptEngine.NAME; + } + + @Override + public String description() { + return "Mock script engine for " + DateHistogramIT.class; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(ExtractFieldScriptEngine.class); + } + + } + + /** + * This mock script returns the field that is specified by name in the script body + */ + public static class ExtractFieldScriptEngine implements ScriptEngineService { + + public static final String NAME = "extract_field"; + + @Override + public void close() throws IOException { + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + throw new UnsupportedOperationException(); + } + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + return new SearchScript() { + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + + return new LeafSearchScript() { + + @Override + public Object unwrap(Object value) { + return null; + } + + @Override + public void setNextVar(String name, Object value) { + } + + @Override + public Object run() { + String fieldName = (String) compiledScript.compiled(); + return leafLookup.doc().get(fieldName); + } + + @Override + public void setScorer(Scorer scorer) { + } + + @Override + public void setSource(Map source) { + } + + @Override + public void setDocument(int doc) { + if (leafLookup != null) { + leafLookup.setDocument(doc); + } + } + + @Override + public long runAsLong() { + throw new UnsupportedOperationException(); + } + + @Override + public float runAsFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public double runAsDouble() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + } + } + + /** + * Mock plugin for the {@link FieldValueScriptEngine} + */ + public static class FieldValueScriptPlugin extends Plugin { + + @Override + public String name() { + return FieldValueScriptEngine.NAME; + } + + @Override + public String description() { + return "Mock script engine for " + DateHistogramIT.class; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(FieldValueScriptEngine.class); + } + + } + + /** + * This mock script returns the field value and adds one month to the returned date + */ + public static class FieldValueScriptEngine implements ScriptEngineService { + + public static final String NAME = "field_value"; + + @Override + public void close() throws IOException { + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + throw new UnsupportedOperationException(); + } + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + return new SearchScript() { + + private Map vars = new HashMap<>(2); + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + + return new LeafSearchScript() { + + @Override + public Object unwrap(Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public void setNextVar(String name, Object value) { + vars.put(name, value); + } + + @Override + public Object run() { + throw new UnsupportedOperationException(); + } + + @Override + public void setScorer(Scorer scorer) { + } + + @Override + public void setSource(Map source) { + } + + @Override + public void setDocument(int doc) { + if (leafLookup != null) { + leafLookup.setDocument(doc); + } + } + + @Override + public long runAsLong() { + return new DateTime((long) vars.get("_value"), DateTimeZone.UTC).plusMonths(1).getMillis(); + } + + @Override + public float runAsFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public double runAsDouble() { + return new DateTime(new Double((double) vars.get("_value")).longValue(), DateTimeZone.UTC).plusMonths(1).getMillis(); + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + } + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 08e07677fa3..bb22361ebde 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -31,7 +31,6 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.List; @@ -84,8 +83,7 @@ public class DateHistogramOffsetIT extends ESIntegTestCase { indexRandom(true, reqs); } - @Test - public void singleValue_WithPositiveOffset() throws Exception { + public void testSingleValueWithPositiveOffset() throws Exception { prepareIndex(date("2014-03-11T00:00:00+00:00"), 5, 1, 0); SearchResponse response = client().prepareSearch("idx2") @@ -107,8 +105,7 @@ public class DateHistogramOffsetIT extends ESIntegTestCase { checkBucketFor(buckets.get(1), new DateTime(2014, 3, 11, 2, 0, DateTimeZone.UTC), 3l); } - @Test - public void singleValue_WithNegativeOffset() throws Exception { + public void testSingleValueWithNegativeOffset() throws Exception { prepareIndex(date("2014-03-11T00:00:00+00:00"), 5, -1, 0); SearchResponse response = client().prepareSearch("idx2") @@ -133,8 +130,7 @@ public class DateHistogramOffsetIT extends ESIntegTestCase { /** * Set offset so day buckets start at 6am. Index first 12 hours for two days, with one day gap. */ - @Test - public void singleValue_WithOffset_MinDocCount() throws Exception { + public void testSingleValueWithOffsetMinDocCount() throws Exception { prepareIndex(date("2014-03-11T00:00:00+00:00"), 12, 1, 0); prepareIndex(date("2014-03-14T00:00:00+00:00"), 12, 1, 13); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java index 47de43df20d..b447580c7e3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.metrics.avg.Avg; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -91,8 +90,7 @@ public class FilterIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void simple() throws Exception { + public void testSimple() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(filter("tag1").filter(termQuery("tag", "tag1"))) .execute().actionGet(); @@ -108,8 +106,7 @@ public class FilterIT extends ESIntegTestCase { // See NullPointer issue when filters are empty: // https://github.com/elasticsearch/elasticsearch/issues/8438 - @Test - public void emptyFilterDeclarations() throws Exception { + public void testEmptyFilterDeclarations() throws Exception { QueryBuilder emptyFilter = new BoolQueryBuilder(); SearchResponse response = client().prepareSearch("idx").addAggregation(filter("tag1").filter(emptyFilter)).execute().actionGet(); @@ -120,8 +117,7 @@ public class FilterIT extends ESIntegTestCase { assertThat(filter.getDocCount(), equalTo((long) numDocs)); } - @Test - public void withSubAggregation() throws Exception { + public void testWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(filter("tag1") .filter(termQuery("tag", "tag1")) @@ -149,9 +145,7 @@ public class FilterIT extends ESIntegTestCase { assertThat((double) filter.getProperty("avg_value.value"), equalTo((double) sum / numTag1Docs)); } - @Test - public void withContextBasedSubAggregation() throws Exception { - + public void testWithContextBasedSubAggregation() throws Exception { try { client().prepareSearch("idx") .addAggregation(filter("tag1") @@ -162,12 +156,12 @@ public class FilterIT extends ESIntegTestCase { fail("expected execution to fail - an attempt to have a context based numeric sub-aggregation, but there is not value source" + "context which the sub-aggregation can inherit"); - } catch (ElasticsearchException ese) { + } catch (ElasticsearchException e) { + assertThat(e.getMessage(), is("all shards failed")); } } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java index e3ff3411010..42e19674095 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.metrics.avg.Avg; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -108,8 +107,7 @@ public class FiltersIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void simple() throws Exception { + public void testSimple() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation( filters("tags") @@ -136,8 +134,7 @@ public class FiltersIT extends ESIntegTestCase { // See NullPointer issue when filters are empty: // https://github.com/elasticsearch/elasticsearch/issues/8438 - @Test - public void emptyFilterDeclarations() throws Exception { + public void testEmptyFilterDeclarations() throws Exception { QueryBuilder emptyFilter = new BoolQueryBuilder(); SearchResponse response = client().prepareSearch("idx") .addAggregation(filters("tags").filter("all", emptyFilter).filter("tag1", termQuery("tag", "tag1"))).execute() @@ -155,8 +152,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo((long) numTag1Docs)); } - @Test - public void withSubAggregation() throws Exception { + public void testWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation( filters("tags") @@ -209,8 +205,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat((double) propertiesCounts[1], equalTo((double) sum / numTag2Docs)); } - @Test - public void withContextBasedSubAggregation() throws Exception { + public void testWithContextBasedSubAggregation() throws Exception { try { client().prepareSearch("idx") @@ -225,12 +220,12 @@ public class FiltersIT extends ESIntegTestCase { fail("expected execution to fail - an attempt to have a context based numeric sub-aggregation, but there is not value source" + "context which the sub-aggregation can inherit"); - } catch (ElasticsearchException ese) { + } catch (ElasticsearchException e) { + assertThat(e.getMessage(), is("all shards failed")); } } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) @@ -251,8 +246,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(all.getDocCount(), is(0l)); } - @Test - public void simple_nonKeyed() throws Exception { + public void testSimpleNonKeyed() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation( filters("tags") @@ -280,8 +274,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo((long) numTag2Docs)); } - @Test - public void otherBucket() throws Exception { + public void testOtherBucket() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -311,8 +304,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo((long) numOtherDocs)); } - @Test - public void otherNamedBucket() throws Exception { + public void testOtherNamedBucket() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -342,8 +334,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo((long) numOtherDocs)); } - @Test - public void other_nonKeyed() throws Exception { + public void testOtherNonKeyed() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation( filters("tags").otherBucket(true) @@ -375,8 +366,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo((long) numOtherDocs)); } - @Test - public void otherWithSubAggregation() throws Exception { + public void testOtherWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation( filters("tags").otherBucket(true) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index 6bbbdb501c0..c76e3681eb8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -18,8 +18,11 @@ */ package org.elasticsearch.search.aggregations.bucket; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -28,8 +31,8 @@ import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -54,6 +57,7 @@ import static org.hamcrest.core.IsNull.nullValue; */ @ESIntegTestCase.SuiteScopeTestCase public class GeoDistanceIT extends ESIntegTestCase { + private Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); private IndexRequestBuilder indexCity(String idx, String name, String... latLons) throws Exception { XContentBuilder source = jsonBuilder().startObject().field("city", name); @@ -68,7 +72,8 @@ public class GeoDistanceIT extends ESIntegTestCase { @Override public void setupSuiteScopeCluster() throws Exception { - prepareCreate("idx") + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + prepareCreate("idx").setSettings(settings) .addMapping("type", "location", "type=geo_point", "city", "type=string,index=not_analyzed") .execute().actionGet(); @@ -110,7 +115,8 @@ public class GeoDistanceIT extends ESIntegTestCase { } } indexRandom(true, cities); - prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer", "location", "type=geo_point").execute().actionGet(); + prepareCreate("empty_bucket_idx") + .addMapping("type", "value", "type=integer", "location", "type=geo_point").execute().actionGet(); List builders = new ArrayList<>(); for (int i = 0; i < 2; i++) { builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i).setSource(jsonBuilder() @@ -123,8 +129,7 @@ public class GeoDistanceIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void simple() throws Exception { + public void testSimple() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(geoDistance("amsterdam_rings") .field("location") @@ -146,7 +151,7 @@ public class GeoDistanceIT extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) (String) bucket.getKey(), equalTo("*-500.0")); + assertThat((String) bucket.getKey(), equalTo("*-500.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(0.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(500.0)); assertThat(bucket.getFromAsString(), equalTo("0.0")); @@ -155,7 +160,7 @@ public class GeoDistanceIT extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) (String) bucket.getKey(), equalTo("500.0-1000.0")); + assertThat((String) bucket.getKey(), equalTo("500.0-1000.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(500.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(1000.0)); assertThat(bucket.getFromAsString(), equalTo("500.0")); @@ -172,8 +177,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(1l)); } - @Test - public void simple_WithCustomKeys() throws Exception { + public void testSimpleWithCustomKeys() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(geoDistance("amsterdam_rings") .field("location") @@ -221,8 +225,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(1l)); } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { client().admin().cluster().prepareHealth("idx_unmapped").setWaitForYellowStatus().execute().actionGet(); SearchResponse response = client().prepareSearch("idx_unmapped") @@ -272,8 +275,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0l)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") .addAggregation(geoDistance("amsterdam_rings") .field("location") @@ -321,9 +323,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(1l)); } - - @Test - public void withSubAggregation() throws Exception { + public void testWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(geoDistance("amsterdam_rings") .field("location") @@ -409,8 +409,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat((Terms) propertiesCities[2], sameInstance(cities)); } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) @@ -437,8 +436,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat(buckets.get(0).getDocCount(), equalTo(0l)); } - @Test - public void multiValues() throws Exception { + public void testMultiValues() throws Exception { SearchResponse response = client().prepareSearch("idx-multi") .addAggregation(geoDistance("amsterdam_rings") .field("location") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index e08b6d780bd..eed080071bb 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -20,15 +20,15 @@ package org.elasticsearch.search.aggregations.bucket; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.ObjectIntMap; -import com.carrotsearch.hppc.ObjectObjectHashMap; -import com.carrotsearch.hppc.ObjectObjectMap; import com.carrotsearch.hppc.cursors.ObjectIntCursor; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -36,7 +36,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid.Bucket; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.VersionUtils; import java.util.ArrayList; import java.util.Arrays; @@ -54,10 +54,10 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; @ESIntegTestCase.SuiteScopeTestCase public class GeoHashGridIT extends ESIntegTestCase { + private Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); static ObjectIntMap expectedDocCountsForGeoHash = null; static ObjectIntMap multiValuedExpectedDocCountsForGeoHash = null; - static ObjectObjectMap expectedCentroidsForGeoHash = null; static int numDocs = 100; static String smallestGeoHash = null; @@ -75,50 +75,38 @@ public class GeoHashGridIT extends ESIntegTestCase { return indexCity(index, name, Arrays.asList(latLon)); } - private GeoPoint updateCentroid(GeoPoint centroid, double lat, double lon, final int docCount) { - if (centroid == null) { - return new GeoPoint(lat, lon); - } - final double newLon = centroid.lon() + (lon - centroid.lon()) / docCount; - final double newLat = centroid.lat() + (lat - centroid.lat()) / docCount; - return centroid.reset(newLat, newLon); - } - @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx_unmapped"); - assertAcked(prepareCreate("idx") + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + + assertAcked(prepareCreate("idx").setSettings(settings) .addMapping("type", "location", "type=geo_point", "city", "type=string,index=not_analyzed")); List cities = new ArrayList<>(); Random random = getRandom(); expectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); - expectedCentroidsForGeoHash = new ObjectObjectHashMap<>(numDocs *2); for (int i = 0; i < numDocs; i++) { //generate random point double lat = (180d * random.nextDouble()) - 90d; double lng = (360d * random.nextDouble()) - 180d; - String randomGeoHash = XGeoHashUtils.stringEncode(lng, lat, XGeoHashUtils.PRECISION); + String randomGeoHash = GeoHashUtils.stringEncode(lng, lat, GeoHashUtils.PRECISION); //Index at the highest resolution cities.add(indexCity("idx", randomGeoHash, lat + ", " + lng)); expectedDocCountsForGeoHash.put(randomGeoHash, expectedDocCountsForGeoHash.getOrDefault(randomGeoHash, 0) + 1); - expectedCentroidsForGeoHash.put(randomGeoHash, updateCentroid(expectedCentroidsForGeoHash.getOrDefault(randomGeoHash, - null), lat, lng, expectedDocCountsForGeoHash.get(randomGeoHash))); //Update expected doc counts for all resolutions.. - for (int precision = XGeoHashUtils.PRECISION - 1; precision > 0; precision--) { - String hash = XGeoHashUtils.stringEncode(lng, lat, precision); + for (int precision = GeoHashUtils.PRECISION - 1; precision > 0; precision--) { + String hash = GeoHashUtils.stringEncode(lng, lat, precision); if ((smallestGeoHash == null) || (hash.length() < smallestGeoHash.length())) { smallestGeoHash = hash; } expectedDocCountsForGeoHash.put(hash, expectedDocCountsForGeoHash.getOrDefault(hash, 0) + 1); - expectedCentroidsForGeoHash.put(hash, updateCentroid(expectedCentroidsForGeoHash.getOrDefault(hash, - null), lat, lng, expectedDocCountsForGeoHash.get(hash))); } } indexRandom(true, cities); - assertAcked(prepareCreate("multi_valued_idx") + assertAcked(prepareCreate("multi_valued_idx").setSettings(settings) .addMapping("type", "location", "type=geo_point", "city", "type=string,index=not_analyzed")); cities = new ArrayList<>(); @@ -132,8 +120,8 @@ public class GeoHashGridIT extends ESIntegTestCase { double lng = (360d * random.nextDouble()) - 180d; points.add(lat + "," + lng); // Update expected doc counts for all resolutions.. - for (int precision = XGeoHashUtils.PRECISION; precision > 0; precision--) { - final String geoHash = XGeoHashUtils.stringEncode(lng, lat, precision); + for (int precision = GeoHashUtils.PRECISION; precision > 0; precision--) { + final String geoHash = GeoHashUtils.stringEncode(lng, lat, precision); geoHashes.add(geoHash); } } @@ -147,10 +135,8 @@ public class GeoHashGridIT extends ESIntegTestCase { ensureSearchable(); } - - @Test - public void simple() throws Exception { - for (int precision = 1; precision <= XGeoHashUtils.PRECISION; precision++) { + public void testSimple() throws Exception { + for (int precision = 1; precision <= GeoHashUtils.PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") .addAggregation(geohashGrid("geohashgrid") .field("location") @@ -170,23 +156,18 @@ public class GeoHashGridIT extends ESIntegTestCase { long bucketCount = cell.getDocCount(); int expectedBucketCount = expectedDocCountsForGeoHash.get(geohash); - GeoPoint centroid = cell.getCentroid(); - GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); assertNotSame(bucketCount, 0); assertEquals("Geohash " + geohash + " has wrong doc count ", expectedBucketCount, bucketCount); - assertEquals("Geohash " + geohash + " has wrong centroid ", - expectedCentroid, centroid); GeoPoint geoPoint = (GeoPoint) propertiesKeys[i]; - assertThat(XGeoHashUtils.stringEncode(geoPoint.lon(), geoPoint.lat(), precision), equalTo(geohash)); + assertThat(GeoHashUtils.stringEncode(geoPoint.lon(), geoPoint.lat(), precision), equalTo(geohash)); assertThat((long) propertiesDocCounts[i], equalTo(bucketCount)); } } } - @Test - public void multivalued() throws Exception { - for (int precision = 1; precision <= XGeoHashUtils.PRECISION; precision++) { + public void testMultivalued() throws Exception { + for (int precision = 1; precision <= GeoHashUtils.PRECISION; precision++) { SearchResponse response = client().prepareSearch("multi_valued_idx") .addAggregation(geohashGrid("geohashgrid") .field("location") @@ -209,11 +190,10 @@ public class GeoHashGridIT extends ESIntegTestCase { } } - @Test - public void filtered() throws Exception { + public void testFiltered() throws Exception { GeoBoundingBoxQueryBuilder bbox = new GeoBoundingBoxQueryBuilder("location"); bbox.setCorners(smallestGeoHash, smallestGeoHash).queryName("bbox"); - for (int precision = 1; precision <= XGeoHashUtils.PRECISION; precision++) { + for (int precision = 1; precision <= GeoHashUtils.PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") .addAggregation( AggregationBuilders.filter("filtered").filter(bbox) @@ -243,9 +223,8 @@ public class GeoHashGridIT extends ESIntegTestCase { } } - @Test - public void unmapped() throws Exception { - for (int precision = 1; precision <= XGeoHashUtils.PRECISION; precision++) { + public void testUnmapped() throws Exception { + for (int precision = 1; precision <= GeoHashUtils.PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx_unmapped") .addAggregation(geohashGrid("geohashgrid") .field("location") @@ -261,9 +240,8 @@ public class GeoHashGridIT extends ESIntegTestCase { } - @Test - public void partiallyUnmapped() throws Exception { - for (int precision = 1; precision <= XGeoHashUtils.PRECISION; precision++) { + public void testPartiallyUnmapped() throws Exception { + for (int precision = 1; precision <= GeoHashUtils.PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") .addAggregation(geohashGrid("geohashgrid") .field("location") @@ -286,9 +264,8 @@ public class GeoHashGridIT extends ESIntegTestCase { } } - @Test public void testTopMatch() throws Exception { - for (int precision = 1; precision <= XGeoHashUtils.PRECISION; precision++) { + for (int precision = 1; precision <= GeoHashUtils.PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") .addAggregation(geohashGrid("geohashgrid") .field("location") @@ -319,10 +296,9 @@ public class GeoHashGridIT extends ESIntegTestCase { } } - @Test // making sure this doesn't runs into an OOME - public void sizeIsZero() { - for (int precision = 1; precision <= XGeoHashUtils.PRECISION; precision++) { + public void testSizeIsZero() { + for (int precision = 1; precision <= GeoHashUtils.PRECISION; precision++) { final int size = randomBoolean() ? 0 : randomIntBetween(1, Integer.MAX_VALUE); final int shardSize = randomBoolean() ? -1 : 0; SearchResponse response = client().prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java index a313d157214..9cebfeb9824 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java @@ -18,9 +18,6 @@ */ package org.elasticsearch.search.aggregations.bucket; -import java.util.ArrayList; -import java.util.List; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -28,7 +25,9 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; @@ -72,8 +71,7 @@ public class GlobalIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void withStatsSubAggregator() throws Exception { + public void testWithStatsSubAggregator() throws Exception { SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.termQuery("tag", "tag1")) .addAggregation(global("global") @@ -105,11 +103,8 @@ public class GlobalIT extends ESIntegTestCase { assertThat(stats.getSum(), equalTo((double) sum)); } - @Test - public void nonTopLevel() throws Exception { - + public void testNonTopLevel() throws Exception { try { - client().prepareSearch("idx") .setQuery(QueryBuilders.termQuery("tag", "tag1")) .addAggregation(global("global") @@ -119,7 +114,8 @@ public class GlobalIT extends ESIntegTestCase { fail("expected to fail executing non-top-level global aggregator. global aggregations are only allowed as top level" + "aggregations"); - } catch (ElasticsearchException ese) { + } catch (ElasticsearchException e) { + assertThat(e.getMessage(), is("all shards failed")); } } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java index 9b5df37cd8c..924ba7283f8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.search.aggregations.metrics.avg.Avg; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -88,8 +87,7 @@ public class MissingIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("unmapped_idx") .addAggregation(missing("missing_tag").field("tag")) .execute().actionGet(); @@ -103,8 +101,7 @@ public class MissingIT extends ESIntegTestCase { assertThat(missing.getDocCount(), equalTo((long) numDocsUnmapped)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx", "unmapped_idx") .addAggregation(missing("missing_tag").field("tag")) .execute().actionGet(); @@ -118,8 +115,7 @@ public class MissingIT extends ESIntegTestCase { assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped)); } - @Test - public void simple() throws Exception { + public void testSimple() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(missing("missing_tag").field("tag")) .execute().actionGet(); @@ -133,8 +129,7 @@ public class MissingIT extends ESIntegTestCase { assertThat(missing.getDocCount(), equalTo((long) numDocsMissing)); } - @Test - public void withSubAggregation() throws Exception { + public void testWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx", "unmapped_idx") .addAggregation(missing("missing_tag").field("tag") .subAggregation(avg("avg_value").field("value"))) @@ -165,9 +160,7 @@ public class MissingIT extends ESIntegTestCase { assertThat((double) missing.getProperty("avg_value.value"), equalTo((double) sum / (numDocsMissing + numDocsUnmapped))); } - @Test - public void withInheritedSubMissing() throws Exception { - + public void testWithInheritedSubMissing() throws Exception { SearchResponse response = client().prepareSearch("idx", "unmapped_idx") .addAggregation(missing("top_missing").field("tag") .subAggregation(missing("sub_missing"))) @@ -188,8 +181,7 @@ public class MissingIT extends ESIntegTestCase { assertThat(subMissing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped)); } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java index 3825901d54e..bfd68372542 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.metrics.MetricsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.Avg; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; @@ -148,23 +147,19 @@ public class NaNSortingIT extends ESIntegTestCase { assertCorrectlySorted(terms, asc, agg); } - @Test - public void stringTerms() { + public void testStringTerms() { testTerms("string_value"); } - @Test - public void longTerms() { + public void testLongTerms() { testTerms("long_value"); } - @Test - public void doubleTerms() { + public void testDoubleTerms() { testTerms("double_value"); } - @Test - public void longHistogram() { + public void testLongHistogram() { final boolean asc = randomBoolean(); SubAggregation agg = randomFrom(SubAggregation.values()); SearchResponse response = client().prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java index 7842fd847a0..a1f4b20dc1c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java @@ -36,7 +36,6 @@ import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -166,8 +165,7 @@ public class NestedIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void simple() throws Exception { + public void testSimple() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(nested("nested").path("nested") .subAggregation(stats("nested_value_stats").field("nested.value"))) @@ -205,8 +203,7 @@ public class NestedIT extends ESIntegTestCase { assertThat(stats.getAvg(), equalTo((double) sum / count)); } - @Test - public void nonExistingNestedField() throws Exception { + public void testNonExistingNestedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .addAggregation(nested("nested").path("value") .subAggregation(stats("nested_value_stats").field("nested.value"))) @@ -218,8 +215,7 @@ public class NestedIT extends ESIntegTestCase { assertThat(nested.getDocCount(), is(0l)); } - @Test - public void nestedWithSubTermsAgg() throws Exception { + public void testNestedWithSubTermsAgg() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(nested("nested").path("nested") .subAggregation(terms("values").field("nested.value").size(100) @@ -270,8 +266,7 @@ public class NestedIT extends ESIntegTestCase { assertThat((LongTerms) nested.getProperty("values"), sameInstance(values)); } - @Test - public void nestedAsSubAggregation() throws Exception { + public void testNestedAsSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("top_values").field("value").size(100) .collectMode(aggCollectionMode) @@ -299,8 +294,7 @@ public class NestedIT extends ESIntegTestCase { } } - @Test - public void nestNestedAggs() throws Exception { + public void testNestNestedAggs() throws Exception { SearchResponse response = client().prepareSearch("idx_nested_nested_aggs") .addAggregation(nested("level1").path("nested1") .subAggregation(terms("a").field("nested1.a") @@ -335,9 +329,7 @@ public class NestedIT extends ESIntegTestCase { assertThat(sum.getValue(), equalTo(2d)); } - - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) @@ -356,8 +348,7 @@ public class NestedIT extends ESIntegTestCase { assertThat(nested.getDocCount(), is(0l)); } - @Test - public void nestedOnObjectField() throws Exception { + public void testNestedOnObjectField() throws Exception { try { client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -369,7 +360,6 @@ public class NestedIT extends ESIntegTestCase { } } - @Test // Test based on: https://github.com/elasticsearch/elasticsearch/issues/9280 public void testParentFilterResolvedCorrectly() throws Exception { XContentBuilder mapping = jsonBuilder().startObject().startObject("provider").startObject("properties") @@ -468,8 +458,7 @@ public class NestedIT extends ESIntegTestCase { assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty } - @Test - public void nestedSameDocIdProcessedMultipleTime() throws Exception { + public void testNestedSameDocIdProcessedMultipleTime() throws Exception { assertAcked( prepareCreate("idx4") .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java index 75751c1fa48..44bd22af1c9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; @@ -35,8 +34,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; public class ParentIdAggIT extends ESIntegTestCase { - - @Test public void testParentIdAggregation() throws IOException { XContentBuilder mapping = jsonBuilder().startObject() .startObject("childtype") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java index 2a7be3b8b0a..88000630437 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -137,8 +136,7 @@ public class ReverseNestedIT extends ESIntegTestCase { indexRandom(false, client().prepareIndex("idx", "type2").setRouting("1").setSource(source)); } - @Test - public void simple_reverseNestedToRoot() throws Exception { + public void testSimpleReverseNestedToRoot() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type1") .addAggregation(nested("nested1").path("nested1") .subAggregation( @@ -326,8 +324,7 @@ public class ReverseNestedIT extends ESIntegTestCase { assertThat(tagsBuckets.get(3).getDocCount(), equalTo(1l)); } - @Test - public void simple_nested1ToRootToNested2() throws Exception { + public void testSimpleNested1ToRootToNested2() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type2") .addAggregation(nested("nested1").path("nested1") .subAggregation( @@ -349,8 +346,7 @@ public class ReverseNestedIT extends ESIntegTestCase { assertThat(nested.getDocCount(), equalTo(27l)); } - @Test - public void simple_reverseNestedToNested1() throws Exception { + public void testSimpleReverseNestedToNested1() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type2") .addAggregation(nested("nested1").path("nested1.nested2") .subAggregation( @@ -452,23 +448,26 @@ public class ReverseNestedIT extends ESIntegTestCase { assertThat(tagsBuckets.get(1).getKeyAsString(), equalTo("f")); } - @Test(expected = SearchPhaseExecutionException.class) - public void testReverseNestedAggWithoutNestedAgg() throws Exception { - client().prepareSearch("idx") - .addAggregation(terms("field2").field("nested1.nested2.field2") - .collectMode(randomFrom(SubAggCollectionMode.values())) - .subAggregation( - reverseNested("nested1_to_field1") - .subAggregation( - terms("field1").field("nested1.field1") - .collectMode(randomFrom(SubAggCollectionMode.values())) - ) - ) - ).get(); + public void testReverseNestedAggWithoutNestedAgg() { + try { + client().prepareSearch("idx") + .addAggregation(terms("field2").field("nested1.nested2.field2") + .collectMode(randomFrom(SubAggCollectionMode.values())) + .subAggregation( + reverseNested("nested1_to_field1") + .subAggregation( + terms("field1").field("nested1.field1") + .collectMode(randomFrom(SubAggCollectionMode.values())) + ) + ) + ).get(); + fail("Expected SearchPhaseExecutionException"); + } catch (SearchPhaseExecutionException e) { + assertThat(e.getMessage(), is("all shards failed")); + } } - @Test - public void nonExistingNestedField() throws Exception { + public void testNonExistingNestedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(nested("nested2").path("nested1.nested2").subAggregation(reverseNested("incorrect").path("nested3"))) @@ -482,7 +481,6 @@ public class ReverseNestedIT extends ESIntegTestCase { assertThat(reverseNested.getDocCount(), is(0l)); } - @Test public void testSameParentDocHavingMultipleBuckets() throws Exception { XContentBuilder mapping = jsonBuilder().startObject().startObject("product").field("dynamic", "strict").startObject("properties") .startObject("id").field("type", "long").endObject() diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java index 58987220909..2535ca33b72 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collection; @@ -58,7 +57,7 @@ public class SamplerIT extends ESIntegTestCase { return randomBoolean() ? null : randomFrom(SamplerAggregator.ExecutionMode.values()).toString(); } - + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, NUM_SHARDS, SETTING_NUMBER_OF_REPLICAS, 0).addMapping( @@ -70,7 +69,7 @@ public class SamplerIT extends ESIntegTestCase { .addMapping("book", "name", "type=string,index=analyzed", "genre", "type=string,index=not_analyzed", "price", "type=float")); ensureGreen(); - String data[] = { + String data[] = { // "id,cat,name,price,inStock,author_t,series_t,sequence_i,genre_s", "0553573403,book,A Game of Thrones,7.99,true,George R.R. Martin,A Song of Ice and Fire,1,fantasy", "0553579908,book,A Clash of Kings,7.99,true,George R.R. Martin,A Song of Ice and Fire,2,fantasy", @@ -84,7 +83,7 @@ public class SamplerIT extends ESIntegTestCase { "080508049X,book,The Black Cauldron,5.99,true,Lloyd Alexander,The Chronicles of Prydain,2,fantasy" }; - + for (int i = 0; i < data.length; i++) { String[] parts = data[i].split(","); client().prepareIndex("test", "book", "" + i).setSource("author", parts[5], "name", parts[2], "genre", parts[8], "price",Float.parseFloat(parts[3])).get(); @@ -92,12 +91,11 @@ public class SamplerIT extends ESIntegTestCase { } client().admin().indices().refresh(new RefreshRequest("test")).get(); } - - @Test - public void issue10719() throws Exception { + + public void testIssue10719() throws Exception { // Tests that we can refer to nested elements under a sample in a path // statement - boolean asc = randomBoolean(); + boolean asc = randomBoolean(); SearchResponse response = client().prepareSearch("test").setTypes("book").setSearchType(SearchType.QUERY_AND_FETCH) .addAggregation(terms("genres") .field("genre") @@ -125,8 +123,7 @@ public class SamplerIT extends ESIntegTestCase { } - @Test - public void noDiversity() throws Exception { + public void testNoDiversity() throws Exception { SamplerAggregationBuilder sampleAgg = new SamplerAggregationBuilder("sample").shardSize(100); sampleAgg.subAggregation(new TermsBuilder("authors").field("author")); SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_AND_FETCH) @@ -143,8 +140,7 @@ public class SamplerIT extends ESIntegTestCase { assertThat(maxBooksPerAuthor, equalTo(3l)); } - @Test - public void simpleDiversity() throws Exception { + public void testSimpleDiversity() throws Exception { int MAX_DOCS_PER_AUTHOR = 1; SamplerAggregationBuilder sampleAgg = new SamplerAggregationBuilder("sample").shardSize(100); sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint()); @@ -160,14 +156,13 @@ public class SamplerIT extends ESIntegTestCase { Sampler sample = response.getAggregations().get("sample"); Terms authors = sample.getAggregations().get("authors"); Collection testBuckets = authors.getBuckets(); - + for (Terms.Bucket testBucket : testBuckets) { assertThat(testBucket.getDocCount(), lessThanOrEqualTo((long) NUM_SHARDS * MAX_DOCS_PER_AUTHOR)); - } + } } - @Test - public void nestedDiversity() throws Exception { + public void testNestedDiversity() throws Exception { // Test multiple samples gathered under buckets made by a parent agg int MAX_DOCS_PER_AUTHOR = 1; TermsBuilder rootTerms = new TermsBuilder("genres").field("genre"); @@ -193,8 +188,7 @@ public class SamplerIT extends ESIntegTestCase { } } - @Test - public void nestedSamples() throws Exception { + public void testNestedSamples() throws Exception { // Test samples nested under samples int MAX_DOCS_PER_AUTHOR = 1; int MAX_DOCS_PER_GENRE = 2; @@ -226,8 +220,7 @@ public class SamplerIT extends ESIntegTestCase { } } - @Test - public void unmappedChildAggNoDiversity() throws Exception { + public void testUnmappedChildAggNoDiversity() throws Exception { SamplerAggregationBuilder sampleAgg = new SamplerAggregationBuilder("sample").shardSize(100); sampleAgg.subAggregation(new TermsBuilder("authors").field("author")); SearchResponse response = client().prepareSearch("idx_unmapped") @@ -244,10 +237,7 @@ public class SamplerIT extends ESIntegTestCase { assertThat(authors.getBuckets().size(), equalTo(0)); } - - - @Test - public void partiallyUnmappedChildAggNoDiversity() throws Exception { + public void testPartiallyUnmappedChildAggNoDiversity() throws Exception { SamplerAggregationBuilder sampleAgg = new SamplerAggregationBuilder("sample").shardSize(100); sampleAgg.subAggregation(new TermsBuilder("authors").field("author")); SearchResponse response = client().prepareSearch("idx_unmapped", "test") @@ -264,8 +254,7 @@ public class SamplerIT extends ESIntegTestCase { assertThat(authors.getBuckets().size(), greaterThan(0)); } - @Test - public void partiallyUnmappedDiversifyField() throws Exception { + public void testPartiallyUnmappedDiversifyField() throws Exception { // One of the indexes is missing the "author" field used for // diversifying results SamplerAggregationBuilder sampleAgg = new SamplerAggregationBuilder("sample").shardSize(100).field("author").maxDocsPerValue(1); @@ -280,9 +269,8 @@ public class SamplerIT extends ESIntegTestCase { assertThat(authors.getBuckets().size(), greaterThan(0)); } - @Test - public void whollyUnmappedDiversifyField() throws Exception { - //All of the indices are missing the "author" field used for diversifying results + public void testWhollyUnmappedDiversifyField() throws Exception { + //All of the indices are missing the "author" field used for diversifying results int MAX_DOCS_PER_AUTHOR = 1; SamplerAggregationBuilder sampleAgg = new SamplerAggregationBuilder("sample").shardSize(100); sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint()); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java index b71ee5b74ec..d138c0ccd3e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.QueryBuilders; @@ -33,7 +33,6 @@ import org.elasticsearch.search.aggregations.bucket.nested.Nested; import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; @@ -65,7 +64,7 @@ public class ShardReduceIT extends ESIntegTestCase { .startObject() .field("value", value) .field("ip", "10.0.0." + value) - .field("location", XGeoHashUtils.stringEncode(5, 52, XGeoHashUtils.PRECISION)) + .field("location", GeoHashUtils.stringEncode(5, 52, GeoHashUtils.PRECISION)) .field("date", date) .field("term-l", 1) .field("term-d", 1.5) @@ -88,9 +87,7 @@ public class ShardReduceIT extends ESIntegTestCase { ensureSearchable(); } - @Test public void testGlobal() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(global("global") @@ -104,9 +101,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testFilter() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(filter("filter").filter(QueryBuilders.matchAllQuery()) @@ -120,9 +115,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testMissing() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(missing("missing").field("foobar") @@ -136,9 +129,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testGlobalWithFilterWithMissing() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(global("global") @@ -156,9 +147,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testNested() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(nested("nested").path("nested") @@ -172,9 +161,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testStringTerms() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(terms("terms").field("term-s") @@ -189,9 +176,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testLongTerms() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(terms("terms").field("term-l") @@ -206,9 +191,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testDoubleTerms() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(terms("terms").field("term-d") @@ -223,9 +206,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testRange() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(range("range").field("value").addRange("r1", 0, 10) @@ -239,9 +220,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testDateRange() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(dateRange("range").field("date").addRange("r1", "2014-01-01", "2014-01-10") @@ -255,9 +234,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testIpRange() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(ipRange("range").field("ip").addRange("r1", "10.0.0.1", "10.0.0.10") @@ -271,9 +248,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testHistogram() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(histogram("topHisto").field("value").interval(5) @@ -287,9 +262,7 @@ public class ShardReduceIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); } - @Test public void testDateHistogram() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(dateHistogram("topHisto").field("date").interval(DateHistogramInterval.MONTH) @@ -304,9 +277,7 @@ public class ShardReduceIT extends ESIntegTestCase { } - @Test public void testGeoHashGrid() throws Exception { - SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .addAggregation(geohashGrid("grid").field("location") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java index e76f48ae762..78e4f7a099e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java @@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.junit.Test; import java.util.Collection; import java.util.HashMap; @@ -32,9 +31,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.hamcrest.Matchers.equalTo; public class ShardSizeTermsIT extends ShardSizeTestCase { - - @Test - public void noShardSize_string() throws Exception { + public void testNoShardSizeString() throws Exception { createIdx("type=string,index=not_analyzed"); indexData(); @@ -57,8 +54,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void shardSizeEqualsSize_string() throws Exception { + public void testShardSizeEqualsSizeString() throws Exception { createIdx("type=string,index=not_analyzed"); indexData(); @@ -81,8 +77,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void withShardSize_string() throws Exception { + public void testWithShardSizeString() throws Exception { createIdx("type=string,index=not_analyzed"); @@ -106,8 +101,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void withShardSize_string_singleShard() throws Exception { + public void testWithShardSizeStringSingleShard() throws Exception { createIdx("type=string,index=not_analyzed"); @@ -131,8 +125,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void noShardSizeTermOrder_string() throws Exception { + public void testNoShardSizeTermOrderString() throws Exception { createIdx("type=string,index=not_analyzed"); indexData(); @@ -155,9 +148,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void noShardSize_long() throws Exception { - + public void testNoShardSizeLong() throws Exception { createIdx("type=long"); indexData(); @@ -180,9 +171,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void shardSizeEqualsSize_long() throws Exception { - + public void testShardSizeEqualsSizeLong() throws Exception { createIdx("type=long"); indexData(); @@ -205,9 +194,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void withShardSize_long() throws Exception { - + public void testWithShardSizeLong() throws Exception { createIdx("type=long"); indexData(); @@ -230,8 +217,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void withShardSize_long_singleShard() throws Exception { + public void testWithShardSizeLongSingleShard() throws Exception { createIdx("type=long"); @@ -255,9 +241,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void noShardSizeTermOrder_long() throws Exception { - + public void testNoShardSizeTermOrderLong() throws Exception { createIdx("type=long"); indexData(); @@ -280,9 +264,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void noShardSize_double() throws Exception { - + public void testNoShardSizeDouble() throws Exception { createIdx("type=double"); indexData(); @@ -305,9 +287,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void shardSizeEqualsSize_double() throws Exception { - + public void testShardSizeEqualsSizeDouble() throws Exception { createIdx("type=double"); indexData(); @@ -330,9 +310,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void withShardSize_double() throws Exception { - + public void testWithShardSizeDouble() throws Exception { createIdx("type=double"); indexData(); @@ -355,9 +333,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void withShardSize_double_singleShard() throws Exception { - + public void testWithShardSizeDoubleSingleShard() throws Exception { createIdx("type=double"); indexData(); @@ -380,9 +356,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase { } } - @Test - public void noShardSizeTermOrder_double() throws Exception { - + public void testNoShardSizeTermOrderDouble() throws Exception { createIdx("type=double"); indexData(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsBackwardCompatibilityIT.java index 556c012c98b..9273621dee5 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsBackwardCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsBackwardCompatibilityIT.java @@ -18,102 +18,18 @@ */ package org.elasticsearch.search.aggregations.bucket; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; -import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsBuilder; -import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; +import org.elasticsearch.test.search.aggregations.bucket.SharedSignificantTermsTestMethods; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutionException; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.equalTo; - -/** - */ public class SignificantTermsBackwardCompatibilityIT extends ESBackcompatTestCase { - static final String INDEX_NAME = "testidx"; - static final String DOC_TYPE = "doc"; - static final String TEXT_FIELD = "text"; - static final String CLASS_FIELD = "class"; - /** - * Simple upgrade test for streaming significant terms buckets + * Test for streaming significant terms buckets to old es versions. */ - @Test - public void testBucketStreaming() throws IOException, ExecutionException, InterruptedException { - - logger.debug("testBucketStreaming: indexing documents"); - String type = randomBoolean() ? "string" : "long"; - String settings = "{\"index.number_of_shards\": 5, \"index.number_of_replicas\": 0}"; - index01Docs(type, settings); - - logClusterState(); - boolean upgraded; - int upgradedNodesCounter = 1; - do { - logger.debug("testBucketStreaming: upgrading {}st node", upgradedNodesCounter++); - upgraded = backwardsCluster().upgradeOneNode(); - ensureGreen(); - logClusterState(); - checkSignificantTermsAggregationCorrect(); - } while (upgraded); - logger.debug("testBucketStreaming: done testing significant terms while upgrading"); - } - - private void index01Docs(String type, String settings) throws ExecutionException, InterruptedException { - String mappings = "{\"doc\": {\"properties\":{\"" + TEXT_FIELD + "\": {\"type\":\"" + type + "\"},\"" + CLASS_FIELD - + "\": {\"type\":\"string\"}}}}"; - assertAcked(prepareCreate(INDEX_NAME).setSettings(settings).addMapping("doc", mappings)); - String[] gb = {"0", "1"}; - List indexRequestBuilderList = new ArrayList<>(); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "1") - .setSource(TEXT_FIELD, "1", CLASS_FIELD, "1")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "2") - .setSource(TEXT_FIELD, "1", CLASS_FIELD, "1")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "3") - .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "4") - .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "5") - .setSource(TEXT_FIELD, gb, CLASS_FIELD, "1")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "6") - .setSource(TEXT_FIELD, gb, CLASS_FIELD, "0")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "7") - .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); - indexRandom(true, indexRequestBuilderList); - } - - private void checkSignificantTermsAggregationCorrect() { - - SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) - .addAggregation(new TermsBuilder("class").field(CLASS_FIELD).subAggregation( - new SignificantTermsBuilder("sig_terms") - .field(TEXT_FIELD))) - .execute() - .actionGet(); - assertSearchResponse(response); - StringTerms classes = (StringTerms) response.getAggregations().get("class"); - assertThat(classes.getBuckets().size(), equalTo(2)); - for (Terms.Bucket classBucket : classes.getBuckets()) { - Map aggs = classBucket.getAggregations().asMap(); - assertTrue(aggs.containsKey("sig_terms")); - SignificantTerms agg = (SignificantTerms) aggs.get("sig_terms"); - assertThat(agg.getBuckets().size(), equalTo(1)); - String term = agg.iterator().next().getKeyAsString(); - String classTerm = classBucket.getKeyAsString(); - assertTrue(term.equals(classTerm)); - } + public void testAggregateAndCheckFromSeveralShards() throws IOException, ExecutionException, InterruptedException { + SharedSignificantTermsTestMethods.aggregateAndCheckFromSeveralShards(this); } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java index 882f2b7318f..7582d75ca0b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java @@ -36,7 +36,6 @@ import org.elasticsearch.search.aggregations.bucket.significant.heuristics.Perce import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.HashMap; @@ -72,7 +71,7 @@ public class SignificantTermsIT extends ESIntegTestCase { public static final int MUSIC_CATEGORY=1; public static final int OTHER_CATEGORY=2; public static final int SNOWBOARDING_CATEGORY=3; - + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 5, SETTING_NUMBER_OF_REPLICAS, 0).addMapping("fact", @@ -81,7 +80,7 @@ public class SignificantTermsIT extends ESIntegTestCase { createIndex("idx_unmapped"); ensureGreen(); - String data[] = { + String data[] = { "A\t1\tpaul weller was lead singer of the jam before the style council", "B\t1\tpaul weller left the jam to form the style council", "A\t2\tpaul smith is a designer in the fashion industry", @@ -100,9 +99,9 @@ public class SignificantTermsIT extends ESIntegTestCase { "B\t3\tterje haakonsen has credited craig kelly as his snowboard mentor", "A\t3\tterje haakonsen and craig kelly were some of the first snowboarders sponsored by burton snowboards", "B\t3\tlike craig kelly before him terje won the mt baker banked slalom many times - once riding switch", - "A\t3\tterje haakonsen has been a team rider for burton snowboards for over 20 years" + "A\t3\tterje haakonsen has been a team rider for burton snowboards for over 20 years" }; - + for (int i = 0; i < data.length; i++) { String[] parts = data[i].split("\t"); client().prepareIndex("test", "fact", "" + i) @@ -112,8 +111,7 @@ public class SignificantTermsIT extends ESIntegTestCase { client().admin().indices().refresh(new RefreshRequest("test")).get(); } - @Test - public void structuredAnalysis() throws Exception { + public void testStructuredAnalysis() throws Exception { SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "terje")) @@ -127,9 +125,8 @@ public class SignificantTermsIT extends ESIntegTestCase { Number topCategory = (Number) topTerms.getBuckets().iterator().next().getKey(); assertTrue(topCategory.equals(new Long(SNOWBOARDING_CATEGORY))); } - - @Test - public void structuredAnalysisWithIncludeExclude() throws Exception { + + public void testStructuredAnalysisWithIncludeExclude() throws Exception { long[] excludeTerms = { MUSIC_CATEGORY }; SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) @@ -145,8 +142,7 @@ public class SignificantTermsIT extends ESIntegTestCase { assertTrue(topCategory.equals(new Long(OTHER_CATEGORY))); } - @Test - public void includeExclude() throws Exception { + public void testIncludeExclude() throws Exception { SearchResponse response = client().prepareSearch("test") .setQuery(new TermQueryBuilder("_all", "weller")) .addAggregation(new SignificantTermsBuilder("mySignificantTerms").field("description").executionHint(randomExecutionHint()) @@ -180,9 +176,8 @@ public class SignificantTermsIT extends ESIntegTestCase { assertThat(terms, hasSize(1)); assertThat(terms.contains("weller"), is(true)); } - - @Test - public void includeExcludeExactValues() throws Exception { + + public void testIncludeExcludeExactValues() throws Exception { String []incExcTerms={"weller","nosuchterm"}; SearchResponse response = client().prepareSearch("test") .setQuery(new TermQueryBuilder("_all", "weller")) @@ -210,10 +205,9 @@ public class SignificantTermsIT extends ESIntegTestCase { } assertThat(terms, hasSize(1)); assertThat(terms.contains("weller"), is(true)); - } - - @Test - public void unmapped() throws Exception { + } + + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "terje")) @@ -223,12 +217,11 @@ public class SignificantTermsIT extends ESIntegTestCase { .execute() .actionGet(); assertSearchResponse(response); - SignificantTerms topTerms = response.getAggregations().get("mySignificantTerms"); + SignificantTerms topTerms = response.getAggregations().get("mySignificantTerms"); assertThat(topTerms.getBuckets().size(), equalTo(0)); } - @Test - public void textAnalysis() throws Exception { + public void testTextAnalysis() throws Exception { SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "terje")) @@ -242,8 +235,7 @@ public class SignificantTermsIT extends ESIntegTestCase { checkExpectedStringTermsFound(topTerms); } - @Test - public void textAnalysisGND() throws Exception { + public void testTextAnalysisGND() throws Exception { SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "terje")) @@ -257,8 +249,7 @@ public class SignificantTermsIT extends ESIntegTestCase { checkExpectedStringTermsFound(topTerms); } - @Test - public void textAnalysisChiSquare() throws Exception { + public void testTextAnalysisChiSquare() throws Exception { SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "terje")) @@ -272,8 +263,7 @@ public class SignificantTermsIT extends ESIntegTestCase { checkExpectedStringTermsFound(topTerms); } - @Test - public void textAnalysisPercentageScore() throws Exception { + public void testTextAnalysisPercentageScore() throws Exception { SearchResponse response = client() .prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) @@ -289,16 +279,15 @@ public class SignificantTermsIT extends ESIntegTestCase { checkExpectedStringTermsFound(topTerms); } - @Test - public void badFilteredAnalysis() throws Exception { + public void testBadFilteredAnalysis() throws Exception { // Deliberately using a bad choice of filter here for the background context in order - // to test robustness. + // to test robustness. // We search for the name of a snowboarder but use music-related content (fact_category:1) // as the background source of term statistics. SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "terje")) - .setFrom(0).setSize(60).setExplain(true) + .setFrom(0).setSize(60).setExplain(true) .addAggregation(new SignificantTermsBuilder("mySignificantTerms").field("description") .minDocCount(2).backgroundFilter(QueryBuilders.termQuery("fact_category", 1))) .execute() @@ -316,15 +305,13 @@ public class SignificantTermsIT extends ESIntegTestCase { } } assertTrue(hasMissingBackgroundTerms); - } - - - @Test - public void filteredAnalysis() throws Exception { + } + + public void testFilteredAnalysis() throws Exception { SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "weller")) - .setFrom(0).setSize(60).setExplain(true) + .setFrom(0).setSize(60).setExplain(true) .addAggregation(new SignificantTermsBuilder("mySignificantTerms").field("description") .minDocCount(1).backgroundFilter(QueryBuilders.termsQuery("description", "paul"))) .execute() @@ -335,16 +322,15 @@ public class SignificantTermsIT extends ESIntegTestCase { for (Bucket topTerm : topTerms) { topWords.add(topTerm.getKeyAsString()); } - //The word "paul" should be a constant of all docs in the background set and therefore not seen as significant + //The word "paul" should be a constant of all docs in the background set and therefore not seen as significant assertFalse(topWords.contains("paul")); - //"Weller" is the only Paul who was in The Jam and therefore this should be identified as a differentiator from the background of all other Pauls. + //"Weller" is the only Paul who was in The Jam and therefore this should be identified as a differentiator from the background of all other Pauls. assertTrue(topWords.contains("jam")); - } + } - @Test - public void nestedAggs() throws Exception { + public void testNestedAggs() throws Exception { String[][] expectedKeywordsByCategory={ - { "paul", "weller", "jam", "style", "council" }, + { "paul", "weller", "jam", "style", "council" }, { "paul", "smith" }, { "craig", "kelly", "terje", "haakonsen", "burton" }}; SearchResponse response = client().prepareSearch("test") @@ -369,11 +355,9 @@ public class SignificantTermsIT extends ESIntegTestCase { assertTrue(expectedKeyword + " missing from category keywords", foundTopWords.contains(expectedKeyword)); } } - } + } - - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped", "test") .setSearchType(SearchType.QUERY_AND_FETCH) .setQuery(new TermQueryBuilder("_all", "terje")) @@ -421,7 +405,6 @@ public class SignificantTermsIT extends ESIntegTestCase { checkExpectedStringTermsFound(topTerms); } - @Test public void testMutualInformation() throws Exception { SearchResponse response = client().prepareSearch("test") .setSearchType(SearchType.QUERY_AND_FETCH) diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SignificantTermsSignificanceScoreTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java similarity index 77% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SignificantTermsSignificanceScoreTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index 5e6625a26bd..1f77ca5bb64 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SignificantTermsSignificanceScoreTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -16,15 +16,13 @@ * specific language governing permissions and limitations * under the License. */ - -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -33,86 +31,59 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptModule; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; import org.elasticsearch.search.aggregations.bucket.script.NativeSignificanceScoreScriptNoParams; import org.elasticsearch.search.aggregations.bucket.script.NativeSignificanceScoreScriptWithParams; -import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorFactory; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsBuilder; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ScriptHeuristic; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicBuilder; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicStreams; +import org.elasticsearch.search.aggregations.bucket.significant.heuristics.*; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.search.aggregations.bucket.SharedSignificantTermsTestMethods; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutionException; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; /** * */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) -public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { +public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase { static final String INDEX_NAME = "testidx"; static final String DOC_TYPE = "doc"; static final String TEXT_FIELD = "text"; static final String CLASS_FIELD = "class"; - @Override - public Settings nodeSettings(int nodeOrdinal) { - return settingsBuilder() - .put(super.nodeSettings(nodeOrdinal)) - .put("path.conf", this.getDataPath("conf")) - .build(); - } - @Override protected Collection> nodePlugins() { - return pluginList(CustomSignificanceHeuristicPlugin.class, GroovyPlugin.class); + return pluginList(CustomSignificanceHeuristicPlugin.class); } public String randomExecutionHint() { return randomBoolean() ? null : randomFrom(SignificantTermsAggregatorFactory.ExecutionMode.values()).toString(); } - @Test public void testPlugin() throws Exception { String type = randomBoolean() ? "string" : "long"; String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; - index01Docs(type, settings); + SharedSignificantTermsTestMethods.index01Docs(type, settings, this); SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) .addAggregation(new TermsBuilder("class") .field(CLASS_FIELD) @@ -177,6 +148,10 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { public static class CustomSignificanceHeuristicPlugin extends Plugin { + static { + SignificanceHeuristicStreams.registerStream(SimpleHeuristic.STREAM); + } + @Override public String name() { return "test-plugin-significance-heuristic"; @@ -189,7 +164,6 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { public void onModule(SearchModule significanceModule) { significanceModule.registerHeuristicParser(SimpleHeuristic.SimpleHeuristicParser.class); - significanceModule.registerStream(SimpleHeuristic.STREAM); } public void onModule(ScriptModule module) { module.registerScript(NativeSignificanceScoreScriptNoParams.NATIVE_SIGNIFICANCE_SCORE_SCRIPT_NO_PARAMS, NativeSignificanceScoreScriptNoParams.Factory.class); @@ -259,13 +233,10 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { } } - - @Test public void testXContentResponse() throws Exception { - String type = randomBoolean() ? "string" : "long"; String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; - index01Docs(type, settings); + SharedSignificantTermsTestMethods.index01Docs(type, settings, this); SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) .addAggregation(new TermsBuilder("class").field(CLASS_FIELD).subAggregation(new SignificantTermsBuilder("sig_terms").field(TEXT_FIELD))) .execute() @@ -295,7 +266,6 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { } - @Test public void testDeletesIssue7951() throws Exception { String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; String mappings = "{\"doc\": {\"properties\":{\"text\": {\"type\":\"string\",\"index\":\"not_analyzed\"}}}}"; @@ -338,11 +308,10 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { .actionGet(); } - @Test public void testBackgroundVsSeparateSet() throws Exception { String type = randomBoolean() ? "string" : "long"; String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}"; - index01Docs(type, settings); + SharedSignificantTermsTestMethods.index01Docs(type, settings, this); testBackgroundVsSeparateSet(new MutualInformation.MutualInformationBuilder(true, true), new MutualInformation.MutualInformationBuilder(true, false)); testBackgroundVsSeparateSet(new ChiSquare.ChiSquareBuilder(true, true), new ChiSquare.ChiSquareBuilder(true, false)); testBackgroundVsSeparateSet(new GND.GNDBuilder(true), new GND.GNDBuilder(false)); @@ -403,29 +372,6 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { assertThat(score11Background, equalTo(score11SeparateSets)); } - private void index01Docs(String type, String settings) throws ExecutionException, InterruptedException { - String mappings = "{\"doc\": {\"properties\":{\"text\": {\"type\":\"" + type + "\"}}}}"; - assertAcked(prepareCreate(INDEX_NAME).setSettings(settings).addMapping("doc", mappings)); - String[] gb = {"0", "1"}; - List indexRequestBuilderList = new ArrayList<>(); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "1") - .setSource(TEXT_FIELD, "1", CLASS_FIELD, "1")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "2") - .setSource(TEXT_FIELD, "1", CLASS_FIELD, "1")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "3") - .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "4") - .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "5") - .setSource(TEXT_FIELD, gb, CLASS_FIELD, "1")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "6") - .setSource(TEXT_FIELD, gb, CLASS_FIELD, "0")); - indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "7") - .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); - indexRandom(true, false, indexRequestBuilderList); - } - - @Test public void testScoresEqualForPositiveAndNegative() throws Exception { indexEqualTestData(); testScoresEqualForPositiveAndNegative(new MutualInformation.MutualInformationBuilder(true, true)); @@ -491,7 +437,6 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { indexRandom(true, false, indexRequestBuilders); } - @Test public void testScriptScore() throws ExecutionException, InterruptedException, IOException { indexRandomFrequencies01(randomBoolean() ? "string" : "long"); ScriptHeuristic.ScriptHeuristicBuilder scriptHeuristicBuilder = getScriptSignificanceHeuristicBuilder(); @@ -512,92 +457,15 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { } } - @Test - public void testNoNumberFormatExceptionWithDefaultScriptingEngine() throws ExecutionException, InterruptedException, IOException { - assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1))); - index("test", "doc", "1", "{\"field\":\"a\"}"); - index("test", "doc", "11", "{\"field\":\"a\"}"); - index("test", "doc", "2", "{\"field\":\"b\"}"); - index("test", "doc", "22", "{\"field\":\"b\"}"); - index("test", "doc", "3", "{\"field\":\"a b\"}"); - index("test", "doc", "33", "{\"field\":\"a b\"}"); - ScriptHeuristic.ScriptHeuristicBuilder scriptHeuristicBuilder = new ScriptHeuristic.ScriptHeuristicBuilder(); - scriptHeuristicBuilder.setScript(new Script("_subset_freq/(_superset_freq - _subset_freq + 1)")); - ensureYellow(); - refresh(); - SearchResponse response = client() - .prepareSearch("test") - .addAggregation( - new TermsBuilder("letters").field("field").subAggregation( - new SignificantTermsBuilder("mySignificantTerms").field("field").executionHint(randomExecutionHint()) - .significanceHeuristic(scriptHeuristicBuilder).minDocCount(1).shardSize(2).size(2))).execute() - .actionGet(); - assertSearchResponse(response); - assertThat(((Terms) response.getAggregations().get("letters")).getBuckets().size(), equalTo(2)); - for (Terms.Bucket classBucket : ((Terms) response.getAggregations().get("letters")).getBuckets()) { - assertThat(((SignificantStringTerms) classBucket.getAggregations().get("mySignificantTerms")).getBuckets().size(), equalTo(2)); - for (SignificantTerms.Bucket bucket : ((SignificantTerms) classBucket.getAggregations().get("mySignificantTerms")).getBuckets()) { - assertThat(bucket.getSignificanceScore(), - closeTo((double) bucket.getSubsetDf() / (bucket.getSupersetDf() - bucket.getSubsetDf() + 1), 1.e-6)); - } - } - } - private ScriptHeuristic.ScriptHeuristicBuilder getScriptSignificanceHeuristicBuilder() throws IOException { - Map params = null; Script script = null; - String lang = null; if (randomBoolean()) { + Map params = null; params = new HashMap<>(); params.put("param", randomIntBetween(1, 100)); - } - int randomScriptKind = randomIntBetween(0, 3); - if (randomBoolean()) { - lang = "groovy"; - } - switch (randomScriptKind) { - case 0: { - if (params == null) { - script = new Script("return _subset_freq + _subset_size + _superset_freq + _superset_size"); - } else { - script = new Script("return param*(_subset_freq + _subset_size + _superset_freq + _superset_size)/param", - ScriptType.INLINE, lang, params); - } - break; - } - case 1: { - String scriptString; - if (params == null) { - scriptString = "return _subset_freq + _subset_size + _superset_freq + _superset_size"; - } else { - scriptString = "return param*(_subset_freq + _subset_size + _superset_freq + _superset_size)/param"; - } - client().prepareIndex().setIndex(ScriptService.SCRIPT_INDEX).setType(ScriptService.DEFAULT_LANG).setId("my_script") - .setSource(XContentFactory.jsonBuilder().startObject().field("script", scriptString).endObject()).get(); - refresh(); - script = new Script("my_script", ScriptType.INDEXED, lang, params); - break; - } - case 2: { - if (params == null) { - script = new Script("significance_script_no_params", ScriptType.FILE, lang, null); - } else { - script = new Script("significance_script_with_params", ScriptType.FILE, lang, params); - } - break; - } - case 3: { - logger.info("NATIVE SCRIPT"); - if (params == null) { - script = new Script("native_significance_score_script_no_params", ScriptType.INLINE, "native", null); - } else { - script = new Script("native_significance_score_script_with_params", ScriptType.INLINE, "native", params); - } - lang = "native"; - if (randomBoolean()) { - } - break; - } + script = new Script("native_significance_score_script_with_params", ScriptType.INLINE, "native", params); + } else { + script = new Script("native_significance_score_script_no_params", ScriptType.INLINE, "native", null); } ScriptHeuristic.ScriptHeuristicBuilder builder = new ScriptHeuristic.ScriptHeuristicBuilder().setScript(script); @@ -622,4 +490,9 @@ public class SignificantTermsSignificanceScoreTests extends ESIntegTestCase { } indexRandom(true, indexRequestBuilderList); } + + public void testReduceFromSeveralShards() throws IOException, ExecutionException, InterruptedException { + SharedSignificantTermsTestMethods.aggregateAndCheckFromSeveralShards(this); + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java index db6f5be9024..79aa6b2d5c9 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -103,7 +102,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertThat(accurateTerms, notNullValue()); assertThat(accurateTerms.getName(), equalTo("terms")); assertThat(accurateTerms.getDocCountError(), equalTo(0l)); - + Terms testTerms = testResponse.getAggregations().get("terms"); assertThat(testTerms, notNullValue()); assertThat(testTerms.getName(), equalTo("terms")); @@ -111,7 +110,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { Collection testBuckets = testTerms.getBuckets(); assertThat(testBuckets.size(), lessThanOrEqualTo(size)); assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size())); - + for (Terms.Bucket testBucket : testBuckets) { assertThat(testBucket, notNullValue()); Terms.Bucket accurateBucket = accurateTerms.getBucketByKey(testBucket.getKeyAsString()); @@ -121,14 +120,14 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertThat(testBucket.getDocCount() + testBucket.getDocCountError(), greaterThanOrEqualTo(accurateBucket.getDocCount())); assertThat(testBucket.getDocCount() - testBucket.getDocCountError(), lessThanOrEqualTo(accurateBucket.getDocCount())); } - + for (Terms.Bucket accurateBucket: accurateTerms.getBuckets()) { assertThat(accurateBucket, notNullValue()); Terms.Bucket testBucket = accurateTerms.getBucketByKey(accurateBucket.getKeyAsString()); if (testBucket == null) { assertThat(accurateBucket.getDocCount(), lessThanOrEqualTo(testTerms.getDocCountError())); } - + } } @@ -137,7 +136,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertThat(accurateTerms, notNullValue()); assertThat(accurateTerms.getName(), equalTo("terms")); assertThat(accurateTerms.getDocCountError(), equalTo(0l)); - + Terms testTerms = testResponse.getAggregations().get("terms"); assertThat(testTerms, notNullValue()); assertThat(testTerms.getName(), equalTo("terms")); @@ -145,7 +144,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { Collection testBuckets = testTerms.getBuckets(); assertThat(testBuckets.size(), lessThanOrEqualTo(size)); assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size())); - + for (Terms.Bucket testBucket : testBuckets) { assertThat(testBucket, notNullValue()); Terms.Bucket accurateBucket = accurateTerms.getBucketByKey(testBucket.getKeyAsString()); @@ -162,7 +161,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertThat(testTerms.getDocCountError(), equalTo(0l)); Collection testBuckets = testTerms.getBuckets(); assertThat(testBuckets.size(), lessThanOrEqualTo(size)); - + for (Terms.Bucket testBucket : testBuckets) { assertThat(testBucket, notNullValue()); assertThat(testBucket.getDocCountError(), equalTo(0l)); @@ -174,7 +173,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertThat(accurateTerms, notNullValue()); assertThat(accurateTerms.getName(), equalTo("terms")); assertThat(accurateTerms.getDocCountError(), equalTo(0l)); - + Terms testTerms = testResponse.getAggregations().get("terms"); assertThat(testTerms, notNullValue()); assertThat(testTerms.getName(), equalTo("terms")); @@ -182,7 +181,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { Collection testBuckets = testTerms.getBuckets(); assertThat(testBuckets.size(), lessThanOrEqualTo(size)); assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size())); - + for (Terms.Bucket testBucket : testBuckets) { assertThat(testBucket, notNullValue()); Terms.Bucket accurateBucket = accurateTerms.getBucketByKey(testBucket.getKeyAsString()); @@ -192,8 +191,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { } } - @Test - public void stringValueField() throws Exception { + public void testStringValueField() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx").setTypes("type") @@ -207,7 +205,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -223,8 +221,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertDocCountErrorWithinBounds(size, accurateResponse, testResponse); } - @Test - public void stringValueField_singleShard() throws Exception { + public void testStringValueFieldSingleShard() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -238,7 +235,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -254,11 +251,10 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void stringValueField_withRouting() throws Exception { + public void testStringValueFieldWithRouting() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); - + SearchResponse testResponse = client().prepareSearch("idx_with_routing").setTypes("type").setRouting(String.valueOf(between(1, numRoutingValues))) .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -274,8 +270,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountErrorSingleResponse(size, testResponse); } - @Test - public void stringValueField_docCountAsc() throws Exception { + public void testStringValueFieldDocCountAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -290,7 +285,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -307,8 +302,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void stringValueField_termSortAsc() throws Exception { + public void testStringValueFieldTermSortAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -323,7 +317,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -340,8 +334,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void stringValueField_termSortDesc() throws Exception { + public void testStringValueFieldTermSortDesc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -356,7 +349,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -373,8 +366,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void stringValueField_subAggAsc() throws Exception { + public void testStringValueFieldSubAggAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -390,7 +382,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -408,8 +400,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void stringValueField_subAggDesc() throws Exception { + public void testStringValueFieldSubAggDesc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -425,7 +416,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -443,8 +434,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void longValueField() throws Exception { + public void testLongValueField() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx").setTypes("type") @@ -458,7 +448,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -474,8 +464,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertDocCountErrorWithinBounds(size, accurateResponse, testResponse); } - @Test - public void longValueField_singleShard() throws Exception { + public void testLongValueFieldSingleShard() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -489,7 +478,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -505,11 +494,10 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void longValueField_withRouting() throws Exception { + public void testLongValueFieldWithRouting() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); - + SearchResponse testResponse = client().prepareSearch("idx_with_routing").setTypes("type").setRouting(String.valueOf(between(1, numRoutingValues))) .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -525,8 +513,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountErrorSingleResponse(size, testResponse); } - @Test - public void longValueField_docCountAsc() throws Exception { + public void testLongValueFieldDocCountAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -541,7 +528,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -558,8 +545,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void longValueField_termSortAsc() throws Exception { + public void testLongValueFieldTermSortAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -574,7 +560,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -591,8 +577,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void longValueField_termSortDesc() throws Exception { + public void testLongValueFieldTermSortDesc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -607,7 +592,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -624,8 +609,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void longValueField_subAggAsc() throws Exception { + public void testLongValueFieldSubAggAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -641,7 +625,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -659,8 +643,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void longValueField_subAggDesc() throws Exception { + public void testLongValueFieldSubAggDesc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -676,7 +659,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -694,8 +677,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void doubleValueField() throws Exception { + public void testDoubleValueField() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx").setTypes("type") @@ -709,7 +691,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -725,8 +707,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertDocCountErrorWithinBounds(size, accurateResponse, testResponse); } - @Test - public void doubleValueField_singleShard() throws Exception { + public void testDoubleValueFieldSingleShard() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -740,7 +721,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -756,11 +737,10 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void doubleValueField_withRouting() throws Exception { + public void testDoubleValueFieldWithRouting() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); - + SearchResponse testResponse = client().prepareSearch("idx_with_routing").setTypes("type").setRouting(String.valueOf(between(1, numRoutingValues))) .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -776,8 +756,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountErrorSingleResponse(size, testResponse); } - @Test - public void doubleValueField_docCountAsc() throws Exception { + public void testDoubleValueFieldDocCountAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -792,7 +771,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -809,8 +788,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void doubleValueField_termSortAsc() throws Exception { + public void testDoubleValueFieldTermSortAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -825,7 +803,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -842,8 +820,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void doubleValueField_termSortDesc() throws Exception { + public void testDoubleValueFieldTermSortDesc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -858,7 +835,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -875,8 +852,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertNoDocCountError(size, accurateResponse, testResponse); } - @Test - public void doubleValueField_subAggAsc() throws Exception { + public void testDoubleValueFieldSubAggAsc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -892,7 +868,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) @@ -910,8 +886,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { assertUnboundedDocCountError(size, accurateResponse, testResponse); } - @Test - public void doubleValueField_subAggDesc() throws Exception { + public void testDoubleValueFieldSubAggDesc() throws Exception { int size = randomIntBetween(1, 20); int shardSize = randomIntBetween(size, size * 2); SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type") @@ -927,7 +902,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase { .execute().actionGet(); assertSearchResponse(accurateResponse); - + SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java index 03aebd6058c..9a7b337dc9e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -51,9 +50,7 @@ public class TermsShardMinDocCountIT extends ESIntegTestCase { } // see https://github.com/elasticsearch/elasticsearch/issues/5998 - @Test - public void shardMinDocCountSignificantTermsTest() throws Exception { - + public void testShardMinDocCountSignificantTermsTest() throws Exception { String termtype = "string"; if (randomBoolean()) { termtype = "long"; @@ -111,8 +108,7 @@ public class TermsShardMinDocCountIT extends ESIntegTestCase { } // see https://github.com/elasticsearch/elasticsearch/issues/5998 - @Test - public void shardMinDocCountTermsTest() throws Exception { + public void testShardMinDocCountTermsTest() throws Exception { final String [] termTypes = {"string", "long", "integer", "float", "double"}; String termtype = termTypes[randomInt(termTypes.length - 1)]; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java new file mode 100644 index 00000000000..cd7dadd7eeb --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.search.SearchParseException; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.TestSearchContext; + +public class GeoHashGridParserTests extends ESTestCase { + public void testParseValidFromInts() throws Exception { + SearchContext searchContext = new TestSearchContext(); + int precision = randomIntBetween(1, 12); + XContentParser stParser = JsonXContent.jsonXContent.createParser( + "{\"field\":\"my_loc\", \"precision\":" + precision + ", \"size\": 500, \"shard_size\": 550}"); + GeoHashGridParser parser = new GeoHashGridParser(); + // can create a factory + assertNotNull(parser.parse("geohash_grid", stParser, searchContext)); + } + + public void testParseValidFromStrings() throws Exception { + SearchContext searchContext = new TestSearchContext(); + int precision = randomIntBetween(1, 12); + XContentParser stParser = JsonXContent.jsonXContent.createParser( + "{\"field\":\"my_loc\", \"precision\":\"" + precision + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); + GeoHashGridParser parser = new GeoHashGridParser(); + // can create a factory + assertNotNull(parser.parse("geohash_grid", stParser, searchContext)); + } + + public void testParseErrorOnNonIntPrecision() throws Exception { + SearchContext searchContext = new TestSearchContext(); + XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":\"2.0\"}"); + GeoHashGridParser parser = new GeoHashGridParser(); + try { + parser.parse("geohash_grid", stParser, searchContext); + fail(); + } catch (NumberFormatException ex) { + assertEquals("For input string: \"2.0\"", ex.getMessage()); + } + } + + public void testParseErrorOnBooleanPrecision() throws Exception { + SearchContext searchContext = new TestSearchContext(); + XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":false}"); + GeoHashGridParser parser = new GeoHashGridParser(); + try { + parser.parse("geohash_grid", stParser, searchContext); + fail(); + } catch (SearchParseException ex) { + assertEquals("Unexpected token VALUE_BOOLEAN in [geohash_grid].", ex.getMessage()); + } + } + + public void testParseErrorOnPrecisionOutOfRange() throws Exception { + SearchContext searchContext = new TestSearchContext(); + XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":\"13\"}"); + GeoHashGridParser parser = new GeoHashGridParser(); + try { + parser.parse("geohash_grid", stParser, searchContext); + fail(); + } catch (IllegalArgumentException ex) { + assertEquals("Invalid geohash aggregation precision of 13. Must be between 1 and 12.", ex.getMessage()); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index 56eb619cc80..b5ef5d9eb4f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.BucketCollector; @@ -46,7 +45,6 @@ import org.elasticsearch.search.aggregations.SearchContextAggregations; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -57,8 +55,6 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class NestedAggregatorTests extends ESSingleNodeTestCase { - - @Test public void testResetRootDocId() throws Exception { Directory directory = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(null); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java index aea11bab4f9..d20dff0ae05 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java @@ -28,39 +28,31 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.PercentageScore; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ScriptHeuristic; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicBuilder; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParserMapper; -import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicStreams; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.format.ValueFormatter; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TestSearchContext; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.nio.charset.StandardCharsets; +import java.util.*; import static org.elasticsearch.test.VersionUtils.randomVersion; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.*; /** * @@ -79,8 +71,7 @@ public class SignificanceHeuristicTests extends ESTestCase { } // test that stream output can actually be read - does not replace bwc test - @Test - public void streamResponse() throws Exception { + public void testStreamResponse() throws Exception { Version version = randomVersion(random()); InternalSignificantTerms[] sigTerms = getRandomSignificantTerms(getRandomSignificanceheuristic()); @@ -88,33 +79,37 @@ public class SignificanceHeuristicTests extends ESTestCase { ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer); out.setVersion(version); - sigTerms[0].writeTo(out); // read ByteArrayInputStream inBuffer = new ByteArrayInputStream(outBuffer.toByteArray()); InputStreamStreamInput in = new InputStreamStreamInput(inBuffer); in.setVersion(version); - sigTerms[1].readFrom(in); assertTrue(sigTerms[1].significanceHeuristic.equals(sigTerms[0].significanceHeuristic)); + InternalSignificantTerms.Bucket originalBucket = (InternalSignificantTerms.Bucket) sigTerms[0].buckets.get(0); + InternalSignificantTerms.Bucket streamedBucket = (InternalSignificantTerms.Bucket) sigTerms[1].buckets.get(0); + assertThat(originalBucket.getKeyAsString(), equalTo(streamedBucket.getKeyAsString())); + assertThat(originalBucket.getSupersetDf(), equalTo(streamedBucket.getSupersetDf())); + assertThat(originalBucket.getSubsetDf(), equalTo(streamedBucket.getSubsetDf())); + assertThat(streamedBucket.getSubsetSize(), equalTo(10l)); + assertThat(streamedBucket.getSupersetSize(), equalTo(20l)); } InternalSignificantTerms[] getRandomSignificantTerms(SignificanceHeuristic heuristic) { InternalSignificantTerms[] sTerms = new InternalSignificantTerms[2]; ArrayList buckets = new ArrayList<>(); if (randomBoolean()) { - BytesRef term = new BytesRef("123.0"); buckets.add(new SignificantLongTerms.Bucket(1, 2, 3, 4, 123, InternalAggregations.EMPTY, null)); sTerms[0] = new SignificantLongTerms(10, 20, "some_name", null, 1, 1, heuristic, buckets, - (List) Collections.EMPTY_LIST, null); + Collections.emptyList(), null); sTerms[1] = new SignificantLongTerms(); } else { BytesRef term = new BytesRef("someterm"); buckets.add(new SignificantStringTerms.Bucket(term, 1, 2, 3, 4, InternalAggregations.EMPTY)); - sTerms[0] = new SignificantStringTerms(10, 20, "some_name", 1, 1, heuristic, buckets, (List) Collections.EMPTY_LIST, + sTerms[0] = new SignificantStringTerms(10, 20, "some_name", 1, 1, heuristic, buckets, Collections.emptyList(), null); sTerms[1] = new SignificantStringTerms(); } @@ -130,10 +125,59 @@ public class SignificanceHeuristicTests extends ESTestCase { return heuristics.get(randomInt(3)); } + public void testReduce() { + List aggs = createInternalAggregations(); + SignificantTerms reducedAgg = (SignificantTerms) aggs.get(0).doReduce(aggs, null); + assertThat(reducedAgg.getBuckets().size(), equalTo(2)); + assertThat(reducedAgg.getBuckets().get(0).getSubsetDf(), equalTo(8l)); + assertThat(reducedAgg.getBuckets().get(0).getSubsetSize(), equalTo(16l)); + assertThat(reducedAgg.getBuckets().get(0).getSupersetDf(), equalTo(10l)); + assertThat(reducedAgg.getBuckets().get(0).getSupersetSize(), equalTo(30l)); + assertThat(reducedAgg.getBuckets().get(1).getSubsetDf(), equalTo(8l)); + assertThat(reducedAgg.getBuckets().get(1).getSubsetSize(), equalTo(16l)); + assertThat(reducedAgg.getBuckets().get(1).getSupersetDf(), equalTo(10l)); + assertThat(reducedAgg.getBuckets().get(1).getSupersetSize(), equalTo(30l)); + } + + // Create aggregations as they might come from three different shards and return as list. + private List createInternalAggregations() { + + String type = randomBoolean() ? "long" : "string"; + SignificanceHeuristic significanceHeuristic = getRandomSignificanceheuristic(); + + List aggs = new ArrayList<>(); + List terms0Buckets = new ArrayList<>(); + terms0Buckets.add(createBucket(type, 4, 4, 5, 10, 0)); + aggs.add(createAggregation(type, significanceHeuristic, terms0Buckets, 4, 10)); + List terms1Buckets = new ArrayList<>(); + terms0Buckets.add(createBucket(type, 4, 4, 5, 10, 1)); + aggs.add(createAggregation(type, significanceHeuristic, terms1Buckets, 4, 10)); + List terms01Buckets = new ArrayList<>(); + terms0Buckets.add(createBucket(type, 4, 8, 5, 10, 0)); + terms0Buckets.add(createBucket(type, 4, 8, 5, 10, 1)); + aggs.add(createAggregation(type, significanceHeuristic, terms01Buckets, 8, 10)); + return aggs; + } + + private InternalSignificantTerms createAggregation(String type, SignificanceHeuristic significanceHeuristic, List buckets, long subsetSize, long supersetSize) { + if (type.equals("string")) { + return new SignificantStringTerms(subsetSize, supersetSize, "sig_terms", 2, -1, significanceHeuristic, buckets, new ArrayList(), new HashMap()); + } else { + return new SignificantLongTerms(subsetSize, supersetSize, "sig_terms", ValueFormatter.RAW, 2, -1, significanceHeuristic, buckets, new ArrayList(), new HashMap()); + } + } + + private InternalSignificantTerms.Bucket createBucket(String type, long subsetDF, long subsetSize, long supersetDF, long supersetSize, long label) { + if (type.equals("string")) { + return new SignificantStringTerms.Bucket(new BytesRef(Long.toString(label).getBytes(StandardCharsets.UTF_8)), subsetDF, subsetSize, supersetDF, supersetSize, InternalAggregations.EMPTY); + } else { + return new SignificantLongTerms.Bucket(subsetDF, subsetSize, supersetDF, supersetSize, label, InternalAggregations.EMPTY, ValueFormatter.RAW); + } + } + // test that // 1. The output of the builders can actually be parsed // 2. The parser does not swallow parameters after a significance heuristic was defined - @Test public void testBuilderAndParser() throws Exception { Set parsers = new HashSet<>(); @@ -308,7 +352,6 @@ public class SignificanceHeuristicTests extends ESTestCase { } } - @Test public void testAssertions() throws Exception { testBackgroundAssertions(new MutualInformation(true, true), new MutualInformation(true, false)); testBackgroundAssertions(new ChiSquare(true, true), new ChiSquare(true, false)); @@ -317,8 +360,7 @@ public class SignificanceHeuristicTests extends ESTestCase { testAssertions(JLHScore.INSTANCE); } - @Test - public void basicScoreProperties() { + public void testBasicScoreProperties() { basicScoreProperties(JLHScore.INSTANCE, true); basicScoreProperties(new GND(true), true); basicScoreProperties(PercentageScore.INSTANCE, true); @@ -327,7 +369,6 @@ public class SignificanceHeuristicTests extends ESTestCase { } public void basicScoreProperties(SignificanceHeuristic heuristic, boolean test0) { - assertThat(heuristic.getScore(1, 1, 1, 3), greaterThan(0.0)); assertThat(heuristic.getScore(1, 1, 2, 3), lessThan(heuristic.getScore(1, 1, 1, 3))); assertThat(heuristic.getScore(1, 1, 3, 4), lessThan(heuristic.getScore(1, 1, 2, 4))); @@ -347,8 +388,7 @@ public class SignificanceHeuristicTests extends ESTestCase { assertThat(score, greaterThanOrEqualTo(0.0)); } - @Test - public void scoreMutual() throws Exception { + public void testScoreMutual() throws Exception { SignificanceHeuristic heuristic = new MutualInformation(true, true); assertThat(heuristic.getScore(1, 1, 1, 3), greaterThan(0.0)); assertThat(heuristic.getScore(1, 1, 2, 3), lessThan(heuristic.getScore(1, 1, 1, 3))); @@ -384,7 +424,6 @@ public class SignificanceHeuristicTests extends ESTestCase { assertThat(score, lessThanOrEqualTo(1.0)); } - @Test public void testGNDCornerCases() throws Exception { GND gnd = new GND(true); //term is only in the subset, not at all in the other set but that is because the other set is empty. diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java new file mode 100644 index 00000000000..390e0cf5473 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -0,0 +1,242 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import com.carrotsearch.hppc.ObjectIntHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import com.carrotsearch.hppc.ObjectObjectHashMap; +import com.carrotsearch.hppc.ObjectObjectMap; +import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.geo.RandomGeoGenerator; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +@ESIntegTestCase.SuiteScopeTestCase +public abstract class AbstractGeoTestCase extends ESIntegTestCase { + + protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; + protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; + protected static final String NUMBER_FIELD_NAME = "l_values"; + protected static final String UNMAPPED_IDX_NAME = "idx_unmapped"; + protected static final String IDX_NAME = "idx"; + protected static final String EMPTY_IDX_NAME = "empty_idx"; + protected static final String DATELINE_IDX_NAME = "dateline_idx"; + protected static final String HIGH_CARD_IDX_NAME = "high_card_idx"; + protected static final String IDX_ZERO_NAME = "idx_zero"; + + protected static int numDocs; + protected static int numUniqueGeoPoints; + protected static GeoPoint[] singleValues, multiValues; + protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, singleCentroid, multiCentroid, unmappedCentroid; + protected static ObjectIntMap expectedDocCountsForGeoHash = null; + protected static ObjectObjectMap expectedCentroidsForGeoHash = null; + protected static final double GEOHASH_TOLERANCE = 1E-5D; + + @Override + public void setupSuiteScopeCluster() throws Exception { + createIndex(UNMAPPED_IDX_NAME); + assertAcked(prepareCreate(IDX_NAME) + .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point,geohash_prefix=true,geohash_precision=12", + MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=string,index=not_analyzed")); + + singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + singleBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + multiBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + singleCentroid = new GeoPoint(0, 0); + multiCentroid = new GeoPoint(0, 0); + unmappedCentroid = new GeoPoint(0, 0); + + numDocs = randomIntBetween(6, 20); + numUniqueGeoPoints = randomIntBetween(1, numDocs); + expectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); + expectedCentroidsForGeoHash = new ObjectObjectHashMap<>(numDocs * 2); + + singleValues = new GeoPoint[numUniqueGeoPoints]; + for (int i = 0 ; i < singleValues.length; i++) + { + singleValues[i] = RandomGeoGenerator.randomPoint(random()); + updateBoundsTopLeft(singleValues[i], singleTopLeft); + updateBoundsBottomRight(singleValues[i], singleBottomRight); + } + + multiValues = new GeoPoint[numUniqueGeoPoints]; + for (int i = 0 ; i < multiValues.length; i++) + { + multiValues[i] = RandomGeoGenerator.randomPoint(random()); + updateBoundsTopLeft(multiValues[i], multiTopLeft); + updateBoundsBottomRight(multiValues[i], multiBottomRight); + } + + List builders = new ArrayList<>(); + + GeoPoint singleVal; + final GeoPoint[] multiVal = new GeoPoint[2]; + double newMVLat, newMVLon; + for (int i = 0; i < numDocs; i++) { + singleVal = singleValues[i % numUniqueGeoPoints]; + multiVal[0] = multiValues[i % numUniqueGeoPoints]; + multiVal[1] = multiValues[(i+1) % numUniqueGeoPoints]; + builders.add(client().prepareIndex(IDX_NAME, "type").setSource(jsonBuilder() + .startObject() + .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .startArray(MULTI_VALUED_FIELD_NAME) + .startArray().value(multiVal[0].lon()).value(multiVal[0].lat()).endArray() + .startArray().value(multiVal[1].lon()).value(multiVal[1].lat()).endArray() + .endArray() + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject())); + singleCentroid = singleCentroid.reset(singleCentroid.lat() + (singleVal.lat() - singleCentroid.lat()) / (i+1), + singleCentroid.lon() + (singleVal.lon() - singleCentroid.lon()) / (i+1)); + newMVLat = (multiVal[0].lat() + multiVal[1].lat())/2d; + newMVLon = (multiVal[0].lon() + multiVal[1].lon())/2d; + multiCentroid = multiCentroid.reset(multiCentroid.lat() + (newMVLat - multiCentroid.lat()) / (i+1), + multiCentroid.lon() + (newMVLon - multiCentroid.lon()) / (i+1)); + } + + assertAcked(prepareCreate(EMPTY_IDX_NAME).addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + + assertAcked(prepareCreate(DATELINE_IDX_NAME) + .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=string,index=not_analyzed")); + + GeoPoint[] geoValues = new GeoPoint[5]; + geoValues[0] = new GeoPoint(38, 178); + geoValues[1] = new GeoPoint(12, -179); + geoValues[2] = new GeoPoint(-24, 170); + geoValues[3] = new GeoPoint(32, -175); + geoValues[4] = new GeoPoint(-11, 178); + + for (int i = 0; i < 5; i++) { + builders.add(client().prepareIndex(DATELINE_IDX_NAME, "type").setSource(jsonBuilder() + .startObject() + .array(SINGLE_VALUED_FIELD_NAME, geoValues[i].lon(), geoValues[i].lat()) + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject())); + } + assertAcked(prepareCreate(HIGH_CARD_IDX_NAME).setSettings(Settings.builder().put("number_of_shards", 2)) + .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long,store=true", "tag", "type=string,index=not_analyzed")); + + for (int i = 0; i < 2000; i++) { + singleVal = singleValues[i % numUniqueGeoPoints]; + builders.add(client().prepareIndex(HIGH_CARD_IDX_NAME, "type").setSource(jsonBuilder() + .startObject() + .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .startArray(MULTI_VALUED_FIELD_NAME) + .startArray().value(multiValues[i % numUniqueGeoPoints].lon()).value(multiValues[i % numUniqueGeoPoints].lat()).endArray() + .startArray().value(multiValues[(i + 1) % numUniqueGeoPoints].lon()).value(multiValues[(i + 1) % numUniqueGeoPoints].lat()).endArray() + .endArray() + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject())); + updateGeohashBucketsCentroid(singleVal); + } + + builders.add(client().prepareIndex(IDX_ZERO_NAME, "type").setSource( + jsonBuilder().startObject().array(SINGLE_VALUED_FIELD_NAME, 0.0, 1.0).endObject())); + assertAcked(prepareCreate(IDX_ZERO_NAME).addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + + indexRandom(true, builders); + ensureSearchable(); + + // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same value for NUMBER_FIELD_NAME. This will check that after + // random indexing each document only has 1 value for NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting + // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME) + .order(SortOrder.ASC)).setSize(5000).get(); + assertSearchResponse(response); + long totalHits = response.getHits().totalHits(); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + logger.info("Full high_card_idx Response Content:\n{ {} }", builder.string()); + for (int i = 0; i < totalHits; i++) { + SearchHit searchHit = response.getHits().getAt(i); + assertThat("Hit " + i + " with id: " + searchHit.getId(), searchHit.getIndex(), equalTo("high_card_idx")); + assertThat("Hit " + i + " with id: " + searchHit.getId(), searchHit.getType(), equalTo("type")); + SearchHitField hitField = searchHit.field(NUMBER_FIELD_NAME); + + assertThat("Hit " + i + " has wrong number of values", hitField.getValues().size(), equalTo(1)); + Long value = hitField.getValue(); + assertThat("Hit " + i + " has wrong value", value.intValue(), equalTo(i)); + } + assertThat(totalHits, equalTo(2000l)); + } + + private void updateGeohashBucketsCentroid(final GeoPoint location) { + String hash = GeoHashUtils.stringEncode(location.lon(), location.lat(), GeoHashUtils.PRECISION); + for (int precision = GeoHashUtils.PRECISION; precision > 0; --precision) { + final String h = hash.substring(0, precision); + expectedDocCountsForGeoHash.put(h, expectedDocCountsForGeoHash.getOrDefault(h, 0) + 1); + expectedCentroidsForGeoHash.put(h, updateHashCentroid(h, location)); + } + } + + private GeoPoint updateHashCentroid(String hash, final GeoPoint location) { + GeoPoint centroid = expectedCentroidsForGeoHash.getOrDefault(hash, null); + if (centroid == null) { + return new GeoPoint(location.lat(), location.lon()); + } + final int docCount = expectedDocCountsForGeoHash.get(hash); + final double newLon = centroid.lon() + (location.lon() - centroid.lon()) / docCount; + final double newLat = centroid.lat() + (location.lat() - centroid.lat()) / docCount; + return centroid.reset(newLat, newLon); + } + + private void updateBoundsBottomRight(GeoPoint geoPoint, GeoPoint currentBound) { + if (geoPoint.lat() < currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() > currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } + + private void updateBoundsTopLeft(GeoPoint geoPoint, GeoPoint currentBound) { + if (geoPoint.lat() > currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() < currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java new file mode 100644 index 00000000000..ac146706eb5 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java @@ -0,0 +1,588 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.aggregations.bucket.global.Global; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.metrics.avg.Avg; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + * + */ +public class AvgIT extends AbstractNumericTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ExtractFieldScriptPlugin.class, + FieldValueScriptPlugin.class); + } + + @Override + public void testEmptyAggregation() throws Exception { + + SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") + .setQuery(matchAllQuery()) + .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(avg("avg"))) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l)); + Histogram histo = searchResponse.getAggregations().get("histo"); + assertThat(histo, notNullValue()); + Histogram.Bucket bucket = histo.getBuckets().get(1); + assertThat(bucket, notNullValue()); + + Avg avg = bucket.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(Double.isNaN(avg.getValue()), is(true)); + } + + @Override + public void testUnmapped() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx_unmapped") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("value")) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(0l)); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo(Double.NaN)); + } + + @Override + public void testSingleValuedField() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("value")) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10)); + } + + @Override + public void testSingleValuedFieldGetProperty() throws Exception { + + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) + .addAggregation(global("global").subAggregation(avg("avg").field("value"))).execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Global global = searchResponse.getAggregations().get("global"); + assertThat(global, notNullValue()); + assertThat(global.getName(), equalTo("global")); + assertThat(global.getDocCount(), equalTo(10l)); + assertThat(global.getAggregations(), notNullValue()); + assertThat(global.getAggregations().asMap().size(), equalTo(1)); + + Avg avg = global.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + double expectedAvgValue = (double) (1+2+3+4+5+6+7+8+9+10) / 10; + assertThat(avg.getValue(), equalTo(expectedAvgValue)); + assertThat((Avg) global.getProperty("avg"), equalTo(avg)); + assertThat((double) global.getProperty("avg.value"), equalTo(expectedAvgValue)); + assertThat((double) avg.getProperty("value"), equalTo(expectedAvgValue)); + } + + @Override + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("value")) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10)); + } + + @Override + public void testSingleValuedFieldWithValueScript() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("value") + .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10)); + } + + @Override + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { + Map params = Collections.singletonMap("inc", 1); + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("value") + .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); + } + + public void testSingleValuedField_WithFormatter() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) + .addAggregation(avg("avg").format("#").field("value")).execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10) / 10)); + assertThat(avg.getValueAsString(), equalTo("6")); + } + + @Override + public void testMultiValuedField() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("values")) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12) / 20)); + } + + @Override + public void testMultiValuedFieldWithValueScript() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("values") + .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12) / 20)); + } + + @Override + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { + Map params = Collections.singletonMap("inc", 1); + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg").field("values") + .script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12+12+13) / 20)); + } + + @Override + public void testScriptSingleValued() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg") + .script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10)); + } + + @Override + public void testScriptSingleValuedWithParams() throws Exception { + Map params = Collections.singletonMap("inc", 1); + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg") + .script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); + } + + @Override + public void testScriptMultiValued() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg") + .script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12) / 20)); + } + + @Override + public void testScriptMultiValuedWithParams() throws Exception { + Map params = Collections.singletonMap("inc", 1); + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(avg("avg") + .script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Avg avg = searchResponse.getAggregations().get("avg"); + assertThat(avg, notNullValue()); + assertThat(avg.getName(), equalTo("avg")); + assertThat(avg.getValue(), equalTo((double) (3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12+12+13) / 20)); + } + + /** + * Mock plugin for the {@link ExtractFieldScriptEngine} + */ + public static class ExtractFieldScriptPlugin extends Plugin { + + @Override + public String name() { + return ExtractFieldScriptEngine.NAME; + } + + @Override + public String description() { + return "Mock script engine for " + AvgIT.class; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(ExtractFieldScriptEngine.class); + } + + } + + /** + * This mock script returns the field that is specified by name in the script body + */ + public static class ExtractFieldScriptEngine implements ScriptEngineService { + + public static final String NAME = "extract_field"; + + @Override + public void close() throws IOException { + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + throw new UnsupportedOperationException(); + } + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + final long inc; + if (vars == null || vars.containsKey("inc") == false) { + inc = 0; + } else { + inc = ((Number) vars.get("inc")).longValue(); + } + return new SearchScript() { + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + + return new LeafSearchScript() { + + @Override + public Object unwrap(Object value) { + return null; + } + + @Override + public void setNextVar(String name, Object value) { + } + + @Override + public Object run() { + String fieldName = (String) compiledScript.compiled(); + List values = new ArrayList<>(); + for (Object v : (List) leafLookup.doc().get(fieldName)) { + values.add(((Number) v).longValue() + inc); + } + return values; + } + + @Override + public void setScorer(Scorer scorer) { + } + + @Override + public void setSource(Map source) { + } + + @Override + public void setDocument(int doc) { + if (leafLookup != null) { + leafLookup.setDocument(doc); + } + } + + @Override + public long runAsLong() { + throw new UnsupportedOperationException(); + } + + @Override + public float runAsFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public double runAsDouble() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + } + } + + /** + * Mock plugin for the {@link FieldValueScriptEngine} + */ + public static class FieldValueScriptPlugin extends Plugin { + + @Override + public String name() { + return FieldValueScriptEngine.NAME; + } + + @Override + public String description() { + return "Mock script engine for " + AvgIT.class; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(FieldValueScriptEngine.class); + } + + } + + /** + * This mock script returns the field value and adds one month to the returned date + */ + public static class FieldValueScriptEngine implements ScriptEngineService { + + public static final String NAME = "field_value"; + + @Override + public void close() throws IOException { + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + throw new UnsupportedOperationException(); + } + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + final long inc; + if (vars == null || vars.containsKey("inc") == false) { + inc = 0; + } else { + inc = ((Number) vars.get("inc")).longValue(); + } + return new SearchScript() { + + private Map vars = new HashMap<>(2); + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + + return new LeafSearchScript() { + + @Override + public Object unwrap(Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public void setNextVar(String name, Object value) { + vars.put(name, value); + } + + @Override + public Object run() { + throw new UnsupportedOperationException(); + } + + @Override + public void setScorer(Scorer scorer) { + } + + @Override + public void setSource(Map source) { + } + + @Override + public void setDocument(int doc) { + if (leafLookup != null) { + leafLookup.setDocument(doc); + } + } + + @Override + public long runAsLong() { + return ((Number) vars.get("_value")).longValue() + inc; + } + + @Override + public float runAsFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public double runAsDouble() { + return ((Number) vars.get("_value")).doubleValue() + inc; + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + } + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index 6d2d61e707d..6419e9dcac3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -19,223 +19,57 @@ package org.elasticsearch.search.aggregations.metrics; -import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArray; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBounds; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregator; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.geoBounds; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.sameInstance; +import static org.hamcrest.Matchers.*; /** * */ @ESIntegTestCase.SuiteScopeTestCase -public class GeoBoundsIT extends ESIntegTestCase { +public class GeoBoundsIT extends AbstractGeoTestCase { + private static final String aggName = "geoBounds"; - private static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; - private static final String MULTI_VALUED_FIELD_NAME = "geo_values"; - private static final String NUMBER_FIELD_NAME = "l_values"; - - static int numDocs; - static int numUniqueGeoPoints; - static GeoPoint[] singleValues, multiValues; - static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, unmappedTopLeft, unmappedBottomRight; - - @Override - public void setupSuiteScopeCluster() throws Exception { - assertAcked(prepareCreate("idx") - .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=string,index=not_analyzed")); - createIndex("idx_unmapped"); - - unmappedTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); - unmappedBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); - singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); - singleBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); - multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); - multiBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); - - numDocs = randomIntBetween(6, 20); - numUniqueGeoPoints = randomIntBetween(1, numDocs); - - singleValues = new GeoPoint[numUniqueGeoPoints]; - for (int i = 0 ; i < singleValues.length; i++) - { - singleValues[i] = randomGeoPoint(); - updateBoundsTopLeft(singleValues[i], singleTopLeft); - updateBoundsBottomRight(singleValues[i], singleBottomRight); - } - - multiValues = new GeoPoint[numUniqueGeoPoints]; - for (int i = 0 ; i < multiValues.length; i++) - { - multiValues[i] = randomGeoPoint(); - updateBoundsTopLeft(multiValues[i], multiTopLeft); - updateBoundsBottomRight(multiValues[i], multiBottomRight); - } - - List builders = new ArrayList<>(); - - - for (int i = 0; i < numDocs; i++) { - builders.add(client().prepareIndex("idx", "type").setSource(jsonBuilder() - .startObject() - .array(SINGLE_VALUED_FIELD_NAME, singleValues[i % numUniqueGeoPoints].lon(), singleValues[i % numUniqueGeoPoints].lat()) - .startArray(MULTI_VALUED_FIELD_NAME) - .startArray().value(multiValues[i % numUniqueGeoPoints].lon()).value(multiValues[i % numUniqueGeoPoints].lat()).endArray() - .startArray().value(multiValues[(i+1) % numUniqueGeoPoints].lon()).value(multiValues[(i+1) % numUniqueGeoPoints].lat()).endArray() - .endArray() - .field(NUMBER_FIELD_NAME, i) - .field("tag", "tag" + i) - .endObject())); - } - - assertAcked(prepareCreate("empty_idx").addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point")); - - assertAcked(prepareCreate("idx_dateline") - .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=string,index=not_analyzed")); - - GeoPoint[] geoValues = new GeoPoint[5]; - geoValues[0] = new GeoPoint(38, 178); - geoValues[1] = new GeoPoint(12, -179); - geoValues[2] = new GeoPoint(-24, 170); - geoValues[3] = new GeoPoint(32, -175); - geoValues[4] = new GeoPoint(-11, 178); - - for (int i = 0; i < 5; i++) { - builders.add(client().prepareIndex("idx_dateline", "type").setSource(jsonBuilder() - .startObject() - .array(SINGLE_VALUED_FIELD_NAME, geoValues[i].lon(), geoValues[i].lat()) - .field(NUMBER_FIELD_NAME, i) - .field("tag", "tag" + i) - .endObject())); - } - assertAcked(prepareCreate("high_card_idx").setSettings(Settings.builder().put("number_of_shards", 2)) - .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=string,index=not_analyzed")); - - - for (int i = 0; i < 2000; i++) { - builders.add(client().prepareIndex("high_card_idx", "type").setSource(jsonBuilder() - .startObject() - .array(SINGLE_VALUED_FIELD_NAME, singleValues[i % numUniqueGeoPoints].lon(), singleValues[i % numUniqueGeoPoints].lat()) - .startArray(MULTI_VALUED_FIELD_NAME) - .startArray().value(multiValues[i % numUniqueGeoPoints].lon()).value(multiValues[i % numUniqueGeoPoints].lat()).endArray() - .startArray().value(multiValues[(i+1) % numUniqueGeoPoints].lon()).value(multiValues[(i+1) % numUniqueGeoPoints].lat()).endArray() - .endArray() - .field(NUMBER_FIELD_NAME, i) - .field("tag", "tag" + i) - .endObject())); - } - - builders.add(client().prepareIndex("idx_zero", "type").setSource( - jsonBuilder().startObject().array(SINGLE_VALUED_FIELD_NAME, 0.0, 1.0).endObject())); - assertAcked(prepareCreate("idx_zero").addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point")); - - indexRandom(true, builders); - ensureSearchable(); - - // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same value for NUMBER_FIELD_NAME. This will check that after - // random indexing each document only has 1 value for NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting - // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type - SearchResponse response = client().prepareSearch("high_card_idx").addField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME).order(SortOrder.ASC)).setSize(5000).get(); - assertSearchResponse(response); - long totalHits = response.getHits().totalHits(); - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - response.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - logger.info("Full high_card_idx Response Content:\n{ {} }", builder.string()); - for (int i = 0; i < totalHits; i++) { - SearchHit searchHit = response.getHits().getAt(i); - assertThat("Hit " + i + " with id: " + searchHit.getId(), searchHit.getIndex(), equalTo("high_card_idx")); - assertThat("Hit " + i + " with id: " + searchHit.getId(), searchHit.getType(), equalTo("type")); - SearchHitField hitField = searchHit.field(NUMBER_FIELD_NAME); - - assertThat("Hit " + i + " has wrong number of values", hitField.getValues().size(), equalTo(1)); - Integer value = hitField.getValue(); - assertThat("Hit " + i + " has wrong value", value, equalTo(i)); - } - assertThat(totalHits, equalTo(2000l)); - } - - private void updateBoundsBottomRight(GeoPoint geoPoint, GeoPoint currentBound) { - if (geoPoint.lat() < currentBound.lat()) { - currentBound.resetLat(geoPoint.lat()); - } - if (geoPoint.lon() > currentBound.lon()) { - currentBound.resetLon(geoPoint.lon()); - } - } - - private void updateBoundsTopLeft(GeoPoint geoPoint, GeoPoint currentBound) { - if (geoPoint.lat() > currentBound.lat()) { - currentBound.resetLat(geoPoint.lat()); - } - if (geoPoint.lon() < currentBound.lon()) { - currentBound.resetLon(geoPoint.lon()); - } - } - - private GeoPoint randomGeoPoint() { - return new GeoPoint((randomDouble() * 180) - 90, (randomDouble() * 360) - 180); - } - - @Test - public void singleValuedField() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME) + public void testSingleValuedField() throws Exception { + SearchResponse response = client().prepareSearch(IDX_NAME) + .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) .wrapLongitude(false)) .execute().actionGet(); assertSearchResponse(response); - - GeoBounds geoBounds = response.getAggregations().get("geoBounds"); + GeoBounds geoBounds = response.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(singleTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(singleTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(singleBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(singleBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } - @Test public void testSingleValuedField_getProperty() throws Exception { SearchResponse searchResponse = client() - .prepareSearch("idx") + .prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) .addAggregation( - global("global").subAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false))) + global("global").subAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false))) .execute().actionGet(); assertSearchResponse(searchResponse); @@ -247,105 +81,98 @@ public class GeoBoundsIT extends ESIntegTestCase { assertThat(global.getAggregations(), notNullValue()); assertThat(global.getAggregations().asMap().size(), equalTo(1)); - GeoBounds geobounds = global.getAggregations().get("geoBounds"); + GeoBounds geobounds = global.getAggregations().get(aggName); assertThat(geobounds, notNullValue()); - assertThat(geobounds.getName(), equalTo("geoBounds")); - assertThat((GeoBounds) global.getProperty("geoBounds"), sameInstance(geobounds)); + assertThat(geobounds.getName(), equalTo(aggName)); + assertThat((GeoBounds) global.getProperty(aggName), sameInstance(geobounds)); GeoPoint topLeft = geobounds.topLeft(); GeoPoint bottomRight = geobounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(singleTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(singleTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(singleBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(singleBottomRight.lon())); - assertThat((double) global.getProperty("geoBounds.top"), equalTo(singleTopLeft.lat())); - assertThat((double) global.getProperty("geoBounds.left"), equalTo(singleTopLeft.lon())); - assertThat((double) global.getProperty("geoBounds.bottom"), equalTo(singleBottomRight.lat())); - assertThat((double) global.getProperty("geoBounds.right"), equalTo(singleBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".top"), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".left"), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".bottom"), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".right"), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } - @Test - public void multiValuedField() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(geoBounds("geoBounds").field(MULTI_VALUED_FIELD_NAME) + public void testMultiValuedField() throws Exception { + SearchResponse response = client().prepareSearch(IDX_NAME) + .addAggregation(geoBounds(aggName).field(MULTI_VALUED_FIELD_NAME) .wrapLongitude(false)) .execute().actionGet(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get("geoBounds"); + GeoBounds geoBounds = response.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(multiTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(multiTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(multiBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(multiBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(multiTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(multiTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(multiBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(multiBottomRight.lon(), GEOHASH_TOLERANCE)); } - @Test - public void unmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME) + public void testUnmapped() throws Exception { + SearchResponse response = client().prepareSearch(UNMAPPED_IDX_NAME) + .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) .wrapLongitude(false)) .execute().actionGet(); assertSearchResponse(response); - - GeoBounds geoBounds = response.getAggregations().get("geoBounds"); + GeoBounds geoBounds = response.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft, equalTo(null)); assertThat(bottomRight, equalTo(null)); } - @Test - public void partiallyUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx", "idx_unmapped") - .addAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME) + public void testPartiallyUnmapped() throws Exception { + SearchResponse response = client().prepareSearch(IDX_NAME, UNMAPPED_IDX_NAME) + .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) .wrapLongitude(false)) .execute().actionGet(); assertSearchResponse(response); - - GeoBounds geoBounds = response.getAggregations().get("geoBounds"); + GeoBounds geoBounds = response.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(singleTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(singleTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(singleBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(singleBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } - @Test - public void emptyAggregation() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_idx") + public void testEmptyAggregation() throws Exception { + SearchResponse searchResponse = client().prepareSearch(EMPTY_IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) .wrapLongitude(false)) .execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(0l)); - GeoBounds geoBounds = searchResponse.getAggregations().get("geoBounds"); + GeoBounds geoBounds = searchResponse.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft, equalTo(null)); assertThat(bottomRight, equalTo(null)); } - @Test - public void singleValuedFieldNearDateLine() throws Exception { - SearchResponse response = client().prepareSearch("idx_dateline") - .addAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME) + public void testSingleValuedFieldNearDateLine() throws Exception { + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) .wrapLongitude(false)) .execute().actionGet(); @@ -354,47 +181,44 @@ public class GeoBoundsIT extends ESIntegTestCase { GeoPoint geoValuesTopLeft = new GeoPoint(38, -179); GeoPoint geoValuesBottomRight = new GeoPoint(-24, 178); - GeoBounds geoBounds = response.getAggregations().get("geoBounds"); + GeoBounds geoBounds = response.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(geoValuesTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(geoValuesTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(geoValuesBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(geoValuesBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); } - @Test - public void singleValuedFieldNearDateLineWrapLongitude() throws Exception { + public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { GeoPoint geoValuesTopLeft = new GeoPoint(38, 170); GeoPoint geoValuesBottomRight = new GeoPoint(-24, -175); - - SearchResponse response = client().prepareSearch("idx_dateline") - .addAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(true)) + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(true)) .execute().actionGet(); assertSearchResponse(response); - - GeoBounds geoBounds = response.getAggregations().get("geoBounds"); + + GeoBounds geoBounds = response.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(geoValuesTopLeft.lat())); - assertThat(topLeft.lon(), equalTo(geoValuesTopLeft.lon())); - assertThat(bottomRight.lat(), equalTo(geoValuesBottomRight.lat())); - assertThat(bottomRight.lon(), equalTo(geoValuesBottomRight.lon())); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); } /** * This test forces the {@link GeoBoundsAggregator} to resize the {@link BigArray}s it uses to ensure they are resized correctly */ - @Test - public void singleValuedFieldAsSubAggToHighCardTermsAgg() { - SearchResponse response = client().prepareSearch("high_card_idx") - .addAggregation(terms("terms").field(NUMBER_FIELD_NAME).subAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME) + public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addAggregation(terms("terms").field(NUMBER_FIELD_NAME).subAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) .wrapLongitude(false))) .execute().actionGet(); @@ -409,9 +233,9 @@ public class GeoBoundsIT extends ESIntegTestCase { Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); assertThat("Bucket " + bucket.getKey() + " has wrong number of documents", bucket.getDocCount(), equalTo(1l)); - GeoBounds geoBounds = bucket.getAggregations().get("geoBounds"); + GeoBounds geoBounds = bucket.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); assertThat(geoBounds.topLeft().getLat(), allOf(greaterThanOrEqualTo(-90.0), lessThanOrEqualTo(90.0))); assertThat(geoBounds.topLeft().getLon(), allOf(greaterThanOrEqualTo(-180.0), lessThanOrEqualTo(180.0))); assertThat(geoBounds.bottomRight().getLat(), allOf(greaterThanOrEqualTo(-90.0), lessThanOrEqualTo(90.0))); @@ -419,22 +243,20 @@ public class GeoBoundsIT extends ESIntegTestCase { } } - @Test - public void singleValuedFieldWithZeroLon() throws Exception { - SearchResponse response = client().prepareSearch("idx_zero") - .addAggregation(geoBounds("geoBounds").field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false)).execute().actionGet(); + public void testSingleValuedFieldWithZeroLon() throws Exception { + SearchResponse response = client().prepareSearch(IDX_ZERO_NAME) + .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false)).execute().actionGet(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get("geoBounds"); + GeoBounds geoBounds = response.getAggregations().get(aggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo("geoBounds")); + assertThat(geoBounds.getName(), equalTo(aggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), equalTo(1.0)); - assertThat(topLeft.lon(), equalTo(0.0)); - assertThat(bottomRight.lat(), equalTo(1.0)); - assertThat(bottomRight.lon(), equalTo(0.0)); + assertThat(topLeft.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); } - -} +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java new file mode 100644 index 00000000000..e0d260f5435 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java @@ -0,0 +1,169 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.global.Global; +import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroid; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.List; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.geoCentroid; +import static org.elasticsearch.search.aggregations.AggregationBuilders.geohashGrid; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; + +/** + * Integration Test for GeoCentroid metric aggregator + */ +@ESIntegTestCase.SuiteScopeTestCase +public class GeoCentroidIT extends AbstractGeoTestCase { + private static final String aggName = "geoCentroid"; + + public void testEmptyAggregation() throws Exception { + SearchResponse response = client().prepareSearch(EMPTY_IDX_NAME) + .setQuery(matchAllQuery()) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .execute().actionGet(); + assertSearchResponse(response); + + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(response.getHits().getTotalHits(), equalTo(0l)); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid, equalTo(null)); + } + + public void testUnmapped() throws Exception { + SearchResponse response = client().prepareSearch(UNMAPPED_IDX_NAME) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .execute().actionGet(); + assertSearchResponse(response); + + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid, equalTo(null)); + } + + public void testPartiallyUnmapped() throws Exception { + SearchResponse response = client().prepareSearch(IDX_NAME, UNMAPPED_IDX_NAME) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .execute().actionGet(); + assertSearchResponse(response); + + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); + } + + public void testSingleValuedField() throws Exception { + SearchResponse response = client().prepareSearch(IDX_NAME) + .setQuery(matchAllQuery()) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .execute().actionGet(); + assertSearchResponse(response); + + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); + } + + public void testSingleValueFieldGetProperty() throws Exception { + SearchResponse response = client().prepareSearch(IDX_NAME) + .setQuery(matchAllQuery()) + .addAggregation(global("global").subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME))) + .execute().actionGet(); + assertSearchResponse(response); + + Global global = response.getAggregations().get("global"); + assertThat(global, notNullValue()); + assertThat(global.getName(), equalTo("global")); + assertThat(global.getDocCount(), equalTo((long) numDocs)); + assertThat(global.getAggregations(), notNullValue()); + assertThat(global.getAggregations().asMap().size(), equalTo(1)); + + GeoCentroid geoCentroid = global.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + assertThat((GeoCentroid) global.getProperty(aggName), sameInstance(geoCentroid)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); + assertThat(((GeoPoint) global.getProperty(aggName + ".value")).lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(((GeoPoint) global.getProperty(aggName + ".value")).lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".lat"), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat((double) global.getProperty(aggName + ".lon"), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); + } + + public void testMultiValuedField() throws Exception { + SearchResponse searchResponse = client().prepareSearch(IDX_NAME) + .setQuery(matchAllQuery()) + .addAggregation(geoCentroid(aggName).field(MULTI_VALUED_FIELD_NAME)) + .execute().actionGet(); + assertSearchResponse(searchResponse); + + GeoCentroid geoCentroid = searchResponse.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(multiCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(multiCentroid.lon(), GEOHASH_TOLERANCE)); + } + + public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addAggregation(geohashGrid("geoGrid").field(SINGLE_VALUED_FIELD_NAME) + .subAggregation(geoCentroid(aggName))) + .execute().actionGet(); + assertSearchResponse(response); + + GeoHashGrid grid = response.getAggregations().get("geoGrid"); + assertThat(grid, notNullValue()); + assertThat(grid.getName(), equalTo("geoGrid")); + List buckets = grid.getBuckets(); + for (int i=0; i < buckets.size(); ++i) { + GeoHashGrid.Bucket cell = buckets.get(i); + String geohash = cell.getKeyAsString(); + GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); + GeoCentroid centroidAgg = cell.getAggregations().get(aggName); + assertThat("Geohash " + geohash + " has wrong centroid latitude ", expectedCentroid.lat(), + closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE)); + assertThat("Geohash " + geohash + " has wrong centroid longitude", expectedCentroid.lon(), + closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE)); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java new file mode 100644 index 00000000000..d87de000108 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java @@ -0,0 +1,586 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.aggregations.bucket.global.Global; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +/** + * + */ +public class SumIT extends AbstractNumericTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(ExtractFieldScriptPlugin.class, FieldValueScriptPlugin.class); + } + + @Override + public void testEmptyAggregation() throws Exception { + + SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") + .setQuery(matchAllQuery()) + .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(sum("sum"))) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l)); + Histogram histo = searchResponse.getAggregations().get("histo"); + assertThat(histo, notNullValue()); + Histogram.Bucket bucket = histo.getBuckets().get(1); + assertThat(bucket, notNullValue()); + + Sum sum = bucket.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo(0.0)); + } + + @Override + public void testUnmapped() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx_unmapped") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("value")) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(0l)); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo(0.0)); + } + + @Override + public void testSingleValuedField() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("value")) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); + } + + public void testSingleValuedFieldWithFormatter() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) + .addAggregation(sum("sum").format("0000.0").field("value")).execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10)); + assertThat(sum.getValueAsString(), equalTo("0055.0")); + } + + @Override + public void testSingleValuedFieldGetProperty() throws Exception { + + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) + .addAggregation(global("global").subAggregation(sum("sum").field("value"))).execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Global global = searchResponse.getAggregations().get("global"); + assertThat(global, notNullValue()); + assertThat(global.getName(), equalTo("global")); + assertThat(global.getDocCount(), equalTo(10l)); + assertThat(global.getAggregations(), notNullValue()); + assertThat(global.getAggregations().asMap().size(), equalTo(1)); + + Sum sum = global.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + double expectedSumValue = (double) 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10; + assertThat(sum.getValue(), equalTo(expectedSumValue)); + assertThat((Sum) global.getProperty("sum"), equalTo(sum)); + assertThat((double) global.getProperty("sum.value"), equalTo(expectedSumValue)); + assertThat((double) sum.getProperty("value"), equalTo(expectedSumValue)); + } + + @Override + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("value")) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); + } + + @Override + public void testSingleValuedFieldWithValueScript() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("value").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); + } + + @Override + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { + Map params = new HashMap<>(); + params.put("increment", 1); + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("value").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); + } + + @Override + public void testScriptSingleValued() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); + } + + @Override + public void testScriptSingleValuedWithParams() throws Exception { + Map params = new HashMap<>(); + params.put("inc", 1); + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").script(new Script("value", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 2+3+4+5+6+7+8+9+10+11)); + } + + @Override + public void testScriptMultiValued() throws Exception { + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12)); + } + + @Override + public void testScriptMultiValuedWithParams() throws Exception { + Map params = new HashMap<>(); + params.put("inc", 1); + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + sum("sum").script(new Script("values", ScriptType.INLINE, ExtractFieldScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12+12+13)); + } + + @Override + public void testMultiValuedField() throws Exception { + + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("values")) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12)); + } + + @Override + public void testMultiValuedFieldWithValueScript() throws Exception { + + SearchResponse searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("values").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 2 + 3 + 3 + 4 + 4 + 5 + 5 + 6 + 6 + 7 + 7 + 8 + 8 + 9 + 9 + 10 + 10 + 11 + 11 + 12)); + } + + @Override + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { + Map params = new HashMap<>(); + params.put("increment", 1); + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) + .addAggregation(sum("sum").field("values").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + + Sum sum = searchResponse.getAggregations().get("sum"); + assertThat(sum, notNullValue()); + assertThat(sum.getName(), equalTo("sum")); + assertThat(sum.getValue(), equalTo((double) 2 + 3 + 3 + 4 + 4 + 5 + 5 + 6 + 6 + 7 + 7 + 8 + 8 + 9 + 9 + 10 + 10 + 11 + 11 + 12)); + } + + /** + * Mock plugin for the {@link ExtractFieldScriptEngine} + */ + public static class ExtractFieldScriptPlugin extends Plugin { + + @Override + public String name() { + return ExtractFieldScriptEngine.NAME; + } + + @Override + public String description() { + return "Mock script engine for " + SumIT.class; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(ExtractFieldScriptEngine.class); + } + + } + + /** + * This mock script returns the field that is specified by name in the + * script body + */ + public static class ExtractFieldScriptEngine implements ScriptEngineService { + + public static final String NAME = "extract_field"; + + @Override + public void close() throws IOException { + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + throw new UnsupportedOperationException(); + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + final long inc; + if (vars == null || vars.containsKey("inc") == false) { + inc = 0; + } else { + inc = ((Number) vars.get("inc")).longValue(); + } + return new SearchScript() { + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + + return new LeafSearchScript() { + + @Override + public Object unwrap(Object value) { + return null; + } + + @Override + public void setNextVar(String name, Object value) { + } + + @Override + public Object run() { + String fieldName = (String) compiledScript.compiled(); + List values = new ArrayList<>(); + for (Object v : (List) leafLookup.doc().get(fieldName)) { + values.add(((Number) v).longValue() + inc); + } + return values; + } + + @Override + public void setScorer(Scorer scorer) { + } + + @Override + public void setSource(Map source) { + } + + @Override + public void setDocument(int doc) { + if (leafLookup != null) { + leafLookup.setDocument(doc); + } + } + + @Override + public long runAsLong() { + throw new UnsupportedOperationException(); + } + + @Override + public float runAsFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public double runAsDouble() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + } + } + + /** + * Mock plugin for the {@link FieldValueScriptEngine} + */ + public static class FieldValueScriptPlugin extends Plugin { + + @Override + public String name() { + return FieldValueScriptEngine.NAME; + } + + @Override + public String description() { + return "Mock script engine for " + SumIT.class; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(FieldValueScriptEngine.class); + } + + } + + /** + * This mock script returns the field value and adds one to the returned + * value + */ + public static class FieldValueScriptEngine implements ScriptEngineService { + + public static final String NAME = "field_value"; + + @Override + public void close() throws IOException { + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + throw new UnsupportedOperationException(); + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + final long inc; + if (vars == null || vars.containsKey("inc") == false) { + inc = 0; + } else { + inc = ((Number) vars.get("inc")).longValue(); + } + return new SearchScript() { + + private Map vars = new HashMap<>(2); + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + + return new LeafSearchScript() { + + @Override + public Object unwrap(Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public void setNextVar(String name, Object value) { + vars.put(name, value); + } + + @Override + public Object run() { + throw new UnsupportedOperationException(); + } + + @Override + public void setScorer(Scorer scorer) { + } + + @Override + public void setSource(Map source) { + } + + @Override + public void setDocument(int doc) { + if (leafLookup != null) { + leafLookup.setDocument(doc); + } + } + + @Override + public long runAsLong() { + return ((Number) vars.get("_value")).longValue() + inc; + } + + @Override + public float runAsFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public double runAsDouble() { + return ((Number) vars.get("_value")).doubleValue() + inc; + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + } + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index ae01ea1aafa..65e71fe9c05 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; @@ -47,7 +46,6 @@ import org.elasticsearch.search.highlight.HighlightField; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -89,7 +87,7 @@ public class TopHitsIT extends ESIntegTestCase { private static final String TERMS_AGGS_FIELD = "terms"; private static final String SORT_FIELD = "sort"; - + @Override protected Collection> nodePlugins() { return Collections.singleton(MockScriptEngine.TestPlugin.class); @@ -250,7 +248,6 @@ public class TopHitsIT extends ESIntegTestCase { return bucket.getKeyAsString(); } - @Test public void testBasics() throws Exception { SearchResponse response = client() .prepareSearch("idx") @@ -290,7 +287,6 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test public void testIssue11119() throws Exception { // Test that top_hits aggregation is fed scores if query results size=0 SearchResponse response = client() @@ -349,7 +345,6 @@ public class TopHitsIT extends ESIntegTestCase { } - @Test public void testBreadthFirst() throws Exception { // breadth_first will be ignored since we need scores SearchResponse response = client().prepareSearch("idx").setTypes("type") @@ -381,8 +376,7 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test - public void testBasics_getProperty() throws Exception { + public void testBasicsGetProperty() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(global("global").subAggregation(topHits("hits"))).execute().actionGet(); @@ -401,7 +395,6 @@ public class TopHitsIT extends ESIntegTestCase { } - @Test public void testPagination() throws Exception { int size = randomIntBetween(1, 10); int from = randomIntBetween(0, 10); @@ -447,7 +440,6 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test public void testSortByBucket() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") @@ -487,7 +479,6 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test public void testFieldCollapsing() throws Exception { SearchResponse response = client() .prepareSearch("idx") @@ -531,7 +522,6 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(hits.getAt(0).id(), equalTo("2")); } - @Test public void testFetchFeatures() { SearchResponse response = client().prepareSearch("idx").setTypes("type") .setQuery(matchQuery("text", "text").queryName("test")) @@ -540,8 +530,9 @@ public class TopHitsIT extends ESIntegTestCase { .field(TERMS_AGGS_FIELD) .subAggregation( topHits("hits").setSize(1) - .addHighlightedField("text") + .highlighter(new HighlightBuilder().field("text")) .setExplain(true) + .addField("text") .addFieldDataField("field1") .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())) .setFetchSource("text", null) @@ -578,6 +569,8 @@ public class TopHitsIT extends ESIntegTestCase { SearchHitField field = hit.field("field1"); assertThat(field.getValue().toString(), equalTo("5")); + assertThat(hit.getSource().get("text").toString(), equalTo("some text to entertain")); + field = hit.field("script"); assertThat(field.getValue().toString(), equalTo("5")); @@ -586,7 +579,6 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test public void testInvalidSortField() throws Exception { try { client().prepareSearch("idx").setTypes("type") @@ -603,40 +595,39 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test - public void testFailWithSubAgg() throws Exception { - String source = "{\n" + - " \"aggs\": {\n" + - " \"top-tags\": {\n" + - " \"terms\": {\n" + - " \"field\": \"tags\"\n" + - " },\n" + - " \"aggs\": {\n" + - " \"top_tags_hits\": {\n" + - " \"top_hits\": {},\n" + - " \"aggs\": {\n" + - " \"max\": {\n" + - " \"max\": {\n" + - " \"field\": \"age\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - try { - client().prepareSearch("idx").setTypes("type") - .setSource(new BytesArray(source)) - .get(); - fail(); - } catch (SearchPhaseExecutionException e) { - assertThat(e.toString(), containsString("Aggregator [top_tags_hits] of type [top_hits] cannot accept sub-aggregations")); - } - } + // public void testFailWithSubAgg() throws Exception { + // String source = "{\n" + + // " \"aggs\": {\n" + + // " \"top-tags\": {\n" + + // " \"terms\": {\n" + + // " \"field\": \"tags\"\n" + + // " },\n" + + // " \"aggs\": {\n" + + // " \"top_tags_hits\": {\n" + + // " \"top_hits\": {},\n" + + // " \"aggs\": {\n" + + // " \"max\": {\n" + + // " \"max\": {\n" + + // " \"field\": \"age\"\n" + + // " }\n" + + // " }\n" + + // " }\n" + + // " }\n" + + // " }\n" + + // " }\n" + + // " }\n" + + // "}"; + // try { + // client().prepareSearch("idx").setTypes("type") + // .setSource(new BytesArray(source)) + // .get(); + // fail(); + // } catch (SearchPhaseExecutionException e) { + // assertThat(e.toString(), + // containsString("Aggregator [top_tags_hits] of type [top_hits] cannot accept sub-aggregations")); + // } + // } NORELEASE this needs to be tested in a top_hits aggregations unit test - @Test public void testEmptyIndex() throws Exception { SearchResponse response = client().prepareSearch("empty").setTypes("type") .addAggregation(topHits("hits")) @@ -649,7 +640,6 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(hits.getHits().totalHits(), equalTo(0l)); } - @Test public void testTrackScores() throws Exception { boolean[] trackScores = new boolean[]{true, false}; for (boolean trackScore : trackScores) { @@ -696,7 +686,6 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test public void testTopHitsInNestedSimple() throws Exception { SearchResponse searchResponse = client().prepareSearch("articles") .setQuery(matchQuery("title", "title")) @@ -748,7 +737,6 @@ public class TopHitsIT extends ESIntegTestCase { assertThat((Integer) searchHits.getAt(0).getSource().get("date"), equalTo(4)); } - @Test public void testTopHitsInSecondLayerNested() throws Exception { SearchResponse searchResponse = client().prepareSearch("articles") .setQuery(matchQuery("title", "title")) @@ -849,7 +837,6 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0)); } - @Test public void testNestedFetchFeatures() { String hlType = randomFrom("plain", "fvh", "postings"); HighlightBuilder.Field hlField = new HighlightBuilder.Field("comments.message") @@ -862,7 +849,7 @@ public class TopHitsIT extends ESIntegTestCase { .setQuery(nestedQuery("comments", matchQuery("comments.message", "comment").queryName("test"))) .addAggregation( nested("to-comments").path("comments").subAggregation( - topHits("top-comments").setSize(1).addHighlightedField(hlField).setExplain(true) + topHits("top-comments").setSize(1).highlighter(new HighlightBuilder().field(hlField)).setExplain(true) .addFieldDataField("comments.user") .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())).setFetchSource("message", null) .setVersion(true).addSort("comments.date", SortOrder.ASC))).get(); @@ -902,7 +889,6 @@ public class TopHitsIT extends ESIntegTestCase { assertThat(searchHit.sourceAsMap().get("message").toString(), equalTo("some comment")); } - @Test public void testTopHitsInNested() throws Exception { SearchResponse searchResponse = client().prepareSearch("articles") .addAggregation( @@ -914,7 +900,7 @@ public class TopHitsIT extends ESIntegTestCase { nested("to-comments") .path("comments") .subAggregation(topHits("comments") - .addHighlightedField(new HighlightBuilder.Field("comments.message").highlightQuery(matchQuery("comments.message", "text"))) + .highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("comments.message").highlightQuery(matchQuery("comments.message", "text")))) .addSort("comments.id", SortOrder.ASC)) ) ) @@ -944,7 +930,6 @@ public class TopHitsIT extends ESIntegTestCase { } } - @Test public void testDontExplode() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ValueCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java similarity index 56% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ValueCountTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java index ca2cffce59b..fde7256ad01 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ValueCountTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java @@ -16,22 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.metrics; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; +import org.elasticsearch.script.*; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.io.IOException; +import java.util.*; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -45,13 +45,7 @@ import static org.hamcrest.Matchers.notNullValue; * */ @ESIntegTestCase.SuiteScopeTestCase -public class ValueCountTests extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - +public class ValueCountIT extends ESIntegTestCase { @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -69,8 +63,12 @@ public class ValueCountTests extends ESIntegTestCase { ensureSearchable(); } - @Test - public void unmapped() throws Exception { + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(FieldValueScriptPlugin.class); + } + + public void testUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx_unmapped") .setQuery(matchAllQuery()) .addAggregation(count("count").field("value")) @@ -84,9 +82,7 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(0l)); } - @Test - public void singleValuedField() throws Exception { - + public void testSingleValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(count("count").field("value")) @@ -100,9 +96,7 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(10l)); } - @Test - public void singleValuedField_getProperty() throws Exception { - + public void testSingleValuedFieldGetProperty() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(global("global").subAggregation(count("count").field("value"))).execute().actionGet(); @@ -124,8 +118,7 @@ public class ValueCountTests extends ESIntegTestCase { assertThat((double) valueCount.getProperty("value"), equalTo(10d)); } - @Test - public void singleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") .setQuery(matchAllQuery()) .addAggregation(count("count").field("value")) @@ -139,9 +132,7 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(10l)); } - @Test - public void multiValuedField() throws Exception { - + public void testMultiValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(count("count").field("values")) @@ -155,10 +146,9 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(20l)); } - @Test - public void singleValuedScript() throws Exception { + public void testSingleValuedScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(count("count").script(new Script("doc['value'].value"))).execute().actionGet(); + .addAggregation(count("count").script(new Script("value", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))).execute().actionGet(); assertHitCount(searchResponse, 10); @@ -168,10 +158,9 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(10l)); } - @Test - public void multiValuedScript() throws Exception { + public void testMultiValuedScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(count("count").script(new Script("doc['values'].values"))).execute().actionGet(); + .addAggregation(count("count").script(new Script("values", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))).execute().actionGet(); assertHitCount(searchResponse, 10); @@ -181,12 +170,10 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(20l)); } - @Test - public void singleValuedScriptWithParams() throws Exception { - Map params = new HashMap<>(); - params.put("s", "value"); + public void testSingleValuedScriptWithParams() throws Exception { + Map params = Collections.singletonMap("s", "value"); SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(count("count").script(new Script("doc[s].value", ScriptType.INLINE, null, params))).execute().actionGet(); + .addAggregation(count("count").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))).execute().actionGet(); assertHitCount(searchResponse, 10); @@ -196,12 +183,10 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(10l)); } - @Test - public void multiValuedScriptWithParams() throws Exception { - Map params = new HashMap<>(); - params.put("s", "values"); + public void testMultiValuedScriptWithParams() throws Exception { + Map params = Collections.singletonMap("s", "values"); SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(count("count").script(new Script("doc[s].values", ScriptType.INLINE, null, params))).execute().actionGet(); + .addAggregation(count("count").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, params))).execute().actionGet(); assertHitCount(searchResponse, 10); @@ -211,4 +196,138 @@ public class ValueCountTests extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(20l)); } + /** + * Mock plugin for the {@link FieldValueScriptEngine} + */ + public static class FieldValueScriptPlugin extends Plugin { + + @Override + public String name() { + return FieldValueScriptEngine.NAME; + } + + @Override + public String description() { + return "Mock script engine for " + ValueCountIT.class; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(FieldValueScriptEngine.class); + } + + } + + /** + * This mock script returns the field value. If the parameter map contains a parameter "s", the corresponding is used as field name. + */ + public static class FieldValueScriptEngine implements ScriptEngineService { + + public static final String NAME = "field_value"; + + @Override + public void close() throws IOException { + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return types(); + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + return script; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + throw new UnsupportedOperationException(); + } + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + final String fieldNameParam; + if (vars == null || vars.containsKey("s") == false) { + fieldNameParam = null; + } else { + fieldNameParam = (String) vars.get("s"); + } + + return new SearchScript() { + private Map vars = new HashMap<>(2); + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + + return new LeafSearchScript() { + + @Override + public Object unwrap(Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public void setNextVar(String name, Object value) { + vars.put(name, value); + } + + @Override + public Object run() { + String fieldName = (fieldNameParam != null) ? fieldNameParam : (String) compiledScript.compiled(); + return leafLookup.doc().get(fieldName); + } + + @Override + public void setScorer(Scorer scorer) { + } + + @Override + public void setSource(Map source) { + } + + @Override + public void setDocument(int doc) { + if (leafLookup != null) { + leafLookup.setDocument(doc); + } + } + + @Override + public long runAsLong() { + throw new UnsupportedOperationException(); + } + + @Override + public float runAsFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public double runAsDouble() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + } + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlusTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlusTests.java index 684a4336600..9a00297c57e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlusTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlusTests.java @@ -21,18 +21,16 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; import com.carrotsearch.hppc.BitMixer; import com.carrotsearch.hppc.IntHashSet; + import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.elasticsearch.search.aggregations.metrics.cardinality.HyperLogLogPlusPlus.MAX_PRECISION; import static org.elasticsearch.search.aggregations.metrics.cardinality.HyperLogLogPlusPlus.MIN_PRECISION; import static org.hamcrest.Matchers.closeTo; public class HyperLogLogPlusPlusTests extends ESTestCase { - - @Test - public void encodeDecode() { + public void testEncodeDecode() { final int iters = scaledRandomIntBetween(100000, 500000); // random hashes for (int i = 0; i < iters; ++i) { @@ -56,8 +54,7 @@ public class HyperLogLogPlusPlusTests extends ESTestCase { assertEquals(runLen, HyperLogLogPlusPlus.decodeRunLen(encoded, p1)); } - @Test - public void accuracy() { + public void testAccuracy() { final long bucket = randomInt(20); final int numValues = randomIntBetween(1, 100000); final int maxValue = randomIntBetween(1, randomBoolean() ? 1000: 100000); @@ -77,8 +74,7 @@ public class HyperLogLogPlusPlusTests extends ESTestCase { assertThat((double) e.cardinality(bucket), closeTo(set.size(), 0.1 * set.size())); } - @Test - public void merge() { + public void testMerge() { final int p = randomIntBetween(MIN_PRECISION, MAX_PRECISION); final HyperLogLogPlusPlus single = new HyperLogLogPlusPlus(p, BigArrays.NON_RECYCLING_INSTANCE, 0); final HyperLogLogPlusPlus[] multi = new HyperLogLogPlusPlus[randomIntBetween(2, 100)]; @@ -106,8 +102,7 @@ public class HyperLogLogPlusPlusTests extends ESTestCase { } } - @Test - public void fakeHashes() { + public void testFakeHashes() { // hashes with lots of leading zeros trigger different paths in the code that we try to go through here final int p = randomIntBetween(MIN_PRECISION, MAX_PRECISION); final HyperLogLogPlusPlus counts = new HyperLogLogPlusPlus(p, BigArrays.NON_RECYCLING_INSTANCE, 0); @@ -123,8 +118,7 @@ public class HyperLogLogPlusPlusTests extends ESTestCase { assertEquals(1, counts.cardinality(0)); } - @Test - public void precisionFromThreshold() { + public void testPrecisionFromThreshold() { assertEquals(4, HyperLogLogPlusPlus.precisionFromThreshold(0)); assertEquals(6, HyperLogLogPlusPlus.precisionFromThreshold(10)); assertEquals(10, HyperLogLogPlusPlus.precisionFromThreshold(100)); @@ -133,5 +127,4 @@ public class HyperLogLogPlusPlusTests extends ESTestCase { assertEquals(18, HyperLogLogPlusPlus.precisionFromThreshold(100000)); assertEquals(18, HyperLogLogPlusPlus.precisionFromThreshold(1000000)); } - } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java index a18f1296ddf..ea0eb7fd93a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java @@ -28,17 +28,15 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.avgBucket; - import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.avgBucket; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; @@ -91,8 +89,7 @@ public class AvgBucketIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void testDocCount_topLevel() throws Exception { + public void testDocCountTopLevel() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .extendedBounds((long) minRandomValue, (long) maxRandomValue)) @@ -124,8 +121,7 @@ public class AvgBucketIT extends ESIntegTestCase { assertThat(avgBucketValue.value(), equalTo(avgValue)); } - @Test - public void testDocCount_asSubAgg() throws Exception { + public void testDocCountAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -173,8 +169,7 @@ public class AvgBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevel() throws Exception { + public void testMetricTopLevel() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -208,8 +203,7 @@ public class AvgBucketIT extends ESIntegTestCase { assertThat(avgBucketValue.value(), equalTo(avgValue)); } - @Test - public void testMetric_asSubAgg() throws Exception { + public void testMetricAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -262,8 +256,7 @@ public class AvgBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAggWithInsertZeros() throws Exception { + public void testMetricAsSubAggWithInsertZeros() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -316,7 +309,6 @@ public class AvgBucketIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -336,7 +328,6 @@ public class AvgBucketIT extends ESIntegTestCase { assertThat(avgBucketValue.value(), equalTo(Double.NaN)); } - @Test public void testNested() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumIT.java index 1d844e17af2..6f10e5d91fa 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -86,7 +85,6 @@ public class CumulativeSumIT extends ESIntegTestCase { ensureSearchable(); } - @Test public void testDocCount() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) @@ -116,7 +114,6 @@ public class CumulativeSumIT extends ESIntegTestCase { } - @Test public void testMetric() throws Exception { SearchResponse response = client() .prepareSearch("idx") @@ -149,7 +146,6 @@ public class CumulativeSumIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java index 569830b0a95..3058d1f10d6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java @@ -34,17 +34,16 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.junit.After; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.derivative; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.derivative; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; @@ -105,8 +104,7 @@ public class DateDerivativeIT extends ESIntegTestCase { internalCluster().wipeIndices("idx2"); } - @Test - public void singleValuedField() throws Exception { + public void testSingleValuedField() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -148,8 +146,7 @@ public class DateDerivativeIT extends ESIntegTestCase { assertThat(docCountDeriv.value(), equalTo(1d)); } - @Test - public void singleValuedField_normalised() throws Exception { + public void testSingleValuedFieldNormalised() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -194,8 +191,7 @@ public class DateDerivativeIT extends ESIntegTestCase { assertThat(docCountDeriv.normalizedValue(), closeTo(1d / 29d, 0.00001)); } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -264,8 +260,7 @@ public class DateDerivativeIT extends ESIntegTestCase { assertThat((double) propertiesCounts[2], equalTo(15.0)); } - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -320,8 +315,7 @@ public class DateDerivativeIT extends ESIntegTestCase { assertThat(docCountDeriv.value(), equalTo(-2.0)); } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx_unmapped") .addAggregation( @@ -336,8 +330,7 @@ public class DateDerivativeIT extends ESIntegTestCase { assertThat(deriv.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx", "idx_unmapped") .addAggregation( diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java index fbbb173ee7f..b65a86ac57d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java @@ -36,7 +36,6 @@ import org.elasticsearch.search.aggregations.pipeline.derivative.Derivative; import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -167,8 +166,7 @@ public class DerivativeIT extends ESIntegTestCase { /** * test first and second derivative on the sing */ - @Test - public void docCountDerivative() { + public void testDocCountDerivative() { SearchResponse response = client() .prepareSearch("idx") @@ -208,9 +206,7 @@ public class DerivativeIT extends ESIntegTestCase { /** * test first and second derivative on the sing */ - @Test - public void singleValuedField_normalised() { - + public void testSingleValuedField_normalised() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -248,8 +244,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void singleValueAggDerivative() throws Exception { + public void testSingleValueAggDerivative() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -294,8 +289,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void multiValueAggDerivative() throws Exception { + public void testMultiValueAggDerivative() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -340,8 +334,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx_unmapped") .addAggregation( @@ -356,8 +349,7 @@ public class DerivativeIT extends ESIntegTestCase { assertThat(deriv.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx", "idx_unmapped") .addAggregation( @@ -385,8 +377,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void docCountDerivativeWithGaps() throws Exception { + public void testDocCountDerivativeWithGaps() throws Exception { SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) @@ -414,8 +405,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void docCountDerivativeWithGaps_random() throws Exception { + public void testDocCountDerivativeWithGaps_random() throws Exception { SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx_rnd") .setQuery(matchAllQuery()) @@ -445,8 +435,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void docCountDerivativeWithGaps_insertZeros() throws Exception { + public void testDocCountDerivativeWithGaps_insertZeros() throws Exception { SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) @@ -475,8 +464,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void singleValueAggDerivativeWithGaps() throws Exception { + public void testSingleValueAggDerivativeWithGaps() throws Exception { SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) @@ -517,8 +505,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void singleValueAggDerivativeWithGaps_insertZeros() throws Exception { + public void testSingleValueAggDerivativeWithGaps_insertZeros() throws Exception { SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) @@ -556,8 +543,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void singleValueAggDerivativeWithGaps_random() throws Exception { + public void testSingleValueAggDerivativeWithGaps_random() throws Exception { GapPolicy gapPolicy = randomFrom(GapPolicy.values()); SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx_rnd") @@ -600,8 +586,7 @@ public class DerivativeIT extends ESIntegTestCase { } } - @Test - public void singleValueAggDerivative_invalidPath() throws Exception { + public void testSingleValueAggDerivative_invalidPath() throws Exception { try { client().prepareSearch("idx") .addAggregation( diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java index 3c3d705ecc0..6c7ae2383f0 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java @@ -30,17 +30,15 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucket; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.extendedStatsBucket; - import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.extendedStatsBucket; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; @@ -93,8 +91,7 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void testDocCount_topLevel() throws Exception { + public void testDocCountTopLevel() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .extendedBounds((long) minRandomValue, (long) maxRandomValue)) @@ -135,8 +132,7 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { assertThat(extendedStatsBucketValue.getSumOfSquares(), equalTo(sumOfSquares)); } - @Test - public void testDocCount_asSubAgg() throws Exception { + public void testDocCountAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -193,8 +189,7 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevel() throws Exception { + public void testMetricTopLevel() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -237,8 +232,7 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { assertThat(extendedStatsBucketValue.getSumOfSquares(), equalTo(sumOfSquares)); } - @Test - public void testMetric_asSubAgg() throws Exception { + public void testMetricAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -300,8 +294,7 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAggWithInsertZeros() throws Exception { + public void testMetricAsSubAggWithInsertZeros() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -363,7 +356,6 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -383,8 +375,7 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { assertThat(extendedStatsBucketValue.getAvg(), equalTo(Double.NaN)); } - @Test - public void testBadSigma_asSubAgg() throws Exception { + public void testBadSigmaAsSubAgg() throws Exception { try { SearchResponse response = client() .prepareSearch("idx") @@ -404,7 +395,6 @@ public class ExtendedStatsBucketIT extends ESIntegTestCase { } } - @Test public void testNested() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java index bacb6bda9b6..81b5735012e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -94,8 +93,7 @@ public class MaxBucketIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void testDocCount_topLevel() throws Exception { + public void testDocCountTopLevel() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .extendedBounds((long) minRandomValue, (long) maxRandomValue)) @@ -132,8 +130,7 @@ public class MaxBucketIT extends ESIntegTestCase { assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()]))); } - @Test - public void testDocCount_asSubAgg() throws Exception { + public void testDocCountAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -186,8 +183,7 @@ public class MaxBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevel() throws Exception { + public void testMetricTopLevel() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -226,8 +222,7 @@ public class MaxBucketIT extends ESIntegTestCase { assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()]))); } - @Test - public void testMetric_asSubAgg() throws Exception { + public void testMetricAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -285,8 +280,7 @@ public class MaxBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAggOfSingleBucketAgg() throws Exception { + public void testMetricAsSubAggOfSingleBucketAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -334,8 +328,7 @@ public class MaxBucketIT extends ESIntegTestCase { assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()]))); } - @Test - public void testMetric_asSubAggWithInsertZeros() throws Exception { + public void testMetricAsSubAggWithInsertZeros() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -392,7 +385,6 @@ public class MaxBucketIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -413,7 +405,6 @@ public class MaxBucketIT extends ESIntegTestCase { assertThat(maxBucketValue.keys(), equalTo(new String[0])); } - @Test public void testNested() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java index cb83f60f23a..f02a85f130d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java @@ -29,16 +29,15 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.minBucket; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.minBucket; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; @@ -91,8 +90,7 @@ public class MinBucketIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void testDocCount_topLevel() throws Exception { + public void testDocCountTopLevel() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .extendedBounds((long) minRandomValue, (long) maxRandomValue)) @@ -129,8 +127,7 @@ public class MinBucketIT extends ESIntegTestCase { assertThat(minBucketValue.keys(), equalTo(minKeys.toArray(new String[minKeys.size()]))); } - @Test - public void testDocCount_asSubAgg() throws Exception { + public void testDocCountAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -183,8 +180,7 @@ public class MinBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevel() throws Exception { + public void testMetricTopLevel() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -223,8 +219,7 @@ public class MinBucketIT extends ESIntegTestCase { assertThat(minBucketValue.keys(), equalTo(minKeys.toArray(new String[minKeys.size()]))); } - @Test - public void testMetric_asSubAgg() throws Exception { + public void testMetricAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -282,8 +277,7 @@ public class MinBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAggWithInsertZeros() throws Exception { + public void testMetricAsSubAggWithInsertZeros() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -340,7 +334,6 @@ public class MinBucketIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -361,7 +354,6 @@ public class MinBucketIT extends ESIntegTestCase { assertThat(minBucketValue.keys(), equalTo(new String[0])); } - @Test public void testNested() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java index f7c1d060bd3..c4dc267ec5b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucket; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -94,8 +93,7 @@ public class PercentilesBucketIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void testDocCount_topLevel() throws Exception { + public void testDocCountopLevel() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .extendedBounds((long) minRandomValue, (long) maxRandomValue)) @@ -132,8 +130,7 @@ public class PercentilesBucketIT extends ESIntegTestCase { } - @Test - public void testDocCount_asSubAgg() throws Exception { + public void testDocCountAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -185,8 +182,7 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevel() throws Exception { + public void testMetricTopLevel() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -224,8 +220,7 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevelDefaultPercents() throws Exception { + public void testMetricTopLevelDefaultPercents() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -263,8 +258,7 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAgg() throws Exception { + public void testMetricAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -321,8 +315,7 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAggWithInsertZeros() throws Exception { + public void testMetricAsSubAggWithInsertZeros() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -380,7 +373,6 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -404,7 +396,6 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test public void testWrongPercents() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -432,7 +423,6 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test public void testBadPercents() throws Exception { Double[] badPercents = {-1.0, 110.0}; @@ -453,7 +443,6 @@ public class PercentilesBucketIT extends ESIntegTestCase { } - @Test public void testBadPercents_asSubAgg() throws Exception { Double[] badPercents = {-1.0, 110.0}; @@ -481,7 +470,6 @@ public class PercentilesBucketIT extends ESIntegTestCase { } - @Test public void testNested() throws Exception { SearchResponse response = client() .prepareSearch("idx") @@ -547,7 +535,6 @@ public class PercentilesBucketIT extends ESIntegTestCase { } } - @Test public void testNestedWithDecimal() throws Exception { Double[] percent = {99.9}; SearchResponse response = client() diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java index 866fdc07738..92325ccd81b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java @@ -29,17 +29,15 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucket; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.statsBucket; - import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.statsBucket; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; @@ -92,8 +90,7 @@ public class StatsBucketIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void testDocCount_topLevel() throws Exception { + public void testDocCountTopLevel() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .extendedBounds((long) minRandomValue, (long) maxRandomValue)) @@ -131,8 +128,7 @@ public class StatsBucketIT extends ESIntegTestCase { assertThat(statsBucketValue.getMax(), equalTo(max)); } - @Test - public void testDocCount_asSubAgg() throws Exception { + public void testDocCountAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -186,8 +182,7 @@ public class StatsBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevel() throws Exception { + public void testMetricTopLevel() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -227,8 +222,7 @@ public class StatsBucketIT extends ESIntegTestCase { assertThat(statsBucketValue.getMax(), equalTo(max)); } - @Test - public void testMetric_asSubAgg() throws Exception { + public void testMetricAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -287,8 +281,7 @@ public class StatsBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAggWithInsertZeros() throws Exception { + public void testMetricAsSubAggWithInsertZeros() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -347,7 +340,6 @@ public class StatsBucketIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -367,7 +359,6 @@ public class StatsBucketIT extends ESIntegTestCase { assertThat(statsBucketValue.getAvg(), equalTo(Double.NaN)); } - @Test public void testNested() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java index be11f90fa62..ba13b553d89 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java @@ -28,17 +28,15 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.sumBucket; - import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.sumBucket; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; @@ -91,8 +89,7 @@ public class SumBucketIT extends ESIntegTestCase { ensureSearchable(); } - @Test - public void testDocCount_topLevel() throws Exception { + public void testDocCountTopLevel() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .extendedBounds((long) minRandomValue, (long) maxRandomValue)) @@ -121,8 +118,7 @@ public class SumBucketIT extends ESIntegTestCase { assertThat(sumBucketValue.value(), equalTo(sum)); } - @Test - public void testDocCount_asSubAgg() throws Exception { + public void testDocCountAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -167,8 +163,7 @@ public class SumBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_topLevel() throws Exception { + public void testMetricTopLevel() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation(terms("terms").field("tag").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -199,8 +194,7 @@ public class SumBucketIT extends ESIntegTestCase { assertThat(sumBucketValue.value(), equalTo(bucketSum)); } - @Test - public void testMetric_asSubAgg() throws Exception { + public void testMetricAsSubAgg() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -250,8 +244,7 @@ public class SumBucketIT extends ESIntegTestCase { } } - @Test - public void testMetric_asSubAggWithInsertZeros() throws Exception { + public void testMetricAsSubAggWithInsertZeros() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -301,7 +294,6 @@ public class SumBucketIT extends ESIntegTestCase { } } - @Test public void testNoBuckets() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field("tag").exclude("tag.*").subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -321,7 +313,6 @@ public class SumBucketIT extends ESIntegTestCase { assertThat(sumBucketValue.value(), equalTo(0.0)); } - @Test public void testNested() throws Exception { SearchResponse response = client() .prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java index ac4fcf89aef..90d4437fcea 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java @@ -32,28 +32,41 @@ import org.elasticsearch.search.aggregations.pipeline.BucketHelpers; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregationHelperTests; import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.search.aggregations.pipeline.derivative.Derivative; -import org.elasticsearch.search.aggregations.pipeline.movavg.models.*; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModelBuilder; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.max; +import static org.elasticsearch.search.aggregations.AggregationBuilders.min; +import static org.elasticsearch.search.aggregations.AggregationBuilders.range; import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.derivative; import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.movingAvg; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; import static org.hamcrest.core.IsNull.nullValue; @ESIntegTestCase.SuiteScopeTestCase public class MovAvgIT extends ESIntegTestCase { - private static final String INTERVAL_FIELD = "l_value"; private static final String VALUE_FIELD = "v_value"; - private static final String GAP_FIELD = "g_value"; static int interval; static int numBuckets; @@ -79,6 +92,7 @@ public class MovAvgIT extends ESIntegTestCase { name = s; } + @Override public String toString(){ return name; } @@ -93,6 +107,7 @@ public class MovAvgIT extends ESIntegTestCase { name = s; } + @Override public String toString(){ return name; } @@ -342,8 +357,8 @@ public class MovAvgIT extends ESIntegTestCase { s += vs[i]; b += (vs[i + period] - vs[i]) / period; } - s /= (double) period; - b /= (double) period; + s /= period; + b /= period; last_s = s; // Calculate first seasonal @@ -388,9 +403,7 @@ public class MovAvgIT extends ESIntegTestCase { /** * test simple moving average on single value field */ - @Test - public void simpleSingleValuedField() { - + public void testSimpleSingleValuedField() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -440,9 +453,7 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test - public void linearSingleValuedField() { - + public void testLinearSingleValuedField() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -492,9 +503,7 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test - public void ewmaSingleValuedField() { - + public void testEwmaSingleValuedField() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -544,9 +553,7 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test - public void holtSingleValuedField() { - + public void testHoltSingleValuedField() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -596,9 +603,7 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test - public void HoltWintersValuedField() { - + public void testHoltWintersValuedField() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -652,7 +657,6 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test public void testPredictNegativeKeysAtStart() { SearchResponse response = client() @@ -704,8 +708,6 @@ public class MovAvgIT extends ESIntegTestCase { } } - - @Test public void testSizeZeroWindow() { try { client() @@ -721,13 +723,11 @@ public class MovAvgIT extends ESIntegTestCase { .setBucketsPaths("the_metric")) ).execute().actionGet(); fail("MovingAvg should not accept a window that is zero"); - - } catch (SearchPhaseExecutionException exception) { - // All good + } catch (SearchPhaseExecutionException e) { + assertThat(e.getMessage(), is("all shards failed")); } } - @Test public void testBadParent() { try { client() @@ -748,7 +748,6 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test public void testNegativeWindow() { try { client() @@ -772,14 +771,12 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test public void testNoBucketsInHistogram() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( histogram("histo").field("test").interval(interval) - .extendedBounds(0L, (long) (interval * (numBuckets - 1))) .subAggregation(randomMetric("the_metric", VALUE_FIELD)) .subAggregation(movingAvg("movavg_counts") .window(windowSize) @@ -797,14 +794,12 @@ public class MovAvgIT extends ESIntegTestCase { assertThat(buckets.size(), equalTo(0)); } - @Test public void testNoBucketsInHistogramWithPredict() { int numPredictions = randomIntBetween(1,10); SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( histogram("histo").field("test").interval(interval) - .extendedBounds(0L, (long) (interval * (numBuckets - 1))) .subAggregation(randomMetric("the_metric", VALUE_FIELD)) .subAggregation(movingAvg("movavg_counts") .window(windowSize) @@ -823,7 +818,6 @@ public class MovAvgIT extends ESIntegTestCase { assertThat(buckets.size(), equalTo(0)); } - @Test public void testZeroPrediction() { try { client() @@ -846,7 +840,6 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test public void testNegativePrediction() { try { client() @@ -869,7 +862,6 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test public void testHoltWintersNotEnoughData() { try { SearchResponse response = client() @@ -897,9 +889,7 @@ public class MovAvgIT extends ESIntegTestCase { } - @Test public void testTwoMovAvgsWithPredictions() { - SearchResponse response = client() .prepareSearch("double_predict") .setTypes("type") @@ -1011,7 +1001,6 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test public void testBadModelParams() { try { SearchResponse response = client() @@ -1032,9 +1021,7 @@ public class MovAvgIT extends ESIntegTestCase { } - @Test - public void HoltWintersMinimization() { - + public void testHoltWintersMinimization() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -1122,9 +1109,7 @@ public class MovAvgIT extends ESIntegTestCase { * * We can simulate this by setting the window size == size of histo */ - @Test - public void minimizeNotEnoughData() { - + public void testMinimizeNotEnoughData() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -1179,9 +1164,7 @@ public class MovAvgIT extends ESIntegTestCase { /** * Only some models can be minimized, should throw exception for: simple, linear */ - @Test - public void checkIfNonTunableCanBeMinimized() { - + public void testCheckIfNonTunableCanBeMinimized() { try { client() .prepareSearch("idx").setTypes("type") @@ -1224,9 +1207,7 @@ public class MovAvgIT extends ESIntegTestCase { /** * These models are all minimizable, so they should not throw exceptions */ - @Test - public void checkIfTunableCanBeMinimized() { - + public void testCheckIfTunableCanBeMinimized() { MovAvgModelBuilder[] builders = new MovAvgModelBuilder[]{ new EwmaModel.EWMAModelBuilder(), new HoltLinearModel.HoltLinearModelBuilder(), @@ -1254,9 +1235,7 @@ public class MovAvgIT extends ESIntegTestCase { } } - @Test public void testUnrecognizedParams() { - MovAvgModelBuilder[] builders = new MovAvgModelBuilder[]{ new SimpleModel.SimpleModelBuilder(), new LinearModel.LinearModelBuilder(), @@ -1375,7 +1354,7 @@ public class MovAvgIT extends ESIntegTestCase { return new SimpleModel.SimpleModelBuilder(); } } - + private ValuesSourceMetricsAggregationBuilder randomMetric(String name, String field) { int rand = randomIntBetween(0,3); @@ -1388,7 +1367,7 @@ public class MovAvgIT extends ESIntegTestCase { return avg(name).field(field); default: return avg(name).field(field); - } + } } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java index 11c5e4035d6..da9c8236f99 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java @@ -21,18 +21,24 @@ package org.elasticsearch.search.aggregations.pipeline.moving.avg; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.collect.EvictingQueue; -import org.elasticsearch.search.aggregations.pipeline.movavg.models.*; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel; +import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.text.ParseException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.equalTo; public class MovAvgUnitTests extends ESTestCase { - - @Test public void testSimpleMovAvgModel() { MovAvgModel model = new SimpleModel(); @@ -61,7 +67,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testSimplePredictionModel() { MovAvgModel model = new SimpleModel(); @@ -87,7 +92,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testLinearMovAvgModel() { MovAvgModel model = new LinearModel(); @@ -119,7 +123,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testLinearPredictionModel() { MovAvgModel model = new LinearModel(); @@ -150,7 +153,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testEWMAMovAvgModel() { double alpha = randomDouble(); MovAvgModel model = new EwmaModel(alpha); @@ -185,7 +187,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testEWMAPredictionModel() { double alpha = randomDouble(); MovAvgModel model = new EwmaModel(alpha); @@ -218,7 +219,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testHoltLinearMovAvgModel() { double alpha = randomDouble(); double beta = randomDouble(); @@ -267,7 +267,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testHoltLinearPredictionModel() { double alpha = randomDouble(); double beta = randomDouble(); @@ -313,7 +312,6 @@ public class MovAvgUnitTests extends ESTestCase { } } - @Test public void testHoltWintersMultiplicativePadModel() { double alpha = randomDouble(); double beta = randomDouble(); @@ -353,8 +351,8 @@ public class MovAvgUnitTests extends ESTestCase { s += vs[i]; b += (vs[i + period] - vs[i]) / period; } - s /= (double) period; - b /= (double) period; + s /= period; + b /= period; last_s = s; // Calculate first seasonal @@ -381,7 +379,6 @@ public class MovAvgUnitTests extends ESTestCase { assertThat(Double.compare(expected, actual), equalTo(0)); } - @Test public void testHoltWintersMultiplicativePadPredictionModel() { double alpha = randomDouble(); double beta = randomDouble(); @@ -424,8 +421,8 @@ public class MovAvgUnitTests extends ESTestCase { s += vs[i]; b += (vs[i + period] - vs[i]) / period; } - s /= (double) period; - b /= (double) period; + s /= period; + b /= period; last_s = s; // Calculate first seasonal @@ -455,7 +452,6 @@ public class MovAvgUnitTests extends ESTestCase { } - @Test public void testHoltWintersAdditiveModel() { double alpha = randomDouble(); double beta = randomDouble(); @@ -494,8 +490,8 @@ public class MovAvgUnitTests extends ESTestCase { s += vs[i]; b += (vs[i + period] - vs[i]) / period; } - s /= (double) period; - b /= (double) period; + s /= period; + b /= period; last_s = s; // Calculate first seasonal @@ -522,7 +518,6 @@ public class MovAvgUnitTests extends ESTestCase { assertThat(Double.compare(expected, actual), equalTo(0)); } - @Test public void testHoltWintersAdditivePredictionModel() { double alpha = randomDouble(); double beta = randomDouble(); @@ -564,8 +559,8 @@ public class MovAvgUnitTests extends ESTestCase { s += vs[i]; b += (vs[i + period] - vs[i]) / period; } - s /= (double) period; - b /= (double) period; + s /= period; + b /= period; last_s = s; // Calculate first seasonal @@ -594,9 +589,7 @@ public class MovAvgUnitTests extends ESTestCase { } - @Test public void testNumericValidation() { - List parsers = new ArrayList<>(5); // Simple and Linear don't have any settings to test diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java index ccd4dcbc136..aebd6a7e780 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java @@ -31,16 +31,23 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregationHelperT import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.max; +import static org.elasticsearch.search.aggregations.AggregationBuilders.min; import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.diff; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; import static org.hamcrest.core.IsNull.nullValue; @@ -67,6 +74,7 @@ public class SerialDiffIT extends ESIntegTestCase { name = s; } + @Override public String toString(){ return name; } @@ -218,9 +226,7 @@ public class SerialDiffIT extends ESIntegTestCase { testValues.put(target.toString(), values); } - @Test - public void basicDiff() { - + public void testBasicDiff() { SearchResponse response = client() .prepareSearch("idx").setTypes("type") .addAggregation( @@ -268,8 +274,7 @@ public class SerialDiffIT extends ESIntegTestCase { } } - @Test - public void invalidLagSize() { + public void testInvalidLagSize() { try { client() .prepareSearch("idx").setTypes("type") @@ -283,9 +288,7 @@ public class SerialDiffIT extends ESIntegTestCase { .setBucketsPaths("_count")) ).execute().actionGet(); } catch (SearchPhaseExecutionException e) { - // All good + assertThat(e.getMessage(), is("all shards failed")); } } - - } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/PathTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/PathTests.java index 77a3e123629..4602035a40b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/PathTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/PathTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.support; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -32,8 +31,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class PathTests extends ESTestCase { - - @Test public void testInvalidPaths() throws Exception { assertInvalidPath("[foo]", "brackets at the beginning of the token expression"); assertInvalidPath("foo[bar", "open brackets without closing at the token expression"); @@ -44,7 +41,6 @@ public class PathTests extends ESTestCase { assertInvalidPath("foo.", "dot separator at the end of the token expression"); } - @Test public void testValidPaths() throws Exception { assertValidPath("foo>bar", tokens().add("foo").add("bar")); assertValidPath("foo.bar", tokens().add("foo", "bar")); @@ -81,7 +77,6 @@ public class PathTests extends ESTestCase { } private static class Tokens { - private List tokens = new ArrayList<>(); Tokens add(String name) { @@ -101,8 +96,5 @@ public class PathTests extends ESTestCase { AggregationPath.PathElement[] toArray() { return tokens.toArray(new AggregationPath.PathElement[tokens.size()]); } - - } - } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index d1a97c7616c..18e93656562 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.support.values.ScriptBytesValues; import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues; import org.elasticsearch.search.aggregations.support.values.ScriptLongValues; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.Map; @@ -36,10 +35,10 @@ import java.util.Map; public class ScriptValuesTests extends ESTestCase { private static class FakeSearchScript implements LeafSearchScript { - + private final Object[][] values; int index; - + FakeSearchScript(Object[][] values) { this.values = values; index = -1; @@ -94,8 +93,7 @@ public class ScriptValuesTests extends ESTestCase { } - @Test - public void longs() { + public void testLongs() { final Object[][] values = new Long[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Long[] longs = new Long[randomInt(8)]; @@ -116,8 +114,7 @@ public class ScriptValuesTests extends ESTestCase { } } - @Test - public void doubles() { + public void testDoubles() { final Object[][] values = new Double[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Double[] doubles = new Double[randomInt(8)]; @@ -138,8 +135,7 @@ public class ScriptValuesTests extends ESTestCase { } } - @Test - public void bytes() { + public void testBytes() { final String[][] values = new String[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { String[] strings = new String[randomInt(8)]; diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java index cea6f92f3a6..8d401e5e2e6 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java @@ -19,13 +19,12 @@ package org.elasticsearch.search.basic; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -36,23 +35,18 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; * shards possibly not active at all (cause they haven't allocated) will still work. */ public class SearchWhileCreatingIndexIT extends ESIntegTestCase { - - @Test public void testIndexCausesIndexCreation() throws Exception { searchWhileCreatingIndex(false, 1); // 1 replica in our default... } - @Test public void testNoReplicas() throws Exception { searchWhileCreatingIndex(true, 0); } - @Test public void testOneReplica() throws Exception { searchWhileCreatingIndex(true, 1); } - @Test public void testTwoReplicas() throws Exception { searchWhileCreatingIndex(true, 2); } diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java index 86b27f25e7d..69c4bbdbd11 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -44,20 +43,16 @@ import static org.hamcrest.Matchers.is; @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SearchWhileRelocatingIT extends ESIntegTestCase { - - @Test @Nightly public void testSearchAndRelocateConcurrently0Replicas() throws Exception { testSearchAndRelocateConcurrently(0); } - @Test @Nightly public void testSearchAndRelocateConcurrently1Replicas() throws Exception { testSearchAndRelocateConcurrently(1); } - @Test public void testSearchAndRelocateConcurrentlyRanodmReplicas() throws Exception { testSearchAndRelocateConcurrently(randomIntBetween(0, 1)); } @@ -77,7 +72,7 @@ public class SearchWhileRelocatingIT extends ESIntegTestCase { .endObject().endObject())); } indexRandom(true, indexBuilders.toArray(new IndexRequestBuilder[indexBuilders.size()])); - assertHitCount(client().prepareSearch().get(), (long) (numDocs)); + assertHitCount(client().prepareSearch().get(), (numDocs)); final int numIters = scaledRandomIntBetween(5, 20); for (int i = 0; i < numIters; i++) { final AtomicBoolean stop = new AtomicBoolean(false); @@ -98,7 +93,7 @@ public class SearchWhileRelocatingIT extends ESIntegTestCase { // request comes in. It's a small window but a known limitation. // criticalException = sr.getTotalShards() == sr.getSuccessfulShards() || sr.getFailedShards() > 0; - assertHitCount(sr, (long) (numDocs)); + assertHitCount(sr, (numDocs)); criticalException = true; final SearchHits sh = sr.getHits(); assertThat("Expected hits to be the same size the actual hits array", sh.getTotalHits(), diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java index 556bf136c30..052676301f2 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java @@ -32,12 +32,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.engine.MockEngineSupport; -import org.elasticsearch.test.engine.MockEngineSupportModule; import org.elasticsearch.test.engine.ThrowingLeafReaderWrapper; import java.io.IOException; @@ -159,8 +159,8 @@ public class SearchWithRandomExceptionsIT extends ESIntegTestCase { public String description() { return "a mock reader wrapper that throws random exceptions for testing"; } - public void onModule(MockEngineSupportModule module) { - module.wrapperImpl = RandomExceptionDirectoryReaderWrapper.class; + public void onModule(MockEngineFactoryPlugin.MockEngineReaderModule module) { + module.setReaderClass(RandomExceptionDirectoryReaderWrapper.class); } } diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java index 457f63d54e5..95c1a807935 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.store.MockFSDirectoryService; +import org.elasticsearch.test.store.MockFSIndexStore; import java.io.IOException; import java.util.concurrent.ExecutionException; @@ -103,7 +104,7 @@ public class SearchWithRandomIOExceptionsIT extends ESIntegTestCase { } else { Settings.Builder settings = settingsBuilder() .put("index.number_of_replicas", randomIntBetween(0, 1)) - .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) + .put(MockFSIndexStore.CHECK_INDEX_ON_CLOSE, false) .put(MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE, exceptionRate) .put(MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE_ON_OPEN, exceptionOnOpenRate); // we cannot expect that the index will be valid logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap()); diff --git a/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java b/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java index e5cc0ee7b9a..4586612b007 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java @@ -22,34 +22,36 @@ package org.elasticsearch.search.basic; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.GeohashCellQuery; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; -import static org.elasticsearch.client.Requests.*; +import static org.elasticsearch.client.Requests.clusterHealthRequest; +import static org.elasticsearch.client.Requests.refreshRequest; +import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class TransportSearchFailuresIT extends ESIntegTestCase { - @Override protected int maximumNumberOfReplicas() { return 1; } - @Test public void testFailedSearchWithWrongQuery() throws Exception { logger.info("Start Testing failed search with wrong query"); assertAcked(prepareCreate("test", 1, settingsBuilder().put("routing.hash.type", "simple"))); @@ -66,7 +68,9 @@ public class TransportSearchFailuresIT extends ESIntegTestCase { assertThat(refreshResponse.getFailedShards(), equalTo(0)); for (int i = 0; i < 5; i++) { try { - SearchResponse searchResponse = client().search(searchRequest("test").source(new BytesArray("{ xxx }"))).actionGet(); + SearchResponse searchResponse = client().search( + searchRequest("test").source(new SearchSourceBuilder().query(new GeohashCellQuery.Builder("foo", "biz")))) + .actionGet(); assertThat(searchResponse.getTotalShards(), equalTo(test.numPrimaries)); assertThat(searchResponse.getSuccessfulShards(), equalTo(0)); assertThat(searchResponse.getFailedShards(), equalTo(test.numPrimaries)); @@ -78,11 +82,15 @@ public class TransportSearchFailuresIT extends ESIntegTestCase { } allowNodes("test", 2); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes(">=2").execute().actionGet().isTimedOut(), equalTo(false)); + assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes(">=2").execute() + .actionGet().isTimedOut(), equalTo(false)); logger.info("Running Cluster Health"); - ClusterHealthResponse clusterHealth = client().admin().cluster().health(clusterHealthRequest("test") - .waitForYellowStatus().waitForRelocatingShards(0).waitForActiveShards(test.totalNumShards)).actionGet(); + ClusterHealthResponse clusterHealth = client() + .admin() + .cluster() + .health(clusterHealthRequest("test").waitForYellowStatus().waitForRelocatingShards(0) + .waitForActiveShards(test.totalNumShards)).actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), anyOf(equalTo(ClusterHealthStatus.YELLOW), equalTo(ClusterHealthStatus.GREEN))); @@ -95,7 +103,9 @@ public class TransportSearchFailuresIT extends ESIntegTestCase { for (int i = 0; i < 5; i++) { try { - SearchResponse searchResponse = client().search(searchRequest("test").source(new BytesArray("{ xxx }"))).actionGet(); + SearchResponse searchResponse = client().search( + searchRequest("test").source(new SearchSourceBuilder().query(new GeohashCellQuery.Builder("foo", "biz")))) + .actionGet(); assertThat(searchResponse.getTotalShards(), equalTo(test.numPrimaries)); assertThat(searchResponse.getSuccessfulShards(), equalTo(0)); assertThat(searchResponse.getFailedShards(), equalTo(test.numPrimaries)); diff --git a/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java b/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java index a9b41ce4025..6d239a8cdb0 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java @@ -25,10 +25,10 @@ import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.GeohashCellQuery; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.script.ScriptScoreFunctionBuilder; import org.elasticsearch.script.Script; @@ -41,7 +41,6 @@ import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.HashSet; @@ -52,7 +51,6 @@ import static org.elasticsearch.action.search.SearchType.DFS_QUERY_AND_FETCH; import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; - import static org.elasticsearch.client.Requests.createIndexRequest; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -122,7 +120,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { .endObject(); } - @Test public void testDfsQueryThenFetch() throws Exception { Settings.Builder settingsBuilder = settingsBuilder() .put(indexSettings()) @@ -131,7 +128,7 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { .settings(settingsBuilder)) .actionGet(); ensureGreen(); - + // we need to have age (ie number of repeats of "test" term) high enough // to produce the same 8-bit norm for all docs here, so that // the tf is basically the entire score (assuming idf is fixed, which @@ -162,7 +159,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertEquals(100, total); } - @Test public void testDfsQueryThenFetchWithSort() throws Exception { prepareData(); @@ -187,7 +183,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertEquals(100, total); } - @Test public void testQueryThenFetch() throws Exception { prepareData(); @@ -212,7 +207,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertEquals(100, total); } - @Test public void testQueryThenFetchWithFrom() throws Exception { Set fullExpectedIds = prepareData(); @@ -241,7 +235,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertThat(collectedIds, equalTo(fullExpectedIds)); } - @Test public void testQueryThenFetchWithSort() throws Exception { prepareData(); @@ -266,7 +259,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertEquals(100, total); } - @Test public void testQueryAndFetch() throws Exception { prepareData(3); @@ -306,7 +298,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertThat("make sure we got all [" + expectedIds + "]", expectedIds.size(), equalTo(0)); } - @Test public void testDfsQueryAndFetch() throws Exception { prepareData(3); @@ -335,7 +326,7 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { do { searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll("10m").get(); - + assertThat(searchResponse.getHits().totalHits(), equalTo(100l)); assertThat(searchResponse.getHits().hits().length, lessThanOrEqualTo(40)); for (int i = 0; i < searchResponse.getHits().hits().length; i++) { @@ -348,7 +339,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertThat("make sure we got all [" + expectedIds + "]", expectedIds.size(), equalTo(0)); } - @Test public void testSimpleFacets() throws Exception { prepareData(); @@ -370,7 +360,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { assertThat(all.getDocCount(), equalTo(100l)); } - @Test public void testFailedSearchWithWrongQuery() throws Exception { prepareData(); @@ -378,7 +367,8 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { logger.info("Start Testing failed search with wrong query"); try { - SearchResponse searchResponse = client().search(searchRequest("test").source(new BytesArray("{ xxx }"))).actionGet(); + SearchResponse searchResponse = client().search( + searchRequest("test").source(new SearchSourceBuilder().query(new GeohashCellQuery.Builder("foo", "biz")))).actionGet(); assertThat(searchResponse.getTotalShards(), equalTo(test.numPrimaries)); assertThat(searchResponse.getSuccessfulShards(), equalTo(0)); assertThat(searchResponse.getFailedShards(), equalTo(test.numPrimaries)); @@ -388,9 +378,8 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { // all is well } logger.info("Done Testing failed search"); - } + } - @Test public void testFailedSearchWithWrongFrom() throws Exception { prepareData(); @@ -421,7 +410,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { logger.info("Done Testing failed search"); } - @Test public void testFailedMultiSearchWithWrongQuery() throws Exception { prepareData(); @@ -445,8 +433,7 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase { logger.info("Done Testing failed search"); } - @Test - public void testFailedMultiSearchWithWrongQuery_withFunctionScore() throws Exception { + public void testFailedMultiSearchWithWrongQueryWithFunctionScore() throws Exception { prepareData(); logger.info("Start Testing failed multi search with a wrong query"); diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 80a683c5021..b80810fc6d5 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -16,75 +16,416 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.search.builder; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.Injector; +import org.elasticsearch.common.inject.ModulesBuilder; +import org.elasticsearch.common.inject.multibindings.Multibinder; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.AbstractQueryTestCase; +import org.elasticsearch.index.query.EmptyQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.functionscore.ScoreFunctionParser; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.fetch.innerhits.InnerHitsBuilder; +import org.elasticsearch.search.fetch.innerhits.InnerHitsBuilder.InnerHit; +import org.elasticsearch.search.fetch.source.FetchSourceContext; +import org.elasticsearch.search.highlight.HighlightBuilderTests; +import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.SuggestBuilders; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.threadpool.ThreadPoolModule; +import org.junit.AfterClass; +import org.junit.BeforeClass; import java.io.IOException; +import java.util.ArrayList; import java.util.List; -import java.util.Map; +import java.util.concurrent.TimeUnit; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; public class SearchSourceBuilderTests extends ESTestCase { + private static Injector injector; - SearchSourceBuilder builder = new SearchSourceBuilder(); + private static NamedWriteableRegistry namedWriteableRegistry; - @Test // issue #6632 - public void testThatSearchSourceBuilderIncludesExcludesAreAppliedCorrectly() throws Exception { - builder.fetchSource("foo", null); - assertIncludes(builder, "foo"); - assertExcludes(builder); + private static IndicesQueriesRegistry indicesQueriesRegistry; - builder.fetchSource(null, "foo"); - assertIncludes(builder); - assertExcludes(builder, "foo"); - - builder.fetchSource(null, new String[]{"foo"}); - assertIncludes(builder); - assertExcludes(builder, "foo"); - - builder.fetchSource(new String[]{"foo"}, null); - assertIncludes(builder, "foo"); - assertExcludes(builder); - - builder.fetchSource("foo", "bar"); - assertIncludes(builder, "foo"); - assertExcludes(builder, "bar"); - - builder.fetchSource(new String[]{"foo"}, new String[]{"bar", "baz"}); - assertIncludes(builder, "foo"); - assertExcludes(builder, "bar", "baz"); + @BeforeClass + public static void init() throws IOException { + Settings settings = Settings.settingsBuilder() + .put("name", SearchSourceBuilderTests.class.toString()) + .put("path.home", createTempDir()) + .build(); + injector = new ModulesBuilder().add( + new SettingsModule(settings, new SettingsFilter(settings)), + new ThreadPoolModule(new ThreadPool(settings)), + new IndicesModule() { + @Override + public void configure() { + // skip services + bindQueryParsersExtension(); + } + }, + new AbstractModule() { + @Override + protected void configure() { + Multibinder.newSetBinder(binder(), ScoreFunctionParser.class); + bind(NamedWriteableRegistry.class).asEagerSingleton(); + } + } + ).createInjector(); + indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class); + namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class); } - private void assertIncludes(SearchSourceBuilder builder, String... elems) throws IOException { - assertFieldValues(builder, "includes", elems); + @AfterClass + public static void afterClass() throws Exception { + terminate(injector.getInstance(ThreadPool.class)); + injector = null; + namedWriteableRegistry = null; + indicesQueriesRegistry = null; } - private void assertExcludes(SearchSourceBuilder builder, String... elems) throws IOException { - assertFieldValues(builder, "excludes", elems); - } - - private void assertFieldValues(SearchSourceBuilder builder, String fieldName, String... elems) throws IOException { - Map map = getSourceMap(builder); - - assertThat(map, hasKey(fieldName)); - assertThat(map.get(fieldName), is(instanceOf(List.class))); - List castedList = (List) map.get(fieldName); - assertThat(castedList, hasSize(elems.length)); - assertThat(castedList, hasItems(elems)); - } - - private Map getSourceMap(SearchSourceBuilder builder) throws IOException { - Map data; - try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.toString())) { - data = parser.map(); + protected final SearchSourceBuilder createSearchSourceBuilder() throws IOException { + SearchSourceBuilder builder = new SearchSourceBuilder(); + if (randomBoolean()) { + builder.from(randomIntBetween(0, 10000)); } - assertThat(data, hasKey("_source")); - return (Map) data.get("_source"); + if (randomBoolean()) { + builder.size(randomIntBetween(0, 10000)); + } + if (randomBoolean()) { + builder.explain(randomBoolean()); + } + if (randomBoolean()) { + builder.version(randomBoolean()); + } + if (randomBoolean()) { + builder.trackScores(randomBoolean()); + } + if (randomBoolean()) { + builder.minScore(randomFloat() * 1000); + } + if (randomBoolean()) { + builder.timeout(new TimeValue(randomIntBetween(1, 100), randomFrom(TimeUnit.values()))); + } + if (randomBoolean()) { + builder.terminateAfter(randomIntBetween(1, 100000)); + } + // if (randomBoolean()) { + // builder.defaultRescoreWindowSize(randomIntBetween(1, 100)); + // } + if (randomBoolean()) { + int fieldsSize = randomInt(25); + List fields = new ArrayList<>(fieldsSize); + for (int i = 0; i < fieldsSize; i++) { + fields.add(randomAsciiOfLengthBetween(5, 50)); + } + builder.fields(fields); + } + if (randomBoolean()) { + int fieldDataFieldsSize = randomInt(25); + for (int i = 0; i < fieldDataFieldsSize; i++) { + builder.fieldDataField(randomAsciiOfLengthBetween(5, 50)); + } + } + if (randomBoolean()) { + int scriptFieldsSize = randomInt(25); + for (int i = 0; i < scriptFieldsSize; i++) { + if (randomBoolean()) { + builder.scriptField(randomAsciiOfLengthBetween(5, 50), new Script("foo"), randomBoolean()); + } else { + builder.scriptField(randomAsciiOfLengthBetween(5, 50), new Script("foo")); + } + } + } + if (randomBoolean()) { + FetchSourceContext fetchSourceContext; + int branch = randomInt(5); + String[] includes = new String[randomIntBetween(0, 20)]; + for (int i = 0; i < includes.length; i++) { + includes[i] = randomAsciiOfLengthBetween(5, 20); + } + String[] excludes = new String[randomIntBetween(0, 20)]; + for (int i = 0; i < excludes.length; i++) { + excludes[i] = randomAsciiOfLengthBetween(5, 20); + } + switch (branch) { + case 0: + fetchSourceContext = new FetchSourceContext(randomBoolean()); + break; + case 1: + fetchSourceContext = new FetchSourceContext(includes, excludes); + break; + case 2: + fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)); + break; + case 3: + fetchSourceContext = new FetchSourceContext(true, includes, excludes); + break; + case 4: + fetchSourceContext = new FetchSourceContext(includes); + break; + case 5: + fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20)); + break; + default: + throw new IllegalStateException(); + } + builder.fetchSource(fetchSourceContext); + } + if (randomBoolean()) { + int size = randomIntBetween(0, 20); + List statsGroups = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + statsGroups.add(randomAsciiOfLengthBetween(5, 20)); + } + builder.stats(statsGroups); + } + if (randomBoolean()) { + int indexBoostSize = randomIntBetween(1, 10); + for (int i = 0; i < indexBoostSize; i++) { + builder.indexBoost(randomAsciiOfLengthBetween(5, 20), randomFloat() * 10); + } + } + if (randomBoolean()) { + // NORELEASE make RandomQueryBuilder work outside of the + // AbstractQueryTestCase + // builder.query(RandomQueryBuilder.createQuery(getRandom())); + builder.query(QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20))); + } + if (randomBoolean()) { + // NORELEASE make RandomQueryBuilder work outside of the + // AbstractQueryTestCase + // builder.postFilter(RandomQueryBuilder.createQuery(getRandom())); + builder.postFilter(QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20))); + } + if (randomBoolean()) { + int numSorts = randomIntBetween(1, 5); + for (int i = 0; i < numSorts; i++) { + int branch = randomInt(5); + switch (branch) { + case 0: + builder.sort(SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values()))); + break; + case 1: + builder.sort(SortBuilders.geoDistanceSort(randomAsciiOfLengthBetween(5, 20)) + .geohashes(AbstractQueryTestCase.randomGeohash(1, 12)).order(randomFrom(SortOrder.values()))); + break; + case 2: + builder.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values()))); + break; + case 3: + builder.sort(SortBuilders.scriptSort(new Script("foo"), "number").order(randomFrom(SortOrder.values()))); + break; + case 4: + builder.sort(randomAsciiOfLengthBetween(5, 20)); + break; + case 5: + builder.sort(randomAsciiOfLengthBetween(5, 20), randomFrom(SortOrder.values())); + break; + } + } + } + if (randomBoolean()) { + builder.highlighter(HighlightBuilderTests.randomHighlighterBuilder()); + } + if (randomBoolean()) { + // NORELEASE need a random suggest builder method + builder.suggest(new SuggestBuilder().setText(randomAsciiOfLengthBetween(1, 5)).addSuggestion( + SuggestBuilders.termSuggestion(randomAsciiOfLengthBetween(1, 5)))); + } + if (randomBoolean()) { + // NORELEASE need a random inner hits builder method + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + InnerHit innerHit = new InnerHit(); + innerHit.field(randomAsciiOfLengthBetween(5, 20)); + innerHitsBuilder.addNestedInnerHits(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20), innerHit); + builder.innerHits(innerHitsBuilder); + } + if (randomBoolean()) { + int numRescores = randomIntBetween(1, 5); + for (int i = 0; i < numRescores; i++) { + // NORELEASE need a random rescore builder method + RescoreBuilder rescoreBuilder = new RescoreBuilder(); + rescoreBuilder.rescorer(RescoreBuilder.queryRescorer(QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), + randomAsciiOfLengthBetween(5, 20)))); + builder.addRescorer(rescoreBuilder); + } + } + if (randomBoolean()) { + // NORELEASE need a random aggregation builder method + builder.aggregation(AggregationBuilders.avg(randomAsciiOfLengthBetween(5, 20))); + } + if (true) { + // NORELEASE need a method to randomly build content for ext + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + xContentBuilder.field("term_vectors_fetch", randomAsciiOfLengthBetween(5, 20)); + xContentBuilder.endObject(); + builder.ext(xContentBuilder); + } + return builder; } + public void testFromXContent() throws IOException { + SearchSourceBuilder testSearchSourceBuilder = createSearchSourceBuilder(); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testSearchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertParseSearchSource(testSearchSourceBuilder, builder.bytes()); + } + + private void assertParseSearchSource(SearchSourceBuilder testBuilder, BytesReference searchSourceAsBytes) throws IOException { + XContentParser parser = XContentFactory.xContent(searchSourceAsBytes).createParser(searchSourceAsBytes); + QueryParseContext parseContext = createParseContext(parser); + parseContext.reset(parser); + if (randomBoolean()) { + parser.nextToken(); // sometimes we move it on the START_OBJECT to test the embedded case + } + SearchSourceBuilder newBuilder = SearchSourceBuilder.parseSearchSource(parser, parseContext); + assertNotSame(testBuilder, newBuilder); + assertEquals(testBuilder, newBuilder); + assertEquals(testBuilder.hashCode(), newBuilder.hashCode()); + } + + private static QueryParseContext createParseContext(XContentParser parser) { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.reset(parser); + context.parseFieldMatcher(ParseFieldMatcher.STRICT); + return context; + } + + public void testSerialization() throws IOException { + SearchSourceBuilder testBuilder = createSearchSourceBuilder(); + try (BytesStreamOutput output = new BytesStreamOutput()) { + testBuilder.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + SearchSourceBuilder deserializedBuilder = SearchSourceBuilder.readSearchSourceFrom(in); + assertEquals(deserializedBuilder, testBuilder); + assertEquals(deserializedBuilder.hashCode(), testBuilder.hashCode()); + assertNotSame(deserializedBuilder, testBuilder); + } + } + } + + public void testEqualsAndHashcode() throws IOException { + SearchSourceBuilder firstBuilder = createSearchSourceBuilder(); + assertFalse("source builder is equal to null", firstBuilder.equals(null)); + assertFalse("source builder is equal to incompatible type", firstBuilder.equals("")); + assertTrue("source builder is not equal to self", firstBuilder.equals(firstBuilder)); + assertThat("same source builder's hashcode returns different values if called multiple times", firstBuilder.hashCode(), + equalTo(firstBuilder.hashCode())); + + SearchSourceBuilder secondBuilder = copyBuilder(firstBuilder); + assertTrue("source builder is not equal to self", secondBuilder.equals(secondBuilder)); + assertTrue("source builder is not equal to its copy", firstBuilder.equals(secondBuilder)); + assertTrue("source builder is not symmetric", secondBuilder.equals(firstBuilder)); + assertThat("source builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(firstBuilder.hashCode())); + + SearchSourceBuilder thirdBuilder = copyBuilder(secondBuilder); + assertTrue("source builder is not equal to self", thirdBuilder.equals(thirdBuilder)); + assertTrue("source builder is not equal to its copy", secondBuilder.equals(thirdBuilder)); + assertThat("source builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode())); + assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder)); + assertThat("source builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode())); + assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder)); + assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder)); + } + + //we use the streaming infra to create a copy of the query provided as argument + protected SearchSourceBuilder copyBuilder(SearchSourceBuilder builder) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + builder.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + return SearchSourceBuilder.readSearchSourceFrom(in); + } + } + } + + public void testParseIncludeExclude() throws IOException { + { + String restContent = " { \"_source\": { \"includes\": \"include\", \"excludes\": \"*.field2\"}}"; + try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { + SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser)); + assertArrayEquals(new String[]{"*.field2" }, searchSourceBuilder.fetchSource().excludes()); + assertArrayEquals(new String[]{"include" }, searchSourceBuilder.fetchSource().includes()); + } + } + { + String restContent = " { \"_source\": false}"; + try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { + SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser)); + assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().excludes()); + assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().includes()); + assertFalse(searchSourceBuilder.fetchSource().fetchSource()); + } + } + } + + public void testParseSort() throws IOException { + { + String restContent = " { \"sort\": \"foo\"}"; + try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { + SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser)); + assertEquals(1, searchSourceBuilder.sorts().size()); + assertEquals("{\"foo\":{}}", searchSourceBuilder.sorts().get(0).toUtf8()); + } + } + + { + String restContent = "{\"sort\" : [\n" + + " { \"post_date\" : {\"order\" : \"asc\"}},\n" + + " \"user\",\n" + + " { \"name\" : \"desc\" },\n" + + " { \"age\" : \"desc\" },\n" + + " \"_score\"\n" + + " ]}"; + try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { + SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser)); + assertEquals(5, searchSourceBuilder.sorts().size()); + assertEquals("{\"post_date\":{\"order\":\"asc\"}}", searchSourceBuilder.sorts().get(0).toUtf8()); + assertEquals("\"user\"", searchSourceBuilder.sorts().get(1).toUtf8()); + assertEquals("{\"name\":\"desc\"}", searchSourceBuilder.sorts().get(2).toUtf8()); + assertEquals("{\"age\":\"desc\"}", searchSourceBuilder.sorts().get(3).toUtf8()); + assertEquals("\"_score\"", searchSourceBuilder.sorts().get(4).toUtf8()); + } + } + } + + public void testEmptyPostFilter() throws IOException { + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.postFilter(EmptyQueryBuilder.PROTOTYPE); + String query = "{ \"post_filter\": {} }"; + assertParseSearchSource(builder, new BytesArray(query)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java index 09f33f60ed5..4be2b36fbe6 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java @@ -21,19 +21,16 @@ package org.elasticsearch.search.child; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.explain.ExplainResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.cache.IndexCacheModule; -import org.elasticsearch.index.mapper.MergeMappingException; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.query.HasChildQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -44,41 +41,66 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.hasParentQuery; +import static org.elasticsearch.index.query.QueryBuilders.idsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; +import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.weightFactorFunction; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; /** * */ @ClusterScope(scope = Scope.SUITE) public class ChildQuerySearchIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)) // aggressive filter caching so that we can assert on the filter cache size - .put(IndexCacheModule.QUERY_CACHE_TYPE, IndexCacheModule.INDEX_QUERY_CACHE) - .put(IndexCacheModule.QUERY_CACHE_EVERYTHING, true) + .put(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE) + .put(IndexModule.QUERY_CACHE_EVERYTHING, true) .build(); } - @Test public void testSelfReferentialIsForbidden() { try { prepareCreate("test").addMapping("type", "_parent", "type=type").get(); @@ -90,8 +112,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test - public void multiLevelChild() throws Exception { + public void testMultiLevelChild() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") .addMapping("child", "_parent", "type=parent") @@ -144,7 +165,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).id(), equalTo("gc1")); } - @Test // see #2744 public void test2744() throws IOException { assertAcked(prepareCreate("test") @@ -164,8 +184,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } - @Test - public void simpleChildQuery() throws Exception { + public void testSimpleChildQuery() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") .addMapping("child", "_parent", "type=parent")); @@ -181,7 +200,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { refresh(); // TEST FETCHING _parent from child - SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).addFields("_parent").execute() + SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).fields("_parent").execute() .actionGet(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); @@ -189,7 +208,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).field("_parent").value().toString(), equalTo("p1")); // TEST matching on parent - searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent", "p1")).addFields("_parent").get(); + searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent", "p1")).fields("_parent").get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo(2l)); assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2"))); @@ -197,7 +216,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("c1"), equalTo("c2"))); assertThat(searchResponse.getHits().getAt(1).field("_parent").value().toString(), equalTo("p1")); - searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent:p1")).addFields("_parent").get(); + searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent:p1")).fields("_parent").get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo(2l)); assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2"))); @@ -237,9 +256,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(1).id(), equalTo("c2")); } - @Test - // See: https://github.com/elasticsearch/elasticsearch/issues/3290 - public void testCachingBug_withFqueryFilter() throws Exception { + // Issue #3290 + public void testCachingBugWithFqueryFilter() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") .addMapping("child", "_parent", "type=parent")); @@ -278,7 +296,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test public void testHasParentFilter() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -328,8 +345,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test - public void simpleChildQueryWithFlush() throws Exception { + public void testSimpleChildQueryWithFlush() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") .addMapping("child", "_parent", "type=parent")); @@ -392,7 +408,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1"))); } - @Test public void testScopedFacet() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -430,7 +445,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(termsFacet.getBuckets().get(1).getDocCount(), equalTo(1L)); } - @Test public void testDeletedParent() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -466,7 +480,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).sourceAsString(), containsString("\"p_value1_updated\"")); } - @Test public void testDfsSearchType() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -493,7 +506,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertNoFailures(searchResponse); } - @Test public void testHasChildAndHasParentFailWhenSomeSegmentsDontContainAnyParentOrChildDocs() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -518,7 +530,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); } - @Test public void testCountApiUsage() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -530,24 +541,23 @@ public class ChildQuerySearchIT extends ESIntegTestCase { client().prepareIndex("test", "child", "c1").setSource("c_field", "1").setParent(parentId).get(); refresh(); - CountResponse countResponse = client().prepareCount("test").setQuery(hasChildQuery("child", termQuery("c_field", "1")).scoreMode(ScoreMode.Max)) + SearchResponse countResponse = client().prepareSearch("test").setSize(0).setQuery(hasChildQuery("child", termQuery("c_field", "1")).scoreMode(ScoreMode.Max)) .get(); assertHitCount(countResponse, 1l); - countResponse = client().prepareCount("test").setQuery(hasParentQuery("parent", termQuery("p_field", "1")).score(true)) + countResponse = client().prepareSearch("test").setSize(0).setQuery(hasParentQuery("parent", termQuery("p_field", "1")).score(true)) .get(); assertHitCount(countResponse, 1l); - countResponse = client().prepareCount("test").setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "1")))) + countResponse = client().prepareSearch("test").setSize(0).setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "1")))) .get(); assertHitCount(countResponse, 1l); - countResponse = client().prepareCount("test").setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("p_field", "1")))) + countResponse = client().prepareSearch("test").setSize(0).setQuery(constantScoreQuery(hasParentQuery("parent", termQuery("p_field", "1")))) .get(); assertHitCount(countResponse, 1l); } - @Test public void testExplainUsage() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -636,8 +646,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { return indexBuilders; } - @Test - public void testScoreForParentChildQueries_withFunctionScore() throws Exception { + public void testScoreForParentChildQueriesWithFunctionScore() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") .addMapping("child", "_parent", "type=parent") @@ -723,8 +732,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(response.getHits().hits()[6].score(), equalTo(5f)); } - @Test - // https://github.com/elasticsearch/elasticsearch/issues/2536 + // Issue #2536 public void testParentChildQueriesCanHandleNoRelevantTypesInIndex() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -758,7 +766,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(response.getHits().totalHits(), equalTo(0l)); } - @Test public void testHasChildAndHasParentFilter_withFilter() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -785,7 +792,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().hits()[0].id(), equalTo("2")); } - @Test public void testHasChildAndHasParentWrappedInAQueryFilter() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -815,7 +821,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertSearchHit(searchResponse, 1, hasId("2")); } - @Test public void testSimpleQueryRewrite() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent", "p_field", "type=string") @@ -862,9 +867,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test - // See also issue: - // https://github.com/elasticsearch/elasticsearch/issues/3144 + // Issue #3144 public void testReIndexingParentAndChildDocuments() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -925,9 +928,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(1).id(), Matchers.anyOf(equalTo("c3"), equalTo("c4"))); } - @Test - // See also issue: - // https://github.com/elasticsearch/elasticsearch/issues/3203 + // Issue #3203 public void testHasChildQueryWithMinimumScore() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -952,7 +953,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).score(), equalTo(3.0f)); } - @Test public void testParentFieldFilter() throws Exception { assertAcked(prepareCreate("test") .setSettings(settingsBuilder().put(indexSettings()) @@ -1019,7 +1019,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertHitCount(response, 2l); } - @Test public void testHasChildNotBeingCached() throws IOException { assertAcked(prepareCreate("test") .addMapping("parent") @@ -1081,8 +1080,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test - // Relates to bug: https://github.com/elasticsearch/elasticsearch/issues/3818 + // Issue #3818 public void testHasChildQueryOnlyReturnsSingleChildType() { assertAcked(prepareCreate("grandissue") .addMapping("grandparent", "name", "type=string") @@ -1135,8 +1133,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test - public void indexChildDocWithNoParentMapping() throws IOException { + public void testIndexChildDocWithNoParentMapping() throws IOException { assertAcked(prepareCreate("test") .addMapping("parent") .addMapping("child1")); @@ -1159,7 +1156,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { refresh(); } - @Test public void testAddingParentToExistingMapping() throws IOException { createIndex("test"); ensureGreen(); @@ -1179,12 +1175,11 @@ public class ChildQuerySearchIT extends ESIntegTestCase { .startObject("_parent").field("type", "parent").endObject() .endObject().endObject()).get(); fail(); - } catch (MergeMappingException e) { + } catch (IllegalArgumentException e) { assertThat(e.toString(), containsString("Merge failed with failures {[The _parent field's type option can't be changed: [null]->[parent]")); } } - @Test public void testHasChildQueryWithNestedInnerObjects() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent", "objects", "type=nested") @@ -1214,19 +1209,18 @@ public class ChildQuerySearchIT extends ESIntegTestCase { ScoreMode scoreMode = randomFrom(ScoreMode.values()); SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(boolQuery().must(QueryBuilders.hasChildQuery("child", termQuery("c_field", "blue")).scoreMode(scoreMode)).filter(notQuery(termQuery("p_field", "3")))) + .setQuery(boolQuery().must(QueryBuilders.hasChildQuery("child", termQuery("c_field", "blue")).scoreMode(scoreMode)).filter(boolQuery().mustNot(termQuery("p_field", "3")))) .get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); searchResponse = client().prepareSearch("test") - .setQuery(boolQuery().must(QueryBuilders.hasChildQuery("child", termQuery("c_field", "red")).scoreMode(scoreMode)).filter(notQuery(termQuery("p_field", "3")))) + .setQuery(boolQuery().must(QueryBuilders.hasChildQuery("child", termQuery("c_field", "red")).scoreMode(scoreMode)).filter(boolQuery().mustNot(termQuery("p_field", "3")))) .get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo(2l)); } - @Test public void testNamedFilters() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -1263,7 +1257,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("test")); } - @Test public void testParentChildQueriesNoParentType() throws Exception { assertAcked(prepareCreate("test") .setSettings(settingsBuilder() @@ -1321,8 +1314,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test - public void testAdd_ParentFieldAfterIndexingParentDocButBeforeIndexingChildDoc() throws Exception { + public void testAddParentFieldAfterIndexingParentDocButBeforeIndexingChildDoc() throws Exception { assertAcked(prepareCreate("test") .setSettings(settingsBuilder() .put(indexSettings()) @@ -1345,7 +1337,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test public void testParentChildCaching() throws Exception { assertAcked(prepareCreate("test") .setSettings( @@ -1363,7 +1354,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { client().prepareIndex("test", "child", "c1").setParent("p1").setSource("c_field", "blue").get(); client().prepareIndex("test", "child", "c2").setParent("p1").setSource("c_field", "red").get(); client().prepareIndex("test", "child", "c3").setParent("p2").setSource("c_field", "red").get(); - client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).setFlush(true).get(); + client().admin().indices().prepareForceMerge("test").setMaxNumSegments(1).setFlush(true).get(); client().prepareIndex("test", "parent", "p3").setSource("p_field", "p_value3").get(); client().prepareIndex("test", "parent", "p4").setSource("p_field", "p_value4").get(); client().prepareIndex("test", "child", "c4").setParent("p3").setSource("c_field", "green").get(); @@ -1394,7 +1385,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); } - @Test public void testParentChildQueriesViaScrollApi() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -1438,8 +1428,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - // https://github.com/elasticsearch/elasticsearch/issues/5783 - @Test + // Issue #5783 public void testQueryBeforeChildType() throws Exception { assertAcked(prepareCreate("test") .addMapping("features") @@ -1453,18 +1442,12 @@ public class ChildQuerySearchIT extends ESIntegTestCase { SearchResponse resp; resp = client().prepareSearch("test") - .setSource(new BytesArray("{\"query\": {\"has_child\": {\"type\": \"posts\", \"query\": {\"match\": {\"field\": \"bar\"}}}}}")).get(); + .setSource(new SearchSourceBuilder().query(QueryBuilders.hasChildQuery("posts", QueryBuilders.matchQuery("field", "bar")))) + .get(); assertHitCount(resp, 1L); - - // Now reverse the order for the type after the query - resp = client().prepareSearch("test") - .setSource(new BytesArray("{\"query\": {\"has_child\": {\"query\": {\"match\": {\"field\": \"bar\"}}, \"type\": \"posts\"}}}")).get(); - assertHitCount(resp, 1L); - } - @Test - // https://github.com/elasticsearch/elasticsearch/issues/6256 + // Issue #6256 public void testParentFieldInMultiMatchField() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1") @@ -1483,7 +1466,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).id(), equalTo("1")); } - @Test public void testTypeIsAppliedInHasParentInnerQuery() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") @@ -1497,12 +1479,12 @@ public class ChildQuerySearchIT extends ESIntegTestCase { indexRandom(true, indexRequests); SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(constantScoreQuery(hasParentQuery("parent", notQuery(termQuery("field1", "a"))))) + .setQuery(constantScoreQuery(hasParentQuery("parent", boolQuery().mustNot(termQuery("field1", "a"))))) .get(); assertHitCount(searchResponse, 0l); searchResponse = client().prepareSearch("test") - .setQuery(hasParentQuery("parent", constantScoreQuery(notQuery(termQuery("field1", "a"))))) + .setQuery(hasParentQuery("parent", constantScoreQuery(boolQuery().mustNot(termQuery("field1", "a"))))) .get(); assertHitCount(searchResponse, 0l); @@ -1574,7 +1556,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { .addSort("_score", SortOrder.DESC).addSort("id", SortOrder.ASC).get(); } - @Test public void testMinMaxChildren() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent", "id", "type=long") @@ -1905,7 +1886,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase { } } - @Test public void testParentFieldToNonExistingType() { assertAcked(prepareCreate("test").addMapping("parent").addMapping("child", "_parent", "type=parent2")); client().prepareIndex("test", "parent", "1").setSource("{}").get(); diff --git a/core/src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java b/core/src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java deleted file mode 100644 index d3b5160db4f..00000000000 --- a/core/src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.compress; - -import org.elasticsearch.Version; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.Priority; -import org.elasticsearch.common.compress.Compressor; -import org.elasticsearch.common.compress.CompressorFactory; -import org.elasticsearch.common.compress.lzf.LZFTestCompressor; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class SearchSourceCompressTests extends ESSingleNodeTestCase { - - @Test - public void testSourceCompressionLZF() throws IOException { - final Compressor defaultCompressor = CompressorFactory.defaultCompressor(); - try { - CompressorFactory.setDefaultCompressor(new LZFTestCompressor()); - verifySource(true); - verifySource(false); - verifySource(null); - } finally { - CompressorFactory.setDefaultCompressor(defaultCompressor); - } - } - - private void verifySource(Boolean compress) throws IOException { - try { - client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (Exception e) { - // ignore - } - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - createIndex("test", settings); - client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); - - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("_source").field("compress", compress).endObject() - .endObject().endObject().string(); - - client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); - - for (int i = 1; i < 100; i++) { - client().prepareIndex("test", "type1", Integer.toString(i)).setSource(buildSource(i)).execute().actionGet(); - } - client().prepareIndex("test", "type1", Integer.toString(10000)).setSource(buildSource(10000)).execute().actionGet(); - - client().admin().indices().prepareRefresh().execute().actionGet(); - - for (int i = 1; i < 100; i++) { - GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet(); - assertThat(getResponse.getSourceAsBytes(), equalTo(buildSource(i).bytes().toBytes())); - } - GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(10000)).execute().actionGet(); - assertThat(getResponse.getSourceAsBytes(), equalTo(buildSource(10000).bytes().toBytes())); - - for (int i = 1; i < 100; i++) { - SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.idsQuery("type1").addIds(Integer.toString(i))).execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); - assertThat(searchResponse.getHits().getAt(0).source(), equalTo(buildSource(i).bytes().toBytes())); - } - } - - private XContentBuilder buildSource(int count) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - StringBuilder sb = new StringBuilder(); - for (int j = 0; j < count; j++) { - sb.append("value").append(j).append(' '); - } - builder.field("field", sb.toString()); - return builder.endObject(); - } -} diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index 480484255ff..1eff57a0567 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fetch; + import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -26,19 +27,19 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHitField; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -50,20 +51,18 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.client.Requests.indexRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.CoreMatchers.equalTo; /** * */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class FetchSubPhasePluginIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(FetchTermVectorsPlugin.class); } - @Test public void testPlugin() throws Exception { client().admin() .indices() @@ -86,14 +85,16 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); - String searchSource = jsonBuilder().startObject() + XContentBuilder extSource = jsonBuilder().startObject() .field("term_vectors_fetch", "test") - .endObject().string(); - SearchResponse response = client().prepareSearch().setSource(new BytesArray(searchSource)).get(); + .endObject(); + SearchResponse response = client().prepareSearch().setSource(new SearchSourceBuilder().ext(extSource)).get(); assertSearchResponse(response); assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("i"), equalTo(2)); - assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("am"), equalTo(2)); - assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("sam"), equalTo(1)); + assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("am"), + equalTo(2)); + assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("sam"), + equalTo(1)); } public static class FetchTermVectorsPlugin extends Plugin { diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java b/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java new file mode 100644 index 00000000000..7405ceef5a5 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsParseElement; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.TestSearchContext; + +import static org.hamcrest.Matchers.containsString; + +public class FieldDataFieldsTests extends ESTestCase { + + public void testValidFieldDataFieldString() throws Exception { + FieldDataFieldsParseElement parseElement = new FieldDataFieldsParseElement(); + + BytesArray data = new BytesArray(new BytesRef("{\"fielddata_fields\": \"foobar\"}")); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + parser.nextToken(); + parser.nextToken(); + parser.nextToken(); + SearchContext context = new TestSearchContext(); + parseElement.parse(parser, context); + } + + public void testValidFieldDataFieldArray() throws Exception { + FieldDataFieldsParseElement parseElement = new FieldDataFieldsParseElement(); + + BytesArray data = new BytesArray(new BytesRef("{\"fielddata_fields\": [ \"foo\", \"bar\", \"baz\"]}}")); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + parser.nextToken(); + parser.nextToken(); + parser.nextToken(); + SearchContext context = new TestSearchContext(); + parseElement.parse(parser, context); + } + + public void testInvalidFieldDataField() throws Exception { + FieldDataFieldsParseElement parseElement = new FieldDataFieldsParseElement(); + + BytesArray data; + if (randomBoolean()) { + data = new BytesArray(new BytesRef("{\"fielddata_fields\": {}}}")); + } else { + data = new BytesArray(new BytesRef("{\"fielddata_fields\": 1.0}}")); + } + XContentParser parser = XContentFactory.xContent(data).createParser(data); + parser.nextToken(); + parser.nextToken(); + parser.nextToken(); + SearchContext context = new TestSearchContext(); + try { + parseElement.parse(parser, context); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), containsString("Expected either a VALUE_STRING or an START_ARRAY but got ")); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/fetch/innerhits/NestedChildrenFilterTests.java b/core/src/test/java/org/elasticsearch/search/fetch/innerhits/NestedChildrenFilterTests.java index d946f11b813..f00b72bfa8f 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/innerhits/NestedChildrenFilterTests.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/innerhits/NestedChildrenFilterTests.java @@ -29,18 +29,17 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Filter; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.store.Directory; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext.NestedInnerHits.NestedChildrenQuery; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -50,8 +49,6 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class NestedChildrenFilterTests extends ESTestCase { - - @Test public void testNestedChildrenFilter() throws Exception { int numParentDocs = scaledRandomIntBetween(0, 32); int maxChildDocsPerParent = scaledRandomIntBetween(8, 16); @@ -80,10 +77,11 @@ public class NestedChildrenFilterTests extends ESTestCase { IndexSearcher searcher = new IndexSearcher(reader); FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term("type", "parent"))); - Filter childFilter = new QueryWrapperFilter(new TermQuery(new Term("type", "child"))); + Query childFilter = new TermQuery(new Term("type", "child")); int checkedParents = 0; + final Weight parentsWeight = searcher.createNormalizedWeight(new TermQuery(new Term("type", "parent")), false); for (LeafReaderContext leaf : reader.leaves()) { - DocIdSetIterator parents = new QueryWrapperFilter(new TermQuery(new Term("type", "parent"))).getDocIdSet(leaf, null).iterator(); + DocIdSetIterator parents = parentsWeight.scorer(leaf); for (int parentDoc = parents.nextDoc(); parentDoc != DocIdSetIterator.NO_MORE_DOCS ; parentDoc = parents.nextDoc()) { int expectedChildDocs = leaf.reader().document(parentDoc).getField("num_child_docs").numericValue().intValue(); hitContext.reset(null, leaf, parentDoc, searcher); diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java index 983fb52bba9..e4d44ec73af 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java @@ -19,45 +19,53 @@ package org.elasticsearch.search.functionscore; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.junit.Test; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.concurrent.ExecutionException; import static org.elasticsearch.client.Requests.indexRequest; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.exponentialDecayFunction; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.gaussDecayFunction; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.linearDecayFunction; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; - +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isOneOf; +import static org.hamcrest.Matchers.lessThan; public class DecayFunctionScoreIT extends ESIntegTestCase { - - @Test public void testDistanceScoreGeoLinGaussExp() throws Exception { assertAcked(prepareCreate("test").addMapping( "type1", @@ -156,7 +164,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { assertThat(sh.getAt(1).getId(), equalTo("2")); } - @Test public void testDistanceScoreGeoLinGaussExpWithOffset() throws Exception { assertAcked(prepareCreate("test").addMapping( "type1", @@ -231,7 +238,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { assertThat(sh.getAt(1).score(), equalTo(sh.getAt(0).score())); } - @Test public void testBoostModeSettingWorks() throws Exception { assertAcked(prepareCreate("test").addMapping( "type1", @@ -286,7 +292,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { } - @Test public void testParseGeoPoint() throws Exception { assertAcked(prepareCreate("test").addMapping( "type1", @@ -327,9 +332,7 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { assertThat((double) sh.getAt(0).score(), closeTo(0.30685282, 1.e-5)); } - @Test public void testCombineModes() throws Exception { - assertAcked(prepareCreate("test").addMapping( "type1", jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("test").field("type", "string") @@ -410,7 +413,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { } - @Test(expected = SearchPhaseExecutionException.class) public void testExceptionThrownIfScaleLE0() throws Exception { assertAcked(prepareCreate("test").addMapping( "type1", @@ -429,14 +431,15 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source( searchSource().query( functionScoreQuery(termQuery("test", "value"), gaussDecayFunction("num1", "2013-05-28", "-1d"))))); - - SearchResponse sr = response.actionGet(); - assertOrderedSearchHits(sr, "2", "1"); + try { + response.actionGet(); + fail("Expected SearchPhaseExecutionException"); + } catch (SearchPhaseExecutionException e) { + assertThat(e.getMessage(), is("all shards failed")); + } } - - @Test + public void testParseDateMath() throws Exception { - assertAcked(prepareCreate("test").addMapping( "type1", jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("test").field("type", "string") @@ -457,7 +460,7 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { assertNoFailures(sr); assertOrderedSearchHits(sr, "1", "2"); - + sr = client().search( searchRequest().source( searchSource().query( @@ -468,9 +471,7 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { } - @Test public void testValueMissingLin() throws Exception { - assertAcked(prepareCreate("test").addMapping( "type1", jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("test").field("type", "string") @@ -519,7 +520,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { } - @Test public void testDateWithoutOrigin() throws Exception { DateTime dt = new DateTime(DateTimeZone.UTC); @@ -569,22 +569,27 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { } - @Test public void testManyDocsLin() throws Exception { - assertAcked(prepareCreate("test").addMapping( - "type", - jsonBuilder().startObject().startObject("type").startObject("properties").startObject("test").field("type", "string") - .endObject().startObject("date").field("type", "date").endObject().startObject("num").field("type", "double") - .endObject().startObject("geo").field("type", "geo_point").field("coerce", true).endObject().endObject() - .endObject().endObject())); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + XContentBuilder xContentBuilder = jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("test").field("type", "string").endObject().startObject("date").field("type", "date") + .field("doc_values", true).endObject().startObject("num").field("type", "double") + .field("doc_values", true).endObject().startObject("geo").field("type", "geo_point") + .field("ignore_malformed", true); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("coerce", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type", xContentBuilder.string())); ensureYellow(); int numDocs = 200; List indexBuilders = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { - double lat = 100 + (int) (10.0 * (float) (i) / (float) (numDocs)); + double lat = 100 + (int) (10.0 * (i) / (numDocs)); double lon = 100; - int day = (int) (29.0 * (float) (i) / (float) (numDocs)) + 1; + int day = (int) (29.0 * (i) / (numDocs)) + 1; String dayString = day < 10 ? "0" + Integer.toString(day) : Integer.toString(day); String date = "2013-05-" + dayString; @@ -622,7 +627,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { } } - @Test(expected = SearchPhaseExecutionException.class) public void testParsingExceptionIfFieldDoesNotExist() throws Exception { assertAcked(prepareCreate("test").addMapping( "type", @@ -644,10 +648,14 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { .size(numDocs) .query(functionScoreQuery(termQuery("test", "value"), linearDecayFunction("type1.geo", lonlat, "1000km")) .scoreMode(FiltersFunctionScoreQuery.ScoreMode.MULTIPLY)))); - SearchResponse sr = response.actionGet(); + try { + response.actionGet(); + fail("Expected SearchPhaseExecutionException"); + } catch (SearchPhaseExecutionException e) { + assertThat(e.getMessage(), is("all shards failed")); + } } - @Test(expected = SearchPhaseExecutionException.class) public void testParsingExceptionIfFieldTypeDoesNotMatch() throws Exception { assertAcked(prepareCreate("test").addMapping( "type", @@ -663,10 +671,14 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source( searchSource().query( functionScoreQuery(termQuery("test", "value"), linearDecayFunction("num", 1.0, 0.5)).scoreMode(FiltersFunctionScoreQuery.ScoreMode.MULTIPLY)))); - response.actionGet(); + try { + response.actionGet(); + fail("Expected SearchPhaseExecutionException"); + } catch (SearchPhaseExecutionException e) { + assertThat(e.getMessage(), is("all shards failed")); + } } - @Test public void testNoQueryGiven() throws Exception { assertAcked(prepareCreate("test").addMapping( "type", @@ -686,7 +698,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { response.actionGet(); } - @Test public void testMultiFieldOptions() throws Exception { assertAcked(prepareCreate("test").addMapping( "type1", @@ -772,7 +783,7 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { assertThat(sh.getAt(0).getId(), equalTo("2")); assertThat(sh.getAt(1).getId(), equalTo("1")); - assertThat((double)(1.0 - sh.getAt(0).getScore()), closeTo((double)((1.0 - sh.getAt(1).getScore())/3.0), 1.e-6d)); + assertThat(1.0 - sh.getAt(0).getScore(), closeTo((1.0 - sh.getAt(1).getScore())/3.0, 1.e-6d)); response = client().search( searchRequest().source( searchSource().query( @@ -780,47 +791,6 @@ public class DecayFunctionScoreIT extends ESIntegTestCase { sr = response.actionGet(); assertSearchHits(sr, "1", "2"); sh = sr.getHits(); - assertThat((double) (sh.getAt(0).getScore()), closeTo((double) (sh.getAt(1).getScore()), 1.e-6d)); - } - - @Test - public void errorMessageForFaultyFunctionScoreBody() throws Exception { - assertAcked(prepareCreate("test").addMapping( - "type", - jsonBuilder().startObject().startObject("type").startObject("properties").startObject("test").field("type", "string") - .endObject().startObject("num").field("type", "double").endObject().endObject().endObject().endObject())); - ensureYellow(); - client().index( - indexRequest("test").type("type").source(jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject())) - .actionGet(); - refresh(); - - XContentBuilder query = XContentFactory.jsonBuilder(); - // query that contains a single function and a functions[] array - query.startObject().startObject("function_score").field("weight", "1").startArray("functions").startObject().startObject("script_score").field("script", "3").endObject().endObject().endArray().endObject().endObject(); - try { - client().search( - searchRequest().source( - searchSource().query(query))).actionGet(); - fail("Search should result in SearchPhaseExecutionException"); - } catch (SearchPhaseExecutionException e) { - logger.info(e.shardFailures()[0].reason()); - assertThat(e.shardFailures()[0].reason(), containsString("already found [weight], now encountering [functions].")); - } - - query = XContentFactory.jsonBuilder(); - // query that contains a single function (but not boost factor) and a functions[] array - query.startObject().startObject("function_score").startObject("random_score").field("seed", 3).endObject().startArray("functions").startObject().startObject("random_score").field("seed", 3).endObject().endObject().endArray().endObject().endObject(); - try { - client().search( - searchRequest().source( - searchSource().query(query))).actionGet(); - fail("Search should result in SearchPhaseExecutionException"); - } catch (SearchPhaseExecutionException e) { - logger.info(e.shardFailures()[0].reason()); - assertThat(e.shardFailures()[0].reason(), containsString("already found [random_score], now encountering [functions]")); - assertThat(e.shardFailures()[0].reason(), not(containsString("did you mean [boost] instead?"))); - - } + assertThat((double) (sh.getAt(0).getScore()), closeTo((sh.getAt(1).getScore()), 1.e-6d)); } } diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index eb7903f665a..39ce61f6c73 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -27,7 +27,11 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.*; +import org.elasticsearch.script.AbstractDoubleSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ExplainableSearchScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -35,7 +39,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -55,15 +58,12 @@ import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class ExplainableScriptIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(ExplainableScriptPlugin.class); } - @Test public void testNativeExplainScript() throws InterruptedException, IOException, ExecutionException { - List indexRequests = new ArrayList<>(); for (int i = 0; i < 20; i++) { indexRequests.add(client().prepareIndex("test", "type").setId(Integer.toString(i)).setSource( diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreBackwardCompatibilityIT.java index 8bf957d283e..db5f1ed70e1 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreBackwardCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreBackwardCompatibilityIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.script.Script; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -34,22 +33,23 @@ import java.util.concurrent.ExecutionException; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; -import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.gaussDecayFunction; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.weightFactorFunction; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; /** */ public class FunctionScoreBackwardCompatibilityIT extends ESBackcompatTestCase { - /** - * Simple upgrade test for function score + * Simple upgrade test for function score. */ - @Test public void testSimpleFunctionScoreParsingWorks() throws IOException, ExecutionException, InterruptedException { - assertAcked(prepareCreate("test").addMapping( "type1", jsonBuilder().startObject() diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java index 419861d2c77..5331dfe6ff9 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java @@ -23,21 +23,22 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; /** * Tests for the {@code field_value_factor} function in a function_score query. */ public class FunctionScoreFieldValueIT extends ESIntegTestCase { - - @Test public void testFieldValueFactor() throws IOException { assertAcked(prepareCreate("test").addMapping( "type1", diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java index 4dab3c3f6cc..b428d911dd5 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.junit.Test; import java.util.Collection; @@ -52,13 +51,11 @@ import static org.hamcrest.Matchers.equalTo; */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 1) public class FunctionScorePluginIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(CustomDistanceScorePlugin.class); } - @Test public void testPlugin() throws Exception { client().admin() .indices() diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index e906ac6b187..861701a9f6a 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -40,7 +40,6 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.rescore.RescoreBuilder.QueryRescorer; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.Comparator; @@ -48,15 +47,24 @@ import java.util.Comparator; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFourthHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThirdHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; /** * */ public class QueryRescorerIT extends ESIntegTestCase { - - @Test public void testEnforceWindowSize() { createIndex("test"); // this @@ -73,8 +81,9 @@ public class QueryRescorerIT extends ESIntegTestCase { .setQuery(QueryBuilders.matchAllQuery()) .setRescorer(RescoreBuilder.queryRescorer( QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(), - ScoreFunctionBuilders.weightFactorFunction(100)).boostMode(CombineFunction.REPLACE)).setQueryWeight(0.0f).setRescoreQueryWeight(1.0f)) - .setRescoreWindow(1).setSize(randomIntBetween(2, 10)).execute().actionGet(); + ScoreFunctionBuilders.weightFactorFunction(100)).boostMode(CombineFunction.REPLACE)) + .setQueryWeight(0.0f).setRescoreQueryWeight(1.0f), 1).setSize(randomIntBetween(2, 10)).execute() + .actionGet(); assertSearchResponse(searchResponse); assertFirstHit(searchResponse, hasScore(100.f)); int numDocsWith100AsAScore = 0; @@ -89,7 +98,6 @@ public class QueryRescorerIT extends ESIntegTestCase { } } - @Test public void testRescorePhrase() throws Exception { assertAcked(prepareCreate("test") .addMapping( @@ -106,8 +114,9 @@ public class QueryRescorerIT extends ESIntegTestCase { refresh(); SearchResponse searchResponse = client().prepareSearch() .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) - .setRescorer(RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "quick brown").slop(2).boost(4.0f)).setRescoreQueryWeight(2)) - .setRescoreWindow(5).execute().actionGet(); + .setRescorer( + RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "quick brown").slop(2).boost(4.0f)) + .setRescoreQueryWeight(2), 5).execute().actionGet(); assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); @@ -116,8 +125,8 @@ public class QueryRescorerIT extends ESIntegTestCase { searchResponse = client().prepareSearch() .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) - .setRescorer(RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "the quick brown").slop(3))) - .setRescoreWindow(5).execute().actionGet(); + .setRescorer(RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "the quick brown").slop(3)), 5) + .execute().actionGet(); assertHitCount(searchResponse, 3); assertFirstHit(searchResponse, hasId("1")); @@ -126,8 +135,8 @@ public class QueryRescorerIT extends ESIntegTestCase { searchResponse = client().prepareSearch() .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) - .setRescorer(RescoreBuilder.queryRescorer((QueryBuilders.matchPhraseQuery("field1", "the quick brown")))) - .setRescoreWindow(5).execute().actionGet(); + .setRescorer(RescoreBuilder.queryRescorer((QueryBuilders.matchPhraseQuery("field1", "the quick brown"))), 5).execute() + .actionGet(); assertHitCount(searchResponse, 3); assertFirstHit(searchResponse, hasId("1")); @@ -135,7 +144,6 @@ public class QueryRescorerIT extends ESIntegTestCase { assertThirdHit(searchResponse, hasId("3")); } - @Test public void testMoreDocs() throws Exception { Builder builder = Settings.builder(); builder.put("index.analysis.analyzer.synonym.tokenizer", "whitespace"); @@ -173,7 +181,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .setSize(5) .setRescorer( RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)) - .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f)).setRescoreWindow(20).execute().actionGet(); + .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 20).execute().actionGet(); assertThat(searchResponse.getHits().hits().length, equalTo(5)); assertHitCount(searchResponse, 9); @@ -189,7 +197,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) .setRescorer( RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)) - .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f)).setRescoreWindow(20).execute().actionGet(); + .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 20).execute().actionGet(); assertThat(searchResponse.getHits().hits().length, equalTo(5)); assertHitCount(searchResponse, 9); @@ -206,7 +214,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) .setRescorer( RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)) - .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f)).setRescoreWindow(20).execute().actionGet(); + .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 20).execute().actionGet(); assertThat(searchResponse.getHits().hits().length, equalTo(5)); assertHitCount(searchResponse, 9); @@ -214,7 +222,6 @@ public class QueryRescorerIT extends ESIntegTestCase { } // Tests a rescore window smaller than number of hits: - @Test public void testSmallRescoreWindow() throws Exception { Builder builder = Settings.builder(); builder.put("index.analysis.analyzer.synonym.tokenizer", "whitespace"); @@ -256,7 +263,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .setSize(5) .setRescorer( RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)) - .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f)).setRescoreWindow(2).execute().actionGet(); + .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 2).execute().actionGet(); // Only top 2 hits were re-ordered: assertThat(searchResponse.getHits().hits().length, equalTo(4)); assertHitCount(searchResponse, 4); @@ -273,7 +280,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .setSize(5) .setRescorer( RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)) - .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f)).setRescoreWindow(3).execute().actionGet(); + .setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 3).execute().actionGet(); // Only top 3 hits were re-ordered: assertThat(searchResponse.getHits().hits().length, equalTo(4)); @@ -285,7 +292,6 @@ public class QueryRescorerIT extends ESIntegTestCase { } // Tests a rescorer that penalizes the scores: - @Test public void testRescorerMadeScoresWorse() throws Exception { Builder builder = Settings.builder(); builder.put("index.analysis.analyzer.synonym.tokenizer", "whitespace"); @@ -327,7 +333,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .setSize(5) .setRescorer( RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)) - .setQueryWeight(1.0f).setRescoreQueryWeight(-1f)).setRescoreWindow(3).execute().actionGet(); + .setQueryWeight(1.0f).setRescoreQueryWeight(-1f), 3).execute().actionGet(); // 6 and 1 got worse, and then the hit (2) outside the rescore window were sorted ahead: assertFirstHit(searchResponse, hasId("3")); @@ -398,7 +404,6 @@ public class QueryRescorerIT extends ESIntegTestCase { } } - @Test // forces QUERY_THEN_FETCH because of https://github.com/elasticsearch/elasticsearch/issues/4829 public void testEquivalence() throws Exception { // no dummy docs since merges can change scores while we run queries. @@ -423,15 +428,28 @@ public class QueryRescorerIT extends ESIntegTestCase { QueryBuilders .constantScoreQuery(QueryBuilders.matchPhraseQuery("field1", intToEnglish).slop(3))) .setQueryWeight(1.0f) - .setRescoreQueryWeight(0.0f)) // no weight - so we basically use the same score as the actual query - .setRescoreWindow(rescoreWindow).execute().actionGet(); +.setRescoreQueryWeight(0.0f), rescoreWindow) // no + // weight + // - + // so + // we + // basically + // use + // the + // same + // score + // as + // the + // actual + // query + .execute().actionGet(); SearchResponse plain = client().prepareSearch() .setSearchType(SearchType.QUERY_THEN_FETCH) .setPreference("test") // ensure we hit the same shards for tie-breaking .setQuery(QueryBuilders.matchQuery("field1", query).operator(Operator.OR)).setFrom(0).setSize(resultSize) .execute().actionGet(); - + // check equivalence assertEquivalent(query, plain, rescored); @@ -448,8 +466,8 @@ public class QueryRescorerIT extends ESIntegTestCase { QueryBuilders .constantScoreQuery(QueryBuilders.matchPhraseQuery("field1", "not in the index").slop(3))) .setQueryWeight(1.0f) - .setRescoreQueryWeight(1.0f)) - .setRescoreWindow(rescoreWindow).execute().actionGet(); +.setRescoreQueryWeight(1.0f), rescoreWindow).execute() + .actionGet(); // check equivalence assertEquivalent(query, plain, rescored); @@ -464,13 +482,12 @@ public class QueryRescorerIT extends ESIntegTestCase { RescoreBuilder .queryRescorer( QueryBuilders.matchPhraseQuery("field1", intToEnglish).slop(0)) - .setQueryWeight(1.0f).setRescoreQueryWeight(1.0f)).setRescoreWindow(2 * rescoreWindow).execute().actionGet(); + .setQueryWeight(1.0f).setRescoreQueryWeight(1.0f), 2 * rescoreWindow).execute().actionGet(); // check equivalence or if the first match differs we check if the phrase is a substring of the top doc assertEquivalentOrSubstringMatch(intToEnglish, plain, rescored); } } - @Test public void testExplain() throws Exception { assertAcked(prepareCreate("test") .addMapping( @@ -495,7 +512,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) .setRescorer( RescoreBuilder.queryRescorer(QueryBuilders.matchPhraseQuery("field1", "the quick brown").slop(2).boost(4.0f)) - .setQueryWeight(0.5f).setRescoreQueryWeight(0.4f)).setRescoreWindow(5).setExplain(true).execute() + .setQueryWeight(0.5f).setRescoreQueryWeight(0.4f), 5).setExplain(true).execute() .actionGet(); assertHitCount(searchResponse, 3); assertFirstHit(searchResponse, hasId("1")); @@ -531,7 +548,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .prepareSearch() .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) - .setRescorer(innerRescoreQuery).setRescoreWindow(5).setExplain(true).execute() + .setRescorer(innerRescoreQuery, 5).setExplain(true).execute() .actionGet(); assertHitCount(searchResponse, 3); assertFirstHit(searchResponse, hasId("1")); @@ -554,8 +571,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .prepareSearch() .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) - .addRescorer(innerRescoreQuery).setRescoreWindow(5) - .addRescorer(outerRescoreQuery).setRescoreWindow(10) + .addRescorer(innerRescoreQuery, 5).addRescorer(outerRescoreQuery, 10) .setExplain(true).get(); assertHitCount(searchResponse, 3); assertFirstHit(searchResponse, hasId("1")); @@ -571,7 +587,6 @@ public class QueryRescorerIT extends ESIntegTestCase { } } - @Test public void testScoring() throws Exception { int numDocs = indexRandomNumbers("keyword"); @@ -615,8 +630,7 @@ public class QueryRescorerIT extends ESIntegTestCase { ScoreFunctionBuilders.weightFactorFunction(0.2f)).boostMode(CombineFunction.REPLACE))) .setFrom(0) .setSize(10) - .setRescorer(rescoreQuery) - .setRescoreWindow(50).execute().actionGet(); +.setRescorer(rescoreQuery, 50).execute().actionGet(); assertHitCount(rescored, 4); @@ -665,7 +679,6 @@ public class QueryRescorerIT extends ESIntegTestCase { } } - @Test public void testMultipleRescores() throws Exception { int numDocs = indexRandomNumbers("keyword", 1, true); QueryRescorer eightIsGreat = RescoreBuilder.queryRescorer( @@ -677,14 +690,14 @@ public class QueryRescorerIT extends ESIntegTestCase { .setScoreMode("total"); // First set the rescore window large enough that both rescores take effect - SearchRequestBuilder request = client().prepareSearch().setRescoreWindow(numDocs); - request.addRescorer(eightIsGreat).addRescorer(sevenIsBetter); + SearchRequestBuilder request = client().prepareSearch(); + request.addRescorer(eightIsGreat, numDocs).addRescorer(sevenIsBetter, numDocs); SearchResponse response = request.get(); assertFirstHit(response, hasId("7")); assertSecondHit(response, hasId("8")); // Now squash the second rescore window so it never gets to see a seven - response = request.setSize(1).clearRescorers().addRescorer(eightIsGreat).addRescorer(sevenIsBetter, 1).get(); + response = request.setSize(1).clearRescorers().addRescorer(eightIsGreat, numDocs).addRescorer(sevenIsBetter, 1).get(); assertFirstHit(response, hasId("8")); // We have no idea what the second hit will be because we didn't get a chance to look for seven @@ -695,7 +708,7 @@ public class QueryRescorerIT extends ESIntegTestCase { QueryRescorer oneToo = RescoreBuilder.queryRescorer( QueryBuilders.functionScoreQuery(QueryBuilders.queryStringQuery("*one*"), ScoreFunctionBuilders.weightFactorFunction(1000.0f)) .boostMode(CombineFunction.REPLACE)).setScoreMode("total"); - request.clearRescorers().addRescorer(ninetyIsGood).addRescorer(oneToo, 10); + request.clearRescorers().addRescorer(ninetyIsGood, numDocs).addRescorer(oneToo, 10); response = request.setSize(2).get(); assertFirstHit(response, hasId("91")); assertFirstHit(response, hasScore(2001.0f)); @@ -745,8 +758,7 @@ public class QueryRescorerIT extends ESIntegTestCase { request.setQuery(QueryBuilders.termQuery("text", "hello")); request.setFrom(1); request.setSize(4); - request.addRescorer(RescoreBuilder.queryRescorer(QueryBuilders.matchAllQuery())); - request.setRescoreWindow(50); + request.addRescorer(RescoreBuilder.queryRescorer(QueryBuilders.matchAllQuery()), 50); assertEquals(4, request.get().getHits().hits().length); } diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java index b61b38993ee..b354edfc60e 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java @@ -19,17 +19,21 @@ package org.elasticsearch.search.geo; +import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.GeoValidationMethod; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.VersionUtils; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.geoBoundingBoxQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; @@ -38,13 +42,16 @@ import static org.hamcrest.Matchers.equalTo; * */ public class GeoBoundingBoxIT extends ESIntegTestCase { - - @Test - public void simpleBoundingBoxTest() throws Exception { + public void testSimpleBoundingBoxTest() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -109,12 +116,16 @@ public class GeoBoundingBoxIT extends ESIntegTestCase { } } - @Test - public void limitsBoundingBoxTest() throws Exception { + public void testLimitsBoundingBox() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -212,12 +223,16 @@ public class GeoBoundingBoxIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).id(), equalTo("9")); } - @Test - public void limit2BoundingBoxTest() throws Exception { + public void testLimit2BoundingBox() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -263,12 +278,16 @@ public class GeoBoundingBoxIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); } - @Test - public void completeLonRangeTest() throws Exception { + public void testCompleteLonRange() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index 3cb70253f81..dfa28947ca2 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -29,27 +29,32 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.query.SpatialArgs; import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.query.UnsupportedSpatialOperation; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; +import org.apache.lucene.util.GeoProjectionUtils; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.builders.LineStringBuilder; import org.elasticsearch.common.geo.builders.MultiPolygonBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; -import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.GeohashCellQuery; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.junit.BeforeClass; -import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; @@ -57,7 +62,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -75,10 +79,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirs import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.*; /** * @@ -112,12 +113,10 @@ public class GeoFilterIT extends ESIntegTestCase { return out.toByteArray(); } - @Test public void testShapeBuilders() { - try { // self intersection polygon - ShapeBuilder.newPolygon() + ShapeBuilders.newPolygon() .point(-10, -10) .point(10, 10) .point(-10, 10) @@ -128,19 +127,19 @@ public class GeoFilterIT extends ESIntegTestCase { } // polygon with hole - ShapeBuilder.newPolygon() + ShapeBuilders.newPolygon() .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) - .hole() + .hole(new LineStringBuilder() .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5) - .close().close().build(); + .close()).close().build(); try { // polygon with overlapping hole - ShapeBuilder.newPolygon() + ShapeBuilders.newPolygon() .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) - .hole() + .hole(new LineStringBuilder() .point(-5, -5).point(-5, 11).point(5, 11).point(5, -5) - .close().close().build(); + .close()).close().build(); fail("Self intersection not detected"); } catch (InvalidShapeException e) { @@ -148,14 +147,14 @@ public class GeoFilterIT extends ESIntegTestCase { try { // polygon with intersection holes - ShapeBuilder.newPolygon() + ShapeBuilders.newPolygon() .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) - .hole() + .hole(new LineStringBuilder() .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5) - .close() - .hole() + .close()) + .hole(new LineStringBuilder() .point(-5, -6).point(5, -6).point(5, -4).point(-5, -4) - .close() + .close()) .close().build(); fail("Intersection of holes not detected"); } catch (InvalidShapeException e) { @@ -163,7 +162,7 @@ public class GeoFilterIT extends ESIntegTestCase { try { // Common line in polygon - ShapeBuilder.newPolygon() + ShapeBuilders.newPolygon() .point(-10, -10) .point(-10, 10) .point(-5, 10) @@ -176,62 +175,34 @@ public class GeoFilterIT extends ESIntegTestCase { } catch (InvalidShapeException e) { } -// Not specified -// try { -// // two overlapping polygons within a multipolygon -// ShapeBuilder.newMultiPolygon() -// .polygon() -// .point(-10, -10) -// .point(-10, 10) -// .point(10, 10) -// .point(10, -10) -// .close() -// .polygon() -// .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5) -// .close().build(); -// fail("Polygon intersection not detected"; -// } catch (InvalidShapeException e) {} - // Multipolygon: polygon with hole and polygon within the whole - ShapeBuilder.newMultiPolygon() - .polygon() - .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) - .hole() - .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5) - .close() - .close() - .polygon() - .point(-4, -4).point(-4, 4).point(4, 4).point(4, -4) - .close() + ShapeBuilders + .newMultiPolygon() + .polygon(new PolygonBuilder() + .point(-10, -10) + .point(-10, 10) + .point(10, 10) + .point(10, -10) + .hole(new LineStringBuilder().point(-5, -5) + .point(-5, 5) + .point(5, 5) + .point(5, -5) + .close()) + .close()) + .polygon(new PolygonBuilder() + .point(-4, -4) + .point(-4, 4) + .point(4, 4) + .point(4, -4) + .close()) .build(); - -// Not supported -// try { -// // Multipolygon: polygon with hole and polygon within the hole but overlapping -// ShapeBuilder.newMultiPolygon() -// .polygon() -// .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) -// .hole() -// .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5) -// .close() -// .close() -// .polygon() -// .point(-4, -4).point(-4, 6).point(4, 6).point(4, -4) -// .close() -// .build(); -// fail("Polygon intersection not detected"; -// } catch (InvalidShapeException e) {} - } - @Test public void testShapeRelations() throws Exception { - assertTrue( "Intersect relation is not supported", intersectSupport); assertTrue("Disjoint relation is not supported", disjointSupport); assertTrue("within relation is not supported", withinSupport); - String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("polygon") @@ -251,16 +222,14 @@ public class GeoFilterIT extends ESIntegTestCase { // Create a multipolygon with two polygons. The first is an rectangle of size 10x10 // with a hole of size 5x5 equidistant from all sides. This hole in turn contains // the second polygon of size 4x4 equidistant from all sites - MultiPolygonBuilder polygon = ShapeBuilder.newMultiPolygon() - .polygon() - .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) - .hole() - .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5) - .close() - .close() - .polygon() - .point(-4, -4).point(-4, 4).point(4, 4).point(4, -4) - .close(); + MultiPolygonBuilder polygon = ShapeBuilders.newMultiPolygon() + .polygon(new PolygonBuilder() + .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) + .hole(new LineStringBuilder() + .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5).close()) + .close()) + .polygon(new PolygonBuilder() + .point(-4, -4).point(-4, 4).point(4, 4).point(4, -4).close()); BytesReference data = jsonBuilder().startObject().field("area", polygon).endObject().bytes(); @@ -270,7 +239,7 @@ public class GeoFilterIT extends ESIntegTestCase { // Point in polygon SearchResponse result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(3, 3))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(3, 3))) .execute().actionGet(); assertHitCount(result, 1); assertFirstHit(result, hasId("1")); @@ -278,7 +247,7 @@ public class GeoFilterIT extends ESIntegTestCase { // Point in polygon hole result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(4.5, 4.5))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(4.5, 4.5))) .execute().actionGet(); assertHitCount(result, 0); @@ -289,7 +258,7 @@ public class GeoFilterIT extends ESIntegTestCase { // Point on polygon border result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(10.0, 5.0))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(10.0, 5.0))) .execute().actionGet(); assertHitCount(result, 1); assertFirstHit(result, hasId("1")); @@ -297,7 +266,7 @@ public class GeoFilterIT extends ESIntegTestCase { // Point on hole border result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(5.0, 2.0))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(5.0, 2.0))) .execute().actionGet(); assertHitCount(result, 1); assertFirstHit(result, hasId("1")); @@ -306,25 +275,24 @@ public class GeoFilterIT extends ESIntegTestCase { // Point not in polygon result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoDisjointQuery("area", ShapeBuilder.newPoint(3, 3))) + .setPostFilter(QueryBuilders.geoDisjointQuery("area", ShapeBuilders.newPoint(3, 3))) .execute().actionGet(); assertHitCount(result, 0); // Point in polygon hole result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoDisjointQuery("area", ShapeBuilder.newPoint(4.5, 4.5))) + .setPostFilter(QueryBuilders.geoDisjointQuery("area", ShapeBuilders.newPoint(4.5, 4.5))) .execute().actionGet(); assertHitCount(result, 1); assertFirstHit(result, hasId("1")); } // Create a polygon that fills the empty area of the polygon defined above - PolygonBuilder inverse = ShapeBuilder.newPolygon() + PolygonBuilder inverse = ShapeBuilders.newPolygon() .point(-5, -5).point(-5, 5).point(5, 5).point(5, -5) - .hole() - .point(-4, -4).point(-4, 4).point(4, 4).point(4, -4) - .close() + .hole(new LineStringBuilder() + .point(-4, -4).point(-4, 4).point(4, 4).point(4, -4).close()) .close(); data = jsonBuilder().startObject().field("area", inverse).endObject().bytes(); @@ -334,22 +302,21 @@ public class GeoFilterIT extends ESIntegTestCase { // re-check point on polygon hole result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(4.5, 4.5))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(4.5, 4.5))) .execute().actionGet(); assertHitCount(result, 1); assertFirstHit(result, hasId("2")); // Create Polygon with hole and common edge - PolygonBuilder builder = ShapeBuilder.newPolygon() + PolygonBuilder builder = ShapeBuilders.newPolygon() .point(-10, -10).point(-10, 10).point(10, 10).point(10, -10) - .hole() - .point(-5, -5).point(-5, 5).point(10, 5).point(10, -5) - .close() + .hole(new LineStringBuilder() + .point(-5, -5).point(-5, 5).point(10, 5).point(10, -5).close()) .close(); if (withinSupport) { // Polygon WithIn Polygon - builder = ShapeBuilder.newPolygon() + builder = ShapeBuilders.newPolygon() .point(-30, -30).point(-30, 30).point(30, 30).point(30, -30).close(); result = client().prepareSearch() @@ -360,7 +327,7 @@ public class GeoFilterIT extends ESIntegTestCase { } // Create a polygon crossing longitude 180. - builder = ShapeBuilder.newPolygon() + builder = ShapeBuilders.newPolygon() .point(170, -10).point(190, -10).point(190, 10).point(170, 10) .close(); @@ -369,9 +336,9 @@ public class GeoFilterIT extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); // Create a polygon crossing longitude 180 with hole. - builder = ShapeBuilder.newPolygon() + builder = ShapeBuilders.newPolygon() .point(170, -10).point(190, -10).point(190, 10).point(170, 10) - .hole().point(175, -5).point(185, -5).point(185, 5).point(175, 5).close() + .hole(new LineStringBuilder().point(175, -5).point(185, -5).point(185, 5).point(175, 5).close()) .close(); data = jsonBuilder().startObject().field("area", builder).endObject().bytes(); @@ -380,51 +347,53 @@ public class GeoFilterIT extends ESIntegTestCase { result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(174, -4))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(174, -4))) .execute().actionGet(); assertHitCount(result, 1); result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(-174, -4))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(-174, -4))) .execute().actionGet(); assertHitCount(result, 1); result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(180, -4))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(180, -4))) .execute().actionGet(); assertHitCount(result, 0); result = client().prepareSearch() .setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilder.newPoint(180, -6))) + .setPostFilter(QueryBuilders.geoIntersectionQuery("area", ShapeBuilders.newPoint(180, -6))) .execute().actionGet(); assertHitCount(result, 1); } - @Test - public void bulktest() throws Exception { + public void testBulk() throws Exception { byte[] bulkAction = unZipData("/org/elasticsearch/search/geo/gzippedmap.gz"); - - String mapping = XContentFactory.jsonBuilder() + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() .startObject("country") .startObject("properties") .startObject("pin") - .field("type", "geo_point") - .field("lat_lon", true) - .field("store", true) + .field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.field("store", true) .endObject() .startObject("location") .field("type", "geo_shape") .endObject() .endObject() .endObject() - .endObject() - .string(); + .endObject(); - client().admin().indices().prepareCreate("countries").addMapping("country", mapping).execute().actionGet(); + client().admin().indices().prepareCreate("countries").setSettings(settings) + .addMapping("country", xContentBuilder.string()).execute().actionGet(); BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null).execute().actionGet(); for (BulkItemResponse item : bulk.getItems()) { @@ -458,24 +427,27 @@ public class GeoFilterIT extends ESIntegTestCase { GeoPoint point = new GeoPoint(); for (SearchHit hit : distance.getHits()) { String name = hit.getId(); - point.resetFromString(hit.fields().get("pin").getValue().toString()); + if (version.before(Version.V_2_2_0)) { + point.resetFromString(hit.fields().get("pin").getValue().toString()); + } else { + point.resetFromIndexHash(hit.fields().get("pin").getValue()); + } double dist = distance(point.getLat(), point.getLon(), 51.11, 9.851); assertThat("distance to '" + name + "'", dist, lessThanOrEqualTo(425000d)); assertThat(name, anyOf(equalTo("CZ"), equalTo("DE"), equalTo("BE"), equalTo("NL"), equalTo("LU"))); if (key.equals(name)) { - assertThat(dist, equalTo(0d)); + assertThat(dist, closeTo(0d, 0.1d)); } } } - @Test public void testGeohashCellFilter() throws IOException { String geohash = randomhash(10); logger.info("Testing geohash_cell filter for [{}]", geohash); - Collection neighbors = XGeoHashUtils.neighbors(geohash); - Collection parentNeighbors = XGeoHashUtils.neighbors(geohash.substring(0, geohash.length() - 1)); + Collection neighbors = GeoHashUtils.neighbors(geohash); + Collection parentNeighbors = GeoHashUtils.neighbors(geohash.substring(0, geohash.length() - 1)); logger.info("Neighbors {}", neighbors); logger.info("Parent Neighbors {}", parentNeighbors); @@ -517,79 +489,56 @@ public class GeoFilterIT extends ESIntegTestCase { expectedCounts.put(geoHashCellQuery("pin", point).neighbors(true).precision(precision), 1L + neighbors.size()); - logger.info("random testing of setting"); List filterBuilders = new ArrayList<>(expectedCounts.keySet()); - for (int j = filterBuilders.size() * 2 * randomIntBetween(1, 5); j > 0; j--) { - Collections.shuffle(filterBuilders, getRandom()); - for (GeohashCellQuery.Builder builder : filterBuilders) { - try { - long expectedCount = expectedCounts.get(builder); - SearchResponse response = client().prepareSearch("locations").setQuery(QueryBuilders.matchAllQuery()) - .setPostFilter(builder).setSize((int) expectedCount).get(); - assertHitCount(response, expectedCount); - String[] expectedIds = expectedResults.get(builder); - if (expectedIds == null) { - ArrayList ids = new ArrayList<>(); - for (SearchHit hit : response.getHits()) { - ids.add(hit.id()); - } - expectedResults.put(builder, ids.toArray(Strings.EMPTY_ARRAY)); - continue; + for (GeohashCellQuery.Builder builder : filterBuilders) { + try { + long expectedCount = expectedCounts.get(builder); + SearchResponse response = client().prepareSearch("locations").setQuery(QueryBuilders.matchAllQuery()) + .setPostFilter(builder).setSize((int) expectedCount).get(); + assertHitCount(response, expectedCount); + String[] expectedIds = expectedResults.get(builder); + if (expectedIds == null) { + ArrayList ids = new ArrayList<>(); + for (SearchHit hit : response.getHits()) { + ids.add(hit.id()); } - - assertSearchHits(response, expectedIds); - - } catch (AssertionError error) { - throw new AssertionError(error.getMessage() + "\n geohash_cell filter:" + builder, error); + expectedResults.put(builder, ids.toArray(Strings.EMPTY_ARRAY)); + continue; } + assertSearchHits(response, expectedIds); + } catch (AssertionError error) { + throw new AssertionError(error.getMessage() + "\n geohash_cell filter:" + builder, error); } } - - logger.info("Testing lat/lon format"); - String pointTest1 = "{\"geohash_cell\": {\"pin\": {\"lat\": " + point.lat() + ",\"lon\": " + point.lon() + "},\"precision\": " + precision + ",\"neighbors\": true}}"; - SearchResponse results3 = client().prepareSearch("locations").setQuery(QueryBuilders.matchAllQuery()).setPostFilter(pointTest1).execute().actionGet(); - assertHitCount(results3, neighbors.size() + 1); - - - logger.info("Testing String format"); - String pointTest2 = "{\"geohash_cell\": {\"pin\": \"" + point.lat() + "," + point.lon() + "\",\"precision\": " + precision + ",\"neighbors\": true}}"; - SearchResponse results4 = client().prepareSearch("locations").setQuery(QueryBuilders.matchAllQuery()).setPostFilter(pointTest2).execute().actionGet(); - assertHitCount(results4, neighbors.size() + 1); - - logger.info("Testing Array format"); - String pointTest3 = "{\"geohash_cell\": {\"pin\": [" + point.lon() + "," + point.lat() + "],\"precision\": " + precision + ",\"neighbors\": true}}"; - SearchResponse results5 = client().prepareSearch("locations").setQuery(QueryBuilders.matchAllQuery()).setPostFilter(pointTest3).execute().actionGet(); - assertHitCount(results5, neighbors.size() + 1); } - @Test public void testNeighbors() { // Simple root case - assertThat(XGeoHashUtils.addNeighbors("7", new ArrayList()), containsInAnyOrder("4", "5", "6", "d", "e", "h", "k", "s")); + assertThat(GeoHashUtils.addNeighbors("7", new ArrayList()), containsInAnyOrder("4", "5", "6", "d", "e", "h", "k", "s")); // Root cases (Outer cells) - assertThat(XGeoHashUtils.addNeighbors("0", new ArrayList()), containsInAnyOrder("1", "2", "3", "p", "r")); - assertThat(XGeoHashUtils.addNeighbors("b", new ArrayList()), containsInAnyOrder("8", "9", "c", "x", "z")); - assertThat(XGeoHashUtils.addNeighbors("p", new ArrayList()), containsInAnyOrder("n", "q", "r", "0", "2")); - assertThat(XGeoHashUtils.addNeighbors("z", new ArrayList()), containsInAnyOrder("8", "b", "w", "x", "y")); + assertThat(GeoHashUtils.addNeighbors("0", new ArrayList()), containsInAnyOrder("1", "2", "3", "p", "r")); + assertThat(GeoHashUtils.addNeighbors("b", new ArrayList()), containsInAnyOrder("8", "9", "c", "x", "z")); + assertThat(GeoHashUtils.addNeighbors("p", new ArrayList()), containsInAnyOrder("n", "q", "r", "0", "2")); + assertThat(GeoHashUtils.addNeighbors("z", new ArrayList()), containsInAnyOrder("8", "b", "w", "x", "y")); // Root crossing dateline - assertThat(XGeoHashUtils.addNeighbors("2", new ArrayList()), containsInAnyOrder("0", "1", "3", "8", "9", "p", "r", "x")); - assertThat(XGeoHashUtils.addNeighbors("r", new ArrayList()), containsInAnyOrder("0", "2", "8", "n", "p", "q", "w", "x")); + assertThat(GeoHashUtils.addNeighbors("2", new ArrayList()), containsInAnyOrder("0", "1", "3", "8", "9", "p", "r", "x")); + assertThat(GeoHashUtils.addNeighbors("r", new ArrayList()), containsInAnyOrder("0", "2", "8", "n", "p", "q", "w", "x")); // level1: simple case - assertThat(XGeoHashUtils.addNeighbors("dk", new ArrayList()), containsInAnyOrder("d5", "d7", "de", "dh", "dj", "dm", "ds", "dt")); + assertThat(GeoHashUtils.addNeighbors("dk", new ArrayList()), containsInAnyOrder("d5", "d7", "de", "dh", "dj", "dm", "ds", "dt")); // Level1: crossing cells - assertThat(XGeoHashUtils.addNeighbors("d5", new ArrayList()), containsInAnyOrder("d4", "d6", "d7", "dh", "dk", "9f", "9g", "9u")); - assertThat(XGeoHashUtils.addNeighbors("d0", new ArrayList()), containsInAnyOrder("d1", "d2", "d3", "9b", "9c", "6p", "6r", "3z")); + assertThat(GeoHashUtils.addNeighbors("d5", new ArrayList()), containsInAnyOrder("d4", "d6", "d7", "dh", "dk", "9f", "9g", "9u")); + assertThat(GeoHashUtils.addNeighbors("d0", new ArrayList()), containsInAnyOrder("d1", "d2", "d3", "9b", "9c", "6p", "6r", "3z")); } public static double distance(double lat1, double lon1, double lat2, double lon2) { - return GeoUtils.EARTH_SEMI_MAJOR_AXIS * DistanceUtils.distHaversineRAD( + return GeoProjectionUtils.SEMIMAJOR_AXIS * DistanceUtils.distHaversineRAD( DistanceUtils.toRadians(lat1), DistanceUtils.toRadians(lon1), DistanceUtils.toRadians(lat2), @@ -607,7 +556,7 @@ public class GeoFilterIT extends ESIntegTestCase { RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy(tree, "area"); Shape shape = SpatialContext.GEO.makePoint(0, 0); SpatialArgs args = new SpatialArgs(relation, shape); - strategy.makeFilter(args); + strategy.makeQuery(args); return true; } catch (UnsupportedSpatialOperation e) { e.printStackTrace(); diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java index 248c62bf16e..69b93018d8d 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java @@ -19,13 +19,16 @@ package org.elasticsearch.search.geo; +import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.VersionUtils; import java.util.ArrayList; import java.util.List; @@ -43,11 +46,15 @@ public class GeoPolygonIT extends ESIntegTestCase { @Override protected void setupSuiteScopeCluster() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true) - .startObject("fielddata").field("format", randomNumericFieldDataFormat()).endObject().endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); indexRandom(true, client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -87,8 +94,7 @@ public class GeoPolygonIT extends ESIntegTestCase { ensureSearchable("test"); } - @Test - public void simplePolygonTest() throws Exception { + public void testSimplePolygon() throws Exception { List points = new ArrayList<>(); points.add(new GeoPoint(40.7, -74.0)); points.add(new GeoPoint(40.7, -74.1)); @@ -105,8 +111,7 @@ public class GeoPolygonIT extends ESIntegTestCase { } } - @Test - public void simpleUnclosedPolygon() throws Exception { + public void testSimpleUnclosedPolygon() throws Exception { List points = new ArrayList<>(); points.add(new GeoPoint(40.7, -74.0)); points.add(new GeoPoint(40.7, -74.1)); diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java similarity index 60% rename from core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java rename to core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 670d31739b1..847e03e5c44 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -19,26 +19,22 @@ package org.elasticsearch.search.geo; -import org.apache.lucene.util.LuceneTestCase; +import com.spatial4j.core.shape.Rectangle; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; +import org.elasticsearch.common.geo.builders.LineStringBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.geo.RandomShapeGenerator; -import org.junit.Test; import java.io.IOException; import java.util.Locale; @@ -46,32 +42,29 @@ import java.util.Locale; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.geoIntersectionQuery; import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.geo.RandomShapeGenerator.createGeometryCollectionWithin; +import static org.elasticsearch.test.geo.RandomShapeGenerator.xRandomPoint; +import static org.elasticsearch.test.geo.RandomShapeGenerator.xRandomRectangle; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; -public class GeoShapeIntegrationIT extends ESIntegTestCase { - - @Test +public class GeoShapeQueryTests extends ESSingleNodeTestCase { public void testNullShape() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") .field("type", "geo_shape") .endObject().endObject() .endObject().endObject().string(); - assertAcked(prepareCreate("test").addMapping("type1", mapping)); + client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet(); ensureGreen(); - indexRandom(false, client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}")); + client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}").setRefresh(true) + .execute().actionGet(); GetResponse result = client().prepareGet("test", "type1", "aNullshape").execute().actionGet(); assertThat(result.getField("location"), nullValue()); } - @Test public void testIndexPointsFilterRectangle() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -79,30 +72,28 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { .field("tree", "quadtree") .endObject().endObject() .endObject().endObject().string(); - assertAcked(prepareCreate("test").addMapping("type1", mapping)); + client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet(); ensureGreen(); - indexRandom(true, + client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("name", "Document 1") + .startObject("location") + .field("type", "point") + .startArray("coordinates").value(-30).value(-30).endArray() + .endObject() + .endObject()).setRefresh(true).execute().actionGet(); - client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() - .field("name", "Document 1") - .startObject("location") - .field("type", "point") - .startArray("coordinates").value(-30).value(-30).endArray() - .endObject() - .endObject()), + client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() + .field("name", "Document 2") + .startObject("location") + .field("type", "point") + .startArray("coordinates").value(-45).value(-50).endArray() + .endObject() + .endObject()).setRefresh(true).execute().actionGet(); - client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() - .field("name", "Document 2") - .startObject("location") - .field("type", "point") - .startArray("coordinates").value(-45).value(-50).endArray() - .endObject() - .endObject())); + ShapeBuilder shape = ShapeBuilders.newEnvelope().topLeft(-45, 45).bottomRight(45, -45); - ShapeBuilder shape = ShapeBuilder.newEnvelope().topLeft(-45, 45).bottomRight(45, -45); - - SearchResponse searchResponse = client().prepareSearch() + SearchResponse searchResponse = client().prepareSearch("test").setTypes("type1") .setQuery(geoIntersectionQuery("location", shape)) .execute().actionGet(); @@ -111,7 +102,7 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); - searchResponse = client().prepareSearch() + searchResponse = client().prepareSearch("test").setTypes("type1") .setQuery(geoShapeQuery("location", shape)) .execute().actionGet(); @@ -121,19 +112,17 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); } - @Test public void testEdgeCases() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") .field("type", "geo_shape") .field("tree", "quadtree") .endObject().endObject() .endObject().endObject().string(); - assertAcked(prepareCreate("test").addMapping("type1", mapping)); + client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet(); ensureGreen(); - indexRandom(true, client().prepareIndex("test", "type1", "blakely").setSource(jsonBuilder().startObject() + client().prepareIndex("test", "type1", "blakely").setSource(jsonBuilder().startObject() .field("name", "Blakely Island") .startObject("location") .field("type", "polygon") @@ -144,14 +133,13 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { .startArray().value(-122.83).value(48.57).endArray() // close the polygon .endArray().endArray() .endObject() - .endObject())); + .endObject()).setRefresh(true).execute().actionGet(); - - ShapeBuilder query = ShapeBuilder.newEnvelope().topLeft(-122.88, 48.62).bottomRight(-122.82, 48.54); + ShapeBuilder query = ShapeBuilders.newEnvelope().topLeft(-122.88, 48.62).bottomRight(-122.82, 48.54); // This search would fail if both geoshape indexing and geoshape filtering // used the bottom-level optimization in SpatialPrefixTree#recursiveGetNodes. - SearchResponse searchResponse = client().prepareSearch() + SearchResponse searchResponse = client().prepareSearch("test").setTypes("type1") .setQuery(geoIntersectionQuery("location", query)) .execute().actionGet(); @@ -161,7 +149,6 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).id(), equalTo("blakely")); } - @Test public void testIndexedShapeReference() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -169,24 +156,23 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { .field("tree", "quadtree") .endObject().endObject() .endObject().endObject().string(); - assertAcked(prepareCreate("test").addMapping("type1", mapping)); + client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet(); createIndex("shapes"); ensureGreen(); - ShapeBuilder shape = ShapeBuilder.newEnvelope().topLeft(-45, 45).bottomRight(45, -45); + ShapeBuilder shape = ShapeBuilders.newEnvelope().topLeft(-45, 45).bottomRight(45, -45); - indexRandom(true, - client().prepareIndex("shapes", "shape_type", "Big_Rectangle").setSource(jsonBuilder().startObject() - .field("shape", shape).endObject()), - client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + client().prepareIndex("shapes", "shape_type", "Big_Rectangle").setSource(jsonBuilder().startObject() + .field("shape", shape).endObject()).setRefresh(true).execute().actionGet(); + client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() .field("name", "Document 1") .startObject("location") .field("type", "point") .startArray("coordinates").value(-30).value(-30).endArray() .endObject() - .endObject())); + .endObject()).setRefresh(true).execute().actionGet(); - SearchResponse searchResponse = client().prepareSearch("test") + SearchResponse searchResponse = client().prepareSearch("test").setTypes("type1") .setQuery(geoIntersectionQuery("location", "Big_Rectangle", "shape_type")) .execute().actionGet(); @@ -205,15 +191,14 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); } - @Test public void testReusableBuilder() throws IOException { - ShapeBuilder polygon = ShapeBuilder.newPolygon() + ShapeBuilder polygon = ShapeBuilders.newPolygon() .point(170, -10).point(190, -10).point(190, 10).point(170, 10) - .hole().point(175, -5).point(185, -5).point(185, 5).point(175, 5).close() + .hole(new LineStringBuilder().point(175, -5).point(185, -5).point(185, 5).point(175, 5).close()) .close(); assertUnmodified(polygon); - ShapeBuilder linestring = ShapeBuilder.newLineString() + ShapeBuilder linestring = ShapeBuilders.newLineString() .point(170, -10).point(190, -10).point(190, 10).point(170, 10); assertUnmodified(linestring); } @@ -225,58 +210,19 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { assertThat(before, equalTo(after)); } - @Test - public void testParsingMultipleShapes() throws Exception { - String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("location1") - .field("type", "geo_shape") - .endObject() - .startObject("location2") - .field("type", "geo_shape") - .endObject() - .endObject() - .endObject() - .endObject() - .string(); - - assertAcked(prepareCreate("test").addMapping("type1", mapping)); - ensureYellow(); - - String p1 = "\"location1\" : {\"type\":\"polygon\", \"coordinates\":[[[-10,-10],[10,-10],[10,10],[-10,10],[-10,-10]]]}"; - String p2 = "\"location2\" : {\"type\":\"polygon\", \"coordinates\":[[[-20,-20],[20,-20],[20,20],[-20,20],[-20,-20]]]}"; - String o1 = "{" + p1 + ", " + p2 + "}"; - - indexRandom(true, client().prepareIndex("test", "type1", "1").setSource(o1)); - - String filter = "{\"geo_shape\": {\"location2\": {\"indexed_shape\": {" - + "\"id\": \"1\"," - + "\"type\": \"type1\"," - + "\"index\": \"test\"," - + "\"path\": \"location2\"" - + "}}}}"; - - SearchResponse result = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).setPostFilter(filter).execute().actionGet(); - assertSearchResponse(result); - assertHitCount(result, 1); - } - - @Test public void testShapeFetchingPath() throws Exception { createIndex("shapes"); - assertAcked(prepareCreate("test").addMapping("type", "location", "type=geo_shape")); + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape").execute().actionGet(); String location = "\"location\" : {\"type\":\"polygon\", \"coordinates\":[[[-10,-10],[10,-10],[10,10],[-10,10],[-10,-10]]]}"; - indexRandom(true, - client().prepareIndex("shapes", "type", "1") + + client().prepareIndex("shapes", "type", "1") .setSource( String.format( Locale.ROOT, "{ %s, \"1\" : { %s, \"2\" : { %s, \"3\" : { %s } }} }", location, location, location, location ) - ), - client().prepareIndex("test", "type", "1") + ).setRefresh(true).execute().actionGet(); + client().prepareIndex("test", "type", "1") .setSource(jsonBuilder().startObject().startObject("location") .field("type", "polygon") .startArray("coordinates").startArray() @@ -286,8 +232,7 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { .startArray().value(-20).value(20).endArray() .startArray().value(-20).value(-20).endArray() .endArray().endArray() - .endObject().endObject())); - ensureSearchable("test", "shapes"); + .endObject().endObject()).setRefresh(true).execute().actionGet(); GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", "1", "type").relation(ShapeRelation.INTERSECTS) .indexedShapeIndex("shapes") @@ -345,36 +290,58 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { assertHitCount(result, 1); } - @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch/issues/9904") - @Test public void testShapeFilterWithRandomGeoCollection() throws Exception { // Create a random geometry collection. GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(getRandom()); logger.info("Created Random GeometryCollection containing " + gcb.numShapes() + " shapes"); - createIndex("randshapes"); - assertAcked(prepareCreate("test").addMapping("type", "location", "type=geo_shape")); + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); - indexRandom(true, client().prepareIndex("test", "type", "1").setSource(docSource)); - - ensureSearchable("test"); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefresh(true).execute().actionGet(); ShapeBuilder filterShape = (gcb.getShapeAt(randomIntBetween(0, gcb.numShapes() - 1))); GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", filterShape); filter.relation(ShapeRelation.INTERSECTS); - SearchResponse result = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()) + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) .setPostFilter(filter).get(); assertSearchResponse(result); assertHitCount(result, 1); } - @Test + public void testContainsShapeQuery() throws Exception { + // Create a random geometry collection. + Rectangle mbr = xRandomRectangle(getRandom(), xRandomPoint(getRandom())); + GeometryCollectionBuilder gcb = createGeometryCollectionWithin(getRandom(), mbr); + + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); + + XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefresh(true).execute().actionGet(); + + // index the mbr of the collection + EnvelopeBuilder env = new EnvelopeBuilder().topLeft(mbr.getMinX(), mbr.getMaxY()).bottomRight(mbr.getMaxX(), mbr.getMinY()); + docSource = env.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "2").setSource(docSource).setRefresh(true).execute().actionGet(); + + ShapeBuilder filterShape = (gcb.getShapeAt(randomIntBetween(0, gcb.numShapes() - 1))); + GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", filterShape) + .relation(ShapeRelation.INTERSECTS); + SearchResponse response = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) + .setPostFilter(filter).get(); + assertSearchResponse(response); + + assertThat(response.getHits().totalHits(), greaterThan(0L)); + } + public void testShapeFilterWithDefinedGeoCollection() throws Exception { createIndex("shapes"); - assertAcked(prepareCreate("test").addMapping("type", "location", "type=geo_shape")); + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); XContentBuilder docSource = jsonBuilder().startObject().startObject("location") .field("type", "geometrycollection") @@ -398,95 +365,39 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { .endObject() .endArray() .endObject().endObject(); - indexRandom(true, - client().prepareIndex("test", "type", "1") - .setSource(docSource)); - ensureSearchable("test"); + client().prepareIndex("test", "type", "1") + .setSource(docSource).setRefresh(true).execute().actionGet(); GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery( "location", - ShapeBuilder.newGeometryCollection() + ShapeBuilders.newGeometryCollection() .polygon( - ShapeBuilder.newPolygon().point(99.0, -1.0).point(99.0, 3.0).point(103.0, 3.0).point(103.0, -1.0) + ShapeBuilders.newPolygon().point(99.0, -1.0).point(99.0, 3.0).point(103.0, 3.0).point(103.0, -1.0) .point(99.0, -1.0))).relation(ShapeRelation.INTERSECTS); - SearchResponse result = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()) + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) .setPostFilter(filter).get(); assertSearchResponse(result); assertHitCount(result, 1); filter = QueryBuilders.geoShapeQuery( "location", - ShapeBuilder.newGeometryCollection().polygon( - ShapeBuilder.newPolygon().point(199.0, -11.0).point(199.0, 13.0).point(193.0, 13.0).point(193.0, -11.0) + ShapeBuilders.newGeometryCollection().polygon( + ShapeBuilders.newPolygon().point(199.0, -11.0).point(199.0, 13.0).point(193.0, 13.0).point(193.0, -11.0) .point(199.0, -11.0))).relation(ShapeRelation.INTERSECTS); - result = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()) + result = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) .setPostFilter(filter).get(); assertSearchResponse(result); assertHitCount(result, 0); - filter = QueryBuilders.geoShapeQuery("location", ShapeBuilder.newGeometryCollection() - .polygon(ShapeBuilder.newPolygon().point(99.0, -1.0).point(99.0, 3.0).point(103.0, 3.0).point(103.0, -1.0).point(99.0, -1.0)) + filter = QueryBuilders.geoShapeQuery("location", ShapeBuilders.newGeometryCollection() + .polygon(ShapeBuilders.newPolygon().point(99.0, -1.0).point(99.0, 3.0).point(103.0, 3.0).point(103.0, -1.0).point(99.0, -1.0)) .polygon( - ShapeBuilder.newPolygon().point(199.0, -11.0).point(199.0, 13.0).point(193.0, 13.0).point(193.0, -11.0) + ShapeBuilders.newPolygon().point(199.0, -11.0).point(199.0, 13.0).point(193.0, 13.0).point(193.0, -11.0) .point(199.0, -11.0))).relation(ShapeRelation.INTERSECTS); - result = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()) + result = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) .setPostFilter(filter).get(); assertSearchResponse(result); assertHitCount(result, 1); } - /** - * Test that orientation parameter correctly persists across cluster restart - */ - public void testOrientationPersistence() throws Exception { - String idxName = "orientation"; - String mapping = XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject().string(); - - // create index - assertAcked(prepareCreate(idxName).addMapping("shape", mapping)); - - mapping = XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject().string(); - - assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping)); - ensureGreen(idxName, idxName+"2"); - - internalCluster().fullRestart(); - ensureGreen(idxName, idxName+"2"); - - // left orientation test - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName)); - IndexService indexService = indicesService.indexService(idxName); - MappedFieldType fieldType = indexService.mapperService().smartNameFieldType("location"); - assertThat(fieldType, instanceOf(GeoShapeFieldMapper.GeoShapeFieldType.class)); - - GeoShapeFieldMapper.GeoShapeFieldType gsfm = (GeoShapeFieldMapper.GeoShapeFieldType)fieldType; - ShapeBuilder.Orientation orientation = gsfm.orientation(); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); - - // right orientation test - indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName+"2")); - indexService = indicesService.indexService(idxName+"2"); - fieldType = indexService.mapperService().smartNameFieldType("location"); - assertThat(fieldType, instanceOf(GeoShapeFieldMapper.GeoShapeFieldType.class)); - - gsfm = (GeoShapeFieldMapper.GeoShapeFieldType)fieldType; - orientation = gsfm.orientation(); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); - assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); - } - - @Test public void testPointsOnly() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -498,31 +409,25 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { .endObject().endObject() .endObject().endObject().string(); - assertAcked(prepareCreate("geo_points_only").addMapping("type1", mapping)); + client().admin().indices().prepareCreate("geo_points_only").addMapping("type1", mapping).execute().actionGet(); ensureGreen(); ShapeBuilder shape = RandomShapeGenerator.createShape(random()); try { - index("geo_points_only", "type1", "1", jsonBuilder().startObject().field("location", shape).endObject()); + client().prepareIndex("geo_points_only", "type1", "1") + .setSource(jsonBuilder().startObject().field("location", shape).endObject()) + .setRefresh(true).execute().actionGet(); } catch (MapperParsingException e) { // RandomShapeGenerator created something other than a POINT type, verify the correct exception is thrown assertThat(e.getCause().getMessage(), containsString("is configured for points only")); return; } - refresh(); // test that point was inserted - SearchResponse response = client().prepareSearch() + SearchResponse response = client().prepareSearch("geo_points_only").setTypes("type1") .setQuery(geoIntersectionQuery("location", shape)) .execute().actionGet(); assertEquals(1, response.getHits().getTotalHits()); } - - private String findNodeName(String index) { - ClusterState state = client().admin().cluster().prepareState().get().getState(); - IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0); - String nodeId = shard.assignedShards().get(0).currentNodeId(); - return state.getNodes().get(nodeId).name(); - } } diff --git a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java index 7c1f1632022..07045cc9176 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.Collection; @@ -56,16 +55,14 @@ public class CustomHighlighterSearchIT extends ESIntegTestCase { ensureYellow(); } - @Test public void testThatCustomHighlightersAreSupported() throws IOException { SearchResponse searchResponse = client().prepareSearch("test").setTypes("test") .setQuery(QueryBuilders.matchAllQuery()) - .addHighlightedField("name").setHighlighterType("test-custom") + .highlighter(new HighlightBuilder().field("name").highlighterType("test-custom")) .execute().actionGet(); assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1")); } - @Test public void testThatCustomHighlighterCanBeConfiguredPerField() throws Exception { HighlightBuilder.Field highlightConfig = new HighlightBuilder.Field("name"); highlightConfig.highlighterType("test-custom"); @@ -75,39 +72,32 @@ public class CustomHighlighterSearchIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch("test").setTypes("test") .setQuery(QueryBuilders.matchAllQuery()) - .addHighlightedField(highlightConfig) + .highlighter(new HighlightBuilder().field(highlightConfig)) .execute().actionGet(); assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1")); assertHighlight(searchResponse, 0, "name", 1, equalTo("field:myFieldOption:someValue")); } - @Test public void testThatCustomHighlighterCanBeConfiguredGlobally() throws Exception { Map options = new HashMap<>(); options.put("myGlobalOption", "someValue"); - SearchResponse searchResponse = client().prepareSearch("test").setTypes("test") - .setQuery(QueryBuilders.matchAllQuery()) - .setHighlighterOptions(options) - .setHighlighterType("test-custom") - .addHighlightedField("name") + SearchResponse searchResponse = client().prepareSearch("test").setTypes("test").setQuery(QueryBuilders.matchAllQuery()) + .highlighter(new HighlightBuilder().field("name").highlighterType("test-custom").options(options)) .execute().actionGet(); assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1")); assertHighlight(searchResponse, 0, "name", 1, equalTo("field:myGlobalOption:someValue")); } - @Test public void testThatCustomHighlighterReceivesFieldsInOrder() throws Exception { SearchResponse searchResponse = client().prepareSearch("test").setTypes("test") .setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).should(QueryBuilders .termQuery("name", "arbitrary"))) - .setHighlighterType("test-custom") - .addHighlightedField("name") - .addHighlightedField("other_name") - .addHighlightedField("other_other_name") - .setHighlighterExplicitFieldOrder(true) + .highlighter( + new HighlightBuilder().highlighterType("test-custom").field("name").field("other_name").field("other_other_name") + .useExplicitFieldOrder(true)) .get(); assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1")); diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java new file mode 100644 index 00000000000..2ac5895c9eb --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java @@ -0,0 +1,685 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.highlight; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperBuilders; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.index.query.IdsQueryBuilder; +import org.elasticsearch.index.query.IdsQueryParser; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryParser; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryParser; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermQueryParser; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.search.highlight.HighlightBuilder.Field; +import org.elasticsearch.search.highlight.HighlightBuilder.Order; +import org.elasticsearch.search.highlight.SearchContextHighlight.FieldOptions; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class HighlightBuilderTests extends ESTestCase { + + private static final int NUMBER_OF_TESTBUILDERS = 20; + private static NamedWriteableRegistry namedWriteableRegistry; + private static IndicesQueriesRegistry indicesQueriesRegistry; + + /** + * setup for the whole base test class + */ + @BeforeClass + public static void init() { + namedWriteableRegistry = new NamedWriteableRegistry(); + @SuppressWarnings("rawtypes") + Set injectedQueryParsers = new HashSet<>(); + injectedQueryParsers.add(new MatchAllQueryParser()); + injectedQueryParsers.add(new IdsQueryParser()); + injectedQueryParsers.add(new TermQueryParser()); + indicesQueriesRegistry = new IndicesQueriesRegistry(Settings.settingsBuilder().build(), injectedQueryParsers, namedWriteableRegistry); + } + + @AfterClass + public static void afterClass() throws Exception { + namedWriteableRegistry = null; + indicesQueriesRegistry = null; + } + + /** + * Test serialization and deserialization of the highlighter builder + */ + public void testSerialization() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + HighlightBuilder original = randomHighlighterBuilder(); + HighlightBuilder deserialized = serializedCopy(original); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + } + + /** + * Test equality and hashCode properties + */ + public void testEqualsAndHashcode() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + HighlightBuilder firstBuilder = randomHighlighterBuilder(); + assertFalse("highlighter is equal to null", firstBuilder.equals(null)); + assertFalse("highlighter is equal to incompatible type", firstBuilder.equals("")); + assertTrue("highlighter is not equal to self", firstBuilder.equals(firstBuilder)); + assertThat("same highlighter's hashcode returns different values if called multiple times", firstBuilder.hashCode(), + equalTo(firstBuilder.hashCode())); + assertThat("different highlighters should not be equal", mutate(firstBuilder), not(equalTo(firstBuilder))); + + HighlightBuilder secondBuilder = serializedCopy(firstBuilder); + assertTrue("highlighter is not equal to self", secondBuilder.equals(secondBuilder)); + assertTrue("highlighter is not equal to its copy", firstBuilder.equals(secondBuilder)); + assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder)); + assertThat("highlighter copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(firstBuilder.hashCode())); + + HighlightBuilder thirdBuilder = serializedCopy(secondBuilder); + assertTrue("highlighter is not equal to self", thirdBuilder.equals(thirdBuilder)); + assertTrue("highlighter is not equal to its copy", secondBuilder.equals(thirdBuilder)); + assertThat("highlighter copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode())); + assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder)); + assertThat("highlighter copy's hashcode is different from original hashcode", firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode())); + assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder)); + assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder)); + } + } + + /** + * creates random highlighter, renders it to xContent and back to new instance that should be equal to original + */ + public void testFromXContent() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + HighlightBuilder highlightBuilder = randomHighlighterBuilder(); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + builder.startObject(); + highlightBuilder.innerXContent(builder); + builder.endObject(); + + XContentParser parser = XContentHelper.createParser(builder.bytes()); + context.reset(parser); + parser.nextToken(); + HighlightBuilder secondHighlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context); + assertNotSame(highlightBuilder, secondHighlightBuilder); + assertEquals(highlightBuilder, secondHighlightBuilder); + assertEquals(highlightBuilder.hashCode(), secondHighlightBuilder.hashCode()); + } + } + + /** + * test that unknown array fields cause exception + */ + public void testUnknownArrayNameExpection() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + String highlightElement = "{\n" + + " \"bad_fieldname\" : [ \"field1\" 1 \"field2\" ]\n" + + "}\n"; + XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + try { + HighlightBuilder.PROTOTYPE.fromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("cannot parse array with name [bad_fieldname]", e.getMessage()); + } + + highlightElement = "{\n" + + " \"fields\" : {\n" + + " \"body\" : {\n" + + " \"bad_fieldname\" : [ \"field1\" , \"field2\" ]\n" + + " }\n" + + " }\n" + + "}\n"; + parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + try { + HighlightBuilder.PROTOTYPE.fromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("cannot parse array with name [bad_fieldname]", e.getMessage()); + } + } + + /** + * test that unknown field name cause exception + */ + public void testUnknownFieldnameExpection() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + String highlightElement = "{\n" + + " \"bad_fieldname\" : \"value\"\n" + + "}\n"; + XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + try { + HighlightBuilder.PROTOTYPE.fromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("unexpected fieldname [bad_fieldname]", e.getMessage()); + } + + highlightElement = "{\n" + + " \"fields\" : {\n" + + " \"body\" : {\n" + + " \"bad_fieldname\" : \"value\"\n" + + " }\n" + + " }\n" + + "}\n"; + parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + try { + HighlightBuilder.PROTOTYPE.fromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("unexpected fieldname [bad_fieldname]", e.getMessage()); + } + } + + /** + * test that unknown field name cause exception + */ + public void testUnknownObjectFieldnameExpection() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + String highlightElement = "{\n" + + " \"bad_fieldname\" : { \"field\" : \"value\" }\n \n" + + "}\n"; + XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + try { + HighlightBuilder.PROTOTYPE.fromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("cannot parse object with name [bad_fieldname]", e.getMessage()); + } + + highlightElement = "{\n" + + " \"fields\" : {\n" + + " \"body\" : {\n" + + " \"bad_fieldname\" : { \"field\" : \"value\" }\n" + + " }\n" + + " }\n" + + "}\n"; + parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + try { + HighlightBuilder.PROTOTYPE.fromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("cannot parse object with name [bad_fieldname]", e.getMessage()); + } + } + + /** + * test that build() outputs a {@link SearchContextHighlight} that is similar to the one + * we would get when parsing the xContent the test highlight builder is rendering out + */ + public void testBuildSearchContextHighlight() throws IOException { + Settings indexSettings = Settings.settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + Index index = new Index(randomAsciiOfLengthBetween(1, 10)); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); + // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter + QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null, indicesQueriesRegistry) { + @Override + public MappedFieldType fieldMapper(String name) { + StringFieldMapper.Builder builder = MapperBuilders.stringField(name); + return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType(); + } + }; + mockShardContext.setMapUnmappedFieldAsString(true); + + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + HighlightBuilder highlightBuilder = randomHighlighterBuilder(); + SearchContextHighlight highlight = highlightBuilder.build(mockShardContext); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + builder.startObject(); + highlightBuilder.innerXContent(builder); + builder.endObject(); + XContentParser parser = XContentHelper.createParser(builder.bytes()); + + SearchContextHighlight parsedHighlight = new HighlighterParseElement().parse(parser, mockShardContext); + assertNotSame(highlight, parsedHighlight); + assertEquals(highlight.globalForceSource(), parsedHighlight.globalForceSource()); + assertEquals(highlight.fields().size(), parsedHighlight.fields().size()); + + Iterator iterator = parsedHighlight.fields().iterator(); + for (org.elasticsearch.search.highlight.SearchContextHighlight.Field field : highlight.fields()) { + org.elasticsearch.search.highlight.SearchContextHighlight.Field otherField = iterator.next(); + assertEquals(field.field(), otherField.field()); + FieldOptions options = field.fieldOptions(); + FieldOptions otherOptions = otherField.fieldOptions(); + assertArrayEquals(options.boundaryChars(), options.boundaryChars()); + assertEquals(options.boundaryMaxScan(), otherOptions.boundaryMaxScan()); + assertEquals(options.encoder(), otherOptions.encoder()); + assertEquals(options.fragmentCharSize(), otherOptions.fragmentCharSize()); + assertEquals(options.fragmenter(), otherOptions.fragmenter()); + assertEquals(options.fragmentOffset(), otherOptions.fragmentOffset()); + assertEquals(options.highlighterType(), otherOptions.highlighterType()); + assertEquals(options.highlightFilter(), otherOptions.highlightFilter()); + assertEquals(options.highlightQuery(), otherOptions.highlightQuery()); + assertEquals(options.matchedFields(), otherOptions.matchedFields()); + assertEquals(options.noMatchSize(), otherOptions.noMatchSize()); + assertEquals(options.numberOfFragments(), otherOptions.numberOfFragments()); + assertEquals(options.options(), otherOptions.options()); + assertEquals(options.phraseLimit(), otherOptions.phraseLimit()); + assertArrayEquals(options.preTags(), otherOptions.preTags()); + assertArrayEquals(options.postTags(), otherOptions.postTags()); + assertEquals(options.requireFieldMatch(), otherOptions.requireFieldMatch()); + assertEquals(options.scoreOrdered(), otherOptions.scoreOrdered()); + } + } + } + + /** + * `tags_schema` is not produced by toXContent in the builder but should be parseable, so this + * adds a simple json test for this. + */ + public void testParsingTagsSchema() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + String highlightElement = "{\n" + + " \"tags_schema\" : \"styled\"\n" + + "}\n"; + XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + HighlightBuilder highlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context); + assertArrayEquals("setting tags_schema 'styled' should alter pre_tags", HighlightBuilder.DEFAULT_STYLED_PRE_TAG, + highlightBuilder.preTags()); + assertArrayEquals("setting tags_schema 'styled' should alter post_tags", HighlightBuilder.DEFAULT_STYLED_POST_TAGS, + highlightBuilder.postTags()); + + highlightElement = "{\n" + + " \"tags_schema\" : \"default\"\n" + + "}\n"; + parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + highlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context); + assertArrayEquals("setting tags_schema 'default' should alter pre_tags", HighlightBuilder.DEFAULT_PRE_TAGS, + highlightBuilder.preTags()); + assertArrayEquals("setting tags_schema 'default' should alter post_tags", HighlightBuilder.DEFAULT_POST_TAGS, + highlightBuilder.postTags()); + + highlightElement = "{\n" + + " \"tags_schema\" : \"somthing_else\"\n" + + "}\n"; + parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + try { + HighlightBuilder.PROTOTYPE.fromXContent(context); + fail("setting unknown tag schema should throw exception"); + } catch (IllegalArgumentException e) { + assertEquals("Unknown tag schema [somthing_else]", e.getMessage()); + } + } + + /** + * test parsing empty highlight or empty fields blocks + */ + public void testParsingEmptyStructure() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + String highlightElement = "{ }"; + XContentParser parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + HighlightBuilder highlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context); + assertEquals("expected plain HighlightBuilder", new HighlightBuilder(), highlightBuilder); + + highlightElement = "{ \"fields\" : { } }"; + parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + highlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context); + assertEquals("defining no field should return plain HighlightBuilder", new HighlightBuilder(), highlightBuilder); + + highlightElement = "{ \"fields\" : { \"foo\" : { } } }"; + parser = XContentFactory.xContent(highlightElement).createParser(highlightElement); + + context.reset(parser); + highlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context); + assertEquals("expected HighlightBuilder with field", new HighlightBuilder().field(new Field("foo")), highlightBuilder); + System.out.println(Math.log(1/(double)(1+1)) + 1.0); + } + + /** + * test ordinals of {@link Order}, since serialization depends on it + */ + public void testValidOrderOrdinals() { + assertThat(Order.NONE.ordinal(), equalTo(0)); + assertThat(Order.SCORE.ordinal(), equalTo(1)); + } + + public void testOrderSerialization() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Order.NONE.writeTo(out); + try (StreamInput in = StreamInput.wrap(out.bytes())) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Order.SCORE.writeTo(out); + try (StreamInput in = StreamInput.wrap(out.bytes())) { + assertThat(in.readVInt(), equalTo(1)); + } + } + } + + protected static XContentBuilder toXContent(HighlightBuilder highlight, XContentType contentType) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (randomBoolean()) { + builder.prettyPrint(); + } + highlight.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + /** + * create random highlight builder that is put under test + */ + public static HighlightBuilder randomHighlighterBuilder() { + HighlightBuilder testHighlighter = new HighlightBuilder(); + setRandomCommonOptions(testHighlighter); + testHighlighter.useExplicitFieldOrder(randomBoolean()); + if (randomBoolean()) { + testHighlighter.encoder(randomFrom(Arrays.asList(new String[]{"default", "html"}))); + } + int numberOfFields = randomIntBetween(1,5); + for (int i = 0; i < numberOfFields; i++) { + Field field = new Field(randomAsciiOfLengthBetween(1, 10)); + setRandomCommonOptions(field); + if (randomBoolean()) { + field.fragmentOffset(randomIntBetween(1, 100)); + } + if (randomBoolean()) { + field.matchedFields(randomStringArray(0, 4)); + } + testHighlighter.field(field); + } + return testHighlighter; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightBuilder) { + if (randomBoolean()) { + // need to set this together, otherwise parsing will complain + highlightBuilder.preTags(randomStringArray(0, 3)); + highlightBuilder.postTags(randomStringArray(0, 3)); + } + if (randomBoolean()) { + highlightBuilder.fragmentSize(randomIntBetween(0, 100)); + } + if (randomBoolean()) { + highlightBuilder.numOfFragments(randomIntBetween(0, 10)); + } + if (randomBoolean()) { + highlightBuilder.highlighterType(randomAsciiOfLengthBetween(1, 10)); + } + if (randomBoolean()) { + highlightBuilder.fragmenter(randomAsciiOfLengthBetween(1, 10)); + } + if (randomBoolean()) { + QueryBuilder highlightQuery; + switch (randomInt(2)) { + case 0: + highlightQuery = new MatchAllQueryBuilder(); + break; + case 1: + highlightQuery = new IdsQueryBuilder(); + break; + default: + case 2: + highlightQuery = new TermQueryBuilder(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); + break; + } + highlightQuery.boost((float) randomDoubleBetween(0, 10, false)); + highlightBuilder.highlightQuery(highlightQuery); + } + if (randomBoolean()) { + if (randomBoolean()) { + highlightBuilder.order(randomFrom(Order.values())); + } else { + // also test the string setter + highlightBuilder.order(randomFrom(Order.values()).toString()); + } + } + if (randomBoolean()) { + highlightBuilder.highlightFilter(randomBoolean()); + } + if (randomBoolean()) { + highlightBuilder.forceSource(randomBoolean()); + } + if (randomBoolean()) { + highlightBuilder.boundaryMaxScan(randomIntBetween(0, 10)); + } + if (randomBoolean()) { + highlightBuilder.boundaryChars(randomAsciiOfLengthBetween(1, 10).toCharArray()); + } + if (randomBoolean()) { + highlightBuilder.noMatchSize(randomIntBetween(0, 10)); + } + if (randomBoolean()) { + highlightBuilder.phraseLimit(randomIntBetween(0, 10)); + } + if (randomBoolean()) { + int items = randomIntBetween(0, 5); + Map options = new HashMap(items); + for (int i = 0; i < items; i++) { + Object value = null; + switch (randomInt(2)) { + case 0: + value = randomAsciiOfLengthBetween(1, 10); + break; + case 1: + value = new Integer(randomInt(1000)); + break; + case 2: + value = new Boolean(randomBoolean()); + break; + } + options.put(randomAsciiOfLengthBetween(1, 10), value); + } + } + if (randomBoolean()) { + highlightBuilder.requireFieldMatch(randomBoolean()); + } + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static void mutateCommonOptions(AbstractHighlighterBuilder highlightBuilder) { + switch (randomIntBetween(1, 16)) { + case 1: + highlightBuilder.preTags(randomStringArray(4, 6)); + break; + case 2: + highlightBuilder.postTags(randomStringArray(4, 6)); + break; + case 3: + highlightBuilder.fragmentSize(randomIntBetween(101, 200)); + break; + case 4: + highlightBuilder.numOfFragments(randomIntBetween(11, 20)); + break; + case 5: + highlightBuilder.highlighterType(randomAsciiOfLengthBetween(11, 20)); + break; + case 6: + highlightBuilder.fragmenter(randomAsciiOfLengthBetween(11, 20)); + break; + case 7: + highlightBuilder.highlightQuery(new TermQueryBuilder(randomAsciiOfLengthBetween(11, 20), randomAsciiOfLengthBetween(11, 20))); + break; + case 8: + if (highlightBuilder.order() == Order.NONE) { + highlightBuilder.order(Order.SCORE); + } else { + highlightBuilder.order(Order.NONE); + } + break; + case 9: + highlightBuilder.highlightFilter(toggleOrSet(highlightBuilder.highlightFilter())); + break; + case 10: + highlightBuilder.forceSource(toggleOrSet(highlightBuilder.forceSource())); + break; + case 11: + highlightBuilder.boundaryMaxScan(randomIntBetween(11, 20)); + break; + case 12: + highlightBuilder.boundaryChars(randomAsciiOfLengthBetween(11, 20).toCharArray()); + break; + case 13: + highlightBuilder.noMatchSize(randomIntBetween(11, 20)); + break; + case 14: + highlightBuilder.phraseLimit(randomIntBetween(11, 20)); + break; + case 15: + int items = 6; + Map options = new HashMap(items); + for (int i = 0; i < items; i++) { + options.put(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); + } + highlightBuilder.options(options); + break; + case 16: + highlightBuilder.requireFieldMatch(toggleOrSet(highlightBuilder.requireFieldMatch())); + break; + } + } + + private static Boolean toggleOrSet(Boolean flag) { + if (flag == null) { + return randomBoolean(); + } else { + return !flag.booleanValue(); + } + } + + private static String[] randomStringArray(int minSize, int maxSize) { + int size = randomIntBetween(minSize, maxSize); + String[] randomStrings = new String[size]; + for (int f = 0; f < size; f++) { + randomStrings[f] = randomAsciiOfLengthBetween(1, 10); + } + return randomStrings; + } + + /** + * mutate the given highlighter builder so the returned one is different in one aspect + */ + private static HighlightBuilder mutate(HighlightBuilder original) throws IOException { + HighlightBuilder mutation = serializedCopy(original); + if (randomBoolean()) { + mutateCommonOptions(mutation); + } else { + switch (randomIntBetween(0, 2)) { + // change settings that only exists on top level + case 0: + mutation.useExplicitFieldOrder(!original.useExplicitFieldOrder()); break; + case 1: + mutation.encoder(original.encoder() + randomAsciiOfLength(2)); break; + case 2: + if (randomBoolean()) { + // add another field + mutation.field(new Field(randomAsciiOfLength(10))); + } else { + // change existing fields + List originalFields = original.fields(); + Field fieldToChange = originalFields.get(randomInt(originalFields.size() - 1)); + if (randomBoolean()) { + fieldToChange.fragmentOffset(randomIntBetween(101, 200)); + } else { + fieldToChange.matchedFields(randomStringArray(5, 10)); + } + } + break; + } + } + return mutation; + } + + private static HighlightBuilder serializedCopy(HighlightBuilder original) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + original.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + return HighlightBuilder.PROTOTYPE.readFrom(in); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java index 4134c4f2941..63378baa721 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java @@ -19,20 +19,22 @@ package org.elasticsearch.search.highlight; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.search.MatchQuery.Type; -import org.elasticsearch.index.search.MatchQuery; import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.search.MatchQuery; +import org.elasticsearch.index.search.MatchQuery.Type; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -40,7 +42,6 @@ import org.elasticsearch.search.highlight.HighlightBuilder.Field; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.HashMap; @@ -49,16 +50,41 @@ import java.util.Map; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.boostingQuery; +import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.fuzzyQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchPhrasePrefixQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; +import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.regexpQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.typeQuery; +import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; import static org.elasticsearch.search.builder.SearchSourceBuilder.highlight; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHighlight; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNotHighlighted; import static org.elasticsearch.test.hamcrest.RegexMatcher.matches; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; public class HighlighterSearchIT extends ESIntegTestCase { - - @Test public void testHighlightingWithWildcardName() throws IOException { // test the kibana case with * as fieldname that will try highlight all fields including meta fields XContentBuilder mappings = jsonBuilder(); @@ -81,12 +107,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { .setSource(jsonBuilder().startObject().field("text", "text").endObject()) .get(); refresh(); - String highlighter = randomFrom(new String[]{"plain", "postings", "fvh"}); - SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))).addHighlightedField(new Field("*").highlighterType(highlighter)).get(); + String highlighter = randomFrom("plain", "postings", "fvh"); + SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))) + .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(highlighter))).get(); assertHighlight(search, 0, "text", 0, equalTo("text")); } - @Test public void testPlainHighlighterWithLongUnanalyzedStringTerm() throws IOException { XContentBuilder mappings = jsonBuilder(); mappings.startObject(); @@ -121,19 +147,21 @@ public class HighlighterSearchIT extends ESIntegTestCase { .setSource(jsonBuilder().startObject().field("long_text", builder.toString()).field("text", "text").endObject()) .get(); refresh(); - String highlighter = randomFrom(new String[]{"plain", "postings", "fvh"}); - SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))).addHighlightedField(new Field("*").highlighterType(highlighter)).get(); + String highlighter = randomFrom("plain", "postings", "fvh"); + SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))) + .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(highlighter))).get(); assertHighlight(search, 0, "text", 0, equalTo("text")); - search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))).addHighlightedField(new Field("long_text").highlighterType(highlighter)).get(); + search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))) + .highlighter(new HighlightBuilder().field(new Field("long_text").highlighterType(highlighter))).get(); assertNoFailures(search); assertThat(search.getHits().getAt(0).getHighlightFields().size(), equalTo(0)); - search = client().prepareSearch().setQuery(prefixQuery("text", "te")).addHighlightedField(new Field("long_text").highlighterType(highlighter)).get(); + search = client().prepareSearch().setQuery(prefixQuery("text", "te")) + .highlighter(new HighlightBuilder().field(new Field("long_text").highlighterType(highlighter))).get(); assertNoFailures(search); assertThat(search.getHits().getAt(0).getHighlightFields().size(), equalTo(0)); } - @Test public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOException { XContentBuilder mappings = jsonBuilder(); mappings.startObject(); @@ -164,16 +192,16 @@ public class HighlighterSearchIT extends ESIntegTestCase { .setSource(jsonBuilder().startObject().field("unstored_text", "text").field("text", "text").endObject()) .get(); refresh(); - String highlighter = randomFrom(new String[]{"plain", "postings", "fvh"}); - SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))).addHighlightedField(new Field("*").highlighterType(highlighter)).get(); + String highlighter = randomFrom("plain", "postings", "fvh"); + SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))) + .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(highlighter))).get(); assertHighlight(search, 0, "text", 0, equalTo("text")); - search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))).addHighlightedField(new Field("unstored_text")).get(); + search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text"))) + .highlighter(new HighlightBuilder().field(new Field("unstored_text"))).get(); assertNoFailures(search); assertThat(search.getHits().getAt(0).getHighlightFields().size(), equalTo(0)); } - - @Test // see #3486 public void testHighTermFrequencyDoc() throws IOException { assertAcked(prepareCreate("test") @@ -187,11 +215,11 @@ public class HighlighterSearchIT extends ESIntegTestCase { .setSource("name", builder.toString()) .get(); refresh(); - SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "abc"))).addHighlightedField("name").get(); + SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "abc"))) + .highlighter(new HighlightBuilder().field("name")).get(); assertHighlight(search, 0, "name", 0, startsWith("abc abc abc abc")); } - @Test public void testNgramHighlightingWithBrokenPositions() throws IOException { assertAcked(prepareCreate("test") .addMapping("test", jsonBuilder() @@ -243,11 +271,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { client().prepareIndex("test", "test", "1") .setSource("name", "ARCOTEL Hotels Deutschland").get(); refresh(); - SearchResponse search = client().prepareSearch("test").setTypes("test").setQuery(matchQuery("name.autocomplete", "deut tel").operator(Operator.OR)).addHighlightedField("name.autocomplete").execute().actionGet(); + SearchResponse search = client().prepareSearch("test").setTypes("test") + .setQuery(matchQuery("name.autocomplete", "deut tel").operator(Operator.OR)) + .highlighter(new HighlightBuilder().field("name.autocomplete")).execute().actionGet(); assertHighlight(search, 0, "name.autocomplete", 0, equalTo("ARCOTEL Hotels Deutschland")); } - @Test public void testMultiPhraseCutoff() throws IOException { /* * MultiPhraseQuery can literally kill an entire node if there are too many terms in the @@ -273,15 +302,25 @@ public class HighlighterSearchIT extends ESIntegTestCase { .setSource("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature") .get(); refresh(); - SearchResponse search = client().prepareSearch().setQuery(matchQuery("body", "Test: http://www.facebook.com ").type(Type.PHRASE)).addHighlightedField("body").execute().actionGet(); + SearchResponse search = client().prepareSearch().setQuery(matchQuery("body", "Test: http://www.facebook.com ").type(Type.PHRASE)) + .highlighter(new HighlightBuilder().field("body")).execute().actionGet(); assertHighlight(search, 0, "body", 0, startsWith("Test: http://www.facebook.com")); - search = client().prepareSearch().setQuery(matchQuery("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature").type(Type.PHRASE)).addHighlightedField("body").execute().actionGet(); - assertHighlight(search, 0, "body", 0, equalTo("Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com")); + search = client() + .prepareSearch() + .setQuery( + matchQuery( + "body", + "Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature") + .type(Type.PHRASE)).highlighter(new HighlightBuilder().field("body")).execute().actionGet(); + assertHighlight( + search, + 0, + "body", + 0, + equalTo("Test: http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com")); } - @Test public void testNgramHighlightingPreLucene42() throws IOException { - assertAcked(prepareCreate("test") .addMapping("test", "name", "type=string,analyzer=name_index_analyzer,search_analyzer=name_search_analyzer," + randomStoreField() + "term_vector=with_positions_offsets", @@ -310,44 +349,49 @@ public class HighlighterSearchIT extends ESIntegTestCase { "name2", "avinci, unilog avinci, logicacmg, logica").get(); refresh(); - SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "logica m"))).addHighlightedField("name").get(); + SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "logica m"))) + .highlighter(new HighlightBuilder().field("name")).get(); assertHighlight(search, 0, "name", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); assertHighlight(search, 1, "name", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); - search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "logica ma"))).addHighlightedField("name").get(); + search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "logica ma"))) + .highlighter(new HighlightBuilder().field("name")).get(); assertHighlight(search, 0, "name", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); assertHighlight(search, 1, "name", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); - search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "logica"))).addHighlightedField("name").get(); + search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name", "logica"))) + .highlighter(new HighlightBuilder().field("name")).get(); assertHighlight(search, 0, "name", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); assertHighlight(search, 0, "name", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); - search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name2", "logica m"))).addHighlightedField("name2").get(); + search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name2", "logica m"))) + .highlighter(new HighlightBuilder().field("name2")).get(); assertHighlight(search, 0, "name2", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); assertHighlight(search, 1, "name2", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); - search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name2", "logica ma"))).addHighlightedField("name2").get(); + search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name2", "logica ma"))) + .highlighter(new HighlightBuilder().field("name2")).get(); assertHighlight(search, 0, "name2", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); assertHighlight(search, 1, "name2", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); - search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name2", "logica"))).addHighlightedField("name2").get(); + search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("name2", "logica"))) + .highlighter(new HighlightBuilder().field("name2")).get(); assertHighlight(search, 0, "name2", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); assertHighlight(search, 1, "name2", 0, anyOf(equalTo("logicacmg ehemals avinci - the know how company"), equalTo("avinci, unilog avinci, logicacmg, logica"))); } - @Test public void testNgramHighlighting() throws IOException { assertAcked(prepareCreate("test") .addMapping("test", @@ -371,26 +415,28 @@ public class HighlighterSearchIT extends ESIntegTestCase { "name2", "logicacmg ehemals avinci - the know how company").get(); refresh(); ensureGreen(); - SearchResponse search = client().prepareSearch().setQuery(matchQuery("name", "logica m")).addHighlightedField("name").get(); + SearchResponse search = client().prepareSearch().setQuery(matchQuery("name", "logica m")) + .highlighter(new HighlightBuilder().field("name")).get(); assertHighlight(search, 0, "name", 0, equalTo("logicacmg ehemals avinci - the know how company")); - search = client().prepareSearch().setQuery(matchQuery("name", "logica ma")).addHighlightedField("name").get(); + search = client().prepareSearch().setQuery(matchQuery("name", "logica ma")).highlighter(new HighlightBuilder().field("name")).get(); assertHighlight(search, 0, "name", 0, equalTo("logicacmg ehemals avinci - the know how company")); - search = client().prepareSearch().setQuery(matchQuery("name", "logica")).addHighlightedField("name").get(); + search = client().prepareSearch().setQuery(matchQuery("name", "logica")).highlighter(new HighlightBuilder().field("name")).get(); assertHighlight(search, 0, "name", 0, equalTo("logicacmg ehemals avinci - the know how company")); - search = client().prepareSearch().setQuery(matchQuery("name2", "logica m")).addHighlightedField("name2").get(); + search = client().prepareSearch().setQuery(matchQuery("name2", "logica m")).highlighter(new HighlightBuilder().field("name2")) + .get(); assertHighlight(search, 0, "name2", 0, equalTo("logicacmg ehemals avinci - the know how company")); - search = client().prepareSearch().setQuery(matchQuery("name2", "logica ma")).addHighlightedField("name2").get(); + search = client().prepareSearch().setQuery(matchQuery("name2", "logica ma")).highlighter(new HighlightBuilder().field("name2")) + .get(); assertHighlight(search, 0, "name2", 0, equalTo("logicacmg ehemals avinci - the know how company")); - search = client().prepareSearch().setQuery(matchQuery("name2", "logica")).addHighlightedField("name2").get(); + search = client().prepareSearch().setQuery(matchQuery("name2", "logica")).highlighter(new HighlightBuilder().field("name2")).get(); assertHighlight(search, 0, "name2", 0, equalTo("logicacmg ehemals avinci - the know how company")); } - @Test public void testEnsureNoNegativeOffsets() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", @@ -406,25 +452,24 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("long_term", "thisisaverylongwordandmakessurethisfails foo highlighed")) - .addHighlightedField("long_term", 18, 1) + .highlighter(new HighlightBuilder().field("long_term", 18, 1)) .get(); assertHighlight(search, 0, "long_term", 0, 1, equalTo("thisisaverylongwordandmakessurethisfails")); search = client().prepareSearch() .setQuery(matchQuery("no_long_term", "test foo highlighed").type(Type.PHRASE).slop(3)) - .addHighlightedField("no_long_term", 18, 1).setHighlighterPostTags("").setHighlighterPreTags("") + .highlighter(new HighlightBuilder().field("no_long_term", 18, 1).postTags("").preTags("")) .get(); assertNotHighlighted(search, 0, "no_long_term"); search = client().prepareSearch() .setQuery(matchQuery("no_long_term", "test foo highlighed").type(Type.PHRASE).slop(3)) - .addHighlightedField("no_long_term", 30, 1).setHighlighterPostTags("").setHighlighterPreTags("") + .highlighter(new HighlightBuilder().field("no_long_term", 30, 1).postTags("").preTags("")) .get(); assertHighlight(search, 0, "no_long_term", 0, 1, equalTo("a test where foo is highlighed and")); } - @Test public void testSourceLookupHighlightingUsingPlainHighlighter() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -446,7 +491,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "bug")) - .addHighlightedField("title", -1, 0) + .highlighter(new HighlightBuilder().field("title", -1, 0)) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -455,7 +500,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { search = client().prepareSearch() .setQuery(matchQuery("attachments.body", "attachment")) - .addHighlightedField("attachments.body", -1, 0) + .highlighter(new HighlightBuilder().field("attachments.body", -1, 0)) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -464,7 +509,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testSourceLookupHighlightingUsingFastVectorHighlighter() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -486,7 +530,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "bug")) - .addHighlightedField("title", -1, 0) + .highlighter(new HighlightBuilder().field("title", -1, 0)) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -495,7 +539,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { search = client().prepareSearch() .setQuery(matchQuery("attachments.body", "attachment")) - .addHighlightedField("attachments.body", -1, 2) + .highlighter(new HighlightBuilder().field("attachments.body", -1, 2)) .execute().get(); for (int i = 0; i < 5; i++) { @@ -504,7 +548,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testSourceLookupHighlightingUsingPostingsHighlighter() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -528,7 +571,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "bug")) //asking for the whole field to be highlighted - .addHighlightedField("title", -1, 0).get(); + .highlighter(new HighlightBuilder().field("title", -1, 0)).get(); for (int i = 0; i < indexRequestBuilders.length; i++) { assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting bug present in elasticsearch. Hopefully it works.")); @@ -538,7 +581,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { search = client().prepareSearch() .setQuery(matchQuery("title", "bug")) //sentences will be generated out of each value - .addHighlightedField("title").get(); + .highlighter(new HighlightBuilder().field("title")).get(); for (int i = 0; i < indexRequestBuilders.length; i++) { assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting bug present in elasticsearch.")); @@ -547,7 +590,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { search = client().prepareSearch() .setQuery(matchQuery("attachments.body", "attachment")) - .addHighlightedField("attachments.body", -1, 2) + .highlighter(new HighlightBuilder().field("attachments.body", -1, 2)) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -556,7 +599,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testHighlightIssue1994() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string,store=no", "titleTV", "type=string,store=no,term_vector=with_positions_offsets")); @@ -571,8 +613,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "bug")) - .addHighlightedField("title", -1, 2) - .addHighlightedField("titleTV", -1, 2).setHighlighterRequireFieldMatch(false) + .highlighter(new HighlightBuilder().field("title", -1, 2).field("titleTV", -1, 2).requireFieldMatch(false)) .get(); assertHighlight(search, 0, "title", 0, equalTo("This is a test on the highlighting bug present in elasticsearch")); @@ -582,14 +623,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { search = client().prepareSearch() .setQuery(matchQuery("titleTV", "highlight")) - .addHighlightedField("titleTV", -1, 2) + .highlighter(new HighlightBuilder().field("titleTV", -1, 2)) .get(); assertHighlight(search, 0, "titleTV", 0, equalTo("some text to highlight")); assertHighlight(search, 0, "titleTV", 1, 2, equalTo("highlight other text")); } - @Test public void testGlobalHighlightingSettingsOverriddenAtFieldLevel() { createIndex("test"); ensureGreen(); @@ -602,7 +642,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1 and field2 produces different tags"); SearchSourceBuilder source = searchSource() .query(termQuery("field1", "test")) - .highlight(highlight().order("score").preTags("").postTags("").fragmentSize(1).numOfFragments(1) + .highlighter(highlight().order("score").preTags("").postTags("").fragmentSize(1).numOfFragments(1) .field(new HighlightBuilder.Field("field1").numOfFragments(2)) .field(new HighlightBuilder.Field("field2").preTags("").postTags("").fragmentSize(50).requireFieldMatch(false))); @@ -613,7 +653,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("this is another test")); } - @Test //https://github.com/elasticsearch/elasticsearch/issues/5175 + // Issue #5175 public void testHighlightingOnWildcardFields() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", @@ -632,7 +672,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchSourceBuilder source = searchSource() //postings hl doesn't support require_field_match, its field needs to be queried directly .query(termQuery("field-postings", "test")) - .highlight(highlight().field("field*").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field*").preTags("").postTags("").requireFieldMatch(false)); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -641,9 +681,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { assertHighlight(searchResponse, 0, "field-plain", 0, 1, equalTo("This is the test for the plain highlighter")); } - @Test public void testForceSourceWithSourceDisabled() throws Exception { - assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1") .startObject("_source").field("enabled", false).endObject() @@ -661,42 +699,47 @@ public class HighlighterSearchIT extends ESIntegTestCase { //works using stored field SearchResponse searchResponse = client().prepareSearch("test") .setQuery(termQuery("field1", "quick")) - .addHighlightedField(new Field("field1").preTags("").postTags("")) + .highlighter(new HighlightBuilder().field(new Field("field1").preTags("").postTags(""))) .get(); assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The quick brown fox jumps over the lazy dog")); assertFailures(client().prepareSearch("test") .setQuery(termQuery("field1", "quick")) - .addHighlightedField(new Field("field1").preTags("").postTags("").highlighterType("plain").forceSource(true)), + .highlighter( + new HighlightBuilder().field(new Field("field1").preTags("").postTags("") + .highlighterType("plain").forceSource(true))), RestStatus.BAD_REQUEST, containsString("source is forced for fields [field1] but type [type1] has disabled _source")); assertFailures(client().prepareSearch("test") .setQuery(termQuery("field1", "quick")) - .addHighlightedField(new Field("field1").preTags("").postTags("").highlighterType("fvh").forceSource(true)), + .highlighter( + new HighlightBuilder().field(new Field("field1").preTags("").postTags("").highlighterType("fvh") + .forceSource(true))), RestStatus.BAD_REQUEST, containsString("source is forced for fields [field1] but type [type1] has disabled _source")); assertFailures(client().prepareSearch("test") .setQuery(termQuery("field1", "quick")) - .addHighlightedField(new Field("field1").preTags("").postTags("").highlighterType("postings").forceSource(true)), + .highlighter( + new HighlightBuilder().field(new Field("field1").preTags("").postTags("") + .highlighterType("postings").forceSource(true))), RestStatus.BAD_REQUEST, containsString("source is forced for fields [field1] but type [type1] has disabled _source")); SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick")) - .highlight(highlight().forceSource(true).field("field1")); + .highlighter(highlight().forceSource(true).field("field1")); assertFailures(client().prepareSearch("test").setSource(searchSource), RestStatus.BAD_REQUEST, containsString("source is forced for fields [field1] but type [type1] has disabled _source")); searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick")) - .highlight(highlight().forceSource(true).field("field*")); + .highlighter(highlight().forceSource(true).field("field*")); assertFailures(client().prepareSearch("test").setSource(searchSource), RestStatus.BAD_REQUEST, matches("source is forced for fields \\[field\\d, field\\d\\] but type \\[type1\\] has disabled _source")); } - @Test public void testPlainHighlighter() throws Exception { createIndex("test"); ensureGreen(); @@ -708,7 +751,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(termQuery("field1", "test")) - .highlight(highlight().field("field1").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field1").order("score").preTags("").postTags("")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -717,7 +760,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all, highlighting on field1"); source = searchSource() .query(termQuery("_all", "test")) - .highlight(highlight().field("field1").order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field1").order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -726,7 +769,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all, highlighting on field2"); source = searchSource() .query(termQuery("_all", "quick")) - .highlight(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -735,7 +778,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all, highlighting on field2"); source = searchSource() .query(prefixQuery("_all", "qui")) - .highlight(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -744,7 +787,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all with constant score, highlighting on field2"); source = searchSource() .query(constantScoreQuery(prefixQuery("_all", "qui"))) - .highlight(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -753,13 +796,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all with constant score, highlighting on field2"); source = searchSource() .query(boolQuery().should(constantScoreQuery(prefixQuery("_all", "qui")))) - .highlight(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog")); } - @Test public void testFastVectorHighlighter() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping())); ensureGreen(); @@ -771,7 +813,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(termQuery("field1", "test")) - .highlight(highlight().field("field1", 100, 0).order("score").preTags("").postTags("")); + .highlighter(highlight().field("field1", 100, 0).order("score").preTags("").postTags("")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -780,7 +822,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all, highlighting on field1"); source = searchSource() .query(termQuery("_all", "test")) - .highlight(highlight().field("field1", 100, 0).order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field1", 100, 0).order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -790,7 +832,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all, highlighting on field2"); source = searchSource() .query(termQuery("_all", "quick")) - .highlight(highlight().field("field2", 100, 0).order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field2", 100, 0).order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -800,7 +842,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on _all, highlighting on field2"); source = searchSource() .query(prefixQuery("_all", "qui")) - .highlight(highlight().field("field2", 100, 0).order("score").preTags("").postTags("").requireFieldMatch(false)); + .highlighter(highlight().field("field2", 100, 0).order("score").preTags("").postTags("").requireFieldMatch(false)); searchResponse = client().prepareSearch("test").setSource(source).get(); @@ -809,9 +851,9 @@ public class HighlighterSearchIT extends ESIntegTestCase { } /** - * The FHV can spend a long time highlighting degenerate documents if phraseLimit is not set. + * The FHV can spend a long time highlighting degenerate documents if + * phraseLimit is not set. Its default is now reasonably low. */ - @Test(timeout=120000) public void testFVHManyMatches() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping())); ensureGreen(); @@ -823,22 +865,33 @@ public class HighlighterSearchIT extends ESIntegTestCase { .setSource("field1", value).get(); refresh(); - logger.info("--> highlighting and searching on field1"); + logger.info("--> highlighting and searching on field1 with default phrase limit"); SearchSourceBuilder source = searchSource() .query(termQuery("field1", "t")) - .highlight(highlight().highlighterType("fvh").field("field1", 20, 1).order("score").preTags("").postTags("")); - SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); - assertHighlight(searchResponse, 0, "field1", 0, 1, containsString("t")); - logger.info("--> done"); + .highlighter(highlight().highlighterType("fvh").field("field1", 20, 1).order("score").preTags("").postTags("")); + SearchResponse defaultPhraseLimit = client().search(searchRequest("test").source(source)).actionGet(); + assertHighlight(defaultPhraseLimit, 0, "field1", 0, 1, containsString("t")); + + logger.info("--> highlighting and searching on field1 with large phrase limit"); + source = searchSource() + .query(termQuery("field1", "t")) + .highlighter(highlight().highlighterType("fvh").field("field1", 20, 1).order("score").preTags("").postTags("").phraseLimit(30000)); + SearchResponse largePhraseLimit = client().search(searchRequest("test").source(source)).actionGet(); + assertHighlight(largePhraseLimit, 0, "field1", 0, 1, containsString("t")); + + /* + * I hate comparing times because it can be inconsistent but default is + * in the neighborhood of 300ms and the large phrase limit is in the + * neighborhood of 8 seconds. + */ + assertThat(defaultPhraseLimit.getTookInMillis(), lessThan(largePhraseLimit.getTookInMillis())); } - @Test public void testMatchedFieldsFvhRequireFieldMatch() throws Exception { checkMatchedFieldsCase(true); } - @Test public void testMatchedFieldsFvhNoRequireFieldMatch() throws Exception { checkMatchedFieldsCase(false); } @@ -894,9 +947,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { Field fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25) .highlighterType("fvh").requireFieldMatch(requireFieldMatch); - Field barField = new Field("bar").numOfFragments(1).order("score").fragmentSize(25) - .highlighterType("fvh").requireFieldMatch(requireFieldMatch); - SearchRequestBuilder req = client().prepareSearch("test").addHighlightedField(fooField); + SearchRequestBuilder req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField)); // First check highlighting without any matched fields set SearchResponse resp = req.setQuery(queryStringQuery("running scissors").field("foo")).get(); @@ -908,21 +959,31 @@ public class HighlighterSearchIT extends ESIntegTestCase { // Add the subfield to the list of matched fields but don't match it. Everything should still work // like before we added it. + fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh") + .requireFieldMatch(requireFieldMatch); fooField.matchedFields("foo", "foo.plain"); + req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField)); resp = req.setQuery(queryStringQuery("running scissors").field("foo")).get(); assertHighlight(resp, 0, "foo", 0, equalTo("running with scissors")); + // Now make half the matches come from the stored field and half from just a matched field. resp = req.setQuery(queryStringQuery("foo.plain:running scissors").field("foo")).get(); assertHighlight(resp, 0, "foo", 0, equalTo("running with scissors")); // Now remove the stored field from the matched field list. That should work too. + fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh") + .requireFieldMatch(requireFieldMatch); fooField.matchedFields("foo.plain"); + req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField)); resp = req.setQuery(queryStringQuery("foo.plain:running scissors").field("foo")).get(); assertHighlight(resp, 0, "foo", 0, equalTo("running with scissors")); // Now make sure boosted fields don't blow up when matched fields is both the subfield and stored field. + fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh") + .requireFieldMatch(requireFieldMatch); fooField.matchedFields("foo", "foo.plain"); + req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField)); resp = req.setQuery(queryStringQuery("foo.plain:running^5 scissors").field("foo")).get(); assertHighlight(resp, 0, "foo", 0, equalTo("running with scissors")); @@ -949,41 +1010,46 @@ public class HighlighterSearchIT extends ESIntegTestCase { // Speaking of two fields, you can have two fields, only one of which has matchedFields enabled QueryBuilder twoFieldsQuery = queryStringQuery("cats").field("foo").field("foo.plain", 5) .field("bar").field("bar.plain", 5); - resp = req.setQuery(twoFieldsQuery).addHighlightedField(barField).get(); + Field barField = new Field("bar").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh") + .requireFieldMatch(requireFieldMatch); + resp = req.setQuery(twoFieldsQuery).highlighter(new HighlightBuilder().field(fooField).field(barField)).get(); assertHighlight(resp, 0, "foo", 0, equalTo("junk junk cats junk junk")); assertHighlight(resp, 0, "bar", 0, equalTo("cat cat junk junk junk junk")); - // And you can enable matchedField highlighting on both barField.matchedFields("bar", "bar.plain"); - resp = req.get(); + resp = req.setQuery(twoFieldsQuery).highlighter(new HighlightBuilder().field(fooField).field(barField)).get(); assertHighlight(resp, 0, "foo", 0, equalTo("junk junk cats junk junk")); assertHighlight(resp, 0, "bar", 0, equalTo("junk junk cats junk junk")); // Setting a matchedField that isn't searched/doesn't exist is simply ignored. barField.matchedFields("bar", "candy"); - resp = req.get(); + resp = req.setQuery(twoFieldsQuery).highlighter(new HighlightBuilder().field(fooField).field(barField)).get(); assertHighlight(resp, 0, "foo", 0, equalTo("junk junk cats junk junk")); assertHighlight(resp, 0, "bar", 0, equalTo("cat cat junk junk junk junk")); // If the stored field doesn't have a value it doesn't matter what you match, you get nothing. barField.matchedFields("bar", "foo.plain"); - resp = req.setQuery(queryStringQuery("running scissors").field("foo.plain").field("bar")).get(); + resp = req.setQuery(queryStringQuery("running scissors").field("foo.plain").field("bar")) + .highlighter(new HighlightBuilder().field(fooField).field(barField)).get(); assertHighlight(resp, 0, "foo", 0, equalTo("running with scissors")); assertThat(resp.getHits().getAt(0).getHighlightFields(), not(hasKey("bar"))); // If the stored field is found but the matched field isn't then you don't get a result either. fooField.matchedFields("bar.plain"); - resp = req.setQuery(queryStringQuery("running scissors").field("foo").field("foo.plain").field("bar").field("bar.plain")).get(); + resp = req.setQuery(queryStringQuery("running scissors").field("foo").field("foo.plain").field("bar").field("bar.plain")) + .highlighter(new HighlightBuilder().field(fooField).field(barField)).get(); assertThat(resp.getHits().getAt(0).getHighlightFields(), not(hasKey("foo"))); // But if you add the stored field to the list of matched fields then you'll get a result again fooField.matchedFields("foo", "bar.plain"); - resp = req.setQuery(queryStringQuery("running scissors").field("foo").field("foo.plain").field("bar").field("bar.plain")).get(); + resp = req.setQuery(queryStringQuery("running scissors").field("foo").field("foo.plain").field("bar").field("bar.plain")) + .highlighter(new HighlightBuilder().field(fooField).field(barField)).get(); assertHighlight(resp, 0, "foo", 0, equalTo("running with scissors")); assertThat(resp.getHits().getAt(0).getHighlightFields(), not(hasKey("bar"))); // You _can_ highlight fields that aren't subfields of one another. - resp = req.setQuery(queryStringQuery("weird").field("foo").field("foo.plain").field("bar").field("bar.plain")).get(); + resp = req.setQuery(queryStringQuery("weird").field("foo").field("foo.plain").field("bar").field("bar.plain")) + .highlighter(new HighlightBuilder().field(fooField).field(barField)).get(); assertHighlight(resp, 0, "foo", 0, equalTo("weird")); assertHighlight(resp, 0, "bar", 0, equalTo("result")); @@ -991,7 +1057,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { RestStatus.INTERNAL_SERVER_ERROR, containsString("IndexOutOfBoundsException")); } - @Test public void testFastVectorHighlighterManyDocs() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping())); ensureGreen(); @@ -1008,7 +1073,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch() .setSize(COUNT) .setQuery(termQuery("field1", "test")) - .addHighlightedField("field1", 100, 0) + .highlighter(new HighlightBuilder().field("field1", 100, 0)) .get(); for (int i = 0; i < COUNT; i++) { SearchHit hit = searchResponse.getHits().getHits()[i]; @@ -1020,7 +1085,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { searchResponse = client().prepareSearch() .setSize(COUNT) .setQuery(termQuery("_all", "test")) - .addHighlightedField("_all", 100, 0) + .highlighter(new HighlightBuilder().field("_all", 100, 0)) .get(); for (int i = 0; i < COUNT; i++) { SearchHit hit = searchResponse.getHits().getHits()[i]; @@ -1038,7 +1103,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { .endObject().endObject(); } - @Test public void testSameContent() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string,store=yes,term_vector=with_positions_offsets")); @@ -1053,7 +1117,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "bug")) - .addHighlightedField("title", -1, 0) + .highlighter(new HighlightBuilder().field("title", -1, 0)) .get(); for (int i = 0; i < 5; i++) { @@ -1061,7 +1125,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testFastVectorHighlighterOffsetParameter() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string,store=yes,term_vector=with_positions_offsets").get()); @@ -1076,7 +1139,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "bug")) - .addHighlightedField("title", 30, 1, 10) + .highlighter(new HighlightBuilder().field("title", 30, 1, 10)) .get(); for (int i = 0; i < 5; i++) { @@ -1085,7 +1148,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testEscapeHtml() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string,store=yes")); @@ -1100,8 +1162,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "test")) - .setHighlighterEncoder("html") - .addHighlightedField("title", 50, 1, 10) + .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1, 10)) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -1109,8 +1170,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test - public void testEscapeHtml_vector() throws Exception { + public void testEscapeHtmlVector() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string,store=yes,term_vector=with_positions_offsets")); ensureYellow(); @@ -1124,8 +1184,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "test")) - .setHighlighterEncoder("html") - .addHighlightedField("title", 30, 1, 10) + .highlighter(new HighlightBuilder().encoder("html").field("title", 30, 1, 10)) .get(); for (int i = 0; i < 5; i++) { @@ -1133,7 +1192,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testMultiMapperVectorWithStore() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -1148,8 +1206,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { // simple search on body with standard analyzer with a simple field query SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1)) .get(); assertHighlight(search, 0, "title", 0, 1, equalTo("this is a test")); @@ -1157,14 +1214,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { // search on title.key and highlight on title search = client().prepareSearch() .setQuery(matchQuery("title.key", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title.key", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1)) .get(); assertHighlight(search, 0, "title.key", 0, 1, equalTo("this is a test")); } - @Test public void testMultiMapperVectorFromSource() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -1181,8 +1236,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { // simple search on body with standard analyzer with a simple field query SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1)) .get(); assertHighlight(search, 0, "title", 0, 1, equalTo("this is a test")); @@ -1190,14 +1244,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { // search on title.key and highlight on title.key search = client().prepareSearch() .setQuery(matchQuery("title.key", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title.key", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1)) .get(); assertHighlight(search, 0, "title.key", 0, 1, equalTo("this is a test")); } - @Test public void testMultiMapperNoVectorWithStore() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -1214,8 +1266,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { // simple search on body with standard analyzer with a simple field query SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1)) .get(); assertHighlight(search, 0, "title", 0, 1, equalTo("this is a test")); @@ -1223,14 +1274,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { // search on title.key and highlight on title search = client().prepareSearch() .setQuery(matchQuery("title.key", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title.key", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1)) .get(); assertHighlight(search, 0, "title.key", 0, 1, equalTo("this is a test")); } - @Test public void testMultiMapperNoVectorFromSource() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -1246,8 +1295,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { // simple search on body with standard analyzer with a simple field query SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1)) .get(); assertHighlight(search, 0, "title", 0, 1, equalTo("this is a test")); @@ -1255,14 +1303,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { // search on title.key and highlight on title.key search = client().prepareSearch() .setQuery(matchQuery("title.key", "this is a test")) - .setHighlighterEncoder("html") - .addHighlightedField("title.key", 50, 1) + .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1)) .get(); assertHighlight(search, 0, "title.key", 0, 1, equalTo("this is a test")); } - @Test public void testFastVectorHighlighterShouldFailIfNoTermVectors() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string,store=yes,term_vector=no")); @@ -1277,25 +1323,22 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchPhraseQuery("title", "this is a test")) - .addHighlightedField("title", 50, 1, 10) + .highlighter(new HighlightBuilder().field("title", 50, 1, 10)) .get(); assertNoFailures(search); assertFailures(client().prepareSearch() .setQuery(matchPhraseQuery("title", "this is a test")) - .addHighlightedField("title", 50, 1, 10) - .setHighlighterType("fast-vector-highlighter"), + .highlighter(new HighlightBuilder().field("title", 50, 1, 10).highlighterType("fvh")), RestStatus.BAD_REQUEST, containsString("the field [title] should be indexed with term vector with position offsets to be used with fast vector highlighter")); //should not fail if there is a wildcard assertNoFailures(client().prepareSearch() .setQuery(matchPhraseQuery("title", "this is a test")) - .addHighlightedField("tit*", 50, 1, 10) - .setHighlighterType("fast-vector-highlighter").get()); + .highlighter(new HighlightBuilder().field("tit*", 50, 1, 10).highlighterType("fvh")).get()); } - @Test public void testDisableFastVectorHighlighter() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string,store=yes,term_vector=with_positions_offsets,analyzer=classic")); @@ -1310,7 +1353,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchPhraseQuery("title", "test for the workaround")) - .addHighlightedField("title", 50, 1, 10) + .highlighter(new HighlightBuilder().field("title", 50, 1, 10)) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -1321,8 +1364,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { // Using plain highlighter instead of FVH search = client().prepareSearch() .setQuery(matchPhraseQuery("title", "test for the workaround")) - .addHighlightedField("title", 50, 1, 10) - .setHighlighterType("highlighter") + .highlighter(new HighlightBuilder().field("title", 50, 1, 10).highlighterType("plain")) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -1332,8 +1374,9 @@ public class HighlighterSearchIT extends ESIntegTestCase { // Using plain highlighter instead of FVH on the field level search = client().prepareSearch() .setQuery(matchPhraseQuery("title", "test for the workaround")) - .addHighlightedField(new HighlightBuilder.Field("title").highlighterType("highlighter")) - .setHighlighterType("highlighter") + .highlighter( + new HighlightBuilder().field(new HighlightBuilder.Field("title").highlighterType("plain")).highlighterType( + "plain")) .get(); for (int i = 0; i < indexRequestBuilders.length; i++) { @@ -1341,7 +1384,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testFSHHighlightAllMvFragments() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "tags", "type=string,term_vector=with_positions_offsets")); @@ -1354,13 +1396,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("tags", "tag")) - .addHighlightedField("tags", -1, 0).get(); + .highlighter(new HighlightBuilder().field("tags", -1, 0)).get(); assertHighlight(response, 0, "tags", 0, equalTo("this is a really long tag i would like to highlight")); assertHighlight(response, 0, "tags", 1, 2, equalTo("here is another one that is very long and has the tag token near the end")); } - @Test public void testBoostingQuery() { createIndex("test"); ensureGreen(); @@ -1371,14 +1412,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(boostingQuery(termQuery("field2", "brown"), termQuery("field2", "foobar")).negativeBoost(0.5f)) - .highlight(highlight().field("field2").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog")); } - @Test @AwaitsFix(bugUrl="Broken now that BoostingQuery does not extend BooleanQuery anymore") public void testBoostingQueryTermVector() throws IOException { assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping())); @@ -1390,14 +1430,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(boostingQuery(termQuery("field2", "brown"), termQuery("field2", "foobar")).negativeBoost(0.5f)) - .highlight(highlight().field("field2").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog")); } - @Test public void testCommonTermsQuery() { createIndex("test"); ensureGreen(); @@ -1410,13 +1449,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100)) - .highlight(highlight().field("field2").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog")); } - @Test public void testCommonTermsTermVector() throws IOException { assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping())); ensureGreen(); @@ -1425,14 +1463,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100)) - .highlight(highlight().field("field2").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog")); } - @Test public void testPhrasePrefix() throws IOException { Builder builder = settingsBuilder() .put(indexSettings()) @@ -1455,7 +1492,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field0"); SearchSourceBuilder source = searchSource() .query(matchPhrasePrefixQuery("field0", "quick bro")) - .highlight(highlight().field("field0").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field0").order("score").preTags("").postTags("")); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -1464,7 +1501,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); source = searchSource() .query(matchPhrasePrefixQuery("field1", "quick bro")) - .highlight(highlight().field("field1").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field1").order("score").preTags("").postTags("")); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -1481,7 +1518,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field3", "fast bro")) - .highlight(highlight().field("field3").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field3").order("score").preTags("").postTags("")); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -1489,7 +1526,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field4"); source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "the fast bro")) - .highlight(highlight().field("field4").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field4").order("score").preTags("").postTags("")); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); assertHighlight(searchResponse, 0, "field4", 0, 1, anyOf(equalTo("The quick browse button is a fancy thing, right bro?"), equalTo("The quick brown fox jumps over the lazy dog"))); @@ -1497,13 +1534,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field4"); source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "a fast quick blue ca")) - .highlight(highlight().field("field4").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field4").order("score").preTags("").postTags("")); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); assertHighlight(searchResponse, 0, "field4", 0, 1, equalTo("a quick fast blue car")); } - @Test public void testPlainHighlightDifferentFragmenter() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "tags", "type=string")); @@ -1516,29 +1552,31 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("tags", "long tag").type(MatchQuery.Type.PHRASE)) - .addHighlightedField(new HighlightBuilder.Field("tags") - .fragmentSize(-1).numOfFragments(2).fragmenter("simple")).get(); + .highlighter( + new HighlightBuilder().field(new HighlightBuilder.Field("tags").fragmentSize(-1).numOfFragments(2) + .fragmenter("simple"))).get(); assertHighlight(response, 0, "tags", 0, equalTo("this is a really long tag i would like to highlight")); assertHighlight(response, 0, "tags", 1, 2, equalTo("here is another one that is very long tag and has the tag token near the end")); response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("tags", "long tag").type(MatchQuery.Type.PHRASE)) - .addHighlightedField(new HighlightBuilder.Field("tags") - .fragmentSize(-1).numOfFragments(2).fragmenter("span")).get(); + .highlighter( + new HighlightBuilder().field(new HighlightBuilder.Field("tags").fragmentSize(-1).numOfFragments(2) + .fragmenter("span"))).get(); assertHighlight(response, 0, "tags", 0, equalTo("this is a really long tag i would like to highlight")); assertHighlight(response, 0, "tags", 1, 2, equalTo("here is another one that is very long tag and has the tag token near the end")); assertFailures(client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("tags", "long tag").type(MatchQuery.Type.PHRASE)) - .addHighlightedField(new HighlightBuilder.Field("tags") - .fragmentSize(-1).numOfFragments(2).fragmenter("invalid")), + .highlighter( + new HighlightBuilder().field(new HighlightBuilder.Field("tags").fragmentSize(-1).numOfFragments(2) + .fragmenter("invalid"))), RestStatus.BAD_REQUEST, containsString("unknown fragmenter option [invalid] for the field [tags]")); } - @Test public void testPlainHighlighterMultipleFields() { createIndex("test"); ensureGreen(); @@ -1548,14 +1586,15 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("field1", "fox")) - .addHighlightedField(new HighlightBuilder.Field("field1").preTags("<1>").postTags("").requireFieldMatch(true)) - .addHighlightedField(new HighlightBuilder.Field("field2").preTags("<2>").postTags("").requireFieldMatch(false)) + .highlighter( + new HighlightBuilder().field( + new HighlightBuilder.Field("field1").preTags("<1>").postTags("").requireFieldMatch(true)).field( + new HighlightBuilder.Field("field2").preTags("<2>").postTags("").requireFieldMatch(false))) .get(); assertHighlight(response, 0, "field1", 0, 1, equalTo("The quick brown <1>fox")); assertHighlight(response, 0, "field2", 0, 1, equalTo("The slow brown <2>fox")); } - @Test public void testFastVectorHighlighterMultipleFields() { assertAcked(prepareCreate("test") .addMapping("type1", "field1", "type=string,term_vector=with_positions_offsets", "field2", "type=string,term_vector=with_positions_offsets")); @@ -1566,14 +1605,15 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("field1", "fox")) - .addHighlightedField(new HighlightBuilder.Field("field1").preTags("<1>").postTags("").requireFieldMatch(true)) - .addHighlightedField(new HighlightBuilder.Field("field2").preTags("<2>").postTags("").requireFieldMatch(false)) + .highlighter( + new HighlightBuilder().field( + new HighlightBuilder.Field("field1").preTags("<1>").postTags("").requireFieldMatch(true)).field( + new HighlightBuilder.Field("field2").preTags("<2>").postTags("").requireFieldMatch(false))) .get(); assertHighlight(response, 0, "field1", 0, 1, equalTo("The quick brown <1>fox")); assertHighlight(response, 0, "field2", 0, 1, equalTo("The slow brown <2>fox")); } - @Test public void testMissingStoredField() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "highlight_field", "type=string,store=yes")); @@ -1587,13 +1627,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { // This query used to fail when the field to highlight was absent SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("field", "highlight").type(MatchQuery.Type.BOOLEAN)) - .addHighlightedField(new HighlightBuilder.Field("highlight_field") - .fragmentSize(-1).numOfFragments(1).fragmenter("simple")).get(); + .highlighter( + new HighlightBuilder().field(new HighlightBuilder.Field("highlight_field").fragmentSize(-1).numOfFragments(1) + .fragmenter("simple"))).get(); assertThat(response.getHits().hits()[0].highlightFields().isEmpty(), equalTo(true)); } - @Test - // https://github.com/elasticsearch/elasticsearch/issues/3211 + // Issue #3211 public void testNumericHighlighting() throws Exception { assertAcked(prepareCreate("test") .addMapping("test", "text", "type=string,index=analyzed", @@ -1607,21 +1647,16 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("text", "test").type(MatchQuery.Type.BOOLEAN)) - .addHighlightedField("text") - .addHighlightedField("byte") - .addHighlightedField("short") - .addHighlightedField("int") - .addHighlightedField("long") - .addHighlightedField("float") - .addHighlightedField("double") + .highlighter( + new HighlightBuilder().field("text").field("byte").field("short").field("int").field("long").field("float") + .field("double")) .get(); // Highlighting of numeric fields is not supported, but it should not raise errors // (this behavior is consistent with version 0.20) assertHitCount(response, 1l); } - @Test - // https://github.com/elasticsearch/elasticsearch/issues/3200 + // Issue #3200 public void testResetTwice() throws Exception { assertAcked(prepareCreate("test") .setSettings(settingsBuilder() @@ -1637,12 +1672,11 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("text", "test").type(MatchQuery.Type.BOOLEAN)) - .addHighlightedField("text").execute().actionGet(); + .highlighter(new HighlightBuilder().field("text")).execute().actionGet(); // PatternAnalyzer will throw an exception if it is resetted twice assertHitCount(response, 1l); } - @Test public void testHighlightUsesHighlightQuery() throws IOException { assertAcked(prepareCreate("test") .addMapping("type1", "text", "type=string," + randomStoreField() + "term_vector=with_positions_offsets,index_options=offsets")); @@ -1653,8 +1687,9 @@ public class HighlighterSearchIT extends ESIntegTestCase { HighlightBuilder.Field field = new HighlightBuilder.Field("text"); + HighlightBuilder highlightBuilder = new HighlightBuilder().field(field); SearchRequestBuilder search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing")) - .addHighlightedField(field); + .highlighter(highlightBuilder); Matcher searchQueryMatcher = equalTo("Testing the highlight query feature"); field.highlighterType("plain"); @@ -1667,9 +1702,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { response = search.get(); assertHighlight(response, 0, "text", 0, searchQueryMatcher); + field = new HighlightBuilder.Field("text"); Matcher hlQueryMatcher = equalTo("Testing the highlight query feature"); field.highlightQuery(matchQuery("text", "query")); + highlightBuilder = new HighlightBuilder().field(field); + search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing")).highlighter(highlightBuilder); field.highlighterType("fvh"); response = search.get(); @@ -1684,7 +1722,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { assertHighlight(response, 0, "text", 0, hlQueryMatcher); // Make sure the the highlightQuery is taken into account when it is set on the highlight context instead of the field - search.setHighlighterQuery(matchQuery("text", "query")); + highlightBuilder.highlightQuery(matchQuery("text", "query")); field.highlighterType("fvh").highlightQuery(null); response = search.get(); assertHighlight(response, 0, "text", 0, hlQueryMatcher); @@ -1705,7 +1743,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { return ""; } - @Test public void testHighlightNoMatchSize() throws IOException { assertAcked(prepareCreate("test") .addMapping("type1", "text", "type=string," + randomStoreField() + "term_vector=with_positions_offsets,index_options=offsets")); @@ -1720,101 +1757,100 @@ public class HighlighterSearchIT extends ESIntegTestCase { .fragmentSize(21) .numOfFragments(1) .highlighterType("plain"); - SearchResponse response = client().prepareSearch("test").addHighlightedField(field).get(); + SearchResponse response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); // When noMatchSize is set to 0 you also shouldn't get any field.highlighterType("plain").noMatchSize(0); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); // When noMatchSize is between 0 and the size of the string field.highlighterType("plain").noMatchSize(21); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so")); // The FVH also works but the fragment is longer than the plain highlighter because of boundary_max_scan field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some")); // Postings hl also works but the fragment is the whole first sentence (size ignored) field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off.")); // We can also ask for a fragment longer than the input string and get the whole string field.highlighterType("plain").noMatchSize(text.length() * 2); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo(text)); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo(text)); //no difference using postings hl as the noMatchSize is ignored (just needs to be greater than 0) field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off.")); // We can also ask for a fragment exactly the size of the input field and get the whole field field.highlighterType("plain").noMatchSize(text.length()); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo(text)); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo(text)); //no difference using postings hl as the noMatchSize is ignored (just needs to be greater than 0) field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off.")); // You can set noMatchSize globally in the highlighter as well field.highlighterType("plain").noMatchSize(null); - response = client().prepareSearch("test").setHighlighterNoMatchSize(21).addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so")); field.highlighterType("fvh"); - response = client().prepareSearch("test").setHighlighterNoMatchSize(21).addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some")); field.highlighterType("postings"); - response = client().prepareSearch("test").setHighlighterNoMatchSize(21).addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off.")); // We don't break if noMatchSize is less than zero though field.highlighterType("plain").noMatchSize(randomIntBetween(Integer.MIN_VALUE, -1)); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); } - @Test public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException { assertAcked(prepareCreate("test") .addMapping("type1", "text", "type=string," + randomStoreField() + "term_vector=with_positions_offsets,index_options=offsets")); @@ -1831,16 +1867,16 @@ public class HighlighterSearchIT extends ESIntegTestCase { .numOfFragments(1) .highlighterType("plain") .noMatchSize(21); - SearchResponse response = client().prepareSearch("test").addHighlightedField(field).get(); + SearchResponse response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so")); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some")); // Postings hl also works but the fragment is the whole first sentence (size ignored) field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off.")); // And noMatchSize returns nothing when the first entry is empty string! @@ -1851,19 +1887,19 @@ public class HighlighterSearchIT extends ESIntegTestCase { field.highlighterType("plain"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("postings"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); // But if the field was actually empty then you should get no highlighting field @@ -1873,19 +1909,19 @@ public class HighlighterSearchIT extends ESIntegTestCase { field.highlighterType("plain"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("postings"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); // Same for if the field doesn't even exist on the document @@ -1896,38 +1932,37 @@ public class HighlighterSearchIT extends ESIntegTestCase { field.highlighterType("plain"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); response = client().prepareSearch("test") .setQuery(idsQueryBuilder) - .addHighlightedField(field).get(); +.highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "postings"); // Again same if the field isn't mapped field = new HighlightBuilder.Field("unmapped") .highlighterType("plain") .noMatchSize(21); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertNotHighlighted(response, 0, "text"); } - @Test public void testHighlightNoMatchSizeNumberOfFragments() throws IOException { assertAcked(prepareCreate("test") .addMapping("type1", "text", "type=string," + randomStoreField() + "term_vector=with_positions_offsets,index_options=offsets")); @@ -1945,37 +1980,36 @@ public class HighlighterSearchIT extends ESIntegTestCase { .numOfFragments(0) .highlighterType("plain") .noMatchSize(20); - SearchResponse response = client().prepareSearch("test").addHighlightedField(field).get(); + SearchResponse response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("This is the first")); field.highlighterType("fvh"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("This is the first sentence")); // Postings hl also works but the fragment is the whole first sentence (size ignored) field.highlighterType("postings"); - response = client().prepareSearch("test").addHighlightedField(field).get(); + response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 1, equalTo("This is the first sentence.")); //if there's a match we only return the values with matches (whole value as number_of_fragments == 0) MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery("text", "third fifth"); field.highlighterType("plain"); - response = client().prepareSearch("test").setQuery(queryBuilder).addHighlightedField(field).get(); + response = client().prepareSearch("test").setQuery(queryBuilder).highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 2, equalTo("This is the third sentence. This is the fourth sentence.")); assertHighlight(response, 0, "text", 1, 2, equalTo("This is the fifth sentence")); field.highlighterType("fvh"); - response = client().prepareSearch("test").setQuery(queryBuilder).addHighlightedField(field).get(); + response = client().prepareSearch("test").setQuery(queryBuilder).highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 2, equalTo("This is the third sentence. This is the fourth sentence.")); assertHighlight(response, 0, "text", 1, 2, equalTo("This is the fifth sentence")); field.highlighterType("postings"); - response = client().prepareSearch("test").setQuery(queryBuilder).addHighlightedField(field).get(); + response = client().prepareSearch("test").setQuery(queryBuilder).highlighter(new HighlightBuilder().field(field)).get(); assertHighlight(response, 0, "text", 0, 2, equalTo("This is the third sentence. This is the fourth sentence.")); assertHighlight(response, 0, "text", 1, 2, equalTo("This is the fifth sentence")); } - @Test public void testPostingsHighlighter() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -1987,7 +2021,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(termQuery("field1", "test")) - .highlight(highlight().field("field1").preTags("").postTags("")); + .highlighter(highlight().field("field1").preTags("").postTags("")); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a test")); @@ -1995,7 +2029,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on field1, highlighting on field1"); source = searchSource() .query(termQuery("field1", "test")) - .highlight(highlight().field("field1").preTags("").postTags("")); + .highlighter(highlight().field("field1").preTags("").postTags("")); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -2004,7 +2038,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on field2, highlighting on field2"); source = searchSource() .query(termQuery("field2", "quick")) - .highlight(highlight().field("field2").order("score").preTags("").postTags("")); + .highlighter(highlight().field("field2").order("score").preTags("").postTags("")); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -2013,7 +2047,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on field2, highlighting on field2"); source = searchSource() .query(matchPhraseQuery("field2", "quick brown")) - .highlight(highlight().field("field2").preTags("").postTags("")); + .highlighter(highlight().field("field2").preTags("").postTags("")); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -2024,14 +2058,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> searching on field2, highlighting on field2, falling back to the plain highlighter"); source = searchSource() .query(matchPhraseQuery("_all", "quick brown")) - .highlight(highlight().field("field2").preTags("").postTags("").highlighterType("highlighter").requireFieldMatch(false)); + .highlighter(highlight().field("field2").preTags("").postTags("").highlighterType("plain").requireFieldMatch(false)); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy quick dog")); } - @Test public void testPostingsHighlighterMultipleFields() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping()).get()); ensureGreen(); @@ -2041,12 +2074,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("field1", "fox")) - .addHighlightedField(new HighlightBuilder.Field("field1").preTags("<1>").postTags("").requireFieldMatch(true)) + .highlighter( + new HighlightBuilder().field(new HighlightBuilder.Field("field1").preTags("<1>").postTags("") + .requireFieldMatch(true))) .get(); assertHighlight(response, 0, "field1", 0, 1, equalTo("The quick brown <1>fox.")); } - @Test public void testPostingsHighlighterNumberOfFragments() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2059,7 +2093,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(termQuery("field1", "fox")) - .highlight(highlight() + .highlighter(highlight() .field(new HighlightBuilder.Field("field1").numOfFragments(5).preTags("").postTags(""))); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -2074,7 +2108,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { source = searchSource() .query(termQuery("field1", "fox")) - .highlight(highlight() + .highlighter(highlight() .field(new HighlightBuilder.Field("field1").numOfFragments(0).preTags("").postTags(""))); searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -2093,7 +2127,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testMultiMatchQueryHighlight() throws IOException { String[] highlighterTypes = new String[] {"fvh", "plain", "postings"}; XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -2124,7 +2157,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchSourceBuilder source = searchSource() .query(multiMatchQueryBuilder) - .highlight(highlight().highlightQuery(randomBoolean() ? multiMatchQueryBuilder : null).highlighterType(highlighterType) + .highlighter(highlight().highlightQuery(randomBoolean() ? multiMatchQueryBuilder : null).highlighterType(highlighterType) .field(new Field("field1").requireFieldMatch(true).preTags("").postTags(""))); logger.info("Running multi-match type: [" + matchQueryType + "] highlight with type: [" + highlighterType + "]"); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -2134,7 +2167,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test public void testPostingsHighlighterOrderByScore() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2148,7 +2180,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(termQuery("field1", "sentence")) - .highlight(highlight().field("field1").order("score")); + .highlighter(highlight().field("field1").order("score")); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); @@ -2163,7 +2195,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { assertThat(field1.fragments()[4].string(), equalTo("One sentence match here and scored lower since the text is quite long, not that appealing.")); } - @Test public void testPostingsHighlighterEscapeHtml() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "title", "type=string," + randomStoreField() + "index_options=offsets")); @@ -2178,15 +2209,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch() .setQuery(matchQuery("title", "test")) - .setHighlighterEncoder("html") - .addHighlightedField("title").get(); + .highlighter(new HighlightBuilder().field("title").encoder("html")).get(); for (int i = 0; i < indexRequestBuilders.length; i++) { assertHighlight(searchResponse, i, "title", 0, 1, equalTo("This is a html escaping highlighting test for *&?")); } } - @Test public void testPostingsHighlighterMultiMapperWithStore() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1") @@ -2203,7 +2232,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch() //lets make sure we analyze the query and we highlight the resulting terms .setQuery(matchQuery("title", "This is a Test")) - .addHighlightedField("title").get(); +.highlighter(new HighlightBuilder().field("title")).get(); assertHitCount(searchResponse, 1l); SearchHit hit = searchResponse.getHits().getAt(0); @@ -2213,14 +2242,13 @@ public class HighlighterSearchIT extends ESIntegTestCase { // search on title.key and highlight on title searchResponse = client().prepareSearch() .setQuery(matchQuery("title.key", "this is a test")) - .addHighlightedField("title.key").get(); + .highlighter(new HighlightBuilder().field("title.key")).get(); assertHitCount(searchResponse, 1l); //stopwords are now highlighted since we used only whitespace analyzer here assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("this is a test .")); } - @Test public void testPostingsHighlighterMultiMapperFromSource() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -2237,7 +2265,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { // simple search on body with standard analyzer with a simple field query SearchResponse searchResponse = client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .addHighlightedField("title") + .highlighter(new HighlightBuilder().field("title")) .get(); assertHighlight(searchResponse, 0, "title", 0, 1, equalTo("this is a test")); @@ -2245,12 +2273,11 @@ public class HighlighterSearchIT extends ESIntegTestCase { // search on title.key and highlight on title.key searchResponse = client().prepareSearch() .setQuery(matchQuery("title.key", "this is a test")) - .addHighlightedField("title.key").get(); + .highlighter(new HighlightBuilder().field("title.key")).get(); assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("this is a test")); } - @Test public void testPostingsHighlighterShouldFailIfNoOffsets() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -2267,33 +2294,29 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .addHighlightedField("title") + .highlighter(new HighlightBuilder().field("title")) .get(); assertNoFailures(search); assertFailures(client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .addHighlightedField("title") - .setHighlighterType("postings-highlighter"), + .highlighter(new HighlightBuilder().field("title").highlighterType("postings")), RestStatus.BAD_REQUEST, containsString("the field [title] should be indexed with positions and offsets in the postings list to be used with postings highlighter")); assertFailures(client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .addHighlightedField("title") - .setHighlighterType("postings"), + .highlighter(new HighlightBuilder().field("title").highlighterType("postings")), RestStatus.BAD_REQUEST, containsString("the field [title] should be indexed with positions and offsets in the postings list to be used with postings highlighter")); //should not fail if there is a wildcard assertNoFailures(client().prepareSearch() .setQuery(matchQuery("title", "this is a test")) - .addHighlightedField("tit*") - .setHighlighterType("postings").get()); + .highlighter(new HighlightBuilder().field("tit*").highlighterType("postings")).get()); } - @Test public void testPostingsHighlighterBoostingQuery() throws IOException { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2304,13 +2327,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource() .query(boostingQuery(termQuery("field2", "brown"), termQuery("field2", "foobar")).negativeBoost(0.5f)) - .highlight(highlight().field("field2").preTags("").postTags("")); + .highlighter(highlight().field("field2").preTags("").postTags("")); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog!")); } - @Test public void testPostingsHighlighterCommonTermsQuery() throws IOException { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2319,7 +2341,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100)) - .highlight(highlight().field("field2").preTags("").postTags("")); + .highlighter(highlight().field("field2").preTags("").postTags("")); SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet(); assertHitCount(searchResponse, 1l); @@ -2335,7 +2357,6 @@ public class HighlighterSearchIT extends ESIntegTestCase { .endObject().endObject(); } - @Test public void testPostingsHighlighterPrefixQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2345,13 +2366,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field2"); SearchSourceBuilder source = searchSource().query(prefixQuery("field2", "qui")) - .highlight(highlight().field("field2")); + .highlighter(highlight().field("field2")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog!")); } - @Test public void testPostingsHighlighterFuzzyQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2360,13 +2380,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field2"); SearchSourceBuilder source = searchSource().query(fuzzyQuery("field2", "quck")) - .highlight(highlight().field("field2")); + .highlighter(highlight().field("field2")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog!")); } - @Test public void testPostingsHighlighterRegexpQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2375,13 +2394,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field2"); SearchSourceBuilder source = searchSource().query(regexpQuery("field2", "qu[a-l]+k")) - .highlight(highlight().field("field2")); + .highlighter(highlight().field("field2")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog!")); } - @Test public void testPostingsHighlighterWildcardQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2390,20 +2408,19 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field2"); SearchSourceBuilder source = searchSource().query(wildcardQuery("field2", "qui*")) - .highlight(highlight().field("field2")); + .highlighter(highlight().field("field2")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog!")); source = searchSource().query(wildcardQuery("field2", "qu*k")) - .highlight(highlight().field("field2")); + .highlighter(highlight().field("field2")); searchResponse = client().prepareSearch("test").setSource(source).get(); assertHitCount(searchResponse, 1l); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog!")); } - @Test public void testPostingsHighlighterTermRangeQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2412,13 +2429,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field2"); SearchSourceBuilder source = searchSource().query(rangeQuery("field2").gte("aaaa").lt("zzzz")) - .highlight(highlight().field("field2")); + .highlighter(highlight().field("field2")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("aaab")); } - @Test public void testPostingsHighlighterQueryString() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2427,14 +2443,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field2"); SearchSourceBuilder source = searchSource().query(queryStringQuery("qui*").defaultField("field2")) - .highlight(highlight().field("field2")); + .highlighter(highlight().field("field2")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog!")); } - @Test public void testPostingsHighlighterRegexpQueryWithinConstantScoreQuery() throws Exception { - assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2443,14 +2457,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource().query(constantScoreQuery(regexpQuery("field1", "pho[a-z]+"))) - .highlight(highlight().field("field1")); + .highlighter(highlight().field("field1")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The photography word will get highlighted")); } - @Test public void testPostingsHighlighterMultiTermQueryMultipleLevels() throws Exception { - assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2459,17 +2471,15 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource().query(boolQuery() - .should(constantScoreQuery(QueryBuilders.missingQuery("field1"))) + .should(boolQuery().mustNot(QueryBuilders.existsQuery("field1"))) .should(matchQuery("field1", "test")) .should(constantScoreQuery(queryStringQuery("field1:photo*")))) - .highlight(highlight().field("field1")); + .highlighter(highlight().field("field1")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The photography word will get highlighted")); } - @Test public void testPostingsHighlighterPrefixQueryWithinBooleanQuery() throws Exception { - assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2478,14 +2488,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { logger.info("--> highlighting and searching on field1"); SearchSourceBuilder source = searchSource().query(boolQuery().must(prefixQuery("field1", "photo")).should(matchQuery("field1", "test").minimumShouldMatch("0"))) - .highlight(highlight().field("field1")); + .highlighter(highlight().field("field1")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The photography word will get highlighted")); } - @Test public void testPostingsHighlighterQueryStringWithinFilteredQuery() throws Exception { - assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2493,13 +2501,14 @@ public class HighlighterSearchIT extends ESIntegTestCase { refresh(); logger.info("--> highlighting and searching on field1"); - SearchSourceBuilder source = searchSource().query(boolQuery().must(queryStringQuery("field1:photo*")).filter(missingQuery("field_null"))) - .highlight(highlight().field("field1")); + SearchSourceBuilder source = searchSource().query(boolQuery() + .must(queryStringQuery("field1:photo*")) + .mustNot(existsQuery("field_null"))) + .highlighter(highlight().field("field1")); SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get(); assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The photography word will get highlighted")); } - @Test public void testPostingsHighlighterManyDocs() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); ensureGreen(); @@ -2523,10 +2532,10 @@ public class HighlighterSearchIT extends ESIntegTestCase { SearchRequestBuilder searchRequestBuilder = client().prepareSearch() .setSize(COUNT) .setQuery(termQuery("field1", "test")) - .addHighlightedField("field1"); + .highlighter(new HighlightBuilder().field("field1")); SearchResponse searchResponse = searchRequestBuilder.get(); - assertHitCount(searchResponse, (long)COUNT); + assertHitCount(searchResponse, COUNT); assertThat(searchResponse.getHits().hits().length, equalTo(COUNT)); for (SearchHit hit : searchResponse.getHits()) { String prefix = prefixes.get(hit.id()); @@ -2534,14 +2543,12 @@ public class HighlighterSearchIT extends ESIntegTestCase { } } - @Test @AwaitsFix(bugUrl="Broken now that BoostingQuery does not extend BooleanQuery anymore") public void testFastVectorHighlighterPhraseBoost() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping())); phraseBoostTestCase("fvh"); } - @Test public void testPostingsHighlighterPhraseBoost() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping())); phraseBoostTestCase("postings"); @@ -2596,9 +2603,8 @@ public class HighlighterSearchIT extends ESIntegTestCase { phraseBoostTestCaseForClauses(String highlighterType, float boost, QueryBuilder terms, P phrase) { Matcher highlightedMatcher = Matchers.either(containsString("highlight words together")).or( containsString("highlight words together")); - SearchRequestBuilder search = client().prepareSearch("test").setHighlighterRequireFieldMatch(true) - .setHighlighterOrder("score").setHighlighterType(highlighterType) - .addHighlightedField("field1", 100, 1); + SearchRequestBuilder search = client().prepareSearch("test").highlighter( + new HighlightBuilder().field("field1", 100, 1).order("score").highlighterType(highlighterType).requireFieldMatch(true)); // Try with a bool query phrase.boost(boost); diff --git a/core/src/test/java/org/elasticsearch/search/indicesboost/SimpleIndicesBoostSearchIT.java b/core/src/test/java/org/elasticsearch/search/indicesboost/SimpleIndicesBoostSearchIT.java index 589f79a876b..f9245a3d981 100644 --- a/core/src/test/java/org/elasticsearch/search/indicesboost/SimpleIndicesBoostSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/indicesboost/SimpleIndicesBoostSearchIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.search.indicesboost; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.client.Requests.indexRequest; import static org.elasticsearch.client.Requests.searchRequest; @@ -36,8 +35,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class SimpleIndicesBoostSearchIT extends ESIntegTestCase { - - @Test public void testIndicesBoost() throws Exception { assertHitCount(client().prepareSearch().setQuery(termQuery("test", "value")).get(), 0); diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java index 84a315c8585..65fbbd3340d 100644 --- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.innerhits; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -36,10 +36,10 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.fetch.innerhits.InnerHitsBuilder; +import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -48,20 +48,33 @@ import java.util.List; import java.util.Locale; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery; +import static org.elasticsearch.index.query.QueryBuilders.hasParentQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** */ public class InnerHitsIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(MockScriptEngine.TestPlugin.class); } - @Test public void testSimpleNested() throws Exception { assertAcked(prepareCreate("articles").addMapping("article", jsonBuilder().startObject().startObject("article").startObject("properties") .startObject("comments") @@ -96,11 +109,14 @@ public class InnerHitsIT extends ESIntegTestCase { .endObject())); indexRandom(true, requests); + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addNestedInnerHits("comment", "comments", + new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.message", "fox"))); // Inner hits can be defined in two ways: 1) with the query 2) as seperate inner_hit definition SearchRequest[] searchRequests = new SearchRequest[]{ client().prepareSearch("articles").setQuery(nestedQuery("comments", matchQuery("comments.message", "fox")).innerHit(new QueryInnerHits("comment", null))).request(), client().prepareSearch("articles").setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"))) - .addNestedInnerHits("comment", "comments", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.message", "fox"))).request() + .innerHits(innerHitsBuilder).request() }; for (SearchRequest searchRequest : searchRequests) { SearchResponse response = client().search(searchRequest).actionGet(); @@ -119,10 +135,15 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(1).getNestedIdentity().getOffset(), equalTo(1)); } + innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addNestedInnerHits("comment", "comments", + new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.message", "elephant"))); + // Inner hits can be defined in two ways: 1) with the query 2) as + // seperate inner_hit definition searchRequests = new SearchRequest[] { client().prepareSearch("articles") .setQuery(nestedQuery("comments", matchQuery("comments.message", "elephant"))) - .addNestedInnerHits("comment", "comments", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.message", "elephant"))).request(), + .innerHits(innerHitsBuilder).request(), client().prepareSearch("articles") .setQuery(nestedQuery("comments", matchQuery("comments.message", "elephant")).innerHit(new QueryInnerHits("comment", null))).request(), client().prepareSearch("articles") @@ -149,21 +170,23 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(2).getNestedIdentity().getOffset(), equalTo(2)); } InnerHitsBuilder.InnerHit innerHit = new InnerHitsBuilder.InnerHit(); - innerHit.highlightBuilder().field("comments.message"); + innerHit.highlighter(new HighlightBuilder().field("comments.message")); innerHit.setExplain(true); innerHit.addFieldDataField("comments.message"); innerHit.addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())); innerHit.setSize(1); - searchRequests = new SearchRequest[] { - client().prepareSearch("articles") - .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"))) - .addNestedInnerHits("comments", "comments", new InnerHitsBuilder.InnerHit() + innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addNestedInnerHits("comments", "comments", new InnerHitsBuilder.InnerHit() .setQuery(matchQuery("comments.message", "fox")) - .addHighlightedField("comments.message") + .highlighter(new HighlightBuilder().field("comments.message")) .setExplain(true) .addFieldDataField("comments.message") .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())) - .setSize(1)).request(), + .setSize(1)); + searchRequests = new SearchRequest[] { + client().prepareSearch("articles") + .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"))) + .innerHits(innerHitsBuilder).request(), client().prepareSearch("articles") .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox")).innerHit(new QueryInnerHits(null, innerHit))).request() }; @@ -181,7 +204,6 @@ public class InnerHitsIT extends ESIntegTestCase { } } - @Test public void testRandomNested() throws Exception { assertAcked(prepareCreate("idx").addMapping("type", "field1", "type=nested", "field2", "type=nested")); int numDocs = scaledRandomIntBetween(25, 100); @@ -208,11 +230,13 @@ public class InnerHitsIT extends ESIntegTestCase { int size = randomIntBetween(0, numDocs); SearchResponse searchResponse; if (randomBoolean()) { + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addNestedInnerHits("a", "field1", new InnerHitsBuilder.InnerHit().addSort("_doc", SortOrder.DESC).setSize(size)); // Sort order is DESC, because we reverse the inner objects during indexing! + innerHitsBuilder.addNestedInnerHits("b", "field2", new InnerHitsBuilder.InnerHit().addSort("_doc", SortOrder.DESC).setSize(size)); searchResponse = client().prepareSearch("idx") .setSize(numDocs) .addSort("_uid", SortOrder.ASC) - .addNestedInnerHits("a", "field1", new InnerHitsBuilder.InnerHit().addSort("_doc", SortOrder.DESC).setSize(size)) // Sort order is DESC, because we reverse the inner objects during indexing! - .addNestedInnerHits("b", "field2", new InnerHitsBuilder.InnerHit().addSort("_doc", SortOrder.DESC).setSize(size)) + .innerHits(innerHitsBuilder) .get(); } else { BoolQueryBuilder boolQuery = new BoolQueryBuilder(); @@ -256,7 +280,6 @@ public class InnerHitsIT extends ESIntegTestCase { } } - @Test public void testSimpleParentChild() throws Exception { assertAcked(prepareCreate("articles") .addMapping("article", "title", "type=string") @@ -274,10 +297,12 @@ public class InnerHitsIT extends ESIntegTestCase { requests.add(client().prepareIndex("articles", "comment", "6").setParent("2").setSource("message", "elephant scared by mice x y")); indexRandom(true, requests); + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "fox"))); SearchRequest[] searchRequests = new SearchRequest[]{ client().prepareSearch("articles") .setQuery(hasChildQuery("comment", matchQuery("message", "fox"))) - .addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "fox"))) + .innerHits(innerHitsBuilder) .request(), client().prepareSearch("articles") .setQuery(hasChildQuery("comment", matchQuery("message", "fox")).innerHit(new QueryInnerHits("comment", null))) @@ -300,10 +325,12 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(1).type(), equalTo("comment")); } + innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "elephant"))); searchRequests = new SearchRequest[] { client().prepareSearch("articles") .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"))) - .addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "elephant"))) + .innerHits(innerHitsBuilder) .request(), client().prepareSearch("articles") .setQuery(hasChildQuery("comment", matchQuery("message", "elephant")).innerHit(new QueryInnerHits())) @@ -327,22 +354,24 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(2).type(), equalTo("comment")); } InnerHitsBuilder.InnerHit innerHit = new InnerHitsBuilder.InnerHit(); - innerHit.highlightBuilder().field("message"); + innerHit.highlighter(new HighlightBuilder().field("message")); innerHit.setExplain(true); innerHit.addFieldDataField("message"); innerHit.addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())); innerHit.setSize(1); - searchRequests = new SearchRequest[] { - client().prepareSearch("articles") - .setQuery(hasChildQuery("comment", matchQuery("message", "fox"))) - .addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit() + innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit() .setQuery(matchQuery("message", "fox")) - .addHighlightedField("message") + .highlighter(new HighlightBuilder().field("message")) .setExplain(true) .addFieldDataField("message") .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())) - .setSize(1) - ).request(), + .setSize(1)); + searchRequests = new SearchRequest[] { + client().prepareSearch("articles") + .setQuery(hasChildQuery("comment", matchQuery("message", "fox"))) + .innerHits(innerHitsBuilder) + .request(), client().prepareSearch("articles") .setQuery( @@ -361,7 +390,6 @@ public class InnerHitsIT extends ESIntegTestCase { } } - @Test public void testRandomParentChild() throws Exception { assertAcked(prepareCreate("idx") .addMapping("parent") @@ -393,14 +421,16 @@ public class InnerHitsIT extends ESIntegTestCase { indexRandom(true, requestBuilders); int size = randomIntBetween(0, numDocs); + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("a", "child1", new InnerHitsBuilder.InnerHit().addSort("_uid", SortOrder.ASC).setSize(size)); + innerHitsBuilder.addParentChildInnerHits("b", "child2", new InnerHitsBuilder.InnerHit().addSort("_uid", SortOrder.ASC).setSize(size)); SearchResponse searchResponse; if (randomBoolean()) { searchResponse = client().prepareSearch("idx") .setSize(numDocs) .setTypes("parent") .addSort("_uid", SortOrder.ASC) - .addParentChildInnerHits("a", "child1", new InnerHitsBuilder.InnerHit().addSort("_uid", SortOrder.ASC).setSize(size)) - .addParentChildInnerHits("b", "child2", new InnerHitsBuilder.InnerHit().addSort("_uid", SortOrder.ASC).setSize(size)) + .innerHits(innerHitsBuilder) .get(); } else { BoolQueryBuilder boolQuery = new BoolQueryBuilder(); @@ -455,13 +485,15 @@ public class InnerHitsIT extends ESIntegTestCase { } } - @Test + @AwaitsFix(bugUrl = "need validation of type or path defined in InnerHitsBuilder") public void testPathOrTypeMustBeDefined() { createIndex("articles"); ensureGreen("articles"); try { + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("comment", null, new InnerHitsBuilder.InnerHit()); client().prepareSearch("articles") - .addParentChildInnerHits("comment", null, new InnerHitsBuilder.InnerHit()) + .innerHits(innerHitsBuilder) .get(); } catch (Exception e) { assertThat(e.getMessage(), containsString("Failed to build")); @@ -469,7 +501,6 @@ public class InnerHitsIT extends ESIntegTestCase { } - @Test public void testInnerHitsOnHasParent() throws Exception { assertAcked(prepareCreate("stack") .addMapping("question", "body", "type=string") @@ -508,7 +539,6 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(searchHit.getInnerHits().get("question").getAt(0).id(), equalTo("2")); } - @Test public void testParentChildMultipleLayers() throws Exception { assertAcked(prepareCreate("articles") .addMapping("article", "title", "type=string") @@ -525,13 +555,15 @@ public class InnerHitsIT extends ESIntegTestCase { requests.add(client().prepareIndex("articles", "remark", "2").setParent("2").setRouting("2").setSource("message", "bad")); indexRandom(true, requests); + InnerHitsBuilder innerInnerHitsBuilder = new InnerHitsBuilder(); + innerInnerHitsBuilder.addParentChildInnerHits("remark", "remark", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "good"))); + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit() + .setQuery(hasChildQuery("remark", matchQuery("message", "good"))) + .innerHits(innerInnerHitsBuilder)); SearchResponse response = client().prepareSearch("articles") .setQuery(hasChildQuery("comment", hasChildQuery("remark", matchQuery("message", "good")))) - .addParentChildInnerHits("comment", "comment", - new InnerHitsBuilder.InnerHit() - .setQuery(hasChildQuery("remark", matchQuery("message", "good"))) - .addParentChildInnerHits("remark", "remark", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "good"))) - ) + .innerHits(innerHitsBuilder) .get(); assertNoFailures(response); @@ -549,13 +581,15 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(0).getId(), equalTo("1")); assertThat(innerHits.getAt(0).type(), equalTo("remark")); + innerInnerHitsBuilder = new InnerHitsBuilder(); + innerInnerHitsBuilder.addParentChildInnerHits("remark", "remark", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "bad"))); + innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("comment", "comment", new InnerHitsBuilder.InnerHit() + .setQuery(hasChildQuery("remark", matchQuery("message", "bad"))) + .innerHits(innerInnerHitsBuilder)); response = client().prepareSearch("articles") .setQuery(hasChildQuery("comment", hasChildQuery("remark", matchQuery("message", "bad")))) - .addParentChildInnerHits("comment", "comment", - new InnerHitsBuilder.InnerHit() - .setQuery(hasChildQuery("remark", matchQuery("message", "bad"))) - .addParentChildInnerHits("remark", "remark", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("message", "bad"))) - ) + .innerHits(innerHitsBuilder) .get(); assertNoFailures(response); @@ -574,7 +608,6 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(0).type(), equalTo("remark")); } - @Test public void testNestedMultipleLayers() throws Exception { assertAcked(prepareCreate("articles").addMapping("article", jsonBuilder().startObject().startObject("article").startObject("properties") .startObject("comments") @@ -617,12 +650,16 @@ public class InnerHitsIT extends ESIntegTestCase { .endObject())); indexRandom(true, requests); + InnerHitsBuilder innerInnerHitsBuilder = new InnerHitsBuilder(); + innerInnerHitsBuilder.addNestedInnerHits("remark", "comments.remarks", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.remarks.message", "good"))); + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addNestedInnerHits("comment", "comments", new InnerHitsBuilder.InnerHit() + .setQuery(nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "good"))) + .innerHits(innerInnerHitsBuilder) + ); SearchResponse response = client().prepareSearch("articles") .setQuery(nestedQuery("comments", nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "good")))) - .addNestedInnerHits("comment", "comments", new InnerHitsBuilder.InnerHit() - .setQuery(nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "good"))) - .addNestedInnerHits("remark", "comments.remarks", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.remarks.message", "good"))) - ).get(); + .innerHits(innerHitsBuilder).get(); assertNoFailures(response); assertHitCount(response, 1); assertSearchHit(response, 1, hasId("1")); @@ -659,11 +696,15 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("remarks")); assertThat(innerHits.getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0)); + innerInnerHitsBuilder = new InnerHitsBuilder(); + innerInnerHitsBuilder.addNestedInnerHits("remark", "comments.remarks", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.remarks.message", "bad"))); + innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addNestedInnerHits("comment", "comments", new InnerHitsBuilder.InnerHit() + .setQuery(nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad"))) + .innerHits(innerInnerHitsBuilder)); response = client().prepareSearch("articles") .setQuery(nestedQuery("comments", nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad")))) - .addNestedInnerHits("comment", "comments", new InnerHitsBuilder.InnerHit() - .setQuery(nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad"))) - .addNestedInnerHits("remark", "comments.remarks", new InnerHitsBuilder.InnerHit().setQuery(matchQuery("comments.remarks.message", "bad")))) + .innerHits(innerHitsBuilder) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -685,8 +726,7 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerHits.getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0)); } - @Test - // https://github.com/elasticsearch/elasticsearch/issues/9723 + // Issue #9723 public void testNestedDefinedAsObject() throws Exception { assertAcked(prepareCreate("articles").addMapping("article", "comments", "type=nested", "title", "type=string")); @@ -710,7 +750,6 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getChild(), nullValue()); } - @Test public void testNestedInnerHitsWithStoredFieldsAndNoSourceBackcompat() throws Exception { assertAcked(prepareCreate("articles") .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) @@ -746,10 +785,9 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getOffset(), equalTo(0)); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getChild(), nullValue()); - assertThat(String.valueOf((Object)response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).fields().get("comments.message").getValue()), equalTo("fox eat quick")); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).fields().get("comments.message").getValue(), equalTo("fox eat quick")); } - @Test public void testNestedInnerHitsWithHighlightOnStoredFieldBackcompat() throws Exception { assertAcked(prepareCreate("articles") .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) @@ -774,7 +812,7 @@ public class InnerHitsIT extends ESIntegTestCase { .endObject())); indexRandom(true, requests); InnerHitsBuilder.InnerHit builder = new InnerHitsBuilder.InnerHit(); - builder.highlightBuilder().field("comments.message"); + builder.highlighter(new HighlightBuilder().field("comments.message")); SearchResponse response = client().prepareSearch("articles") .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox")).innerHit(new QueryInnerHits(null, builder))) .get(); @@ -789,7 +827,6 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(String.valueOf(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).highlightFields().get("comments.message").getFragments()[0]), equalTo("fox eat quick")); } - @Test public void testNestedInnerHitsWithExcludeSourceBackcompat() throws Exception { assertAcked(prepareCreate("articles").setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) .addMapping("article", jsonBuilder().startObject() @@ -826,10 +863,9 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getField().string(), equalTo("comments")); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getOffset(), equalTo(0)); assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getChild(), nullValue()); - assertThat(String.valueOf((Object)response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).fields().get("comments.message").getValue()), equalTo("fox eat quick")); + assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).fields().get("comments.message").getValue(), equalTo("fox eat quick")); } - @Test public void testNestedInnerHitsHiglightWithExcludeSourceBackcompat() throws Exception { assertAcked(prepareCreate("articles").setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) .addMapping("article", jsonBuilder().startObject() @@ -853,7 +889,7 @@ public class InnerHitsIT extends ESIntegTestCase { .endObject())); indexRandom(true, requests); InnerHitsBuilder.InnerHit builder = new InnerHitsBuilder.InnerHit(); - builder.highlightBuilder().field("comments.message"); + builder.highlighter(new HighlightBuilder().field("comments.message")); SearchResponse response = client().prepareSearch("articles") .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox")).innerHit(new QueryInnerHits(null, builder))) .get(); @@ -868,7 +904,6 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(String.valueOf(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).highlightFields().get("comments.message").getFragments()[0]), equalTo("fox eat quick")); } - @Test public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception { assertAcked(prepareCreate("articles") .addMapping("article", jsonBuilder().startObject() @@ -940,7 +975,6 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).getInnerHits().get("comments.messages").getAt(0).getNestedIdentity().getChild(), nullValue()); } - @Test public void testRoyals() throws Exception { assertAcked( prepareCreate("royals") @@ -965,17 +999,23 @@ public class InnerHitsIT extends ESIntegTestCase { requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king").setSource("{}")); indexRandom(true, requests); - SearchResponse response = client().prepareSearch("royals") - .setTypes("duke") - .addParentChildInnerHits("earls", "earl", new InnerHitsBuilder.InnerHit() + InnerHitsBuilder innerInnerHitsBuilder = new InnerHitsBuilder(); + innerInnerHitsBuilder.addParentChildInnerHits("barons", "baron", new InnerHitsBuilder.InnerHit()); + InnerHitsBuilder innerHitsBuilder = new InnerHitsBuilder(); + innerHitsBuilder.addParentChildInnerHits("earls", "earl", new InnerHitsBuilder.InnerHit() .addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC)) .setSize(4) - .addParentChildInnerHits("barons", "baron", new InnerHitsBuilder.InnerHit()) - ) - .addParentChildInnerHits("princes", "prince", + .innerHits(innerInnerHitsBuilder) + ); + innerInnerHitsBuilder = new InnerHitsBuilder(); + innerInnerHitsBuilder.addParentChildInnerHits("kings", "king", new InnerHitsBuilder.InnerHit()); + innerHitsBuilder.addParentChildInnerHits("princes", "prince", new InnerHitsBuilder.InnerHit() - .addParentChildInnerHits("kings", "king", new InnerHitsBuilder.InnerHit()) - ) + .innerHits(innerInnerHitsBuilder) + ); + SearchResponse response = client().prepareSearch("royals") + .setTypes("duke") + .innerHits(innerHitsBuilder) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("duke")); @@ -1012,8 +1052,7 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(innerInnerHits.getAt(0).getId(), equalTo("king")); } - @Test - public void matchesQueries_nestedInnerHits() throws Exception { + public void testMatchesQueriesNestedInnerHits() throws Exception { XContentBuilder builder = jsonBuilder().startObject() .startObject("type1") .startObject("properties") @@ -1110,8 +1149,7 @@ public class InnerHitsIT extends ESIntegTestCase { } } - @Test - public void matchesQueries_parentChildInnerHits() throws Exception { + public void testMatchesQueriesParentChildInnerHits() throws Exception { assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent")); List requests = new ArrayList<>(); requests.add(client().prepareIndex("index", "parent", "1").setSource("{}")); @@ -1147,7 +1185,6 @@ public class InnerHitsIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2")); } - @Test public void testDontExplode() throws Exception { assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent")); List requests = new ArrayList<>(); diff --git a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java index acd36e124e5..d8c16282e18 100644 --- a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java +++ b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java @@ -24,9 +24,17 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.indicesQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.elasticsearch.index.query.QueryBuilders.wrapperQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItemInArray; @@ -35,9 +43,7 @@ import static org.hamcrest.Matchers.hasItemInArray; * */ public class MatchedQueriesIT extends ESIntegTestCase { - - @Test - public void simpleMatchedQueryFromFilteredQuery() throws Exception { + public void testSimpleMatchedQueryFromFilteredQuery() throws Exception { createIndex("test"); ensureGreen(); @@ -77,8 +83,7 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test - public void simpleMatchedQueryFromTopLevelFilter() throws Exception { + public void testSimpleMatchedQueryFromTopLevelFilter() throws Exception { createIndex("test"); ensureGreen(); @@ -127,8 +132,7 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test - public void simpleMatchedQueryFromTopLevelFilterAndFilteredQuery() throws Exception { + public void testSimpleMatchedQueryFromTopLevelFilterAndFilteredQuery() throws Exception { createIndex("test"); ensureGreen(); @@ -166,7 +170,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test public void testIndicesFilterSupportsName() { createIndex("test1", "test2"); ensureGreen(); @@ -205,7 +208,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test public void testRegExpQuerySupportsName() { createIndex("test1"); ensureGreen(); @@ -227,7 +229,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test public void testPrefixQuerySupportsName() { createIndex("test1"); ensureGreen(); @@ -249,7 +250,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test public void testFuzzyQuerySupportsName() { createIndex("test1"); ensureGreen(); @@ -271,7 +271,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test public void testWildcardQuerySupportsName() { createIndex("test1"); ensureGreen(); @@ -293,7 +292,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test public void testSpanFirstQuerySupportsName() { createIndex("test1"); ensureGreen(); @@ -318,7 +316,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { /** * Test case for issue #4361: https://github.com/elasticsearch/elasticsearch/issues/4361 */ - @Test public void testMatchedWithShould() throws Exception { createIndex("test"); ensureGreen(); @@ -355,7 +352,6 @@ public class MatchedQueriesIT extends ESIntegTestCase { } } - @Test public void testMatchedWithWrapperQuery() throws Exception { createIndex("test"); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java index 0e7001555fe..f281eb3281f 100644 --- a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java +++ b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.morelikethis; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; @@ -32,21 +32,28 @@ import org.elasticsearch.index.query.MoreLikeThisQueryBuilder; import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; -import static org.elasticsearch.index.query.MoreLikeThisQueryBuilder.ids; -import static org.elasticsearch.client.Requests.*; +import static org.elasticsearch.client.Requests.indexAliasesRequest; +import static org.elasticsearch.client.Requests.indexRequest; +import static org.elasticsearch.client.Requests.refreshRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.MoreLikeThisQueryBuilder.ids; import static org.elasticsearch.index.query.QueryBuilders.moreLikeThisQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -54,8 +61,6 @@ import static org.hamcrest.Matchers.notNullValue; * */ public class MoreLikeThisIT extends ESIntegTestCase { - - @Test public void testSimpleMoreLikeThis() throws Exception { logger.info("Creating index test"); assertAcked(prepareCreate("test").addMapping("type1", @@ -77,7 +82,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertHitCount(response, 1l); } - @Test public void testSimpleMoreLikeOnLongField() throws Exception { logger.info("Creating index test"); assertAcked(prepareCreate("test").addMapping("type1", "some_long", "type=long")); @@ -97,7 +101,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertHitCount(response, 0l); } - @Test public void testMoreLikeThisWithAliases() throws Exception { logger.info("Creating index test"); assertAcked(prepareCreate("test").addMapping("type1", @@ -142,7 +145,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertThat(response.getHits().getAt(0).id(), equalTo("3")); } - @Test public void testMoreLikeThisIssue2197() throws Exception { Client client = client(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("bar") @@ -166,8 +168,7 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertThat(response, notNullValue()); } - @Test - // See: https://github.com/elasticsearch/elasticsearch/issues/2489 + // Issue #2489 public void testMoreLikeWithCustomRouting() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("bar") .startObject("properties") @@ -188,8 +189,7 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertThat(response, notNullValue()); } - @Test - // See issue: https://github.com/elasticsearch/elasticsearch/issues/3039 + // Issue #3039 public void testMoreLikeThisIssueRoutingNotSerialized() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("bar") .startObject("properties") @@ -211,8 +211,7 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertThat(response, notNullValue()); } - @Test - // See issue https://github.com/elasticsearch/elasticsearch/issues/3252 + // Issue #3252 public void testNumericField() throws Exception { final String[] numericTypes = new String[]{"byte", "short", "integer", "long"}; prepareCreate("test").addMapping("type", jsonBuilder() @@ -272,7 +271,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test public void testSimpleMoreLikeInclude() throws Exception { logger.info("Creating index test"); assertAcked(prepareCreate("test").addMapping("type1", @@ -332,7 +330,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertHitCount(mltResponse, 3l); } - @Test public void testSimpleMoreLikeThisIdsMultipleTypes() throws Exception { logger.info("Creating index test"); int numOfTypes = randomIntBetween(2, 10); @@ -365,7 +362,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertHitCount(mltResponse, numOfTypes); } - @Test public void testMoreLikeThisMultiValueFields() throws Exception { logger.info("Creating the index ..."); assertAcked(prepareCreate("test") @@ -398,7 +394,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { } } - @Test public void testMinimumShouldMatch() throws ExecutionException, InterruptedException { logger.info("Creating the index ..."); assertAcked(prepareCreate("test") @@ -436,7 +431,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { } } - @Test public void testMoreLikeThisArtificialDocs() throws Exception { int numFields = randomIntBetween(5, 10); @@ -463,7 +457,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertHitCount(response, 1); } - @Test public void testMoreLikeThisMalformedArtificialDocs() throws Exception { logger.info("Creating the index ..."); assertAcked(prepareCreate("test") @@ -530,7 +523,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { assertHitCount(response, 1); } - @Test public void testMoreLikeThisUnlike() throws ExecutionException, InterruptedException, IOException { createIndex("test"); ensureGreen(); @@ -578,7 +570,6 @@ public class MoreLikeThisIT extends ESIntegTestCase { } } - @Test public void testSelectFields() throws IOException, ExecutionException, InterruptedException { assertAcked(prepareCreate("test") .addMapping("type1", "text", "type=string,analyzer=whitespace", "text1", "type=string,analyzer=whitespace")); diff --git a/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java index 53f7e615adf..c7454a5c8b6 100644 --- a/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java @@ -22,17 +22,17 @@ package org.elasticsearch.search.msearch; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; import static org.hamcrest.Matchers.equalTo; /** */ public class SimpleMultiSearchIT extends ESIntegTestCase { - - @Test - public void simpleMultiSearch() { + public void testSimpleMultiSearch() { createIndex("test"); ensureGreen(); client().prepareIndex("test", "type", "1").setSource("field", "xxx").execute().actionGet(); @@ -43,7 +43,7 @@ public class SimpleMultiSearchIT extends ESIntegTestCase { .add(client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "yyy"))) .add(client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery())) .execute().actionGet(); - + for (MultiSearchResponse.Item item : response) { assertNoFailures(item.getResponse()); } diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java index 23f6e6f6c61..39d052400cb 100644 --- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.nested; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.delete.DeleteResponse; @@ -36,18 +36,24 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; public class SimpleNestedIT extends ESIntegTestCase { - - @Test - public void simpleNested() throws Exception { + public void testSimpleNested() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", "nested1", "type=nested").addMapping("type2", "nested1", "type=nested")); ensureGreen(); @@ -156,8 +162,7 @@ public class SimpleNestedIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); } - @Test - public void multiNested() throws Exception { + public void testMultiNested() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("nested1") @@ -226,11 +231,9 @@ public class SimpleNestedIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(0l)); } - @Test // When IncludeNestedDocsQuery is wrapped in a FilteredQuery then a in-finite loop occurs b/c of a bug in IncludeNestedDocsQuery#advance() // This IncludeNestedDocsQuery also needs to be aware of the filter from alias public void testDeleteNestedDocsWithAlias() throws Exception { - assertAcked(prepareCreate("test") .setSettings(settingsBuilder().put(indexSettings()).put("index.referesh_interval", -1).build()) .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -282,9 +285,7 @@ public class SimpleNestedIT extends ESIntegTestCase { assertDocumentCount("test", 6); } - @Test public void testExplain() throws Exception { - assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("nested1") @@ -325,7 +326,6 @@ public class SimpleNestedIT extends ESIntegTestCase { // assertThat(explanation.getDetails()[1].getDescription(), equalTo("Child[1]")); } - @Test public void testSimpleNestedSorting() throws Exception { assertAcked(prepareCreate("test") .setSettings(settingsBuilder() @@ -408,9 +408,7 @@ public class SimpleNestedIT extends ESIntegTestCase { assertThat(searchResponse.getHits().hits()[2].sortValues()[0].toString(), equalTo("2")); } - - @Test - public void testSimpleNestedSorting_withNestedFilterMissing() throws Exception { + public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception { assertAcked(prepareCreate("test") .setSettings(settingsBuilder() .put(indexSettings()) @@ -510,7 +508,6 @@ public class SimpleNestedIT extends ESIntegTestCase { client().prepareClearScroll().addScrollId("_all").get(); } - @Test public void testSortNestedWithNestedFilter() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", XContentFactory.jsonBuilder().startObject() @@ -867,8 +864,7 @@ public class SimpleNestedIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3")); } - @Test - // https://github.com/elasticsearch/elasticsearch/issues/9305 + // Issue #9305 public void testNestedSortingWithNestedFilterAsFilter() throws Exception { assertAcked(prepareCreate("test").addMapping("type", jsonBuilder().startObject().startObject("properties") .startObject("officelocation").field("type", "string").endObject() @@ -1004,7 +1000,6 @@ public class SimpleNestedIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(1).sortValues()[1].toString(), equalTo("fname3")); } - @Test public void testCheckFixedBitSetCache() throws Exception { boolean loadFixedBitSeLazily = randomBoolean(); Settings.Builder settingsBuilder = Settings.builder().put(indexSettings()) diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index a9cd6de06ae..93e94c49b47 100644 --- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -19,12 +19,11 @@ package org.elasticsearch.search.preference; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; @@ -32,12 +31,14 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SearchPreferenceIT extends ESIntegTestCase { - - @Test // see #2896 + // see #2896 public void testStopOneNodePreferenceWithRedState() throws InterruptedException, IOException { assertAcked(prepareCreate("test").setSettings(settingsBuilder().put("index.number_of_shards", cluster().numDataNodes()+2).put("index.number_of_replicas", 0))); ensureGreen(); @@ -67,14 +68,13 @@ public class SearchPreferenceIT extends ESIntegTestCase { assertThat("_only_local", searchResponse.getFailedShards(), greaterThanOrEqualTo(0)); } - @Test - public void noPreferenceRandom() throws Exception { + public void testNoPreferenceRandom() throws Exception { assertAcked(prepareCreate("test").setSettings( //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data settingsBuilder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas())) )); ensureGreen(); - + client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet(); refresh(); @@ -87,8 +87,7 @@ public class SearchPreferenceIT extends ESIntegTestCase { assertThat(firstNodeId, not(equalTo(secondNodeId))); } - @Test - public void simplePreferenceTests() throws Exception { + public void testSimplePreference() throws Exception { client().admin().indices().prepareCreate("test").setSettings("number_of_replicas=1").get(); ensureGreen(); @@ -121,7 +120,6 @@ public class SearchPreferenceIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); } - @Test public void testReplicaPreference() throws Exception { client().admin().indices().prepareCreate("test").setSettings("number_of_replicas=0").get(); ensureGreen(); @@ -146,11 +144,15 @@ public class SearchPreferenceIT extends ESIntegTestCase { assertThat(resp.getHits().totalHits(), equalTo(1l)); } - @Test (expected = IllegalArgumentException.class) public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Exception { createIndex("test"); ensureGreen(); - client().prepareSearch().setQuery(matchAllQuery()).setPreference("_only_node:DOES-NOT-EXIST").execute().actionGet(); + try { + client().prepareSearch().setQuery(matchAllQuery()).setPreference("_only_node:DOES-NOT-EXIST").execute().actionGet(); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("No data node with id[DOES-NOT-EXIST] found")); + } } } diff --git a/core/src/test/java/org/elasticsearch/search/query/ExistsMissingIT.java b/core/src/test/java/org/elasticsearch/search/query/ExistsIT.java similarity index 69% rename from core/src/test/java/org/elasticsearch/search/query/ExistsMissingIT.java rename to core/src/test/java/org/elasticsearch/search/query/ExistsIT.java index 349197d5f48..73906b2ed83 100644 --- a/core/src/test/java/org/elasticsearch/search/query/ExistsMissingIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/ExistsIT.java @@ -43,7 +43,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -public class ExistsMissingIT extends ESIntegTestCase { +public class ExistsIT extends ESIntegTestCase { // TODO: move this to a unit test somewhere... public void testEmptyIndex() throws Exception { @@ -51,11 +51,11 @@ public class ExistsMissingIT extends ESIntegTestCase { ensureYellow("test"); SearchResponse resp = client().prepareSearch("test").setQuery(QueryBuilders.existsQuery("foo")).execute().actionGet(); assertSearchResponse(resp); - resp = client().prepareSearch("test").setQuery(QueryBuilders.missingQuery("foo")).execute().actionGet(); + resp = client().prepareSearch("test").setQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("foo"))).execute().actionGet(); assertSearchResponse(resp); } - public void testExistsMissing() throws Exception { + public void testExists() throws Exception { XContentBuilder mapping = XContentBuilder.builder(JsonXContent.jsonXContent) .startObject() .startObject("type") @@ -145,62 +145,6 @@ public class ExistsMissingIT extends ESIntegTestCase { } throw e; } - - // missing - resp = client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery(fieldName)).execute().actionGet(); - assertSearchResponse(resp); - assertEquals(String.format(Locale.ROOT, "missing(%s, %d) mapping: %s response: %s", fieldName, count, mapping.string(), resp), numDocs - count, resp.getHits().totalHits()); } } - - public void testNullValueUnset() throws Exception { - assertAcked(client().admin().indices().prepareCreate("idx").addMapping("type", "f", "type=string,index=not_analyzed")); - indexRandom(true, - client().prepareIndex("idx", "type", "1").setSource("f", "foo"), - client().prepareIndex("idx", "type", "2").setSource("f", null), - client().prepareIndex("idx", "type", "3").setSource("g", "bar"), - client().prepareIndex("idx", "type", "4").setSource("f", "bar")); - - SearchResponse resp = client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", true, true)).get(); - assertSearchHits(resp, "2", "3"); - - resp = client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", false, true)).get(); - assertSearchHits(resp, "2", "3"); - - resp = client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", true, false)).get(); - assertSearchHits(resp); - - try { - client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", false, false)).get(); - fail("both existence and null_value can't be false"); - } catch (IllegalArgumentException e) { - // expected - } - } - - public void testNullValueSet() throws Exception { - assertAcked(client().admin().indices().prepareCreate("idx").addMapping("type", "f", "type=string,index=not_analyzed,null_value=bar")); - indexRandom(true, - client().prepareIndex("idx", "type", "1").setSource("f", "foo"), - client().prepareIndex("idx", "type", "2").setSource("f", null), - client().prepareIndex("idx", "type", "3").setSource("g", "bar"), - client().prepareIndex("idx", "type", "4").setSource("f", "bar")); - - SearchResponse resp = client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", true, true)).get(); - assertSearchHits(resp, "2", "3", "4"); - - resp = client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", false, true)).get(); - assertSearchHits(resp, "3"); - - resp = client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", true, false)).get(); - assertSearchHits(resp, "2", "4"); - - try { - client().prepareSearch("idx").setQuery(QueryBuilders.missingQuery("f", false, false)).get(); - fail("both existence and null_value can't be false"); - } catch (IllegalArgumentException e) { - // expected - } - } - } diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java index 3110d78820c..235438cc442 100644 --- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java @@ -19,16 +19,17 @@ package org.elasticsearch.search.query; import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.*; + import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.Operator; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.search.MatchQuery; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -36,12 +37,9 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; -import org.junit.Test; import java.io.IOException; -import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -170,7 +168,6 @@ public class MultiMatchQueryIT extends ESIntegTestCase { .endObject().endObject(); } - @Test public void testDefaults() throws ExecutionException, InterruptedException { MatchQuery.Type type = randomBoolean() ? MatchQueryBuilder.DEFAULT_TYPE : MatchQuery.Type.BOOLEAN; SearchResponse searchResponse = client().prepareSearch("test") @@ -210,7 +207,6 @@ public class MultiMatchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("theone")); } - @Test public void testPhraseType() { SearchResponse searchResponse = client().prepareSearch("test") .setQuery(randomizeType(multiMatchQuery("Man the Ultimate", "full_name_phrase", "first_name_phrase", "last_name_phrase", "category_phrase") @@ -230,7 +226,6 @@ public class MultiMatchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 2l); } - @Test public void testSingleField() throws NoSuchFieldException, IllegalAccessException { SearchResponse searchResponse = client().prepareSearch("test") .setQuery(randomizeType(multiMatchQuery("15", "skill"))).get(); @@ -276,10 +271,9 @@ public class MultiMatchQueryIT extends ESIntegTestCase { } - @Test public void testCutoffFreq() throws ExecutionException, InterruptedException { - final long numDocs = client().prepareCount("test") - .setQuery(matchAllQuery()).get().getCount(); + final long numDocs = client().prepareSearch("test").setSize(0) + .setQuery(matchAllQuery()).get().getHits().totalHits(); MatchQuery.Type type = randomBoolean() ? MatchQueryBuilder.DEFAULT_TYPE : MatchQuery.Type.BOOLEAN; Float cutoffFrequency = randomBoolean() ? Math.min(1, numDocs * 1.f / between(10, 20)) : 1.f / between(10, 20); SearchResponse searchResponse = client().prepareSearch("test") @@ -336,12 +330,10 @@ public class MultiMatchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("theother")); } - - @Test public void testEquivalence() { - final int numDocs = (int) client().prepareCount("test") - .setQuery(matchAllQuery()).get().getCount(); + final int numDocs = (int) client().prepareSearch("test").setSize(0) + .setQuery(matchAllQuery()).get().getHits().totalHits(); int numIters = scaledRandomIntBetween(5, 10); for (int i = 0; i < numIters; i++) { { @@ -432,7 +424,6 @@ public class MultiMatchQueryIT extends ESIntegTestCase { } } - @Test public void testCrossFieldMode() throws ExecutionException, InterruptedException { SearchResponse searchResponse = client().prepareSearch("test") .setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name") diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java similarity index 94% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java rename to core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 008a83e4c9b..9918d449657 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -32,7 +32,13 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.Operator; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.WrapperQueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.index.search.MatchQuery; import org.elasticsearch.index.search.MatchQuery.Type; @@ -45,7 +51,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.ISODateTimeFormat; -import org.junit.Test; import java.io.IOException; import java.util.Random; @@ -54,10 +59,52 @@ import java.util.concurrent.ExecutionException; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.fuzzyQuery; +import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery; +import static org.elasticsearch.index.query.QueryBuilders.idsQuery; +import static org.elasticsearch.index.query.QueryBuilders.indicesQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; +import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.regexpQuery; +import static org.elasticsearch.index.query.QueryBuilders.spanMultiTermQueryBuilder; +import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery; +import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery; +import static org.elasticsearch.index.query.QueryBuilders.spanOrQuery; +import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsLookupQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.elasticsearch.index.query.QueryBuilders.typeQuery; +import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; +import static org.elasticsearch.index.query.QueryBuilders.wrapperQuery; import static org.elasticsearch.test.VersionUtils.randomVersion; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThirdHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; public class SearchQueryIT extends ESIntegTestCase { @@ -71,7 +118,6 @@ public class SearchQueryIT extends ESIntegTestCase { return Math.min(2, cluster().numDataNodes() - 1); } - @Test public void testOmitNormsOnAll() throws ExecutionException, InterruptedException, IOException { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1") @@ -103,7 +149,7 @@ public class SearchQueryIT extends ESIntegTestCase { } - @Test // see #3952 + // see #3952 public void testEmptyQueryString() throws ExecutionException, InterruptedException, IOException { createIndex("test"); indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox jumps"), @@ -114,7 +160,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setQuery(queryStringQuery("")).get(), 0l); // return no docs } - @Test // see https://github.com/elasticsearch/elasticsearch/issues/3177 + // see https://github.com/elasticsearch/elasticsearch/issues/3177 public void testIssue3177() { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get(); @@ -122,7 +168,7 @@ public class SearchQueryIT extends ESIntegTestCase { client().prepareIndex("test", "type1", "3").setSource("field1", "value3").get(); ensureGreen(); waitForRelocation(); - optimize(); + forceMerge(); refresh(); assertHitCount( client().prepareSearch() @@ -130,7 +176,7 @@ public class SearchQueryIT extends ESIntegTestCase { .setPostFilter( boolQuery().must( matchAllQuery()).must( - notQuery(boolQuery().must(termQuery("field1", "value1")).must( + boolQuery().mustNot(boolQuery().must(termQuery("field1", "value1")).must( termQuery("field1", "value2"))))).get(), 3l); assertHitCount( @@ -139,24 +185,14 @@ public class SearchQueryIT extends ESIntegTestCase { boolQuery().must( boolQuery().should(termQuery("field1", "value1")).should(termQuery("field1", "value2")) .should(termQuery("field1", "value3"))).filter( - notQuery(boolQuery().must(termQuery("field1", "value1")).must( + boolQuery().mustNot(boolQuery().must(termQuery("field1", "value1")).must( termQuery("field1", "value2"))))).get(), 3l); assertHitCount( - client().prepareSearch().setQuery(matchAllQuery()).setPostFilter(notQuery(termQuery("field1", "value3"))).get(), + client().prepareSearch().setQuery(matchAllQuery()).setPostFilter(boolQuery().mustNot(termQuery("field1", "value3"))).get(), 2l); } - @Test - public void passQueryAsStringTest() throws Exception { - createIndex("test"); - client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").setRefresh(true).get(); - - SearchResponse searchResponse = client().prepareSearch().setQuery("{ \"term\" : { \"field1\" : \"value1_1\" }}").get(); - assertHitCount(searchResponse, 1l); - } - - @Test public void testIndexOptions() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "field1", "type=string,index_options=docs")); @@ -172,7 +208,7 @@ public class SearchQueryIT extends ESIntegTestCase { containsString("field \"field1\" was indexed without position data; cannot run PhraseQuery")); } - @Test // see #3521 + // see #3521 public void testConstantScoreQuery() throws Exception { Random random = getRandom(); createIndex("test"); @@ -206,7 +242,7 @@ public class SearchQueryIT extends ESIntegTestCase { int num = scaledRandomIntBetween(100, 200); IndexRequestBuilder[] builders = new IndexRequestBuilder[num]; for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex("test", "type", "" + i).setSource("f", English.intToEnglish(i)); + builders[i] = client().prepareIndex("test_1", "type", "" + i).setSource("f", English.intToEnglish(i)); } createIndex("test_1"); indexRandom(true, builders); @@ -214,7 +250,7 @@ public class SearchQueryIT extends ESIntegTestCase { int queryRounds = scaledRandomIntBetween(10, 20); for (int i = 0; i < queryRounds; i++) { MatchQueryBuilder matchQuery = matchQuery("f", English.intToEnglish(between(0, num))); - searchResponse = client().prepareSearch("test_1").setQuery(matchQuery).setSize(num).get(); + searchResponse = client().prepareSearch("test_1").setQuery(constantScoreQuery(matchQuery)).setSize(num).get(); long totalHits = searchResponse.getHits().totalHits(); SearchHits hits = searchResponse.getHits(); for (SearchHit searchHit : hits) { @@ -234,7 +270,7 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test // see #3521 + // see #3521 public void testAllDocsQueryString() throws InterruptedException, ExecutionException { createIndex("test"); indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("foo", "bar"), @@ -254,7 +290,6 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test public void testCommonTermsQueryOnAllField() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("type1", "message", "type=string", "comment", "type=string,boost=5.0") @@ -269,7 +304,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertThat(searchResponse.getHits().getHits()[0].getScore(), greaterThan(searchResponse.getHits().getHits()[1].getScore())); } - @Test public void testCommonTermsQuery() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("type1", "field1", "type=string,analyzer=whitespace") @@ -311,10 +345,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); assertFirstHit(searchResponse, hasId("2")); - searchResponse = client().prepareSearch().setQuery("{ \"common\" : { \"field1\" : { \"query\" : \"the lazy fox brown\", \"cutoff_frequency\" : 1, \"minimum_should_match\" : { \"high_freq\" : 4 } } } }").get(); - assertHitCount(searchResponse, 1l); - assertFirstHit(searchResponse, hasId("2")); - // Default searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the lazy fox brown").cutoffFrequency(1)).get(); assertHitCount(searchResponse, 1l); @@ -354,7 +384,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertThirdHit(searchResponse, hasId("2")); } - @Test public void testCommonTermsQueryStackedTokens() throws Exception { assertAcked(prepareCreate("test") .setSettings(settingsBuilder() @@ -403,10 +432,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); assertFirstHit(searchResponse, hasId("2")); - searchResponse = client().prepareSearch().setQuery("{ \"common\" : { \"field1\" : { \"query\" : \"the fast lazy fox brown\", \"cutoff_frequency\" : 1, \"minimum_should_match\" : { \"high_freq\" : 6 } } } }").get(); - assertHitCount(searchResponse, 1l); - assertFirstHit(searchResponse, hasId("2")); - // Default searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the fast lazy fox brown").cutoffFrequency(1)).get(); assertHitCount(searchResponse, 1l); @@ -451,7 +476,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertThirdHit(searchResponse, hasId("2")); } - @Test public void testOmitTermFreqsAndPositions() throws Exception { cluster().wipeTemplates(); // no randomized template for this test -- we are testing bwc compat and set version explicitly this might cause failures if an unsupported feature // is added randomly via an index template. @@ -485,8 +509,7 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test - public void queryStringAnalyzedWildcard() throws Exception { + public void testQueryStringAnalyzedWildcard() throws Exception { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("field1", "value_1", "field2", "value_2").get(); @@ -508,7 +531,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); } - @Test public void testLowercaseExpandedTerms() { createIndex("test"); @@ -529,7 +551,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test //https://github.com/elasticsearch/elasticsearch/issues/3540 + // Issue #3540 public void testDateRangeInQueryString() { //the mapping needs to be provided upfront otherwise we are not sure how many failures we get back //as with dynamic mappings some shards might be lacking behind and parse a different query @@ -557,7 +579,7 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test // https://github.com/elasticsearch/elasticsearch/issues/7880 + // Issue #7880 public void testDateRangeInQueryStringWithTimeZone_7880() { //the mapping needs to be provided upfront otherwise we are not sure how many failures we get back //as with dynamic mappings some shards might be lacking behind and parse a different query @@ -576,7 +598,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); } - @Test // https://github.com/elasticsearch/elasticsearch/issues/10477 + // Issue #10477 public void testDateRangeInQueryStringWithTimeZone_10477() { //the mapping needs to be provided upfront otherwise we are not sure how many failures we get back //as with dynamic mappings some shards might be lacking behind and parse a different query @@ -613,13 +635,11 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test - public void typeFilterTypeIndexedTests() throws Exception { + public void testTypeFilterTypeIndexedTests() throws Exception { typeFilterTests("not_analyzed"); } - @Test - public void typeFilterTypeNotIndexedTests() throws Exception { + public void testTypeFilterTypeNotIndexedTests() throws Exception { typeFilterTests("no"); } @@ -648,13 +668,11 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setTypes("type1", "type2").setQuery(matchAllQuery()).get(), 5l); } - @Test - public void idsQueryTestsIdIndexed() throws Exception { + public void testIdsQueryTestsIdIndexed() throws Exception { idsQueryTests("not_analyzed"); } - @Test - public void idsQueryTestsIdNotIndexed() throws Exception { + public void testIdsQueryTestsIdNotIndexed() throws Exception { idsQueryTests("no"); } @@ -696,17 +714,15 @@ public class SearchQueryIT extends ESIntegTestCase { assertSearchHits(searchResponse, "1", "3"); } - @Test - public void term_indexQueryTestsIndexed() throws Exception { - term_indexQueryTests("not_analyzed"); + public void testTermIndexQueryIndexed() throws Exception { + termIndexQueryTests("not_analyzed"); } - @Test - public void term_indexQueryTestsNotIndexed() throws Exception { - term_indexQueryTests("no"); + public void testTermIndexQueryNotIndexed() throws Exception { + termIndexQueryTests("no"); } - private void term_indexQueryTests(String index) throws Exception { + private void termIndexQueryTests(String index) throws Exception { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); String[] indexNames = { "test1", "test2" }; for (String indexName : indexNames) { @@ -749,8 +765,7 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test - public void filterExistsMissingTests() throws Exception { + public void testFilterExistsMissing() throws Exception { createIndex("test"); indexRandom(true, @@ -789,36 +804,9 @@ public class SearchQueryIT extends ESIntegTestCase { searchResponse = client().prepareSearch().setQuery(existsQuery("obj1")).get(); assertHitCount(searchResponse, 2l); assertSearchHits(searchResponse, "1", "2"); - - searchResponse = client().prepareSearch().setQuery(missingQuery("field1")).get(); - assertHitCount(searchResponse, 2l); - assertSearchHits(searchResponse, "3", "4"); - - searchResponse = client().prepareSearch().setQuery(missingQuery("field1")).get(); - assertHitCount(searchResponse, 2l); - assertSearchHits(searchResponse, "3", "4"); - - searchResponse = client().prepareSearch().setQuery(constantScoreQuery(missingQuery("field1"))).get(); - assertHitCount(searchResponse, 2l); - assertSearchHits(searchResponse, "3", "4"); - - searchResponse = client().prepareSearch().setQuery(queryStringQuery("_missing_:field1")).get(); - assertHitCount(searchResponse, 2l); - assertSearchHits(searchResponse, "3", "4"); - - // wildcard check - searchResponse = client().prepareSearch().setQuery(missingQuery("x*")).get(); - assertHitCount(searchResponse, 2l); - assertSearchHits(searchResponse, "3", "4"); - - // object check - searchResponse = client().prepareSearch().setQuery(missingQuery("obj1")).get(); - assertHitCount(searchResponse, 2l); - assertSearchHits(searchResponse, "3", "4"); } - @Test - public void passQueryOrFilterAsJSONStringTest() throws Exception { + public void testPassQueryOrFilterAsJSONString() throws Exception { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").setRefresh(true).get(); @@ -833,7 +821,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setPostFilter(wrapperFilter).get(), 1l); } - @Test public void testFiltersWithCustomCacheKey() throws Exception { createIndex("test"); @@ -852,7 +839,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); } - @Test public void testMatchQueryNumeric() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", "long", "type=long", "double", "type=double")); @@ -875,7 +861,6 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test public void testMultiMatchQuery() throws Exception { createIndex("test"); @@ -939,7 +924,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("1")); } - @Test public void testMatchQueryZeroTermsQuery() { assertAcked(prepareCreate("test") .addMapping("type1", "field1", "type=string,analyzer=classic", "field2", "type=string,analyzer=classic")); @@ -989,7 +973,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 2l); } - @Test public void testMultiMatchQueryMinShouldMatch() { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("field1", new String[]{"value1", "value2", "value3"}).get(); @@ -1035,7 +1018,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("1")); } - @Test public void testFuzzyQueryString() { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("str", "kimchy", "date", "2012-02-01", "num", 12).get(); @@ -1056,7 +1038,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("1")); } - @Test public void testQuotedQueryStringWithBoost() throws InterruptedException, ExecutionException { float boost = 10.0f; assertAcked(prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 1)); @@ -1080,7 +1061,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertThat((double)searchResponse.getHits().getAt(0).score(), closeTo(boost * searchResponse.getHits().getAt(1).score(), .1)); } - @Test public void testSpecialRangeSyntaxInQueryString() { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("str", "kimchy", "date", "2012-02-01", "num", 12).get(); @@ -1111,7 +1091,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); } - @Test public void testEmptytermsQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "term", "type=string")); @@ -1128,7 +1107,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test public void testTermsQuery() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "str", "type=string", "lng", "type=long", "dbl", "type=double")); @@ -1196,7 +1174,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test public void testTermsLookupFilter() throws Exception { assertAcked(prepareCreate("lookup").addMapping("type", "terms","type=string", "other", "type=string")); assertAcked(prepareCreate("lookup2").addMapping("type", @@ -1284,7 +1261,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test public void testBasicQueryById() throws Exception { createIndex("test"); @@ -1317,7 +1293,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertThat(searchResponse.getHits().hits().length, equalTo(2)); } - @Test public void testNumericTermsAndRanges() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", @@ -1417,7 +1392,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("1")); } - @Test public void testNumericRangeFilter_2826() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", @@ -1456,15 +1430,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 2l); } - @Test - public void testEmptyTopLevelFilter() { - client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).get(); - - SearchResponse searchResponse = client().prepareSearch().setPostFilter("{}").get(); - assertHitCount(searchResponse, 1l); - } - - @Test // see #2926 + // see #2926 public void testMustNot() throws IOException, ExecutionException, InterruptedException { assertAcked(prepareCreate("test") //issue manifested only with shards>=2 @@ -1487,7 +1453,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 2l); } - @Test // see #2994 + // see #2994 public void testSimpleSpan() throws IOException, ExecutionException, InterruptedException { createIndex("test"); @@ -1507,7 +1473,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 3l); } - @Test public void testSpanMultiTermQuery() throws IOException { createIndex("test"); @@ -1539,7 +1504,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(response, 3); } - @Test public void testSpanNot() throws IOException, ExecutionException, InterruptedException { createIndex("test"); @@ -1563,7 +1527,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); } - @Test public void testSimpleDFSQuery() throws IOException { assertAcked(prepareCreate("test") .addMapping("s", jsonBuilder() @@ -1618,7 +1581,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertNoFailures(response); } - @Test public void testMultiFieldQueryString() { client().prepareIndex("test", "s", "1").setSource("field1", "value1", "field2", "value2").setRefresh(true).get(); @@ -1640,7 +1602,6 @@ public class SearchQueryIT extends ESIntegTestCase { } // see #3881 - for extensive description of the issue - @Test public void testMatchQueryWithSynonyms() throws IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) @@ -1671,7 +1632,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 2); } - @Test public void testMatchQueryWithStackedStems() throws IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) @@ -1696,7 +1656,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 2); } - @Test public void testQueryStringWithSynonyms() throws IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) @@ -1729,7 +1688,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 2); } - @Test // see https://github.com/elasticsearch/elasticsearch/issues/3898 + // see #3898 public void testCustomWordDelimiterQueryString() { assertAcked(client().admin().indices().prepareCreate("test") .setSettings("analysis.analyzer.my_analyzer.type", "custom", @@ -1756,7 +1715,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(response, 1l); } - @Test // see https://github.com/elasticsearch/elasticsearch/issues/3797 + // see #3797 public void testMultiMatchLenientIssue3797() { createIndex("test"); @@ -1776,14 +1735,12 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); } - @Test public void testAllFieldEmptyMapping() throws Exception { client().prepareIndex("myindex", "mytype").setId("1").setSource("{}").setRefresh(true).get(); SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get(); assertNoFailures(response); } - @Test public void testAllDisabledButQueried() throws Exception { createIndex("myindex"); assertAcked(client().admin().indices().preparePutMapping("myindex").setType("mytype").setSource( @@ -1794,7 +1751,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(response, 0); } - @Test public void testIndicesQuery() throws Exception { createIndex("index1", "index2", "index3"); @@ -1828,7 +1784,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("1")); } - @Test // https://github.com/elasticsearch/elasticsearch/issues/2416 + // See #2416 public void testIndicesQuerySkipParsing() throws Exception { createIndex("simple"); assertAcked(prepareCreate("related") @@ -1860,7 +1816,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertSearchHits(searchResponse, "1", "2"); } - @Test public void testIndicesQueryMissingIndices() throws IOException, ExecutionException, InterruptedException { createIndex("index1"); createIndex("index2"); @@ -1929,7 +1884,6 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test public void testMinScore() throws ExecutionException, InterruptedException { createIndex("test"); @@ -1946,7 +1900,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertSecondHit(searchResponse, hasId("1")); } - @Test public void testQueryStringWithSlopAndFields() { createIndex("test"); @@ -1975,7 +1928,6 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test public void testDateProvidedAsNumber() throws ExecutionException, InterruptedException { createIndex("test"); assertAcked(client().admin().indices().preparePutMapping("test").setType("type").setSource("field", "type=date,format=epoch_millis").get()); @@ -1984,11 +1936,10 @@ public class SearchQueryIT extends ESIntegTestCase { client().prepareIndex("test", "type", "3").setSource("field", -999999999999L)); - assertHitCount(client().prepareCount("test").setQuery(rangeQuery("field").lte(-1000000000000L)).get(), 2); - assertHitCount(client().prepareCount("test").setQuery(rangeQuery("field").lte(-999999999999L)).get(), 3); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-1000000000000L)).get(), 2); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-999999999999L)).get(), 3); } - @Test public void testRangeQueryWithTimeZone() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", "date", "type=date", "num", "type=integer")); @@ -2083,7 +2034,6 @@ public class SearchQueryIT extends ESIntegTestCase { } } - @Test public void testSearchEmptyDoc() { assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}")); client().prepareIndex("test", "type1", "1").setSource("{}").get(); @@ -2092,7 +2042,7 @@ public class SearchQueryIT extends ESIntegTestCase { assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1l); } - @Test // see #5120 + // see #5120 public void testNGramCopyField() { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) @@ -2158,7 +2108,6 @@ public class SearchQueryIT extends ESIntegTestCase { assertSearchHits(searchResponse, "2"); } - @Test public void testQueryStringParserCache() throws Exception { createIndex("test"); indexRandom(true, false, client().prepareIndex("test", "type", "1").setSource("nameTokens", "xyz")); @@ -2184,26 +2133,4 @@ public class SearchQueryIT extends ESIntegTestCase { assertThat(i + " expected: " + first + " actual: " + actual, Float.compare(first, actual), equalTo(0)); } } - - @Test // see #7686. - public void testIdsQueryWithInvalidValues() throws Exception { - createIndex("test"); - indexRandom(true, false, client().prepareIndex("test", "type", "1").setSource("body", "foo")); - - try { - client().prepareSearch("test") - .setTypes("type") - .setQuery("{\n" + - " \"ids\": {\n" + - " \"values\": [[\"1\"]]\n" + - " }\n" + - "}") - .get(); - fail("query is invalid and should have produced a parse exception"); - } catch (Exception e) { - assertThat("query could not be parsed due to bad format: " + e.toString(), - e.toString().contains("Illegal value for id, expecting a string or number, got: START_ARRAY"), - equalTo(true)); - } - } } diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index bf3e458cb5b..358122f54ec 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -21,29 +21,36 @@ package org.elasticsearch.search.query; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.SimpleQueryStringFlag; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Locale; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; import static org.hamcrest.Matchers.equalTo; /** * Tests for the {@code simple_query_string} query */ public class SimpleQueryStringIT extends ESIntegTestCase { - - @Test public void testSimpleQueryString() throws ExecutionException, InterruptedException { createIndex("test"); indexRandom(true, false, @@ -93,7 +100,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertSearchHits(searchResponse, "5", "6"); } - @Test public void testSimpleQueryStringMinimumShouldMatch() throws Exception { createIndex("test"); ensureGreen("test"); @@ -141,7 +147,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertSearchHits(searchResponse, "6", "7", "8"); } - @Test public void testSimpleQueryStringLowercasing() { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("body", "Professional").get(); @@ -165,7 +170,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertHitCount(searchResponse, 0l); } - @Test public void testQueryStringLocale() { createIndex("test"); client().prepareIndex("test", "type1", "1").setSource("body", "bılly").get(); @@ -186,7 +190,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertSearchHits(searchResponse, "1"); } - @Test public void testNestedFieldSimpleQueryString() throws IOException { assertAcked(prepareCreate("test") .addMapping("type1", jsonBuilder() @@ -226,7 +229,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertSearchHits(searchResponse, "1"); } - @Test public void testSimpleQueryStringFlags() throws ExecutionException, InterruptedException { createIndex("test"); indexRandom(true, @@ -242,11 +244,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertHitCount(searchResponse, 3l); assertSearchHits(searchResponse, "1", "2", "3"); - // Sending a negative 'flags' value is the same as SimpleQueryStringFlag.ALL - searchResponse = client().prepareSearch().setQuery("{\"simple_query_string\": {\"query\": \"foo bar\", \"flags\": -1}}").get(); - assertHitCount(searchResponse, 3l); - assertSearchHits(searchResponse, "1", "2", "3"); - searchResponse = client().prepareSearch().setQuery( simpleQueryStringQuery("foo | bar") .defaultOperator(Operator.AND) @@ -267,26 +264,22 @@ public class SimpleQueryStringIT extends ESIntegTestCase { .flags(SimpleQueryStringFlag.NONE)).get(); assertHitCount(searchResponse, 0l); - searchResponse = client().prepareSearch().setSource(new BytesArray("{\n" + - " \"query\": {\n" + - " \"simple_query_string\": {\n" + - " \"query\": \"foo|bar\",\n" + - " \"default_operator\": \"AND\"," + - " \"flags\": \"NONE\"\n" + - " }\n" + - " }\n" + - "}")).get(); + searchResponse = client() + .prepareSearch() + .setSource( + new SearchSourceBuilder().query(QueryBuilders.simpleQueryStringQuery("foo|bar").defaultOperator(Operator.AND) + .flags(SimpleQueryStringFlag.NONE))).get(); assertHitCount(searchResponse, 1l); - searchResponse = client().prepareSearch().setQuery( - simpleQueryStringQuery("baz | egg*") - .defaultOperator(Operator.AND) - .flags(SimpleQueryStringFlag.WHITESPACE, SimpleQueryStringFlag.PREFIX)).get(); + searchResponse = client() + .prepareSearch() + .setQuery( + simpleQueryStringQuery("baz | egg*").defaultOperator(Operator.AND).flags(SimpleQueryStringFlag.WHITESPACE, + SimpleQueryStringFlag.PREFIX)).get(); assertHitCount(searchResponse, 1l); assertFirstHit(searchResponse, hasId("4")); } - @Test public void testSimpleQueryStringLenient() throws ExecutionException, InterruptedException { createIndex("test1", "test2"); indexRandom(true, client().prepareIndex("test1", "type1", "1").setSource("field", "foo"), @@ -304,7 +297,7 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertSearchHits(searchResponse, "1"); } - @Test // see: https://github.com/elasticsearch/elasticsearch/issues/7967 + // Issue #7967 public void testLenientFlagBeingTooLenient() throws Exception { indexRandom(true, client().prepareIndex("test", "doc", "1").setSource("num", 1, "body", "foo bar baz"), @@ -319,7 +312,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertSearchHits(resp, "1"); } - @Test public void testSimpleQueryStringAnalyzeWildcard() throws ExecutionException, InterruptedException, IOException { String mapping = XContentFactory.jsonBuilder() .startObject() @@ -343,5 +335,4 @@ public class SimpleQueryStringIT extends ESIntegTestCase { assertHitCount(searchResponse, 1l); assertSearchHits(searchResponse, "1"); } - } diff --git a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java index efa26f25f7d..720d51508f7 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Arrays; @@ -46,9 +45,7 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class DuelScrollIT extends ESIntegTestCase { - - @Test - public void testDuel_queryThenFetch() throws Exception { + public void testDuelQueryThenFetch() throws Exception { TestContext context = create(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH); SearchResponse control = client().prepareSearch("index") @@ -103,8 +100,7 @@ public class DuelScrollIT extends ESIntegTestCase { clearScroll(scrollId); } - @Test - public void testDuel_queryAndFetch() throws Exception { + public void testDuelQueryAndFetch() throws Exception { // *_QUERY_AND_FETCH search types are tricky: the ordering can be incorrect, since it returns num_shards * (from + size) // a subsequent scroll call can return hits that should have been in the hits of the first scroll call. diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java index 4aeb4161fde..bb81f28d15f 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java @@ -19,7 +19,12 @@ package org.elasticsearch.search.scroll; -import org.elasticsearch.action.search.*; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Priority; import org.elasticsearch.common.bytes.BytesArray; @@ -40,22 +45,32 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.junit.Test; import java.io.IOException; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; /** * */ public class SearchScrollIT extends ESIntegTestCase { - - @Test public void testSimpleScrollQueryThenFetch() throws Exception { client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3)).execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -107,7 +122,6 @@ public class SearchScrollIT extends ESIntegTestCase { } } - @Test public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws Exception { client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3)).execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -181,7 +195,6 @@ public class SearchScrollIT extends ESIntegTestCase { } } - @Test public void testScrollAndUpdateIndex() throws Exception { client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.number_of_shards", 5)).execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -193,11 +206,11 @@ public class SearchScrollIT extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(), equalTo(500l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "test")).execute().actionGet().getCount(), equalTo(500l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "test")).execute().actionGet().getCount(), equalTo(500l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "update")).execute().actionGet().getCount(), equalTo(0l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "update")).execute().actionGet().getCount(), equalTo(0l)); + assertThat(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(500l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "test")).execute().actionGet().getHits().totalHits(), equalTo(500l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "test")).execute().actionGet().getHits().totalHits(), equalTo(500l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "update")).execute().actionGet().getHits().totalHits(), equalTo(0l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "update")).execute().actionGet().getHits().totalHits(), equalTo(0l)); SearchResponse searchResponse = client().prepareSearch() .setQuery(queryStringQuery("user:kimchy")) @@ -216,17 +229,16 @@ public class SearchScrollIT extends ESIntegTestCase { } while (searchResponse.getHits().hits().length > 0); client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(), equalTo(500l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "test")).execute().actionGet().getCount(), equalTo(0l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "test")).execute().actionGet().getCount(), equalTo(0l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "update")).execute().actionGet().getCount(), equalTo(500l)); - assertThat(client().prepareCount().setQuery(termQuery("message", "update")).execute().actionGet().getCount(), equalTo(500l)); + assertThat(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet().getHits().totalHits(), equalTo(500l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "test")).execute().actionGet().getHits().totalHits(), equalTo(0l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "test")).execute().actionGet().getHits().totalHits(), equalTo(0l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "update")).execute().actionGet().getHits().totalHits(), equalTo(500l)); + assertThat(client().prepareSearch().setSize(0).setQuery(termQuery("message", "update")).execute().actionGet().getHits().totalHits(), equalTo(500l)); } finally { clearScroll(searchResponse.getScrollId()); } } - @Test public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception { client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3)).execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -303,7 +315,6 @@ public class SearchScrollIT extends ESIntegTestCase { assertThrows(client().prepareSearchScroll(searchResponse2.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)), RestStatus.NOT_FOUND); } - @Test public void testClearNonExistentScrollId() throws Exception { createIndex("idx"); ClearScrollResponse response = client().prepareClearScroll() @@ -317,7 +328,6 @@ public class SearchScrollIT extends ESIntegTestCase { assertToXContentResponse(response, true, response.getNumFreed()); } - @Test public void testClearIllegalScrollId() throws Exception { createIndex("idx"); try { @@ -340,8 +350,7 @@ public class SearchScrollIT extends ESIntegTestCase { } } - @Test - public void testSimpleScrollQueryThenFetch_clearAllScrollIds() throws Exception { + public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception { client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.number_of_shards", 3)).execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -440,7 +449,6 @@ public class SearchScrollIT extends ESIntegTestCase { } } - @Test public void testThatNonExistingScrollIdReturnsCorrectException() throws Exception { client().prepareIndex("index", "type", "1").setSource("field", "value").execute().get(); refresh(); @@ -454,7 +462,6 @@ public class SearchScrollIT extends ESIntegTestCase { assertThrows(internalCluster().transportClient().prepareSearchScroll(searchResponse.getScrollId()), RestStatus.NOT_FOUND); } - @Test public void testStringSortMissingAscTerminates() throws Exception { assertAcked(prepareCreate("test") .setSettings(Settings.settingsBuilder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)) @@ -488,7 +495,6 @@ public class SearchScrollIT extends ESIntegTestCase { assertThat(response.getHits().getHits().length, equalTo(0)); } - @Test public void testParseSearchScrollRequest() throws Exception { BytesReference content = XContentFactory.jsonBuilder() .startObject() @@ -503,7 +509,6 @@ public class SearchScrollIT extends ESIntegTestCase { assertThat(searchScrollRequest.scroll().keepAlive(), equalTo(TimeValue.parseTimeValue("1m", null, "scroll"))); } - @Test public void testParseSearchScrollRequestWithInvalidJsonThrowsException() throws Exception { SearchScrollRequest searchScrollRequest = new SearchScrollRequest(); try { @@ -515,7 +520,6 @@ public class SearchScrollIT extends ESIntegTestCase { } } - @Test public void testParseSearchScrollRequestWithUnknownParamThrowsException() throws Exception { SearchScrollRequest searchScrollRequest = new SearchScrollRequest(); BytesReference invalidContent = XContentFactory.jsonBuilder().startObject() @@ -532,7 +536,6 @@ public class SearchScrollIT extends ESIntegTestCase { } } - @Test public void testParseClearScrollRequest() throws Exception { BytesReference content = XContentFactory.jsonBuilder().startObject() .array("scroll_id", "value_1", "value_2") @@ -542,7 +545,6 @@ public class SearchScrollIT extends ESIntegTestCase { assertThat(clearScrollRequest.scrollIds(), contains("value_1", "value_2")); } - @Test public void testParseClearScrollRequestWithInvalidJsonThrowsException() throws Exception { ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); @@ -555,7 +557,6 @@ public class SearchScrollIT extends ESIntegTestCase { } } - @Test public void testParseClearScrollRequestWithUnknownParamThrowsException() throws Exception { BytesReference invalidContent = XContentFactory.jsonBuilder().startObject() .array("scroll_id", "value_1", "value_2") diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java index a037a6d9aeb..96f2e23bba1 100644 --- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java +++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocatio import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.List; @@ -34,13 +33,14 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; /** */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) public class SearchScrollWithFailingNodesIT extends ESIntegTestCase { - @Override protected int numberOfShards() { return 2; @@ -51,7 +51,6 @@ public class SearchScrollWithFailingNodesIT extends ESIntegTestCase { return 0; } - @Test public void testScanScrollWithShardExceptions() throws Exception { internalCluster().startNode(); internalCluster().startNode(); diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 74e2ff37c3f..27cc3d3cfb8 100644 --- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.internal.DefaultSearchContext; import org.elasticsearch.test.ESIntegTestCase; @@ -41,9 +42,11 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class SimpleSearchIT extends ESIntegTestCase { public void testSearchNullIndex() { @@ -104,6 +107,52 @@ public class SimpleSearchIT extends ESIntegTestCase { assertHitCount(search, 1l); } + public void testIpCidr() throws Exception { + createIndex("test"); + + client().admin().indices().preparePutMapping("test").setType("type1") + .setSource(XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("ip").field("type", "ip").endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + ensureGreen(); + + client().prepareIndex("test", "type1", "1").setSource("ip", "192.168.0.1").execute().actionGet(); + client().prepareIndex("test", "type1", "2").setSource("ip", "192.168.0.2").execute().actionGet(); + client().prepareIndex("test", "type1", "3").setSource("ip", "192.168.0.3").execute().actionGet(); + client().prepareIndex("test", "type1", "4").setSource("ip", "192.168.1.4").execute().actionGet(); + refresh(); + + SearchResponse search = client().prepareSearch() + .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/32"))) + .execute().actionGet(); + assertHitCount(search, 1l); + + search = client().prepareSearch() + .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.0/24"))) + .execute().actionGet(); + assertHitCount(search, 3l); + + search = client().prepareSearch() + .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))) + .execute().actionGet(); + assertHitCount(search, 4l); + + search = client().prepareSearch() + .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))) + .execute().actionGet(); + assertHitCount(search, 4l); + + search = client().prepareSearch() + .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.5/32"))) + .execute().actionGet(); + assertHitCount(search, 0l); + + assertFailures(client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0/0/0/0/0"))), + RestStatus.BAD_REQUEST, + containsString("invalid IPv4/CIDR; expected [a.b.c.d, e] but was [[0, 0, 0, 0, 0]]")); + } + public void testSimpleId() { createIndex("test"); @@ -283,6 +332,18 @@ public class SimpleSearchIT extends ESIntegTestCase { .setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW * 10).get(), 1); } + public void testQueryNumericFieldWithRegex() throws Exception { + assertAcked(prepareCreate("idx").addMapping("type", "num", "type=integer")); + ensureGreen("idx"); + + try { + client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get(); + fail("SearchPhaseExecutionException should have been thrown"); + } catch (SearchPhaseExecutionException ex) { + assertThat(ex.getCause().getCause().getMessage(), equalTo("Cannot use regular expression to filter numeric field [num]")); + } + } + private void assertWindowFails(SearchRequestBuilder search) { try { search.get(); diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java index 84295b7b198..e39fd81b28d 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java @@ -17,10 +17,8 @@ * under the License. */ - package org.elasticsearch.search.sort; - import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -29,13 +27,10 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexService; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.TestSearchContext; -import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; public class SortParserTests extends ESSingleNodeTestCase { - - @Test public void testGeoDistanceSortParserManyPointsNoException() throws Exception { XContentBuilder mapping = jsonBuilder(); mapping.startObject().startObject("type").startObject("properties").startObject("location").field("type", "geo_point").endObject().endObject().endObject().endObject(); diff --git a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java index d6a8e860329..76086da4de5 100644 --- a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java +++ b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java @@ -21,15 +21,12 @@ package org.elasticsearch.search.source; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.IsEqual.equalTo; public class SourceFetchingIT extends ESIntegTestCase { - - @Test public void testSourceDefaultBehavior() { createIndex("test"); ensureGreen(); @@ -48,7 +45,6 @@ public class SourceFetchingIT extends ESIntegTestCase { } - @Test public void testSourceFiltering() { createIndex("test"); ensureGreen(); @@ -82,7 +78,6 @@ public class SourceFetchingIT extends ESIntegTestCase { * Test Case for #5132: Source filtering with wildcards broken when given multiple patterns * https://github.com/elasticsearch/elasticsearch/issues/5132 */ - @Test public void testSourceWithWildcardFiltering() { createIndex("test"); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java index 7c1b443a0db..76f20c85516 100644 --- a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java +++ b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java @@ -22,14 +22,11 @@ package org.elasticsearch.search.stats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.search.stats.SearchStats.Stats; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.Map; public class SearchStatsUnitTests extends ESTestCase { - - @Test // https://github.com/elasticsearch/elasticsearch/issues/7644 public void testShardLevelSearchGroupStats() throws Exception { // let's create two dummy search stats with groups diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index 0b5e0d88a11..0f5ac1a522f 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -20,23 +20,25 @@ package org.elasticsearch.search.suggest; import com.carrotsearch.hppc.ObjectLongHashMap; import com.carrotsearch.randomizedtesting.generators.RandomStrings; +import org.apache.lucene.analysis.TokenStreamToAutomaton; +import org.apache.lucene.search.suggest.document.ContextSuggestField; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.percolator.PercolatorService; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -44,18 +46,12 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder.FuzzyOptionsBuilder; +import org.elasticsearch.search.suggest.completion.context.*; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.ExecutionException; +import java.util.*; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -65,56 +61,297 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.*; @SuppressCodecs("*") // requires custom completion format public class CompletionSuggestSearchIT extends ESIntegTestCase { - private final String INDEX = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); private final String TYPE = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); private final String FIELD = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); private final CompletionMappingBuilder completionMappingBuilder = new CompletionMappingBuilder(); - @Test - public void testSimple() throws Exception { - createIndexAndMapping(completionMappingBuilder); - String[][] input = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, - {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, - {"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, - {"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies - for (int i = 0; i < input.length; i++) { - client().prepareIndex(INDEX, TYPE, "" + i) + public void testPrefix() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) .setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(input[i]).endArray() - .endObject() - .endObject() - ) - .execute().actionGet(); + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i) + .endObject() + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"); + } + + public void testRegex() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugg" + i + "estion") + .field("weight", i) + .endObject() + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("sugg.*es"); + assertSuggestions("foo", prefix, "sugg10estion", "sugg9estion", "sugg8estion", "sugg7estion", "sugg6estion"); + } + + public void testFuzzy() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugxgestion" + i) + .field("weight", i) + .endObject() + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE); + assertSuggestions("foo", prefix, "sugxgestion10", "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6"); + } + + public void testEarlyTermination() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = atLeast(100); + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + (numDocs - i)) + .field("weight", numDocs - i) + .endObject() + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + int size = randomIntBetween(3, 10); + String[] outputs = new String[size]; + for (int i = 0; i < size; i++) { + outputs[i] = "suggestion" + (numDocs - i); + } + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sug").size(size); + assertSuggestions("foo", prefix, outputs); + + CompletionSuggestionBuilder regex = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("su[g|s]g").size(size); + assertSuggestions("foo", regex, outputs); + + CompletionSuggestionBuilder fuzzyPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE).size(size); + assertSuggestions("foo", fuzzyPrefix, outputs); + } + + public void testSuggestWithNumericPayload() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source= jsonBuilder() + .startObject() + .field(FIELD, "suggestion" + i) + .field("count", i) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(numDocs).payload("count"); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet(); + assertNoFailures(suggestResponse); + CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); + assertThat(options.getOptions().size(), equalTo(numDocs)); + for (CompletionSuggestion.Entry.Option option : options) { + Map> payloads = option.getPayload(); + assertThat(payloads.keySet(), contains("count")); + } + } + + public void testMalformedRequestPayload() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + SuggestRequest request = new SuggestRequest(INDEX); + XContentBuilder suggest = jsonBuilder().startObject() + .startObject("bad-payload") + .field("prefix", "sug") + .startObject("completion") + .field("field", FIELD) + .startArray("payload") + .startObject() + .field("payload", "field") + .endObject() + .endArray() + .endObject() + .endObject().endObject(); + request.suggest(suggest.bytes()); + ensureGreen(); + + SuggestResponse suggestResponse = client().suggest(request).get(); + assertThat(suggestResponse.getSuccessfulShards(), equalTo(0)); + for (ShardOperationFailedException exception : suggestResponse.getShardFailures()) { + assertThat(exception.reason(), containsString("ParsingException[[completion] failed to parse field [payload]]; nested: IllegalStateException[expected value but got [START_OBJECT]]")); + } + } + + public void testMissingPayloadField() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + List indexRequestBuilders = Arrays.asList( + client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "suggestion", "test_field", "test"), + client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, "suggestion") + ); + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").payload("test_field"); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet(); + assertNoFailures(suggestResponse); + CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); + assertThat(options.getOptions().size(), equalTo(2)); + for (CompletionSuggestion.Entry.Option option : options.getOptions()) { + assertThat(option.getPayload().keySet(), contains("test_field")); + } + } + + public void testPayload() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + List indexRequestBuilders = new ArrayList<>(); + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggest") + .field("weight", 1) + .endObject() + .field("title", "title1") + .field("count", 1) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "1").setSource(source)); + source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion") + .field("weight", 2) + .endObject() + .field("title", "title2") + .field("count", 2) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "2").setSource(source)); + indexRandom(true, indexRequestBuilders); + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").payload("title", "count"); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet(); + assertNoFailures(suggestResponse); + CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + List options = completionSuggestion.getEntries().get(0).getOptions(); + assertThat(options.size(), equalTo(2)); + assertThat(options.get(0).getText().toString(), equalTo("suggestion")); + assertThat(options.get(0).getScore(), equalTo(2f)); + assertThat(options.get(1).getText().toString(), equalTo("suggest")); + assertThat(options.get(1).getScore(), equalTo(1f)); + + Map> firstPayload = options.get(0).getPayload(); + assertThat(firstPayload.keySet(), containsInAnyOrder("title", "count")); + assertThat((String) firstPayload.get("title").get(0), equalTo("title2")); + assertThat((long) firstPayload.get("count").get(0), equalTo(2l)); + + Map> secondPayload = options.get(1).getPayload(); + assertThat(secondPayload.keySet(), containsInAnyOrder("title", "count")); + assertThat((String) secondPayload.get("title").get(0), equalTo("title1")); + assertThat((long) secondPayload.get("count").get(0), equalTo(1l)); + } + + public void testSuggestWithPayload() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = randomIntBetween(10, 100); + int numPayloadFields = randomIntBetween(2, 5); + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i) + .endObject(); + for (int j = 0; j < numPayloadFields; j++) { + source.field("test_field" + j, j + "value" + i); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + int suggestionSize = randomIntBetween(1, numDocs); + int numRequestedPayloadFields = randomIntBetween(2, numPayloadFields); + String[] payloadFields = new String[numRequestedPayloadFields]; + for (int i = 0; i < numRequestedPayloadFields; i++) { + payloadFields[i] = "test_field" + i; + } + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(suggestionSize).payload(payloadFields); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet(); + assertNoFailures(suggestResponse); + CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); + assertThat(options.getOptions().size(), equalTo(suggestionSize)); + int id = numDocs; + for (CompletionSuggestion.Entry.Option option : options) { + assertThat(option.getText().toString(), equalTo("suggestion" + id)); + assertThat(option.getPayload().size(), equalTo(numRequestedPayloadFields)); + for (int i = 0; i < numRequestedPayloadFields; i++) { + List fieldValue = option.getPayload().get("test_field" + i); + assertNotNull(fieldValue); + assertThat(fieldValue.size(), equalTo(1)); + assertThat((String)fieldValue.get(0), equalTo(i + "value" + id)); + } + id--; } - - refresh(); - - assertSuggestionsNotInOrder("f", "Foo Fighters", "Firestarter", "Foo Fighters Generator", "Foo Fighters Learn to Fly"); - assertSuggestionsNotInOrder("t", "The Prodigy", "Turbonegro", "Turbonegro Get it on", "The Prodigy Firestarter"); } - @Test public void testSuggestFieldWithPercolateApi() throws Exception { createIndexAndMapping(completionMappingBuilder); - String[][] input = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, + String[][] inputs = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, {"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, {"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies - for (int i = 0; i < input.length; i++) { + for (int i = 0; i < inputs.length; i++) { + XContentBuilder source = jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input"); + for (String input : inputs[i]) { + source.value(input); + } + source.endArray() + .endObject() + .endObject(); client().prepareIndex(INDEX, TYPE, "" + i) - .setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(input[i]).endArray() - .endObject() - .endObject() - ) - .execute().actionGet(); + .setSource(source).execute().actionGet(); } client().prepareIndex(INDEX, PercolatorService.TYPE_NAME, "4") @@ -129,20 +366,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertThat(response.getCount(), equalTo(1l)); } - @Test - public void testBasicPrefixSuggestion() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - for (int i = 0; i < 2; i++) { - createData(i == 0); - assertSuggestions("f", "Firestarter - The Prodigy", "Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters"); - assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro"); - assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro"); - assertSuggestions("t", "The Prodigy", "Firestarter - The Prodigy", "Get it on - Turbonegro", "Turbonegro"); - } - } - - @Test public void testThatWeightsAreWorking() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -162,7 +385,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions("the", "the", "The the", "The Verve", "The Prodigy"); } - @Test public void testThatWeightMustBeAnInteger() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -179,7 +401,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - @Test public void testThatWeightCanBeAString() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -206,7 +427,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } - @Test public void testThatWeightMustNotBeANonNumberString() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -223,7 +443,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - @Test public void testThatWeightAsStringMustBeInt() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -241,127 +460,20 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - @Test public void testThatInputCanBeAStringInsteadOfAnArray() throws Exception { createIndexAndMapping(completionMappingBuilder); client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .field("input", "Foo Fighters") - .field("output", "Boo Fighters") - .endObject().endObject() + .startObject().startObject(FIELD) + .field("input", "Foo Fighters") + .endObject().endObject() ).get(); refresh(); - assertSuggestions("f", "Boo Fighters"); + assertSuggestions("f", "Foo Fighters"); } - @Test - public void testThatPayloadsAreArbitraryJsonObjects() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .startObject("payload").field("foo", "bar").startArray("test").value("spam").value("eggs").endArray().endObject() - .endObject().endObject() - ).get(); - - refresh(); - - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) - ).execute().actionGet(); - - assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); - assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); - CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; - assertThat(prefixOption.getPayload(), is(notNullValue())); - - // parse JSON - Map jsonMap = prefixOption.getPayloadAsMap(); - assertThat(jsonMap.size(), is(2)); - assertThat(jsonMap.get("foo").toString(), is("bar")); - assertThat(jsonMap.get("test"), is(instanceOf(List.class))); - List listValues = (List) jsonMap.get("test"); - assertThat(listValues, hasItems("spam", "eggs")); - } - - @Test - public void testPayloadAsNumeric() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .field("payload", 1) - .endObject().endObject() - ).get(); - - refresh(); - - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) - ).execute().actionGet(); - - assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); - assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); - CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; - assertThat(prefixOption.getPayload(), is(notNullValue())); - - assertThat(prefixOption.getPayloadAsLong(), equalTo(1l)); - } - - @Test - public void testPayloadAsString() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .field("payload", "test") - .endObject().endObject() - ).get(); - - refresh(); - - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) - ).execute().actionGet(); - - assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); - assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); - CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; - assertThat(prefixOption.getPayload(), is(notNullValue())); - - assertThat(prefixOption.getPayloadAsString(), equalTo("test")); - } - - @Test(expected = MapperException.class) - public void testThatExceptionIsThrownWhenPayloadsAreDisabledButInIndexRequest() throws Exception { - completionMappingBuilder.payloads(false); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .startArray("payload").value("spam").value("eggs").endArray() - .endObject().endObject() - ).get(); - } - - @Test public void testDisabledPreserveSeparators() throws Exception { completionMappingBuilder.preserveSeparators(false); createIndexAndMapping(completionMappingBuilder); @@ -385,7 +497,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions("foof", "Foof", "Foo Fighters"); } - @Test public void testEnabledPreserveSeparators() throws Exception { completionMappingBuilder.preserveSeparators(true); createIndexAndMapping(completionMappingBuilder); @@ -407,24 +518,21 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions("foof", "Foof"); } - @Test public void testThatMultipleInputsAreSupported() throws Exception { createIndexAndMapping(completionMappingBuilder); client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() .startObject().startObject(FIELD) .startArray("input").value("Foo Fighters").value("Fu Fighters").endArray() - .field("output", "The incredible Foo Fighters") .endObject().endObject() ).get(); refresh(); - assertSuggestions("foo", "The incredible Foo Fighters"); - assertSuggestions("fu", "The incredible Foo Fighters"); + assertSuggestions("foo", "Foo Fighters"); + assertSuggestions("fu", "Fu Fighters"); } - @Test public void testThatShortSyntaxIsWorking() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -440,7 +548,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions("f", "Firestarter"); } - @Test public void testThatDisablingPositionIncrementsWorkForStopwords() throws Exception { // analyzer which removes stopwords... so may not be the simple one completionMappingBuilder.searchAnalyzer("classic").indexAnalyzer("classic").preservePositionIncrements(false); @@ -457,7 +564,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions("b", "The Beatles"); } - @Test public void testThatSynonymsWork() throws Exception { Settings.Builder settingsBuilder = settingsBuilder() .put("analysis.analyzer.suggest_analyzer_synonyms.type", "custom") @@ -480,7 +586,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions("r", "Foo Fighters"); } - @Test public void testThatUpgradeToMultiFieldTypeWorks() throws Exception { final XContentBuilder mapping = jsonBuilder() .startObject() @@ -524,7 +629,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); } - @Test public void testThatUpgradeToMultiFieldsWorks() throws Exception { final XContentBuilder mapping = jsonBuilder() .startObject() @@ -567,7 +671,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); } - @Test public void testThatFuzzySuggesterWorks() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -580,17 +683,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirv").size(10) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirv").size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirw").size(10) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirw", Fuzziness.ONE).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } - @Test public void testThatFuzzySuggesterSupportsEditDistances() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -604,18 +706,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { // edit distance 1 SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Norw").size(10) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Norw", Fuzziness.ONE).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); // edit distance 2 suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Norw").size(10).setFuzziness(Fuzziness.TWO) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Norw", Fuzziness.TWO).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } - @Test public void testThatFuzzySuggesterSupportsTranspositions() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -628,17 +729,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(false).setFuzziness(Fuzziness.ONE) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriv", new FuzzyOptionsBuilder().setTranspositions(false)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(true).setFuzziness(Fuzziness.ONE) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriv", Fuzziness.ONE).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } - @Test public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -651,17 +751,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nriva").size(10).setFuzzyMinLength(6) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriva", new FuzzyOptionsBuilder().setFuzzyMinLength(6)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nrivan").size(10).setFuzzyMinLength(6) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nrivan", new FuzzyOptionsBuilder().setFuzzyMinLength(6)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } - @Test public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -674,17 +773,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirw").size(10).setFuzzyPrefixLength(4) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirw", new FuzzyOptionsBuilder().setFuzzyPrefixLength(4)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirvo").size(10).setFuzzyPrefixLength(4) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirvo", new FuzzyOptionsBuilder().setFuzzyPrefixLength(4)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } - @Test public void testThatFuzzySuggesterIsUnicodeAware() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -697,32 +795,32 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); // suggestion with a character, which needs unicode awareness - CompletionSuggestionFuzzyBuilder completionSuggestionBuilder = - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("öööи").size(10).setUnicodeAware(true); + org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder completionSuggestionBuilder = + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(true)).size(10); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "ööööö"); // removing unicode awareness leads to no result - completionSuggestionBuilder.setUnicodeAware(false); + completionSuggestionBuilder = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(false)).size(10); suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); // increasing edit distance instead of unicode awareness works again, as this is only a single character - completionSuggestionBuilder.setFuzziness(Fuzziness.TWO); + completionSuggestionBuilder = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(false).setFuzziness(Fuzziness.TWO)).size(10); suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "ööööö"); } - @Test public void testThatStatsAreWorking() throws Exception { String otherField = "testOtherField"; - - createIndex(INDEX); - + client().admin().indices().prepareCreate(INDEX) + .setSettings(Settings.settingsBuilder().put("index.number_of_replicas", 0).put("index.number_of_shards", 2)) + .execute().actionGet(); + ensureGreen(); PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() .startObject(TYPE).startObject("properties") - .startObject(FIELD.toString()) + .startObject(FIELD) .field("type", "completion").field("analyzer", "simple") .endObject() .startObject(otherField) @@ -733,8 +831,14 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertThat(putMappingResponse.isAcknowledged(), is(true)); // Index two entities - client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()).get(); - client().prepareIndex(INDEX, TYPE, "2").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()).get(); + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()).get(); + client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()).get(); + + refresh(); + ensureGreen(); + // load the fst index into ram + client().prepareSuggest(INDEX).addSuggestion(SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("f")).get(); + client().prepareSuggest(INDEX).addSuggestion(SuggestBuilders.completionSuggestion("foo").field(otherField).prefix("f")).get(); // Get all stats IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).get(); @@ -756,7 +860,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertThat(regexSizeInBytes, is(totalSizeInBytes)); } - @Test public void testThatSortingOnCompletionFieldReturnsUsefulException() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -776,7 +879,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - @Test public void testThatSuggestStopFilterWorks() throws Exception { Settings.Builder settingsBuilder = settingsBuilder() .put("index.analysis.analyzer.stoptest.tokenizer", "standard") @@ -817,29 +919,36 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions("feed the t"); } - @Test(expected = MapperParsingException.class) public void testThatIndexingInvalidFieldsInCompletionFieldResultsInException() throws Exception { CompletionMappingBuilder completionMappingBuilder = new CompletionMappingBuilder(); createIndexAndMapping(completionMappingBuilder); - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("FRIGGININVALID").value("Nirvana").endArray() - .endObject().endObject()).get(); + try { + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("FRIGGININVALID").value("Nirvana").endArray() + .endObject().endObject()).get(); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("failed to parse")); + } } - public void assertSuggestions(String suggestion, String... suggestions) { - String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggestion).size(10) + public void assertSuggestions(String suggestionName, SuggestBuilder.SuggestionBuilder suggestBuilder, String... suggestions) { + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestBuilder ).execute().actionGet(); - assertSuggestions(suggestResponse, suggestionName, suggestions); + + } + public void assertSuggestions(String suggestion, String... suggestions) { + String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggestion).size(10); + assertSuggestions(suggestionName, suggestionBuilder, suggestions); } public void assertSuggestionsNotInOrder(String suggestString, String... suggestions) { - String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10); + String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggestString).size(10) ).execute().actionGet(); @@ -847,11 +956,11 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { assertSuggestions(suggestResponse, false, suggestionName, suggestions); } - private void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) { + static void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) { assertSuggestions(suggestResponse, true, name, suggestions); } - private void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) { + private static void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) { assertAllSuccessful(suggestResponse); List suggestionNames = new ArrayList<>(); @@ -881,7 +990,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - private List getNames(Suggest.Suggestion.Entry suggestEntry) { + private static List getNames(Suggest.Suggestion.Entry suggestEntry) { List names = new ArrayList<>(); for (Suggest.Suggestion.Entry.Option entry : suggestEntry.getOptions()) { names.add(entry.getText().string()); @@ -890,20 +999,44 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } private void createIndexAndMappingAndSettings(Settings settings, CompletionMappingBuilder completionMappingBuilder) throws IOException { + XContentBuilder mapping = jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .field("analyzer", completionMappingBuilder.indexAnalyzer) + .field("search_analyzer", completionMappingBuilder.searchAnalyzer) + .field("preserve_separators", completionMappingBuilder.preserveSeparators) + .field("preserve_position_increments", completionMappingBuilder.preservePositionIncrements); + + if (completionMappingBuilder.contextMappings != null) { + mapping = mapping.startArray("contexts"); + for (Map.Entry contextMapping : completionMappingBuilder.contextMappings.entrySet()) { + mapping = mapping.startObject() + .field("name", contextMapping.getValue().name()) + .field("type", contextMapping.getValue().type().name()); + switch (contextMapping.getValue().type()) { + case CATEGORY: + mapping = mapping.field("path", ((CategoryContextMapping) contextMapping.getValue()).getFieldName()); + break; + case GEO: + mapping = mapping + .field("path", ((GeoContextMapping) contextMapping.getValue()).getFieldName()) + .field("precision", ((GeoContextMapping) contextMapping.getValue()).getPrecision()); + break; + } + + mapping = mapping.endObject(); + } + + mapping = mapping.endArray(); + } + mapping = mapping.endObject() + .endObject().endObject() + .endObject(); + assertAcked(client().admin().indices().prepareCreate(INDEX) .setSettings(Settings.settingsBuilder().put(indexSettings()).put(settings)) - .addMapping(TYPE, jsonBuilder().startObject() - .startObject(TYPE).startObject("properties") - .startObject(FIELD) - .field("type", "completion") - .field("analyzer", completionMappingBuilder.indexAnalyzer) - .field("search_analyzer", completionMappingBuilder.searchAnalyzer) - .field("payloads", completionMappingBuilder.payloads) - .field("preserve_separators", completionMappingBuilder.preserveSeparators) - .field("preserve_position_increments", completionMappingBuilder.preservePositionIncrements) - .endObject() - .endObject().endObject() - .endObject()) + .addMapping(TYPE, mapping) .get()); ensureYellow(); } @@ -912,48 +1045,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { createIndexAndMappingAndSettings(Settings.EMPTY, completionMappingBuilder); } - private void createData(boolean optimize) throws IOException, InterruptedException, ExecutionException { - String[][] input = {{"Foo Fighters"}, {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; - String[] surface = {"Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters", "The Prodigy", "Firestarter - The Prodigy", "Turbonegro", "Get it on - Turbonegro"}; - int[] weight = {10, 9, 8, 12, 11, 6, 7}; - IndexRequestBuilder[] builders = new IndexRequestBuilder[input.length]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex(INDEX, TYPE, "" + i) - .setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(input[i]).endArray() - .field("output", surface[i]) - .startObject("payload").field("id", i).endObject() - .field("weight", 1) // WE FORCEFULLY INDEX A BOGUS WEIGHT - .endObject() - .endObject() - ); - } - indexRandom(false, builders); - - for (int i = 0; i < builders.length; i++) { // add them again to make sure we deduplicate on the surface form - builders[i] = client().prepareIndex(INDEX, TYPE, "n" + i) - .setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(input[i]).endArray() - .field("output", surface[i]) - .startObject("payload").field("id", i).endObject() - .field("weight", weight[i]) - .endObject() - .endObject() - ); - } - indexRandom(false, builders); - - client().admin().indices().prepareRefresh(INDEX).execute().actionGet(); - if (optimize) { - // make sure merging works just fine - client().admin().indices().prepareFlush(INDEX).execute().actionGet(); - client().admin().indices().prepareOptimize(INDEX).setMaxNumSegments(randomIntBetween(1, 5)).get(); - } - } - - @Test // see #3555 + // see #3555 public void testPrunedSegments() throws IOException { createIndexAndMappingAndSettings(settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build(), completionMappingBuilder); @@ -967,7 +1059,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { .field("somefield", "somevalue") .endObject() ).get(); // we have 2 docs in a segment... - OptimizeResponse actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).execute().actionGet(); + ForceMergeResponse actionGet = client().admin().indices().prepareForceMerge().setFlush(true).setMaxNumSegments(1).execute().actionGet(); assertAllSuccessful(actionGet); refresh(); // update the first one and then merge.. the target segment will have no value in FIELD @@ -976,12 +1068,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { .field("somefield", "somevalue") .endObject() ).get(); - actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).execute().actionGet(); + actionGet = client().admin().indices().prepareForceMerge().setFlush(true).setMaxNumSegments(1).execute().actionGet(); assertAllSuccessful(actionGet); refresh(); assertSuggestions("b"); - assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount())); + assertThat(2l, equalTo(client().prepareSearch(INDEX).setSize(0).get().getHits().totalHits())); for (IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) { ShardSegments[] shards = seg.getShards(); for (ShardSegments shardSegments : shards) { @@ -990,45 +1082,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - @Test - public void testMaxFieldLength() throws IOException { - client().admin().indices().prepareCreate(INDEX).get(); - ensureGreen(); - int iters = scaledRandomIntBetween(10, 20); - for (int i = 0; i < iters; i++) { - int maxInputLen = between(3, 50); - String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(maxInputLen + 1, maxInputLen + scaledRandomIntBetween(2, 50)), (char) 0x01); - assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() - .startObject(TYPE).startObject("properties") - .startObject(FIELD) - .field("type", "completion") - .field("max_input_length", maxInputLen) - // upgrade mapping each time - .field("analyzer", "keyword") - .endObject() - .endObject().endObject() - .endObject())); - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(str).endArray() - .field("output", "foobar") - .endObject().endObject() - ).setRefresh(true).get(); - // need to flush and refresh, because we keep changing the same document - // we have to make sure that segments without any live documents are deleted - flushAndRefresh(); - int prefixLen = CompletionFieldMapper.correctSubStringLen(str, between(1, maxInputLen - 1)); - assertSuggestions(str.substring(0, prefixLen), "foobar"); - if (maxInputLen + 1 < str.length()) { - int offset = Character.isHighSurrogate(str.charAt(maxInputLen - 1)) ? 2 : 1; - int correctSubStringLen = CompletionFieldMapper.correctSubStringLen(str, maxInputLen + offset); - String shortenedSuggestion = str.substring(0, correctSubStringLen); - assertSuggestions(shortenedSuggestion); - } - } - } - - @Test // see #3596 public void testVeryLongInput() throws IOException { assertAcked(client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, jsonBuilder().startObject() @@ -1044,14 +1097,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() .startObject().startObject(FIELD) .startArray("input").value(longString).endArray() - .field("output", "foobar") .endObject().endObject() ).setRefresh(true).get(); } // see #3648 - @Test(expected = MapperParsingException.class) public void testReservedChars() throws IOException { assertAcked(client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, jsonBuilder().startObject() .startObject(TYPE).startObject("properties") @@ -1063,15 +1114,20 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { ensureYellow(); // can cause stack overflow without the default max_input_length String string = "foo" + (char) 0x00 + "bar"; - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(string).endArray() - .field("output", "foobar") - .endObject().endObject() - ).setRefresh(true).get(); + try { + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(string).endArray() + .field("output", "foobar") + .endObject().endObject() + ).setRefresh(true).get(); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("failed to parse")); + } } - @Test // see #5930 + // see #5930 public void testIssue5930() throws IOException { assertAcked(client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, jsonBuilder().startObject() .startObject(TYPE).startObject("properties") @@ -1083,9 +1139,9 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { ensureYellow(); String string = "foo bar"; client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject() - .field(FIELD, string) - .endObject() + .startObject() + .field(FIELD, string) + .endObject() ).setRefresh(true).get(); try { @@ -1099,7 +1155,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } // see issue #6399 - @Test public void testIndexingUnrelatedNullValue() throws Exception { String mapping = jsonBuilder() .startObject() @@ -1117,7 +1172,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { ensureGreen(); client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "strings make me happy", FIELD + "_1", "nulls make me sad") - .setRefresh(true).get(); + .setRefresh(true).get(); try { client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, null, FIELD + "_1", "nulls make me sad") @@ -1127,25 +1182,36 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { // make sure that the exception has the name of the field causing the error assertTrue(e.getDetailedMessage().contains(FIELD)); } + } + public static boolean isReservedChar(char c) { + switch (c) { + case '\u001F': + case TokenStreamToAutomaton.HOLE: + case 0x0: + case ContextSuggestField.CONTEXT_SEPARATOR: + return true; + default: + return false; + } } private static String replaceReservedChars(String input, char replacement) { char[] charArray = input.toCharArray(); for (int i = 0; i < charArray.length; i++) { - if (CompletionFieldMapper.isReservedChar(charArray[i])) { + if (isReservedChar(charArray[i])) { charArray[i] = replacement; } } return new String(charArray); } - private static class CompletionMappingBuilder { - private String searchAnalyzer = "simple"; - private String indexAnalyzer = "simple"; - private Boolean payloads = getRandom().nextBoolean(); - private Boolean preserveSeparators = getRandom().nextBoolean(); - private Boolean preservePositionIncrements = getRandom().nextBoolean(); + static class CompletionMappingBuilder { + String searchAnalyzer = "simple"; + String indexAnalyzer = "simple"; + Boolean preserveSeparators = getRandom().nextBoolean(); + Boolean preservePositionIncrements = getRandom().nextBoolean(); + LinkedHashMap contextMappings = null; public CompletionMappingBuilder searchAnalyzer(String searchAnalyzer) { this.searchAnalyzer = searchAnalyzer; @@ -1155,10 +1221,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { this.indexAnalyzer = indexAnalyzer; return this; } - public CompletionMappingBuilder payloads(Boolean payloads) { - this.payloads = payloads; - return this; - } public CompletionMappingBuilder preserveSeparators(Boolean preserveSeparators) { this.preserveSeparators = preserveSeparators; return this; @@ -1167,5 +1229,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { this.preservePositionIncrements = preservePositionIncrements; return this; } + + public CompletionMappingBuilder context(LinkedHashMap contextMappings) { + this.contextMappings = contextMappings; + return this; + } } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionTokenStreamTests.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionTokenStreamTests.java deleted file mode 100644 index d7280c89ef4..00000000000 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionTokenStreamTests.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.suggest; - -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.core.SimpleAnalyzer; -import org.apache.lucene.analysis.synonym.SynonymFilter; -import org.apache.lucene.analysis.synonym.SynonymMap; -import org.apache.lucene.analysis.synonym.SynonymMap.Builder; -import org.apache.lucene.analysis.tokenattributes.*; -import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CharsRef; -import org.apache.lucene.util.IntsRef; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream.ByteTermAttribute; -import org.elasticsearch.test.ESTokenStreamTestCase; -import org.junit.Test; - -import java.io.IOException; -import java.io.StringReader; -import java.util.Set; - -import static org.hamcrest.Matchers.equalTo; - -public class CompletionTokenStreamTests extends ESTokenStreamTestCase { - - final XAnalyzingSuggester suggester = new XAnalyzingSuggester(new SimpleAnalyzer()); - - @Test - public void testSuggestTokenFilter() throws Exception { - Tokenizer tokenStream = new MockTokenizer(MockTokenizer.WHITESPACE, true); - tokenStream.setReader(new StringReader("mykeyword")); - BytesRef payload = new BytesRef("Surface keyword|friggin payload|10"); - TokenStream suggestTokenStream = new ByteTermAttrToCharTermAttrFilter(new CompletionTokenStream(tokenStream, payload, new CompletionTokenStream.ToFiniteStrings() { - @Override - public Set toFiniteStrings(TokenStream stream) throws IOException { - return suggester.toFiniteStrings(stream); - } - })); - assertTokenStreamContents(suggestTokenStream, new String[] {"mykeyword"}, null, null, new String[] {"Surface keyword|friggin payload|10"}, new int[] { 1 }, null, null); - } - - @Test - public void testSuggestTokenFilterWithSynonym() throws Exception { - Builder builder = new SynonymMap.Builder(true); - builder.add(new CharsRef("mykeyword"), new CharsRef("mysynonym"), true); - - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); - tokenizer.setReader(new StringReader("mykeyword")); - SynonymFilter filter = new SynonymFilter(tokenizer, builder.build(), true); - - BytesRef payload = new BytesRef("Surface keyword|friggin payload|10"); - TokenStream suggestTokenStream = new ByteTermAttrToCharTermAttrFilter(new CompletionTokenStream(filter, payload, new CompletionTokenStream.ToFiniteStrings() { - @Override - public Set toFiniteStrings(TokenStream stream) throws IOException { - return suggester.toFiniteStrings(stream); - } - })); - assertTokenStreamContents(suggestTokenStream, new String[] {"mysynonym", "mykeyword"}, null, null, new String[] {"Surface keyword|friggin payload|10", "Surface keyword|friggin payload|10"}, new int[] { 2, 0 }, null, null); - } - - @Test - public void testValidNumberOfExpansions() throws IOException { - Builder builder = new SynonymMap.Builder(true); - for (int i = 0; i < 256; i++) { - builder.add(new CharsRef("" + (i+1)), new CharsRef("" + (1000 + (i+1))), true); - } - StringBuilder valueBuilder = new StringBuilder(); - for (int i = 0 ; i < 8 ; i++) { - valueBuilder.append(i+1); - valueBuilder.append(" "); - } - MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); - tokenizer.setReader(new StringReader(valueBuilder.toString())); - SynonymFilter filter = new SynonymFilter(tokenizer, builder.build(), true); - - TokenStream suggestTokenStream = new CompletionTokenStream(filter, new BytesRef("Surface keyword|friggin payload|10"), new CompletionTokenStream.ToFiniteStrings() { - @Override - public Set toFiniteStrings(TokenStream stream) throws IOException { - Set finiteStrings = suggester.toFiniteStrings(stream); - return finiteStrings; - } - }); - - suggestTokenStream.reset(); - ByteTermAttribute attr = suggestTokenStream.addAttribute(ByteTermAttribute.class); - PositionIncrementAttribute posAttr = suggestTokenStream.addAttribute(PositionIncrementAttribute.class); - int maxPos = 0; - int count = 0; - while(suggestTokenStream.incrementToken()) { - count++; - assertNotNull(attr.getBytesRef()); - assertTrue(attr.getBytesRef().length > 0); - maxPos += posAttr.getPositionIncrement(); - } - suggestTokenStream.close(); - assertEquals(count, 256); - assertEquals(count, maxPos); - - } - - @Test(expected = IllegalArgumentException.class) - public void testInValidNumberOfExpansions() throws IOException { - Builder builder = new SynonymMap.Builder(true); - for (int i = 0; i < 256; i++) { - builder.add(new CharsRef("" + (i+1)), new CharsRef("" + (1000 + (i+1))), true); - } - StringBuilder valueBuilder = new StringBuilder(); - for (int i = 0 ; i < 9 ; i++) { // 9 -> expands to 512 - valueBuilder.append(i+1); - valueBuilder.append(" "); - } - MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); - tokenizer.setReader(new StringReader(valueBuilder.toString())); - SynonymFilter filter = new SynonymFilter(tokenizer, builder.build(), true); - - TokenStream suggestTokenStream = new CompletionTokenStream(filter, new BytesRef("Surface keyword|friggin payload|10"), new CompletionTokenStream.ToFiniteStrings() { - @Override - public Set toFiniteStrings(TokenStream stream) throws IOException { - Set finiteStrings = suggester.toFiniteStrings(stream); - return finiteStrings; - } - }); - - suggestTokenStream.reset(); - suggestTokenStream.incrementToken(); - suggestTokenStream.close(); - - } - - @Test - public void testSuggestTokenFilterProperlyDelegateInputStream() throws Exception { - Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); - tokenizer.setReader(new StringReader("mykeyword")); - BytesRef payload = new BytesRef("Surface keyword|friggin payload|10"); - TokenStream suggestTokenStream = new ByteTermAttrToCharTermAttrFilter(new CompletionTokenStream(tokenizer, payload, new CompletionTokenStream.ToFiniteStrings() { - @Override - public Set toFiniteStrings(TokenStream stream) throws IOException { - return suggester.toFiniteStrings(stream); - } - })); - TermToBytesRefAttribute termAtt = suggestTokenStream.getAttribute(TermToBytesRefAttribute.class); - assertNotNull(termAtt.getBytesRef()); - suggestTokenStream.reset(); - - while (suggestTokenStream.incrementToken()) { - assertThat(termAtt.getBytesRef().utf8ToString(), equalTo("mykeyword")); - } - suggestTokenStream.end(); - suggestTokenStream.close(); - } - - - public final static class ByteTermAttrToCharTermAttrFilter extends TokenFilter { - private ByteTermAttribute byteAttr = addAttribute(ByteTermAttribute.class); - private PayloadAttribute payload = addAttribute(PayloadAttribute.class); - private TypeAttribute type = addAttribute(TypeAttribute.class); - private CharTermAttribute charTermAttribute = addAttribute(CharTermAttribute.class); - protected ByteTermAttrToCharTermAttrFilter(TokenStream input) { - super(input); - } - - @Override - public boolean incrementToken() throws IOException { - if (input.incrementToken()) { - BytesRef bytesRef = byteAttr.getBytesRef(); - // we move them over so we can assert them more easily in the tests - type.setType(payload.getPayload().utf8ToString()); - return true; - } - return false; - } - - } -} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java new file mode 100644 index 00000000000..ae6ec51ac36 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -0,0 +1,681 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.suggest; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.suggest.CompletionSuggestSearchIT.CompletionMappingBuilder; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; +import org.elasticsearch.search.suggest.completion.context.*; +import org.elasticsearch.test.ESIntegTestCase; + +import java.io.IOException; +import java.util.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +@SuppressCodecs("*") // requires custom completion format +public class ContextCompletionSuggestSearchIT extends ESIntegTestCase { + + private final String INDEX = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + private final String TYPE = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + private final String FIELD = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + + @Override + protected int numberOfReplicas() { + return 0; + } + + public void testContextPrefix() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + boolean addAnotherContext = randomBoolean(); + if (addAnotherContext) { + map.put("type", ContextBuilder.category("type").field("type").build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2); + if (addAnotherContext) { + source.field("type", "type" + i % 3); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + public void testContextRegex() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + boolean addAnotherContext = randomBoolean(); + if (addAnotherContext) { + map.put("type", ContextBuilder.category("type").field("type").build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugg" + i + "estion") + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2); + if (addAnotherContext) { + source.field("type", "type" + i % 3); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("sugg.*es"); + assertSuggestions("foo", prefix, "sugg9estion", "sugg8estion", "sugg7estion", "sugg6estion", "sugg5estion"); + } + + public void testContextFuzzy() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + boolean addAnotherContext = randomBoolean(); + if (addAnotherContext) { + map.put("type", ContextBuilder.category("type").field("type").build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugxgestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2); + if (addAnotherContext) { + source.field("type", "type" + i % 3); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE); + assertSuggestions("foo", prefix, "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6", "sugxgestion5"); + } + + public void testSingleContextFiltering() throws Exception { + CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build(); + LinkedHashMap map = new LinkedHashMap(Collections.singletonMap("cat", contextMapping)); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat0").build()); + + assertSuggestions("foo", prefix, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0"); + } + + public void testSingleContextBoosting() throws Exception { + CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build(); + LinkedHashMap map = new LinkedHashMap(Collections.singletonMap("cat", contextMapping)); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .categoryContexts("cat", + CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(), + CategoryQueryContext.builder().setCategory("cat1").build() + ); + assertSuggestions("foo", prefix, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion2"); + } + + public void testSingleContextMultipleContexts() throws Exception { + CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build(); + LinkedHashMap map = new LinkedHashMap(Collections.singletonMap("cat", contextMapping)); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List contexts = Arrays.asList("type1", "type2", "type3", "type4"); + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", contexts) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + public void testMultiContextFiltering() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + map.put("type", ContextBuilder.category("type").field("type").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .field("type", "type" + i % 4) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + + // filter only on context cat + CompletionSuggestionBuilder catFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + catFilterSuggest.categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat0").build()); + assertSuggestions("foo", catFilterSuggest, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0"); + + // filter only on context type + CompletionSuggestionBuilder typeFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + typeFilterSuggest.categoryContexts("type", CategoryQueryContext.builder().setCategory("type2").build(), + CategoryQueryContext.builder().setCategory("type1").build()); + assertSuggestions("foo", typeFilterSuggest, "suggestion9", "suggestion6", "suggestion5", "suggestion2", "suggestion1"); + + CompletionSuggestionBuilder multiContextFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + // query context order should never matter + if (randomBoolean()) { + multiContextFilterSuggest.categoryContexts("type", CategoryQueryContext.builder().setCategory("type2").build()); + multiContextFilterSuggest.categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat2").build()); + } else { + multiContextFilterSuggest.categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat2").build()); + multiContextFilterSuggest.categoryContexts("type", CategoryQueryContext.builder().setCategory("type2").build()); + } + assertSuggestions("foo", multiContextFilterSuggest, "suggestion6", "suggestion2"); + } + + @AwaitsFix(bugUrl = "multiple context boosting is broken, as a suggestion, contexts pair is treated as (num(context) entries)") + public void testMultiContextBoosting() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + map.put("type", ContextBuilder.category("type").field("type").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .field("type", "type" + i % 4) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + + // boost only on context cat + CompletionSuggestionBuilder catBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + catBoostSuggest.categoryContexts("cat", + CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(), + CategoryQueryContext.builder().setCategory("cat1").build()); + assertSuggestions("foo", catBoostSuggest, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion2"); + + // boost only on context type + CompletionSuggestionBuilder typeBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + typeBoostSuggest.categoryContexts("type", + CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(), + CategoryQueryContext.builder().setCategory("type1").setBoost(4).build()); + assertSuggestions("foo", typeBoostSuggest, "suggestion9", "suggestion5", "suggestion6", "suggestion1", "suggestion2"); + + // boost on both contexts + CompletionSuggestionBuilder multiContextBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + // query context order should never matter + if (randomBoolean()) { + multiContextBoostSuggest.categoryContexts("type", + CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(), + CategoryQueryContext.builder().setCategory("type1").setBoost(4).build()); + multiContextBoostSuggest.categoryContexts("cat", + CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(), + CategoryQueryContext.builder().setCategory("cat1").build()); + } else { + multiContextBoostSuggest.categoryContexts("cat", + CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(), + CategoryQueryContext.builder().setCategory("cat1").build()); + multiContextBoostSuggest.categoryContexts("type", + CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(), + CategoryQueryContext.builder().setCategory("type1").setBoost(4).build()); + } + assertSuggestions("foo", multiContextBoostSuggest, "suggestion9", "suggestion6", "suggestion5", "suggestion2", "suggestion1"); + } + + public void testMissingContextValue() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + map.put("type", ContextBuilder.category("type").field("type").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject(); + if (randomBoolean()) { + source.field("cat", "cat" + i % 2); + } + if (randomBoolean()) { + source.field("type", "type" + i % 4); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + public void testSeveralContexts() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + final int numContexts = randomIntBetween(2, 5); + for (int i = 0; i < numContexts; i++) { + map.put("type" + i, ContextBuilder.category("type" + i).field("type" + i).build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = randomIntBetween(10, 200); + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", numDocs - i) + .endObject(); + for (int c = 0; c < numContexts; c++) { + source.field("type"+c, "type" + c +i % 4); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion0", "suggestion1", "suggestion2", "suggestion3", "suggestion4"); + } + + public void testSimpleGeoPrefix() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", GeoHashUtils.stringEncode(1.2, 1.3)) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + public void testGeoFiltering() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + GeoPoint[] geoPoints = new GeoPoint[] {new GeoPoint("ezs42e44yx96"), new GeoPoint("u4pruydqqvj8")}; + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", (i % 2 == 0) ? geoPoints[0].getGeohash() : geoPoints[1].getGeohash()) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + + CompletionSuggestionBuilder geoFilteringPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .geoContexts("geo", GeoQueryContext.builder().setGeoPoint(new GeoPoint(geoPoints[0])).build()); + + assertSuggestions("foo", geoFilteringPrefix, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0"); + } + + public void testGeoBoosting() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + GeoPoint[] geoPoints = new GeoPoint[] {new GeoPoint("ezs42e44yx96"), new GeoPoint("u4pruydqqvj8")}; + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", (i % 2 == 0) ? geoPoints[0].getGeohash() : geoPoints[1].getGeohash()) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + + GeoQueryContext context1 = GeoQueryContext.builder().setGeoPoint(geoPoints[0]).setBoost(2).build(); + GeoQueryContext context2 = GeoQueryContext.builder().setGeoPoint(geoPoints[1]).build(); + CompletionSuggestionBuilder geoBoostingPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .geoContexts("geo", context1, context2); + + assertSuggestions("foo", geoBoostingPrefix, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion7"); + } + + public void testGeoPointContext() throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .startObject("geo") + .field("lat", 52.22) + .field("lon", 4.53) + .endObject() + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .geoContexts("geo", GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.2263, 4.543)).build()); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + public void testGeoNeighbours() throws Exception { + String geohash = "gcpv"; + List neighbours = new ArrayList<>(); + neighbours.add("gcpw"); + neighbours.add("gcpy"); + neighbours.add("u10n"); + neighbours.add("gcpt"); + neighbours.add("u10j"); + neighbours.add("gcps"); + neighbours.add("gcpu"); + neighbours.add("u10h"); + + LinkedHashMap map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").precision(4).build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", randomFrom(neighbours)) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + ensureYellow(INDEX); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + + CompletionSuggestionBuilder geoNeighbourPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .geoContexts("geo", GeoQueryContext.builder().setGeoPoint(GeoPoint.fromGeohash(geohash)).build()); + + assertSuggestions("foo", geoNeighbourPrefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + public void testGeoField() throws Exception { + + XContentBuilder mapping = jsonBuilder(); + mapping.startObject(); + mapping.startObject(TYPE); + mapping.startObject("properties"); + mapping.startObject("pin"); + mapping.field("type", "geo_point"); + mapping.endObject(); + mapping.startObject(FIELD); + mapping.field("type", "completion"); + mapping.field("analyzer", "simple"); + + mapping.startArray("contexts"); + mapping.startObject(); + mapping.field("name", "st"); + mapping.field("type", "geo"); + mapping.field("path", "pin"); + mapping.field("precision", 5); + mapping.endObject(); + mapping.endArray(); + + mapping.endObject(); + mapping.endObject(); + mapping.endObject(); + mapping.endObject(); + + assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping)); + ensureYellow(); + + XContentBuilder source1 = jsonBuilder() + .startObject() + .latlon("pin", 52.529172, 13.407333) + .startObject(FIELD) + .array("input", "Hotel Amsterdam in Berlin") + .endObject() + .endObject(); + client().prepareIndex(INDEX, TYPE, "1").setSource(source1).execute().actionGet(); + + XContentBuilder source2 = jsonBuilder() + .startObject() + .latlon("pin", 52.363389, 4.888695) + .startObject(FIELD) + .array("input", "Hotel Berlin in Amsterdam") + .endObject() + .endObject(); + client().prepareIndex(INDEX, TYPE, "2").setSource(source2).execute().actionGet(); + + refresh(); + + String suggestionName = randomAsciiOfLength(10); + CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text("h").size(10) + .geoContexts("st", GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.52, 13.4)).build()); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(context).get(); + + assertEquals(suggestResponse.getSuggest().size(), 1); + assertEquals("Hotel Amsterdam in Berlin", suggestResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string()); + } + + public void assertSuggestions(String suggestionName, SuggestBuilder.SuggestionBuilder suggestBuilder, String... suggestions) { + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestBuilder + ).execute().actionGet(); + CompletionSuggestSearchIT.assertSuggestions(suggestResponse, suggestionName, suggestions); + } + + private void createIndexAndMapping(CompletionMappingBuilder completionMappingBuilder) throws IOException { + createIndexAndMappingAndSettings(Settings.EMPTY, completionMappingBuilder); + } + private void createIndexAndMappingAndSettings(Settings settings, CompletionMappingBuilder completionMappingBuilder) throws IOException { + XContentBuilder mapping = jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .field("analyzer", completionMappingBuilder.indexAnalyzer) + .field("search_analyzer", completionMappingBuilder.searchAnalyzer) + .field("preserve_separators", completionMappingBuilder.preserveSeparators) + .field("preserve_position_increments", completionMappingBuilder.preservePositionIncrements); + + if (completionMappingBuilder.contextMappings != null) { + mapping = mapping.startArray("contexts"); + for (Map.Entry contextMapping : completionMappingBuilder.contextMappings.entrySet()) { + mapping = mapping.startObject() + .field("name", contextMapping.getValue().name()) + .field("type", contextMapping.getValue().type().name()); + switch (contextMapping.getValue().type()) { + case CATEGORY: + final String fieldName = ((CategoryContextMapping) contextMapping.getValue()).getFieldName(); + if (fieldName != null) { + mapping = mapping.field("path", fieldName); + } + break; + case GEO: + final String name = ((GeoContextMapping) contextMapping.getValue()).getFieldName(); + mapping = mapping + .field("precision", ((GeoContextMapping) contextMapping.getValue()).getPrecision()); + if (name != null) { + mapping.field("path", name); + } + break; + } + + mapping = mapping.endObject(); + } + + mapping = mapping.endArray(); + } + mapping = mapping.endObject() + .endObject().endObject() + .endObject(); + + assertAcked(client().admin().indices().prepareCreate(INDEX) + .setSettings(Settings.settingsBuilder().put(indexSettings()).put(settings)) + .addMapping(TYPE, mapping) + .get()); + ensureYellow(); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearchIT.java deleted file mode 100644 index 48048b7d8b2..00000000000 --- a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearchIT.java +++ /dev/null @@ -1,1064 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.suggest; - -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; - -import org.apache.lucene.util.XGeoHashUtils; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.search.suggest.Suggest.Suggestion; -import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry; -import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option; -import org.elasticsearch.search.suggest.completion.CompletionSuggestion; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; -import org.elasticsearch.search.suggest.context.ContextBuilder; -import org.elasticsearch.search.suggest.context.ContextMapping; -import org.elasticsearch.test.ESIntegTestCase; -import org.hamcrest.Matchers; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion; -import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertDistance; -import static org.hamcrest.Matchers.containsString; - -@SuppressCodecs("*") // requires custom completion format -public class ContextSuggestSearchIT extends ESIntegTestCase { - - private static final String INDEX = "test"; - private static final String TYPE = "testType"; - private static final String FIELD = "testField"; - - private static final String[][] HEROS = { - { "Afari, Jamal", "Jamal Afari", "Jamal" }, - { "Allerdyce, St. John", "Allerdyce, John", "St. John", "St. John Allerdyce" }, - { "Beaubier, Jean-Paul", "Jean-Paul Beaubier", "Jean-Paul" }, - { "Beaubier, Jeanne-Marie", "Jeanne-Marie Beaubier", "Jeanne-Marie" }, - { "Braddock, Elizabeth \"Betsy\"", "Betsy", "Braddock, Elizabeth", "Elizabeth Braddock", "Elizabeth" }, - { "Cody Mushumanski gun Man", "the hunter", "gun man", "Cody Mushumanski" }, - { "Corbo, Adrian", "Adrian Corbo", "Adrian" }, - { "Corbo, Jared", "Jared Corbo", "Jared" }, - { "Creel, Carl \"Crusher\"", "Creel, Carl", "Crusher", "Carl Creel", "Carl" }, - { "Crichton, Lady Jacqueline Falsworth", "Lady Jacqueline Falsworth Crichton", "Lady Jacqueline Falsworth", - "Jacqueline Falsworth" }, { "Crichton, Kenneth", "Kenneth Crichton", "Kenneth" }, - { "MacKenzie, Al", "Al MacKenzie", "Al" }, - { "MacPherran, Mary \"Skeeter\"", "Mary MacPherran \"Skeeter\"", "MacPherran, Mary", "Skeeter", "Mary MacPherran" }, - { "MacTaggert, Moira", "Moira MacTaggert", "Moira" }, { "Rasputin, Illyana", "Illyana Rasputin", "Illyana" }, - { "Rasputin, Mikhail", "Mikhail Rasputin", "Mikhail" }, { "Rasputin, Piotr", "Piotr Rasputin", "Piotr" }, - { "Smythe, Alistair", "Alistair Smythe", "Alistair" }, { "Smythe, Spencer", "Spencer Smythe", "Spencer" }, - { "Whitemane, Aelfyre", "Aelfyre Whitemane", "Aelfyre" }, { "Whitemane, Kofi", "Kofi Whitemane", "Kofi" } }; - - @Test - public void testBasicGeo() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st").precision("5km").neighbors(true)))); - ensureYellow(); - - XContentBuilder source1 = jsonBuilder() - .startObject() - .startObject(FIELD) - .array("input", "Hotel Amsterdam", "Amsterdam") - .field("output", "Hotel Amsterdam in Berlin") - .startObject("context").latlon("st", 52.529172, 13.407333).endObject() - .endObject() - .endObject(); - client().prepareIndex(INDEX, TYPE, "1").setSource(source1).execute().actionGet(); - - XContentBuilder source2 = jsonBuilder() - .startObject() - .startObject(FIELD) - .array("input", "Hotel Berlin", "Berlin") - .field("output", "Hotel Berlin in Amsterdam") - .startObject("context").latlon("st", 52.363389, 4.888695).endObject() - .endObject() - .endObject(); - client().prepareIndex(INDEX, TYPE, "2").setSource(source2).execute().actionGet(); - - client().admin().indices().prepareRefresh(INDEX).get(); - - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text("h").size(10) - .addGeoLocation("st", 52.52, 13.4); - - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - - assertEquals(suggestResponse.getSuggest().size(), 1); - assertEquals("Hotel Amsterdam in Berlin", suggestResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string()); - } - - @Test - public void testMultiLevelGeo() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st") - .precision(1) - .precision(2) - .precision(3) - .precision(4) - .precision(5) - .precision(6) - .precision(7) - .precision(8) - .precision(9) - .precision(10) - .precision(11) - .precision(12) - .neighbors(true)))); - ensureYellow(); - - XContentBuilder source1 = jsonBuilder() - .startObject() - .startObject(FIELD) - .array("input", "Hotel Amsterdam", "Amsterdam") - .field("output", "Hotel Amsterdam in Berlin") - .startObject("context").latlon("st", 52.529172, 13.407333).endObject() - .endObject() - .endObject(); - client().prepareIndex(INDEX, TYPE, "1").setSource(source1).execute().actionGet(); - - client().admin().indices().prepareRefresh(INDEX).get(); - - for (int precision = 1; precision <= 12; precision++) { - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = new CompletionSuggestionBuilder(suggestionName).field(FIELD).text("h").size(10) - .addGeoLocation("st", 52.529172, 13.407333, precision); - - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - assertEquals(suggestResponse.getSuggest().size(), 1); - assertEquals("Hotel Amsterdam in Berlin", suggestResponse.getSuggest().getSuggestion(suggestionName).iterator().next() - .getOptions().iterator().next().getText().string()); - } - } - - @Test - public void testMappingIdempotency() throws Exception { - List precisions = new ArrayList<>(); - for (int i = 0; i < randomIntBetween(4, 12); i++) { - precisions.add(i+1); - } - Collections.shuffle(precisions, getRandom()); - XContentBuilder mapping = jsonBuilder().startObject().startObject(TYPE) - .startObject("properties").startObject("completion") - .field("type", "completion") - .startObject("context") - .startObject("location") - .field("type", "geo") - .array("precision", (Object[])precisions.toArray(new Integer[precisions.size()])) - .endObject() - .endObject().endObject() - .endObject().endObject(); - - assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping.string())); - ensureYellow(); - - Collections.shuffle(precisions, getRandom()); - mapping = jsonBuilder().startObject().startObject(TYPE) - .startObject("properties").startObject("completion") - .field("type", "completion") - .startObject("context") - .startObject("location") - .field("type", "geo") - .array("precision", (Object[])precisions.toArray(new Integer[precisions.size()])) - .endObject() - .endObject().endObject() - .endObject().endObject(); - assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mapping.string()).get()); - } - - - @Test - public void testGeoField() throws Exception { - - XContentBuilder mapping = jsonBuilder(); - mapping.startObject(); - mapping.startObject(TYPE); - mapping.startObject("properties"); - mapping.startObject("pin"); - mapping.field("type", "geo_point"); - mapping.endObject(); - mapping.startObject(FIELD); - mapping.field("type", "completion"); - mapping.field("analyzer", "simple"); - - mapping.startObject("context"); - mapping.value(ContextBuilder.location("st", 5, true).field("pin").build()); - mapping.endObject(); - - mapping.endObject(); - mapping.endObject(); - mapping.endObject(); - mapping.endObject(); - - assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping)); - ensureYellow(); - - XContentBuilder source1 = jsonBuilder() - .startObject() - .latlon("pin", 52.529172, 13.407333) - .startObject(FIELD) - .array("input", "Hotel Amsterdam", "Amsterdam") - .field("output", "Hotel Amsterdam in Berlin") - .startObject("context").endObject() - .endObject() - .endObject(); - client().prepareIndex(INDEX, TYPE, "1").setSource(source1).execute().actionGet(); - - XContentBuilder source2 = jsonBuilder() - .startObject() - .latlon("pin", 52.363389, 4.888695) - .startObject(FIELD) - .array("input", "Hotel Berlin", "Berlin") - .field("output", "Hotel Berlin in Amsterdam") - .startObject("context").endObject() - .endObject() - .endObject(); - client().prepareIndex(INDEX, TYPE, "2").setSource(source2).execute().actionGet(); - - refresh(); - - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text("h").size(10) - .addGeoLocation("st", 52.52, 13.4); - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - - assertEquals(suggestResponse.getSuggest().size(), 1); - assertEquals("Hotel Amsterdam in Berlin", suggestResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string()); - } - - @Test - public void testSimpleGeo() throws Exception { - String reinickendorf = "u337p3mp11e2"; - String pankow = "u33e0cyyjur4"; - String koepenick = "u33dm4f7fn40"; - String bernau = "u33etnjf1yjn"; - String berlin = "u33dc1v0xupz"; - String mitte = "u33dc0cpke4q"; - String steglitz = "u336m36rjh2p"; - String wilmersdorf = "u336wmw0q41s"; - String spandau = "u336uqek7gh6"; - String tempelhof = "u33d91jh3by0"; - String schoeneberg = "u336xdrkzbq7"; - String treptow = "u33d9unn7fp7"; - - double precision = 100.0; // meters - - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st").precision(precision).neighbors(true)))); - ensureYellow(); - - String[] locations = { reinickendorf, pankow, koepenick, bernau, berlin, mitte, steglitz, wilmersdorf, spandau, tempelhof, - schoeneberg, treptow }; - - String[][] input = { { "pizza - reinickendorf", "pizza", "food" }, { "pizza - pankow", "pizza", "food" }, - { "pizza - koepenick", "pizza", "food" }, { "pizza - bernau", "pizza", "food" }, { "pizza - berlin", "pizza", "food" }, - { "pizza - mitte", "pizza - berlin mitte", "pizza", "food" }, - { "pizza - steglitz", "pizza - Berlin-Steglitz", "pizza", "food" }, { "pizza - wilmersdorf", "pizza", "food" }, - { "pizza - spandau", "spandau bei berlin", "pizza", "food" }, - { "pizza - tempelhof", "pizza - berlin-tempelhof", "pizza", "food" }, - { "pizza - schoeneberg", "pizza - schöneberg", "pizza - berlin schoeneberg", "pizza", "food" }, - { "pizza - treptow", "pizza", "food" } }; - - for (int i = 0; i < locations.length; i++) { - XContentBuilder source = jsonBuilder().startObject().startObject(FIELD).startArray("input").value(input[i]).endArray() - .startObject("context").field("st", locations[i]).endObject().field("payload", locations[i]).endObject().endObject(); - client().prepareIndex(INDEX, TYPE, "" + i).setSource(source).execute().actionGet(); - } - - refresh(); - - assertGeoSuggestionsInRange(berlin, "pizza", precision); - assertGeoSuggestionsInRange(reinickendorf, "pizza", precision); - assertGeoSuggestionsInRange(spandau, "pizza", precision); - assertGeoSuggestionsInRange(koepenick, "pizza", precision); - assertGeoSuggestionsInRange(schoeneberg, "pizza", precision); - assertGeoSuggestionsInRange(tempelhof, "pizza", precision); - assertGeoSuggestionsInRange(bernau, "pizza", precision); - assertGeoSuggestionsInRange(pankow, "pizza", precision); - assertGeoSuggestionsInRange(mitte, "pizza", precision); - assertGeoSuggestionsInRange(steglitz, "pizza", precision); - assertGeoSuggestionsInRange(mitte, "pizza", precision); - assertGeoSuggestionsInRange(wilmersdorf, "pizza", precision); - assertGeoSuggestionsInRange(treptow, "pizza", precision); - } - - @Test - public void testSimplePrefix() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.category("st")))); - ensureYellow(); - - for (int i = 0; i < HEROS.length; i++) { - XContentBuilder source = jsonBuilder().startObject().startObject(FIELD).startArray("input").value(HEROS[i]).endArray() - .startObject("context").field("st", i%3).endObject() - .startObject("payload").field("group", i % 3).field("id", i).endObject() - .endObject().endObject(); - client().prepareIndex(INDEX, TYPE, "" + i).setSource(source).execute().actionGet(); - } - - refresh(); - - assertPrefixSuggestions(0, "a", "Afari, Jamal", "Adrian Corbo", "Adrian"); - assertPrefixSuggestions(0, "b", "Beaubier, Jeanne-Marie"); - assertPrefixSuggestions(0, "c", "Corbo, Adrian", "Crichton, Lady Jacqueline Falsworth"); - assertPrefixSuggestions(0, "mary", "Mary MacPherran \"Skeeter\"", "Mary MacPherran"); - assertPrefixSuggestions(0, "s", "Skeeter", "Smythe, Spencer", "Spencer Smythe", "Spencer"); - assertPrefixSuggestions(1, "s", "St. John", "St. John Allerdyce"); - assertPrefixSuggestions(2, "s", "Smythe, Alistair"); - assertPrefixSuggestions(1, "w", "Whitemane, Aelfyre"); - assertPrefixSuggestions(2, "w", "Whitemane, Kofi"); - } - - @Test - public void testTypeCategoryIsActuallyCalledCategory() throws Exception { - XContentBuilder mapping = jsonBuilder(); - mapping.startObject().startObject(TYPE).startObject("properties") - .startObject("suggest_field").field("type", "completion") - .startObject("context").startObject("color").field("type", "category").endObject().endObject() - .endObject() - .endObject().endObject().endObject(); - assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping)); - ensureYellow(); - XContentBuilder doc1 = jsonBuilder(); - doc1.startObject().startObject("suggest_field") - .field("input", "backpack_red") - .startObject("context").field("color", "red", "all_colors").endObject() - .endObject().endObject(); - XContentBuilder doc2 = jsonBuilder(); - doc2.startObject().startObject("suggest_field") - .field("input", "backpack_green") - .startObject("context").field("color", "green", "all_colors").endObject() - .endObject().endObject(); - - client().prepareIndex(INDEX, TYPE, "1") - .setSource(doc1).execute() - .actionGet(); - client().prepareIndex(INDEX, TYPE, "2") - .setSource(doc2).execute() - .actionGet(); - - refresh(); - getBackpackSuggestionAndCompare("all_colors", "backpack_red", "backpack_green"); - getBackpackSuggestionAndCompare("red", "backpack_red"); - getBackpackSuggestionAndCompare("green", "backpack_green"); - getBackpackSuggestionAndCompare("not_existing_color"); - - } - - private void getBackpackSuggestionAndCompare(String contextValue, String... expectedText) { - Set expected = Sets.newHashSet(expectedText); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion("suggestion").field("suggest_field").text("back").size(10).addContextField("color", contextValue); - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - Suggest suggest = suggestResponse.getSuggest(); - assertEquals(suggest.size(), 1); - for (Suggestion> s : suggest) { - CompletionSuggestion suggestion = (CompletionSuggestion) s; - for (CompletionSuggestion.Entry entry : suggestion) { - List options = entry.getOptions(); - assertEquals(options.size(), expectedText.length); - for (CompletionSuggestion.Entry.Option option : options) { - assertTrue(expected.contains(option.getText().string())); - expected.remove(option.getText().string()); - } - } - } - } - - - @Test - public void testBasic() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, false, ContextBuilder.reference("st", "_type"), ContextBuilder.reference("nd", "_type")))); - ensureYellow(); - - client().prepareIndex(INDEX, TYPE, "1") - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value("my hotel").value("this hotel").endArray() - .startObject("context").endObject() - .field("payload", TYPE + "|" + TYPE).endObject().endObject()).execute() - .actionGet(); - - refresh(); - - assertDoubleFieldSuggestions(TYPE, TYPE, "m", "my hotel"); - } - - @Test - public void testSimpleField() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "category")))); - ensureYellow(); - - for (int i = 0; i < HEROS.length; i++) { - client().prepareIndex(INDEX, TYPE, "" + i) - .setSource( - jsonBuilder().startObject().field("category", Integer.toString(i % 3)).startObject(FIELD).startArray("input") - .value(HEROS[i]).endArray().startObject("context").endObject().field("payload", Integer.toString(i % 3)) - .endObject().endObject()).execute().actionGet(); - } - - refresh(); - - assertFieldSuggestions("0", "a", "Afari, Jamal", "Adrian Corbo", "Adrian"); - assertFieldSuggestions("0", "b", "Beaubier, Jeanne-Marie"); - assertFieldSuggestions("0", "c", "Corbo, Adrian", "Crichton, Lady Jacqueline Falsworth"); - assertFieldSuggestions("0", "mary", "Mary MacPherran \"Skeeter\"", "Mary MacPherran"); - assertFieldSuggestions("0", "s", "Skeeter", "Smythe, Spencer", "Spencer Smythe", "Spencer"); - assertFieldSuggestions("1", "s", "St. John", "St. John Allerdyce"); - assertFieldSuggestions("2", "s", "Smythe, Alistair"); - assertFieldSuggestions("1", "w", "Whitemane, Aelfyre"); - assertFieldSuggestions("2", "w", "Whitemane, Kofi"); - - } - - @Test // see issue #10987 - public void testEmptySuggestion() throws Exception { - String mapping = jsonBuilder() - .startObject() - .startObject(TYPE) - .startObject("properties") - .startObject(FIELD) - .field("type", "completion") - .startObject("context") - .startObject("type_context") - .field("path", "_type") - .field("type", "category") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .string(); - - assertAcked(client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, mapping).get()); - ensureGreen(); - - client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "") - .setRefresh(true).get(); - - } - - @Test - public void testMultiValueField() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "category")))); - ensureYellow(); - - for (int i = 0; i < HEROS.length; i++) { - client().prepareIndex(INDEX, TYPE, "" + i) - .setSource( - jsonBuilder().startObject().startArray("category").value(Integer.toString(i % 3)).value("other").endArray() - .startObject(FIELD).startArray("input").value(HEROS[i]).endArray().startObject("context").endObject() - .field("payload", Integer.toString(i % 3)).endObject().endObject()).execute().actionGet(); - } - - refresh(); - - assertFieldSuggestions("0", "a", "Afari, Jamal", "Adrian Corbo", "Adrian"); - assertFieldSuggestions("0", "b", "Beaubier, Jeanne-Marie"); - assertFieldSuggestions("0", "c", "Corbo, Adrian", "Crichton, Lady Jacqueline Falsworth"); - assertFieldSuggestions("0", "mary", "Mary MacPherran \"Skeeter\"", "Mary MacPherran"); - assertFieldSuggestions("0", "s", "Skeeter", "Smythe, Spencer", "Spencer Smythe", "Spencer"); - assertFieldSuggestions("1", "s", "St. John", "St. John Allerdyce"); - assertFieldSuggestions("2", "s", "Smythe, Alistair"); - assertFieldSuggestions("1", "w", "Whitemane, Aelfyre"); - assertFieldSuggestions("2", "w", "Whitemane, Kofi"); - } - - @Test - public void testMultiContext() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "categoryA"), ContextBuilder.reference("nd", "categoryB")))); - ensureYellow(); - - for (int i = 0; i < HEROS.length; i++) { - client().prepareIndex(INDEX, TYPE, "" + i) - .setSource( - jsonBuilder().startObject().field("categoryA").value("" + (char) ('0' + (i % 3))).field("categoryB") - .value("" + (char) ('A' + (i % 3))).startObject(FIELD).startArray("input").value(HEROS[i]).endArray() - .startObject("context").endObject().field("payload", ((char) ('0' + (i % 3))) + "" + (char) ('A' + (i % 3))) - .endObject().endObject()).execute().actionGet(); - } - - refresh(); - - assertMultiContextSuggestions("0", "A", "a", "Afari, Jamal", "Adrian Corbo", "Adrian"); - assertMultiContextSuggestions("0", "A", "b", "Beaubier, Jeanne-Marie"); - assertMultiContextSuggestions("0", "A", "c", "Corbo, Adrian", "Crichton, Lady Jacqueline Falsworth"); - assertMultiContextSuggestions("0", "A", "mary", "Mary MacPherran \"Skeeter\"", "Mary MacPherran"); - assertMultiContextSuggestions("0", "A", "s", "Skeeter", "Smythe, Spencer", "Spencer Smythe", "Spencer"); - assertMultiContextSuggestions("1", "B", "s", "St. John", "St. John Allerdyce"); - assertMultiContextSuggestions("2", "C", "s", "Smythe, Alistair"); - assertMultiContextSuggestions("1", "B", "w", "Whitemane, Aelfyre"); - assertMultiContextSuggestions("2", "C", "w", "Whitemane, Kofi"); - } - - @Test - public void testMultiContextWithFuzzyLogic() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "categoryA"), ContextBuilder.reference("nd", "categoryB")))); - ensureYellow(); - - for (int i = 0; i < HEROS.length; i++) { - String source = jsonBuilder().startObject().field("categoryA", "" + (char) ('0' + (i % 3))) - .field("categoryB", "" + (char) ('a' + (i % 3))).startObject(FIELD).array("input", HEROS[i]) - .startObject("context").endObject().startObject("payload").field("categoryA", "" + (char) ('0' + (i % 3))) - .field("categoryB", "" + (char) ('a' + (i % 3))).endObject().endObject().endObject().string(); - client().prepareIndex(INDEX, TYPE, "" + i).setSource(source).execute().actionGet(); - } - - refresh(); - - String[] prefix1 = { "0", "1", "2" }; - String[] prefix2 = { "a", "b", "c" }; - String[] prefix3 = { "0", "1" }; - String[] prefix4 = { "a", "b" }; - - assertContextWithFuzzySuggestions(prefix1, prefix2, "mary", "MacKenzie, Al", "MacPherran, Mary", "MacPherran, Mary \"Skeeter\"", - "MacTaggert, Moira", "Mary MacPherran", "Mary MacPherran \"Skeeter\""); - assertContextWithFuzzySuggestions(prefix1, prefix2, "mac", "Mikhail", "Mary MacPherran \"Skeeter\"", "MacTaggert, Moira", - "Moira MacTaggert", "Moira", "MacKenzie, Al", "Mary MacPherran", "Mikhail Rasputin", "MacPherran, Mary", - "MacPherran, Mary \"Skeeter\""); - assertContextWithFuzzySuggestions(prefix3, prefix4, "mary", "MacPherran, Mary", "MacPherran, Mary \"Skeeter\"", - "MacTaggert, Moira", "Mary MacPherran", "Mary MacPherran \"Skeeter\""); - assertContextWithFuzzySuggestions(prefix3, prefix4, "mac", "MacPherran, Mary", "MacPherran, Mary \"Skeeter\"", "MacTaggert, Moira", - "Mary MacPherran", "Mary MacPherran \"Skeeter\"", "Mikhail", "Mikhail Rasputin", "Moira", "Moira MacTaggert"); - } - - @Test - public void testSimpleType() throws Exception { - String[] types = { TYPE + "A", TYPE + "B", TYPE + "C" }; - - CreateIndexRequestBuilder createIndexRequestBuilder = prepareCreate(INDEX); - for (String type : types) { - createIndexRequestBuilder.addMapping(type, createMapping(type, ContextBuilder.reference("st", "_type"))); - } - assertAcked(createIndexRequestBuilder); - ensureYellow(); - - for (int i = 0; i < HEROS.length; i++) { - String type = types[i % types.length]; - client().prepareIndex(INDEX, type, "" + i) - .setSource( - jsonBuilder().startObject().startObject(FIELD).startArray("input").value(HEROS[i]).endArray() - .startObject("context").endObject().field("payload", type).endObject().endObject()).execute().actionGet(); - } - - refresh(); - - assertFieldSuggestions(types[0], "a", "Afari, Jamal", "Adrian Corbo", "Adrian"); - assertFieldSuggestions(types[0], "b", "Beaubier, Jeanne-Marie"); - assertFieldSuggestions(types[0], "c", "Corbo, Adrian", "Crichton, Lady Jacqueline Falsworth"); - assertFieldSuggestions(types[0], "mary", "Mary MacPherran \"Skeeter\"", "Mary MacPherran"); - assertFieldSuggestions(types[0], "s", "Skeeter", "Smythe, Spencer", "Spencer Smythe", "Spencer"); - assertFieldSuggestions(types[1], "s", "St. John", "St. John Allerdyce"); - assertFieldSuggestions(types[2], "s", "Smythe, Alistair"); - assertFieldSuggestions(types[1], "w", "Whitemane, Aelfyre"); - assertFieldSuggestions(types[2], "w", "Whitemane, Kofi"); - } - - @Test // issue 5525, default location didnt work with lat/lon map, and did not set default location appropriately - public void testGeoContextDefaultMapping() throws Exception { - GeoPoint berlinAlexanderplatz = GeoPoint.fromGeohash("u33dc1"); - - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("poi").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("location") - .field("type", "geo") - .field("precision", "500m") - .startObject("default").field("lat", berlinAlexanderplatz.lat()).field("lon", berlinAlexanderplatz.lon()).endObject() - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("poi", xContentBuilder)); - ensureYellow(); - - index(INDEX, "poi", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Alexanderplatz").endObject().endObject()); - refresh(); - - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", berlinAlexanderplatz.lat(), berlinAlexanderplatz.lon()); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz"); - } - - @Test // issue 5525, setting the path of a category context and then indexing a document without that field returned an error - public void testThatMissingPrefixesForContextReturnException() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("service").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("color") - .field("type", "category") - .field("path", "color") - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("service", xContentBuilder)); - ensureYellow(); - - // now index a document with color field - index(INDEX, "service", "1", jsonBuilder().startObject().field("color", "red").startObject("suggest").field("input", "backback").endObject().endObject()); - - // now index a document without a color field - try { - index(INDEX, "service", "2", jsonBuilder().startObject().startObject("suggest").field("input", "backback").endObject().endObject()); - fail("index operation was not supposed to be successful"); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("one or more prefixes needed")); - } - } - - @Test // issue 5525, the geo point parser did not work when the lat/lon values were inside of a value object - public void testThatLocationVenueCanBeParsedAsDocumented() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("poi").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("location") - .field("type", "geo") - .field("precision", "1m") - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("poi", xContentBuilder)); - ensureYellow(); - - SuggestRequest suggestRequest = new SuggestRequest(INDEX); - XContentBuilder builder = jsonBuilder().startObject() - .startObject("suggest") - .field("text", "m") - .startObject("completion") - .field("field", "suggest") - .startObject("context").startObject("location").startObject("value").field("lat", 0).field("lon", 0).endObject().field("precision", "1km").endObject().endObject() - .endObject() - .endObject() - .endObject(); - suggestRequest.suggest(builder.bytes()); - - SuggestResponse suggestResponse = client().suggest(suggestRequest).get(); - assertNoFailures(suggestResponse); - } - - @Test - public void testThatCategoryDefaultWorks() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("item").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("color") - .field("type", "category").field("default", "red") - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); - ensureYellow(); - - index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie red").endObject().endObject()); - index(INDEX, "item", "2", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie blue").startObject("context").field("color", "blue").endObject().endObject().endObject()); - refresh(); - - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("h").size(10).addContextField("color", "red"); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Hoodie red"); - } - - @Test - public void testThatDefaultCategoryAndPathWorks() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("item").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("color") - .field("type", "category") - .field("default", "red") - .field("path", "color") - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); - ensureYellow(); - - index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie red").endObject().endObject()); - index(INDEX, "item", "2", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie blue").endObject().field("color", "blue").endObject()); - refresh(); - - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("h").size(10).addContextField("color", "red"); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Hoodie red"); - } - - @Test - public void testThatGeoPrecisionIsWorking() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("item").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("location") - .field("type", "geo") - .field("precision", 4) // this means geo hashes with a length of four are used, like u345 - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); - ensureYellow(); - - // lets create some locations by geohashes in different cells with the precision 4 - // this means, that poelchaustr is not a neighour to alexanderplatz, but they share the same prefix until the fourth char! - GeoPoint alexanderplatz = GeoPoint.fromGeohash("u33dc1"); - GeoPoint poelchaustr = GeoPoint.fromGeohash("u33du5"); - GeoPoint dahlem = GeoPoint.fromGeohash("u336q"); // berlin dahlem, should be included with that precision - GeoPoint middleOfNoWhere = GeoPoint.fromGeohash("u334"); // location for west from berlin, should not be included in any suggestions - - index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Alexanderplatz").field("weight", 3).startObject("context").startObject("location").field("lat", alexanderplatz.lat()).field("lon", alexanderplatz.lon()).endObject().endObject().endObject().endObject()); - index(INDEX, "item", "2", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Poelchaustr.").field("weight", 2).startObject("context").startObject("location").field("lat", poelchaustr.lat()).field("lon", poelchaustr.lon()).endObject().endObject().endObject().endObject()); - index(INDEX, "item", "3", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Far Away").field("weight", 1).startObject("context").startObject("location").field("lat", middleOfNoWhere.lat()).field("lon", middleOfNoWhere.lon()).endObject().endObject().endObject().endObject()); - index(INDEX, "item", "4", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Dahlem").field("weight", 1).startObject("context").startObject("location").field("lat", dahlem.lat()).field("lon", dahlem.lon()).endObject().endObject().endObject().endObject()); - refresh(); - - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz", "Berlin Poelchaustr.", "Berlin Dahlem"); - } - - @Test - public void testThatNeighborsCanBeExcluded() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("item").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("location") - .field("type", "geo") - .field("precision", 6) - .field("neighbors", false) - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); - ensureYellow(); - - GeoPoint alexanderplatz = GeoPoint.fromGeohash("u33dc1"); - // does not look like it, but is a direct neighbor - // this test would fail, if the precision was set 4, as then both cells would be the same, u33d - GeoPoint cellNeighbourOfAlexanderplatz = GeoPoint.fromGeohash("u33dbc"); - - index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Alexanderplatz").field("weight", 3).startObject("context").startObject("location").field("lat", alexanderplatz.lat()).field("lon", alexanderplatz.lon()).endObject().endObject().endObject().endObject()); - index(INDEX, "item", "2", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Hackescher Markt").field("weight", 2).startObject("context").startObject("location").field("lat", cellNeighbourOfAlexanderplatz.lat()).field("lon", cellNeighbourOfAlexanderplatz.lon()).endObject().endObject().endObject().endObject()); - refresh(); - - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz"); - } - - @Test - public void testThatGeoPathCanBeSelected() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("item").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("location") - .field("type", "geo") - .field("precision", "5m") - .field("path", "loc") - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); - ensureYellow(); - - GeoPoint alexanderplatz = GeoPoint.fromGeohash("u33dc1"); - index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Alexanderplatz").endObject().startObject("loc").field("lat", alexanderplatz.lat()).field("lon", alexanderplatz.lon()).endObject().endObject()); - refresh(); - - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz"); - } - - @Test(expected = MapperParsingException.class) - public void testThatPrecisionIsRequired() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("item").startObject("properties").startObject("suggest") - .field("type", "completion") - .startObject("context").startObject("location") - .field("type", "geo") - .field("path", "loc") - .endObject().endObject() - .endObject().endObject().endObject() - .endObject(); - - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); - } - - @Test - public void testThatLatLonParsingFromSourceWorks() throws Exception { - XContentBuilder xContentBuilder = jsonBuilder().startObject() - .startObject("mappings").startObject("test").startObject("properties").startObject("suggest_geo") - .field("type", "completion") - .startObject("context").startObject("location") - .field("type", "geo") - .field("precision", "1km") - .endObject().endObject() - .endObject().endObject().endObject() - .endObject().endObject(); - - assertAcked(prepareCreate("test").setSource(xContentBuilder.bytes())); - - double latitude = 52.22; - double longitude = 4.53; - String geohash = XGeoHashUtils.stringEncode(longitude, latitude); - - XContentBuilder doc1 = jsonBuilder().startObject().startObject("suggest_geo").field("input", "Hotel Marriot in Amsterdam").startObject("context").startObject("location").field("lat", latitude).field("lon", longitude).endObject().endObject().endObject().endObject(); - index("test", "test", "1", doc1); - XContentBuilder doc2 = jsonBuilder().startObject().startObject("suggest_geo").field("input", "Hotel Marriot in Berlin").startObject("context").startObject("location").field("lat", 53.31).field("lon", 13.24).endObject().endObject().endObject().endObject(); - index("test", "test", "2", doc2); - refresh(); - - XContentBuilder source = jsonBuilder().startObject().startObject("suggestion").field("text", "h").startObject("completion").field("field", "suggest_geo").startObject("context").field("location", geohash).endObject().endObject().endObject().endObject(); - SuggestRequest suggestRequest = new SuggestRequest(INDEX).suggest(source.bytes()); - SuggestResponse suggestResponse = client().suggest(suggestRequest).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Hotel Marriot in Amsterdam"); - - // this is exact the same request, but using lat/lon instead of geohash - source = jsonBuilder().startObject().startObject("suggestion").field("text", "h").startObject("completion").field("field", "suggest_geo").startObject("context").startObject("location").field("lat", latitude).field("lon", longitude).endObject().endObject().endObject().endObject().endObject(); - suggestRequest = new SuggestRequest(INDEX).suggest(source.bytes()); - suggestResponse = client().suggest(suggestRequest).get(); - assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Hotel Marriot in Amsterdam"); - } - - public void assertGeoSuggestionsInRange(String location, String suggest, double precision) throws IOException { - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) - .addGeoLocation("st", location); - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - - Suggest suggest2 = suggestResponse.getSuggest(); - assertTrue(suggest2.iterator().hasNext()); - for (Suggestion> s : suggest2) { - CompletionSuggestion suggestion = (CompletionSuggestion) s; - assertTrue(suggestion.iterator().hasNext()); - for (CompletionSuggestion.Entry entry : suggestion) { - List options = entry.getOptions(); - assertTrue(options.iterator().hasNext()); - for (CompletionSuggestion.Entry.Option option : options) { - String target = option.getPayloadAsString(); - assertDistance(location, target, Matchers.lessThanOrEqualTo(precision)); - } - } - } - } - - public void assertPrefixSuggestions(long prefix, String suggest, String... hits) throws IOException { - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest) - .size(hits.length + 1).addCategory("st", Long.toString(prefix)); - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - ArrayList suggestions = new ArrayList<>(); - Suggest suggest2 = suggestResponse.getSuggest(); - assertTrue(suggest2.iterator().hasNext()); - for (Suggestion> s : suggest2) { - CompletionSuggestion suggestion = (CompletionSuggestion) s; - for (CompletionSuggestion.Entry entry : suggestion) { - List options = entry.getOptions(); - for (CompletionSuggestion.Entry.Option option : options) { - Map payload = option.getPayloadAsMap(); - int group = (Integer) payload.get("group"); - String text = option.getText().string(); - assertEquals(prefix, group); - suggestions.add(text); - } - } - } - assertSuggestionsMatch(suggestions, hits); - } - - public void assertContextWithFuzzySuggestions(String[] prefix1, String[] prefix2, String suggest, String... hits) throws IOException { - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionFuzzyBuilder context = SuggestBuilders.fuzzyCompletionSuggestion(suggestionName).field(FIELD).text(suggest) - .size(hits.length + 10).addContextField("st", prefix1).addContextField("nd", prefix2).setFuzziness(Fuzziness.TWO); - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - - ArrayList suggestions = new ArrayList<>(); - - Suggest suggest2 = suggestResponse.getSuggest(); - assertTrue(suggest2.iterator().hasNext()); - for (Suggestion> s : suggest2) { - CompletionSuggestion suggestion = (CompletionSuggestion) s; - for (CompletionSuggestion.Entry entry : suggestion) { - List options = entry.getOptions(); - for (CompletionSuggestion.Entry.Option option : options) { - Map payload = option.getPayloadAsMap(); - String text = option.getText().string(); - assertThat(prefix1, Matchers.hasItemInArray(payload.get("categoryA"))); - assertThat(prefix2, Matchers.hasItemInArray(payload.get("categoryB"))); - suggestions.add(text); - } - } - } - - assertSuggestionsMatch(suggestions, hits); - } - - public void assertFieldSuggestions(String value, String suggest, String... hits) throws IOException { - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) - .addContextField("st", value); - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - - ArrayList suggestions = new ArrayList<>(); - - Suggest suggest2 = suggestResponse.getSuggest(); - for (Suggestion> s : suggest2) { - CompletionSuggestion suggestion = (CompletionSuggestion) s; - for (CompletionSuggestion.Entry entry : suggestion) { - List options = entry.getOptions(); - for (CompletionSuggestion.Entry.Option option : options) { - String payload = option.getPayloadAsString(); - String text = option.getText().string(); - assertEquals(value, payload); - suggestions.add(text); - } - } - } - assertSuggestionsMatch(suggestions, hits); - } - - public void assertDoubleFieldSuggestions(String field1, String field2, String suggest, String... hits) throws IOException { - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) - .addContextField("st", field1).addContextField("nd", field2); - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - ArrayList suggestions = new ArrayList<>(); - - Suggest suggest2 = suggestResponse.getSuggest(); - for (Suggestion> s : suggest2) { - CompletionSuggestion suggestion = (CompletionSuggestion) s; - for (CompletionSuggestion.Entry entry : suggestion) { - List options = entry.getOptions(); - for (CompletionSuggestion.Entry.Option option : options) { - String payload = option.getPayloadAsString(); - String text = option.getText().string(); - assertEquals(field1 + "|" + field2, payload); - suggestions.add(text); - } - } - } - assertSuggestionsMatch(suggestions, hits); - } - - public void assertMultiContextSuggestions(String value1, String value2, String suggest, String... hits) throws IOException { - String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) - .addContextField("st", value1).addContextField("nd", value2); - - SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); - SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); - ArrayList suggestions = new ArrayList<>(); - - Suggest suggest2 = suggestResponse.getSuggest(); - for (Suggestion> s : suggest2) { - CompletionSuggestion suggestion = (CompletionSuggestion) s; - for (CompletionSuggestion.Entry entry : suggestion) { - List options = entry.getOptions(); - for (CompletionSuggestion.Entry.Option option : options) { - String payload = option.getPayloadAsString(); - String text = option.getText().string(); - assertEquals(value1 + value2, payload); - suggestions.add(text); - } - } - } - assertSuggestionsMatch(suggestions, hits); - } - - private void assertSuggestionsMatch(List suggestions, String... hits) { - boolean[] suggested = new boolean[hits.length]; - Arrays.sort(hits); - Arrays.fill(suggested, false); - int numSuggestions = 0; - - for (String suggestion : suggestions) { - int hitpos = Arrays.binarySearch(hits, suggestion); - - assertEquals(hits[hitpos], suggestion); - assertTrue(hitpos >= 0); - assertTrue(!suggested[hitpos]); - - suggested[hitpos] = true; - numSuggestions++; - - } - assertEquals(hits.length, numSuggestions); - } - - private XContentBuilder createMapping(String type, ContextBuilder... context) throws IOException { - return createMapping(type, false, context); - } - - private XContentBuilder createMapping(String type, boolean preserveSeparators, ContextBuilder... context) throws IOException { - return createMapping(type, "simple", "simple", true, preserveSeparators, true, context); - } - - private XContentBuilder createMapping(String type, String indexAnalyzer, String searchAnalyzer, boolean payloads, boolean preserveSeparators, - boolean preservePositionIncrements, ContextBuilder... contexts) throws IOException { - XContentBuilder mapping = jsonBuilder(); - mapping.startObject(); - mapping.startObject(type); - mapping.startObject("properties"); - mapping.startObject(FIELD); - mapping.field("type", "completion"); - mapping.field("analyzer", indexAnalyzer); - mapping.field("search_analyzer", searchAnalyzer); - mapping.field("payloads", payloads); - mapping.field("preserve_separators", preserveSeparators); - mapping.field("preserve_position_increments", preservePositionIncrements); - - mapping.startObject("context"); - for (ContextBuilder context : contexts) { - mapping.value(context.build()); - } - mapping.endObject(); - - mapping.endObject(); - mapping.endObject(); - mapping.endObject(); - mapping.endObject(); - return mapping; - } -} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java index 245a561c339..281cf6ae18e 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java @@ -20,11 +20,7 @@ package org.elasticsearch.search.suggest; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.CharsRefBuilder; -import org.elasticsearch.common.HasContextAndHeaders; import org.elasticsearch.common.text.StringText; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.IndexQueryParserService; import java.io.IOException; import java.util.Locale; @@ -58,15 +54,11 @@ public class CustomSuggester extends Suggester options = parser.map(); - CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options); - suggestionContext.setField((String) options.get("field")); - return suggestionContext; - } + return (parser, mapperService, fieldData, headersContext) -> { + Map options = parser.map(); + CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options); + suggestionContext.setField((String) options.get("field")); + return suggestionContext; }; } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java index 9b97afc4ad9..18b4fa50e7b 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java @@ -20,14 +20,12 @@ package org.elasticsearch.search.suggest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.io.IOException; import java.util.Collection; @@ -35,7 +33,6 @@ import java.util.List; import java.util.Locale; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -44,13 +41,11 @@ import static org.hamcrest.Matchers.is; */ @ClusterScope(scope= Scope.SUITE, numDataNodes =1) public class CustomSuggesterSearchIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(CustomSuggesterPlugin.class); } - @Test public void testThatCustomSuggestersCanBeRegisteredAndWork() throws Exception { createIndex("test"); client().prepareIndex("test", "test", "1").setSource(jsonBuilder() @@ -59,11 +54,12 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase { .endObject()) .setRefresh(true).execute().actionGet(); ensureYellow(); - + String randomText = randomAsciiOfLength(10); String randomField = randomAsciiOfLength(10); String randomSuffix = randomAsciiOfLength(10); - SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test").setTypes("test").setFrom(0).setSize(1).addSuggestion( + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.addSuggestion( new SuggestBuilder.SuggestionBuilder("someName", "custom") { @Override protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { @@ -73,6 +69,8 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase { } }.text(randomText) ); + SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test").setTypes("test").setFrom(0).setSize(1) + .suggest(suggestBuilder); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProviderV1.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProviderV1.java deleted file mode 100644 index 398310d3a0b..00000000000 --- a/core/src/test/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProviderV1.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.suggest.completion; - -import com.carrotsearch.hppc.ObjectLongHashMap; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; -import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Accountables; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.IntsRef; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.fst.ByteSequenceOutputs; -import org.apache.lucene.util.fst.FST; -import org.apache.lucene.util.fst.PairOutputs; -import org.apache.lucene.util.fst.PairOutputs.Pair; -import org.apache.lucene.util.fst.PositiveIntOutputs; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.completion.AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.LookupFactory; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -import static org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester.HOLE_CHARACTER; - -/** - * This is an older implementation of the AnalyzingCompletionLookupProvider class - * We use this to test for backwards compatibility in our tests, namely - * CompletionPostingsFormatTests - * This ensures upgrades between versions work smoothly - */ -public class AnalyzingCompletionLookupProviderV1 extends CompletionLookupProvider { - - // for serialization - public static final int SERIALIZE_PRESERVE_SEPARATORS = 1; - public static final int SERIALIZE_HAS_PAYLOADS = 2; - public static final int SERIALIZE_PRESERVE_POSITION_INCREMENTS = 4; - - private static final int MAX_SURFACE_FORMS_PER_ANALYZED_FORM = 256; - private static final int MAX_GRAPH_EXPANSIONS = -1; - - public static final String CODEC_NAME = "analyzing"; - public static final int CODEC_VERSION = 1; - - private boolean preserveSep; - private boolean preservePositionIncrements; - private int maxSurfaceFormsPerAnalyzedForm; - private int maxGraphExpansions; - private boolean hasPayloads; - private final XAnalyzingSuggester prototype; - - // important, these are the settings from the old xanalyzingsuggester - public static final int SEP_LABEL = 0xFF; - public static final int END_BYTE = 0x0; - public static final int PAYLOAD_SEP = '\u001f'; - - public AnalyzingCompletionLookupProviderV1(boolean preserveSep, boolean exactFirst, boolean preservePositionIncrements, boolean hasPayloads) { - this.preserveSep = preserveSep; - this.preservePositionIncrements = preservePositionIncrements; - this.hasPayloads = hasPayloads; - this.maxSurfaceFormsPerAnalyzedForm = MAX_SURFACE_FORMS_PER_ANALYZED_FORM; - this.maxGraphExpansions = MAX_GRAPH_EXPANSIONS; - int options = preserveSep ? XAnalyzingSuggester.PRESERVE_SEP : 0; - // needs to fixed in the suggester first before it can be supported - //options |= exactFirst ? XAnalyzingSuggester.EXACT_FIRST : 0; - prototype = new XAnalyzingSuggester(null, null, null, options, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, preservePositionIncrements, - null, false, 1, SEP_LABEL, PAYLOAD_SEP, END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); - } - - @Override - public String getName() { - return "analyzing"; - } - - @Override - public FieldsConsumer consumer(final IndexOutput output) throws IOException { - // TODO write index header? - CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION); - return new FieldsConsumer() { - private Map fieldOffsets = new HashMap<>(); - - @Override - public void close() throws IOException { - try { /* - * write the offsets per field such that we know where - * we need to load the FSTs from - */ - long pointer = output.getFilePointer(); - output.writeVInt(fieldOffsets.size()); - for (Map.Entry entry : fieldOffsets.entrySet()) { - output.writeString(entry.getKey()); - output.writeVLong(entry.getValue()); - } - output.writeLong(pointer); - } finally { - IOUtils.close(output); - } - } - - @Override - public void write(Fields fields) throws IOException { - for (String field : fields) { - Terms terms = fields.terms(field); - if (terms == null) { - continue; - } - TermsEnum termsEnum = terms.iterator(); - PostingsEnum docsEnum = null; - final SuggestPayload spare = new SuggestPayload(); - int maxAnalyzedPathsForOneInput = 0; - final XAnalyzingSuggester.XBuilder builder = new XAnalyzingSuggester.XBuilder(maxSurfaceFormsPerAnalyzedForm, hasPayloads, XAnalyzingSuggester.PAYLOAD_SEP); - int docCount = 0; - while (true) { - BytesRef term = termsEnum.next(); - if (term == null) { - break; - } - docsEnum = termsEnum.postings(docsEnum, PostingsEnum.PAYLOADS); - builder.startTerm(term); - int docFreq = 0; - while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { - for (int i = 0; i < docsEnum.freq(); i++) { - final int position = docsEnum.nextPosition(); - AnalyzingCompletionLookupProviderV1.this.parsePayload(docsEnum.getPayload(), spare); - builder.addSurface(spare.surfaceForm.get(), spare.payload.get(), spare.weight); - // multi fields have the same surface form so we sum up here - maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position + 1); - } - docFreq++; - docCount = Math.max(docCount, docsEnum.docID() + 1); - } - builder.finishTerm(docFreq); - } - /* - * Here we are done processing the field and we can - * buid the FST and write it to disk. - */ - FST> build = builder.build(); - assert build != null || docCount == 0 : "the FST is null but docCount is != 0 actual value: [" + docCount + "]"; - /* - * it's possible that the FST is null if we have 2 segments that get merged - * and all docs that have a value in this field are deleted. This will cause - * a consumer to be created but it doesn't consume any values causing the FSTBuilder - * to return null. - */ - if (build != null) { - fieldOffsets.put(field, output.getFilePointer()); - build.save(output); - /* write some more meta-info */ - output.writeVInt(maxAnalyzedPathsForOneInput); - output.writeVInt(maxSurfaceFormsPerAnalyzedForm); - output.writeInt(maxGraphExpansions); // can be negative - int options = 0; - options |= preserveSep ? SERIALIZE_PRESERVE_SEPARATORS : 0; - options |= hasPayloads ? SERIALIZE_HAS_PAYLOADS : 0; - options |= preservePositionIncrements ? SERIALIZE_PRESERVE_POSITION_INCREMENTS : 0; - output.writeVInt(options); - } - } - } - }; - } - - @Override - public LookupFactory load(IndexInput input) throws IOException { - CodecUtil.checkHeader(input, CODEC_NAME, CODEC_VERSION, CODEC_VERSION); - final Map lookupMap = new HashMap<>(); - input.seek(input.length() - 8); - long metaPointer = input.readLong(); - input.seek(metaPointer); - int numFields = input.readVInt(); - - Map meta = new TreeMap<>(); - for (int i = 0; i < numFields; i++) { - String name = input.readString(); - long offset = input.readVLong(); - meta.put(offset, name); - } - long sizeInBytes = 0; - for (Map.Entry entry : meta.entrySet()) { - input.seek(entry.getKey()); - FST> fst = new FST<>(input, new PairOutputs<>( - PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton())); - int maxAnalyzedPathsForOneInput = input.readVInt(); - int maxSurfaceFormsPerAnalyzedForm = input.readVInt(); - int maxGraphExpansions = input.readInt(); - int options = input.readVInt(); - boolean preserveSep = (options & SERIALIZE_PRESERVE_SEPARATORS) != 0; - boolean hasPayloads = (options & SERIALIZE_HAS_PAYLOADS) != 0; - boolean preservePositionIncrements = (options & SERIALIZE_PRESERVE_POSITION_INCREMENTS) != 0; - sizeInBytes += fst.ramBytesUsed(); - lookupMap.put(entry.getValue(), new AnalyzingSuggestHolder(preserveSep, preservePositionIncrements, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, - hasPayloads, maxAnalyzedPathsForOneInput, fst)); - } - final long ramBytesUsed = sizeInBytes; - return new LookupFactory() { - @Override - public Lookup getLookup(CompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext) { - AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(fieldType.names().indexName()); - if (analyzingSuggestHolder == null) { - return null; - } - int flags = analyzingSuggestHolder.getPreserveSeparator() ? XAnalyzingSuggester.PRESERVE_SEP : 0; - - final Automaton queryPrefix = fieldType.requiresContext() ? ContextQuery.toAutomaton(analyzingSuggestHolder.getPreserveSeparator(), suggestionContext.getContextQueries()) : null; - - XAnalyzingSuggester suggester; - if (suggestionContext.isFuzzy()) { - suggester = new XFuzzySuggester(fieldType.indexAnalyzer(), queryPrefix, fieldType.searchAnalyzer(), flags, - analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions, - suggestionContext.getFuzzyEditDistance(), suggestionContext.isFuzzyTranspositions(), - suggestionContext.getFuzzyPrefixLength(), suggestionContext.getFuzzyMinLength(), false, - analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads, - analyzingSuggestHolder.maxAnalyzedPathsForOneInput, SEP_LABEL, PAYLOAD_SEP, END_BYTE, HOLE_CHARACTER); - } else { - suggester = new XAnalyzingSuggester(fieldType.indexAnalyzer(), queryPrefix, fieldType.searchAnalyzer(), flags, - analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions, - analyzingSuggestHolder.preservePositionIncrements, - analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads, - analyzingSuggestHolder.maxAnalyzedPathsForOneInput, SEP_LABEL, PAYLOAD_SEP, END_BYTE, HOLE_CHARACTER); - } - return suggester; - } - - @Override - public CompletionStats stats(String... fields) { - long sizeInBytes = 0; - ObjectLongHashMap completionFields = null; - if (fields != null && fields.length > 0) { - completionFields = new ObjectLongHashMap<>(fields.length); - } - - for (Map.Entry entry : lookupMap.entrySet()) { - sizeInBytes += entry.getValue().fst.ramBytesUsed(); - if (fields == null || fields.length == 0) { - continue; - } - for (String field : fields) { - // support for getting fields by regex as in fielddata - if (Regex.simpleMatch(field, entry.getKey())) { - long fstSize = entry.getValue().fst.ramBytesUsed(); - completionFields.addTo(field, fstSize); - } - } - } - - return new CompletionStats(sizeInBytes, completionFields); - } - - @Override - AnalyzingSuggestHolder getAnalyzingSuggestHolder(MappedFieldType fieldType) { - return lookupMap.get(fieldType.names().indexName()); - } - - @Override - public long ramBytesUsed() { - return ramBytesUsed; - } - - @Override - public Collection getChildResources() { - return Accountables.namedAccountables("field", lookupMap); - } - }; - } - - /* - // might be readded when we change the current impl, right now not needed - static class AnalyzingSuggestHolder { - final boolean preserveSep; - final boolean preservePositionIncrements; - final int maxSurfaceFormsPerAnalyzedForm; - final int maxGraphExpansions; - final boolean hasPayloads; - final int maxAnalyzedPathsForOneInput; - final FST> fst; - - public AnalyzingSuggestHolder(boolean preserveSep, boolean preservePositionIncrements, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions, - boolean hasPayloads, int maxAnalyzedPathsForOneInput, FST> fst) { - this.preserveSep = preserveSep; - this.preservePositionIncrements = preservePositionIncrements; - this.maxSurfaceFormsPerAnalyzedForm = maxSurfaceFormsPerAnalyzedForm; - this.maxGraphExpansions = maxGraphExpansions; - this.hasPayloads = hasPayloads; - this.maxAnalyzedPathsForOneInput = maxAnalyzedPathsForOneInput; - this.fst = fst; - } - - } - */ - - @Override - public Set toFiniteStrings(TokenStream stream) throws IOException { - return prototype.toFiniteStrings(stream); - } -} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java new file mode 100644 index 00000000000..27a6529bdd8 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java @@ -0,0 +1,290 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.suggest.document.ContextSuggestField; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.index.mapper.*; +import org.elasticsearch.search.suggest.completion.context.CategoryContextMapping; +import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext; +import org.elasticsearch.search.suggest.completion.context.ContextBuilder; +import org.elasticsearch.search.suggest.completion.context.ContextMapping; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.*; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isIn; + +public class CategoryContextMappingTests extends ESSingleNodeTestCase { + + public void testIndexingWithNoContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .array("input", "suggestion1", "suggestion2") + .field("weight", 3) + .endObject() + .startObject() + .array("input", "suggestion3", "suggestion4") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 7); + } + + public void testIndexingWithSimpleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .field("ctx", "ctx1") + .endObject() + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + public void testIndexingWithContextList() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startObject("completion") + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .array("ctx", "ctx1", "ctx2", "ctx3") + .endObject() + .field("weight", 5) + .endObject() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + public void testIndexingWithMultipleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .startObject() + .field("name", "type") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + XContentBuilder builder = jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .startObject("contexts") + .array("ctx", "ctx1", "ctx2", "ctx3") + .array("type", "typr3", "ftg") + .endObject() + .endObject() + .endArray() + .endObject(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", builder.bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + public void testQueryContextParsingBasic() throws Exception { + XContentBuilder builder = jsonBuilder().value("context1"); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(1)); + assertThat(queryContexts.get(0).context, equalTo("context1")); + assertThat(queryContexts.get(0).boost, equalTo(1)); + assertThat(queryContexts.get(0).isPrefix, equalTo(false)); + } + + public void testQueryContextParsingArray() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .value("context1") + .value("context2") + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(2)); + assertThat(queryContexts.get(0).context, equalTo("context1")); + assertThat(queryContexts.get(0).boost, equalTo(1)); + assertThat(queryContexts.get(0).isPrefix, equalTo(false)); + assertThat(queryContexts.get(1).context, equalTo("context2")); + assertThat(queryContexts.get(1).boost, equalTo(1)); + assertThat(queryContexts.get(1).isPrefix, equalTo(false)); + } + + public void testQueryContextParsingObject() throws Exception { + XContentBuilder builder = jsonBuilder().startObject() + .field("context", "context1") + .field("boost", 10) + .field("prefix", true) + .endObject(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(1)); + assertThat(queryContexts.get(0).context, equalTo("context1")); + assertThat(queryContexts.get(0).boost, equalTo(10)); + assertThat(queryContexts.get(0).isPrefix, equalTo(true)); + } + + + public void testQueryContextParsingObjectArray() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .field("context", "context1") + .field("boost", 2) + .field("prefix", true) + .endObject() + .startObject() + .field("context", "context2") + .field("boost", 3) + .field("prefix", false) + .endObject() + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(2)); + assertThat(queryContexts.get(0).context, equalTo("context1")); + assertThat(queryContexts.get(0).boost, equalTo(2)); + assertThat(queryContexts.get(0).isPrefix, equalTo(true)); + assertThat(queryContexts.get(1).context, equalTo("context2")); + assertThat(queryContexts.get(1).boost, equalTo(3)); + assertThat(queryContexts.get(1).isPrefix, equalTo(false)); + } + + public void testQueryContextParsingMixed() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .field("context", "context1") + .field("boost", 2) + .field("prefix", true) + .endObject() + .value("context2") + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(2)); + assertThat(queryContexts.get(0).context, equalTo("context1")); + assertThat(queryContexts.get(0).boost, equalTo(2)); + assertThat(queryContexts.get(0).isPrefix, equalTo(true)); + assertThat(queryContexts.get(1).context, equalTo("context2")); + assertThat(queryContexts.get(1).boost, equalTo(1)); + assertThat(queryContexts.get(1).isPrefix, equalTo(false)); + } + + public void testParsingContextFromDocument() throws Exception { + CategoryContextMapping mapping = ContextBuilder.category("cat").field("category").build(); + ParseContext.Document document = new ParseContext.Document(); + document.add(new StringField("category", "category1", Field.Store.NO)); + Set context = mapping.parseContext(document); + assertThat(context.size(), equalTo(1)); + assertTrue(context.contains("category1")); + } + + static void assertContextSuggestFields(IndexableField[] fields, int expected) { + int actualFieldCount = 0; + for (IndexableField field : fields) { + if (field instanceof ContextSuggestField) { + actualFieldCount++; + } + } + assertThat(actualFieldCount, equalTo(expected)); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionPostingsFormatTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionPostingsFormatTests.java deleted file mode 100644 index ff672fbab50..00000000000 --- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionPostingsFormatTests.java +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.suggest.completion; - -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene53.Lucene53Codec; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.suggest.InputIterator; -import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.search.suggest.Lookup.LookupResult; -import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; -import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LineFileDocs; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType.Names; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.SuggestUtils; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.LookupFactory; -import org.elasticsearch.search.suggest.context.ContextMapping; -import org.elasticsearch.test.ESTestCase; -import org.junit.Test; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -public class CompletionPostingsFormatTests extends ESTestCase { - - Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - static final CompletionFieldMapper.CompletionFieldType FIELD_TYPE = CompletionFieldMapper.Defaults.FIELD_TYPE.clone(); - static final NamedAnalyzer analyzer = new NamedAnalyzer("foo", new StandardAnalyzer()); - static { - FIELD_TYPE.setNames(new Names("foo")); - FIELD_TYPE.setIndexAnalyzer(analyzer); - FIELD_TYPE.setSearchAnalyzer(analyzer); - FIELD_TYPE.freeze(); - } - - @Test - public void testCompletionPostingsFormat() throws IOException { - AnalyzingCompletionLookupProviderV1 providerV1 = new AnalyzingCompletionLookupProviderV1(true, false, true, true); - AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider(true, false, true, true); - List providers = Arrays.asList(providerV1, currentProvider); - - Completion090PostingsFormat.CompletionLookupProvider randomProvider = providers.get(getRandom().nextInt(providers.size())); - RAMDirectory dir = new RAMDirectory(); - writeData(dir, randomProvider); - - IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); - LookupFactory load = currentProvider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); - fieldType.setProvider(currentProvider); - Lookup lookup = load.getLookup(fieldType, new CompletionSuggestionContext(null)); - List result = lookup.lookup("ge", false, 10); - assertThat(result.get(0).key.toString(), equalTo("Generator - Foo Fighters")); - assertThat(result.get(0).payload.utf8ToString(), equalTo("id:10")); - dir.close(); - } - - @Test - public void testProviderBackwardCompatibilityForVersion1() throws IOException { - AnalyzingCompletionLookupProviderV1 providerV1 = new AnalyzingCompletionLookupProviderV1(true, false, true, true); - AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider(true, false, true, true); - - RAMDirectory dir = new RAMDirectory(); - writeData(dir, providerV1); - - IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); - LookupFactory load = currentProvider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); - fieldType.setProvider(currentProvider); - AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder analyzingSuggestHolder = load.getAnalyzingSuggestHolder(fieldType); - assertThat(analyzingSuggestHolder.sepLabel, is(AnalyzingCompletionLookupProviderV1.SEP_LABEL)); - assertThat(analyzingSuggestHolder.payloadSep, is(AnalyzingCompletionLookupProviderV1.PAYLOAD_SEP)); - assertThat(analyzingSuggestHolder.endByte, is(AnalyzingCompletionLookupProviderV1.END_BYTE)); - dir.close(); - } - - @Test - public void testProviderVersion2() throws IOException { - AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider(true, false, true, true); - - RAMDirectory dir = new RAMDirectory(); - writeData(dir, currentProvider); - - IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); - LookupFactory load = currentProvider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); - fieldType.setProvider(currentProvider); - AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder analyzingSuggestHolder = load.getAnalyzingSuggestHolder(fieldType); - assertThat(analyzingSuggestHolder.sepLabel, is(XAnalyzingSuggester.SEP_LABEL)); - assertThat(analyzingSuggestHolder.payloadSep, is(XAnalyzingSuggester.PAYLOAD_SEP)); - assertThat(analyzingSuggestHolder.endByte, is(XAnalyzingSuggester.END_BYTE)); - dir.close(); - } - - @Test - public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, - IllegalAccessException { - final boolean preserveSeparators = getRandom().nextBoolean(); - final boolean preservePositionIncrements = getRandom().nextBoolean(); - final boolean usePayloads = getRandom().nextBoolean(); - final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0; - - XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(), null, new StandardAnalyzer(), - options, 256, -1, preservePositionIncrements, null, false, 1, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); - LineFileDocs docs = new LineFileDocs(getRandom()); - int num = scaledRandomIntBetween(150, 300); - final String[] titles = new String[num]; - final long[] weights = new long[num]; - for (int i = 0; i < titles.length; i++) { - Document nextDoc = docs.nextDoc(); - IndexableField field = nextDoc.getField("title"); - titles[i] = field.stringValue(); - weights[i] = between(0, 100); - - } - docs.close(); - final InputIterator primaryIter = new InputIterator() { - int index = 0; - long currentWeight = -1; - - @Override - public BytesRef next() throws IOException { - if (index < titles.length) { - currentWeight = weights[index]; - return new BytesRef(titles[index++]); - } - return null; - } - - @Override - public long weight() { - return currentWeight; - } - - @Override - public BytesRef payload() { - return null; - } - - @Override - public boolean hasPayloads() { - return false; - } - - @Override - public Set contexts() { - return null; - } - - @Override - public boolean hasContexts() { - return false; - } - - }; - InputIterator iter; - if (usePayloads) { - iter = new InputIterator() { - @Override - public long weight() { - return primaryIter.weight(); - } - - @Override - public BytesRef next() throws IOException { - return primaryIter.next(); - } - - @Override - public BytesRef payload() { - return new BytesRef(Long.toString(weight())); - } - - @Override - public boolean hasPayloads() { - return true; - } - - @Override - public Set contexts() { - return null; - } - - @Override - public boolean hasContexts() { - return false; - } - }; - } else { - iter = primaryIter; - } - reference.build(iter); - - AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider(preserveSeparators, false, preservePositionIncrements, usePayloads); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); - fieldType.setProvider(currentProvider); - final CompletionFieldMapper mapper = new CompletionFieldMapper("foo", fieldType, Integer.MAX_VALUE, indexSettings, FieldMapper.MultiFields.empty(), null); - Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); - if (buildAnalyzingLookup instanceof XAnalyzingSuggester) { - assertEquals(reference.getMaxAnalyzedPathsForOneInput(), ((XAnalyzingSuggester) buildAnalyzingLookup).getMaxAnalyzedPathsForOneInput()); - } - - for (int i = 0; i < titles.length; i++) { - int res = between(1, 10); - final StringBuilder builder = new StringBuilder(); - SuggestUtils.analyze(analyzer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() { - @Override - public void nextToken() throws IOException { - if (builder.length() == 0) { - builder.append(this.charTermAttr.toString()); - } - } - }); - String firstTerm = builder.toString(); - String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length())); - List refLookup = reference.lookup(prefix, false, res); - List lookup = buildAnalyzingLookup.lookup(prefix, false, res); - assertThat(refLookup.toString(),lookup.size(), equalTo(refLookup.size())); - for (int j = 0; j < refLookup.size(); j++) { - assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key)); - assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - " + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value , - lookup.get(j).value, equalTo(refLookup.get(j).value)); - assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload)); - if (usePayloads) { - assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value))); - } - } - } - } - - public Lookup buildAnalyzingLookup(final CompletionFieldMapper mapper, String[] terms, String[] surfaces, long[] weights) - throws IOException { - RAMDirectory dir = new RAMDirectory(); - Codec codec = new Lucene53Codec() { - public PostingsFormat getPostingsFormatForField(String field) { - final PostingsFormat in = super.getPostingsFormatForField(field); - return mapper.fieldType().postingsFormat(in); - } - }; - IndexWriterConfig indexWriterConfig = new IndexWriterConfig(mapper.fieldType().indexAnalyzer()); - - indexWriterConfig.setCodec(codec); - IndexWriter writer = new IndexWriter(dir, indexWriterConfig); - for (int i = 0; i < weights.length; i++) { - Document doc = new Document(); - BytesRef payload = mapper.buildPayload(new BytesRef(surfaces[i]), weights[i], new BytesRef(Long.toString(weights[i]))); - doc.add(mapper.getCompletionField(ContextMapping.EMPTY_CONTEXT, terms[i], payload)); - if (randomBoolean()) { - writer.commit(); - } - writer.addDocument(doc); - } - writer.commit(); - writer.forceMerge(1, true); - writer.commit(); - DirectoryReader reader = DirectoryReader.open(writer, true); - assertThat(reader.leaves().size(), equalTo(1)); - assertThat(reader.leaves().get(0).reader().numDocs(), equalTo(weights.length)); - LeafReaderContext atomicReaderContext = reader.leaves().get(0); - Terms luceneTerms = atomicReaderContext.reader().terms(mapper.fieldType().names().fullName()); - Lookup lookup = ((Completion090PostingsFormat.CompletionTerms) luceneTerms).getLookup(mapper.fieldType(), new CompletionSuggestionContext(null)); - reader.close(); - writer.close(); - dir.close(); - return lookup; - } - @Test - public void testNoDocs() throws IOException { - AnalyzingCompletionLookupProvider provider = new AnalyzingCompletionLookupProvider(true, false, true, true); - RAMDirectory dir = new RAMDirectory(); - IndexOutput output = dir.createOutput("foo.txt", IOContext.DEFAULT); - FieldsConsumer consumer = provider.consumer(output); - consumer.write(new Fields() { - @Override - public Iterator iterator() { - return Arrays.asList("foo").iterator(); - } - - @Override - public Terms terms(String field) throws IOException { - return null; - } - - @Override - public int size() { - return 1; - } - }); - consumer.close(); - output.close(); - - IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); - LookupFactory load = provider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); - fieldType.setProvider(provider); - assertNull(load.getLookup(fieldType, new CompletionSuggestionContext(null))); - dir.close(); - } - - // TODO ADD more unittests - private void writeData(Directory dir, Completion090PostingsFormat.CompletionLookupProvider provider) throws IOException { - IndexOutput output = dir.createOutput("foo.txt", IOContext.DEFAULT); - FieldsConsumer consumer = provider.consumer(output); - final List terms = new ArrayList<>(); - terms.add(new TermPosAndPayload("foofightersgenerator", 256 - 2, provider.buildPayload(new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10")))); - terms.add(new TermPosAndPayload("generator", 256 - 1, provider.buildPayload(new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10")))); - Fields fields = new Fields() { - @Override - public Iterator iterator() { - return Arrays.asList("foo").iterator(); - } - - @Override - public Terms terms(String field) throws IOException { - if (field.equals("foo")) { - return new Terms() { - @Override - public TermsEnum iterator() throws IOException { - final Iterator iterator = terms.iterator(); - return new TermsEnum() { - private TermPosAndPayload current = null; - @Override - public SeekStatus seekCeil(BytesRef text) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void seekExact(long ord) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public BytesRef term() throws IOException { - return current == null ? null : current.term; - } - - @Override - public long ord() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int docFreq() throws IOException { - return current == null ? 0 : 1; - } - - @Override - public long totalTermFreq() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { - final TermPosAndPayload data = current; - return new PostingsEnum() { - boolean done = false; - @Override - public int nextPosition() throws IOException { - return data.pos; - } - - @Override - public int startOffset() throws IOException { - return 0; - } - - @Override - public int endOffset() throws IOException { - return 0; - } - - @Override - public BytesRef getPayload() throws IOException { - return data.payload; - } - - @Override - public int freq() throws IOException { - return 1; - } - - @Override - public int docID() { - if (done) { - return NO_MORE_DOCS; - } - return 0; - } - - @Override - public int nextDoc() throws IOException { - if (done) { - return NO_MORE_DOCS; - } - done = true; - return 0; - } - - @Override - public int advance(int target) throws IOException { - if (done) { - return NO_MORE_DOCS; - } - done = true; - return 0; - } - - @Override - public long cost() { - return 0; - } - }; - } - - @Override - public BytesRef next() throws IOException { - if (iterator.hasNext()) { - current = iterator.next(); - return current.term; - } - current = null; - return null; - } - }; - } - - @Override - public long size() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long getSumTotalTermFreq() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long getSumDocFreq() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int getDocCount() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasFreqs() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasOffsets() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasPositions() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasPayloads() { - throw new UnsupportedOperationException(); - } - }; - } - return null; - } - - @Override - public int size() { - return 0; - } - }; - consumer.write(fields); - consumer.close(); - output.close(); - - } - - private static class TermPosAndPayload { - final BytesRef term; - final int pos; - final BytesRef payload; - - - private TermPosAndPayload(String term, int pos, BytesRef payload) { - this.term = new BytesRef(term); - this.pos = pos; - this.payload = payload; - } - } -} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java new file mode 100644 index 00000000000..60974baee84 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java @@ -0,0 +1,346 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion; + +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.common.inject.matcher.Matchers; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.search.suggest.completion.context.*; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.*; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.search.suggest.completion.CategoryContextMappingTests.assertContextSuggestFields; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isIn; + +public class GeoContextMappingTests extends ESSingleNodeTestCase { + + public void testIndexingWithNoContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "geo") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .array("input", "suggestion1", "suggestion2") + .field("weight", 3) + .endObject() + .startObject() + .array("input", "suggestion3", "suggestion4") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 7); + } + + public void testIndexingWithSimpleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "geo") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .startObject("ctx") + .field("lat", 43.6624803) + .field("lon", -79.3863353) + .endObject() + .endObject() + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + public void testIndexingWithContextList() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "geo") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startObject("completion") + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .startArray("ctx") + .startObject() + .field("lat", 43.6624803) + .field("lon", -79.3863353) + .endObject() + .startObject() + .field("lat", 43.6624718) + .field("lon", -79.3873227) + .endObject() + .endArray() + .endObject() + .field("weight", 5) + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + public void testIndexingWithMultipleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "loc1") + .field("type", "geo") + .endObject() + .startObject() + .field("name", "loc2") + .field("type", "geo") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + XContentBuilder builder = jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .startObject("contexts") + .array("loc1", "ezs42e44yx96") + .array("loc2", "wh0n9447fwrc") + .endObject() + .endObject() + .endArray() + .endObject(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", builder.bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + public void testParsingQueryContextBasic() throws Exception { + XContentBuilder builder = jsonBuilder().value("ezs42e44yx96"); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(1 + 8)); + Collection locations = new ArrayList<>(); + locations.add("ezs42e"); + GeoHashUtils.addNeighbors("ezs42e", GeoContextMapping.DEFAULT_PRECISION, locations); + for (ContextMapping.QueryContext queryContext : queryContexts) { + assertThat(queryContext.context, isIn(locations)); + assertThat(queryContext.boost, equalTo(1)); + assertThat(queryContext.isPrefix, equalTo(false)); + } + } + + public void testParsingQueryContextGeoPoint() throws Exception { + XContentBuilder builder = jsonBuilder().startObject() + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(1 + 8)); + Collection locations = new ArrayList<>(); + locations.add("wh0n94"); + GeoHashUtils.addNeighbors("wh0n94", GeoContextMapping.DEFAULT_PRECISION, locations); + for (ContextMapping.QueryContext queryContext : queryContexts) { + assertThat(queryContext.context, isIn(locations)); + assertThat(queryContext.boost, equalTo(1)); + assertThat(queryContext.isPrefix, equalTo(false)); + } + } + + public void testParsingQueryContextObject() throws Exception { + XContentBuilder builder = jsonBuilder().startObject() + .startObject("context") + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject() + .field("boost", 10) + .array("neighbours", 1, 2, 3) + .endObject(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8)); + Collection locations = new ArrayList<>(); + locations.add("wh0n94"); + locations.add("w"); + GeoHashUtils.addNeighbors("w", 1, locations); + locations.add("wh"); + GeoHashUtils.addNeighbors("wh", 2, locations); + locations.add("wh0"); + GeoHashUtils.addNeighbors("wh0", 3, locations); + for (ContextMapping.QueryContext queryContext : queryContexts) { + assertThat(queryContext.context, isIn(locations)); + assertThat(queryContext.boost, equalTo(10)); + assertThat(queryContext.isPrefix, equalTo(queryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION)); + } + } + + public void testParsingQueryContextObjectArray() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .startObject("context") + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject() + .field("boost", 10) + .array("neighbours", 1, 2, 3) + .endObject() + .startObject() + .startObject("context") + .field("lat", 22.337374) + .field("lon", 92.112583) + .endObject() + .field("boost", 2) + .array("neighbours", 5) + .endObject() + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8 + 1 + 1 + 8)); + Collection firstLocations = new ArrayList<>(); + firstLocations.add("wh0n94"); + firstLocations.add("w"); + GeoHashUtils.addNeighbors("w", 1, firstLocations); + firstLocations.add("wh"); + GeoHashUtils.addNeighbors("wh", 2, firstLocations); + firstLocations.add("wh0"); + GeoHashUtils.addNeighbors("wh0", 3, firstLocations); + Collection secondLocations = new ArrayList<>(); + secondLocations.add("w5cx04"); + secondLocations.add("w5cx0"); + GeoHashUtils.addNeighbors("w5cx0", 5, secondLocations); + for (ContextMapping.QueryContext queryContext : queryContexts) { + if (firstLocations.contains(queryContext.context)) { + assertThat(queryContext.boost, equalTo(10)); + } else if (secondLocations.contains(queryContext.context)) { + assertThat(queryContext.boost, equalTo(2)); + } else { + fail(queryContext.context + " was not expected"); + } + assertThat(queryContext.isPrefix, equalTo(queryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION)); + } + } + + public void testParsingQueryContextMixed() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .startObject("context") + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject() + .field("boost", 10) + .array("neighbours", 1, 2) + .endObject() + .startObject() + .field("lat", 22.337374) + .field("lon", 92.112583) + .endObject() + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + List queryContexts = mapping.parseQueryContext(parser); + assertThat(queryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8)); + Collection firstLocations = new ArrayList<>(); + firstLocations.add("wh0n94"); + firstLocations.add("w"); + GeoHashUtils.addNeighbors("w", 1, firstLocations); + firstLocations.add("wh"); + GeoHashUtils.addNeighbors("wh", 2, firstLocations); + Collection secondLocations = new ArrayList<>(); + secondLocations.add("w5cx04"); + GeoHashUtils.addNeighbors("w5cx04", 6, secondLocations); + for (ContextMapping.QueryContext queryContext : queryContexts) { + if (firstLocations.contains(queryContext.context)) { + assertThat(queryContext.boost, equalTo(10)); + } else if (secondLocations.contains(queryContext.context)) { + assertThat(queryContext.boost, equalTo(1)); + } else { + fail(queryContext.context + " was not expected"); + } + assertThat(queryContext.isPrefix, equalTo(queryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION)); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/context/GeoLocationContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/context/GeoLocationContextMappingTests.java deleted file mode 100644 index b525c4aa4be..00000000000 --- a/core/src/test/java/org/elasticsearch/search/suggest/context/GeoLocationContextMappingTests.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.suggest.context; - -import org.apache.lucene.util.XGeoHashUtils; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextConfig; -import org.elasticsearch.test.ESTestCase; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - -/** - * - */ -public class GeoLocationContextMappingTests extends ESTestCase { - - @Test - public void testThatParsingGeoPointsWorksWithCoercion() throws Exception { - XContentBuilder builder = jsonBuilder().startObject().field("lat", "52").field("lon", "4").endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); - - HashMap config = new HashMap<>(); - config.put("precision", 12); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - mapping.parseQuery("foo", parser); - } - - - @Test - public void testUseWithDefaultGeoHash() throws Exception { - XContentBuilder builder = jsonBuilder().startObject().field("lat", 52d).field("lon", 4d).endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); - - String geohash = XGeoHashUtils.stringEncode(randomIntBetween(-180, +180), randomIntBetween(-90, +90)); - HashMap config = new HashMap<>(); - config.put("precision", 12); - config.put("default", geohash); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - mapping.parseQuery("foo", parser); - } - - @Test - public void testUseWithDefaultLatLon() throws Exception { - XContentBuilder builder = jsonBuilder().startObject().field("lat", 52d).field("lon", 4d).endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); - - HashMap config = new HashMap<>(); - config.put("precision", 12); - HashMap pointAsMap = new HashMap<>(); - pointAsMap.put("lat", 51d); - pointAsMap.put("lon", 0d); - config.put("default", pointAsMap); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - mapping.parseQuery("foo", parser); - } - - @Test - public void testUseWithDefaultBadLatLon() throws Exception { - XContentBuilder builder = jsonBuilder().startObject().field("lat", 52d).field("lon", 4d).endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); - - HashMap config = new HashMap<>(); - config.put("precision", 12); - HashMap pointAsMap = new HashMap<>(); - pointAsMap.put("latitude", 51d); // invalid field names - pointAsMap.put("longitude", 0d); // invalid field names - config.put("default", pointAsMap); - ElasticsearchParseException expected = null; - try { - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - mapping.parseQuery("foo", parser); - - } catch (ElasticsearchParseException e) { - expected = e; - } - assertNotNull(expected); - } - - @Test - public void testUseWithMultiplePrecisions() throws Exception { - XContentBuilder builder = jsonBuilder().startObject().field("lat", 52d).field("lon", 4d).endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); - - HashMap config = new HashMap<>(); - int numElements = randomIntBetween(1, 12); - ArrayList precisions = new ArrayList<>(); - for (int i = 0; i < numElements; i++) { - precisions.add(randomIntBetween(1, 12)); - } - config.put("precision", precisions); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - mapping.parseQuery("foo", parser); - } - - @Test - public void testHashcode() throws Exception { - HashMap config = new HashMap<>(); - if (randomBoolean()) { - config.put("precision", Arrays.asList(1, 2, 3, 4)); - } else { - config.put("precision", randomIntBetween(1, 12)); - } - if (randomBoolean()) { - HashMap pointAsMap = new HashMap<>(); - pointAsMap.put("lat", 51d); - pointAsMap.put("lon", 0d); - config.put("default", pointAsMap); - } - HashMap config2 = new HashMap<>(config); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - GeolocationContextMapping mapping2 = GeolocationContextMapping.load("foo", config2); - - assertEquals(mapping, mapping2); - assertEquals(mapping.hashCode(), mapping2.hashCode()); - } - - @Test - public void testUseWithBadGeoContext() throws Exception { - double lon = 4d; - String badLat = "W"; - XContentBuilder builder = jsonBuilder().startObject().startArray("location").value(4d).value(badLat).endArray().endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); // start of object - parser.nextToken(); // "location" field name - parser.nextToken(); // array - - HashMap config = new HashMap<>(); - config.put("precision", randomIntBetween(1, 12)); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - ElasticsearchParseException expected = null; - try { - ContextConfig geoconfig = mapping.parseContext(null, parser); - } catch (ElasticsearchParseException e) { - expected = e; - } - assertNotNull(expected); - } - - @Test - public void testUseWithLonLatGeoContext() throws Exception { - double lon = 4d; - double lat = 52d; - XContentBuilder builder = jsonBuilder().startObject().startArray("location").value(lon).value(lat).endArray().endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); // start of object - parser.nextToken(); // "location" field name - parser.nextToken(); // array - - HashMap config = new HashMap<>(); - config.put("precision", randomIntBetween(1, 12)); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - mapping.parseContext(null, parser); - } - - public void testUseWithMultiGeoHashGeoContext() throws Exception { - String geohash1 = XGeoHashUtils.stringEncode(randomIntBetween(-180, +180), randomIntBetween(-90, +90)); - String geohash2 = XGeoHashUtils.stringEncode(randomIntBetween(-180, +180), randomIntBetween(-90, +90)); - XContentBuilder builder = jsonBuilder().startObject().startArray("location").value(geohash1).value(geohash2).endArray().endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); // start of object - parser.nextToken(); // "location" field name - parser.nextToken(); // array - - HashMap config = new HashMap<>(); - config.put("precision", randomIntBetween(1, 12)); - GeolocationContextMapping mapping = GeolocationContextMapping.load("foo", config); - ContextConfig parsedContext = mapping.parseContext(null, parser); - } - -} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java index b02c42107b5..812928dee28 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.suggest.phrase; -import java.nio.charset.StandardCharsets; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.Tokenizer; @@ -43,9 +42,12 @@ import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; -import java.io.*; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -57,7 +59,6 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { private final BytesRef preTag = new BytesRef(""); private final BytesRef postTag = new BytesRef(""); - @Test public void testMarvelHeros() throws IOException { RAMDirectory dir = new RAMDirectory(); Map mapping = new HashMap<>(); @@ -97,7 +98,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { DirectoryReader ir = DirectoryReader.open(writer, false); WordScorer wordScorer = new LaplaceScorer(ir, MultiFields.getTerms(ir, "body_ngram"), "body_ngram", 0.95d, new BytesRef(" "), 0.5f); - + NoisyChannelSpellChecker suggester = new NoisyChannelSpellChecker(); DirectSpellChecker spellchecker = new DirectSpellChecker(); spellchecker.setMinQueryLength(1); @@ -108,7 +109,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { assertThat(corrections[0].join(space).utf8ToString(), equalTo("american ace")); assertThat(corrections[0].join(space, preTag, postTag).utf8ToString(), equalTo("american ace")); assertThat(result.cutoffScore, greaterThan(0d)); - + result = suggester.getCorrections(wrapper, new BytesRef("american ame"), generator, 1, 1, ir, "body", wordScorer, 0, 1); corrections = result.corrections; assertThat(corrections.length, equalTo(1)); @@ -128,14 +129,14 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { assertThat(corrections[1].join(space, preTag, postTag).utf8ToString(), equalTo("xor the god jewel")); assertThat(corrections[2].join(space, preTag, postTag).utf8ToString(), equalTo("xorn the god jewel")); assertThat(corrections[3].join(space, preTag, postTag).utf8ToString(), equalTo("xorr the got jewel")); - + corrections = suggester.getCorrections(wrapper, new BytesRef("Xor the Got-Jewel"), generator, 0.5f, 4, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections.length, equalTo(4)); assertThat(corrections[0].join(space).utf8ToString(), equalTo("xorr the god jewel")); assertThat(corrections[1].join(space).utf8ToString(), equalTo("xor the god jewel")); assertThat(corrections[2].join(space).utf8ToString(), equalTo("xorn the god jewel")); assertThat(corrections[3].join(space).utf8ToString(), equalTo("xorr the got jewel")); - + // Test some of the highlighting corner cases suggester = new NoisyChannelSpellChecker(0.85); wordScorer = new LaplaceScorer(ir, MultiFields.getTerms(ir, "body_ngram"), "body_ngram", 0.85d, new BytesRef(" "), 0.5f); @@ -151,7 +152,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { assertThat(corrections[3].join(space, preTag, postTag).utf8ToString(), equalTo("xor teh god jewel")); // test synonyms - + Analyzer analyzer = new Analyzer() { @Override @@ -160,7 +161,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { TokenFilter filter = new LowerCaseFilter(t); try { SolrSynonymParser parser = new SolrSynonymParser(true, false, new WhitespaceAnalyzer()); - ((SolrSynonymParser) parser).parse(new StringReader("usa => usa, america, american\nursa => usa, america, american")); + parser.parse(new StringReader("usa => usa, america, american\nursa => usa, america, american")); filter = new SynonymFilter(filter, parser.build(), true); } catch (Exception e) { throw new RuntimeException(e); @@ -168,7 +169,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { return new TokenStreamComponents(t, filter); } }; - + spellchecker.setAccuracy(0.0f); spellchecker.setMinPrefix(1); spellchecker.setMinQueryLength(1); @@ -177,7 +178,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { corrections = suggester.getCorrections(analyzer, new BytesRef("captian usa"), generator, 2, 4, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections[0].join(space).utf8ToString(), equalTo("captain america")); assertThat(corrections[0].join(space, preTag, postTag).utf8ToString(), equalTo("captain america")); - + generator = new DirectCandidateGenerator(spellchecker, "body", SuggestMode.SUGGEST_MORE_POPULAR, ir, 0.85, 10, null, analyzer, MultiFields.getTerms(ir, "body")); corrections = suggester.getCorrections(analyzer, new BytesRef("captian usw"), generator, 2, 4, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("captain america")); @@ -189,8 +190,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("captain america")); assertThat(corrections[0].join(space, preTag, postTag).utf8ToString(), equalTo("captain america")); } - - @Test + public void testMarvelHerosMultiGenerator() throws IOException { RAMDirectory dir = new RAMDirectory(); Map mapping = new HashMap<>(); @@ -246,23 +246,23 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { DirectCandidateGenerator forward = new DirectCandidateGenerator(spellchecker, "body", SuggestMode.SUGGEST_ALWAYS, ir, 0.95, 10); DirectCandidateGenerator reverse = new DirectCandidateGenerator(spellchecker, "body_reverse", SuggestMode.SUGGEST_ALWAYS, ir, 0.95, 10, wrapper, wrapper, MultiFields.getTerms(ir, "body_reverse")); CandidateGenerator generator = new MultiCandidateGeneratorWrapper(10, forward, reverse); - + Correction[] corrections = suggester.getCorrections(wrapper, new BytesRef("american cae"), generator, 1, 1, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("american ace")); - + generator = new MultiCandidateGeneratorWrapper(5, forward, reverse); corrections = suggester.getCorrections(wrapper, new BytesRef("american ame"), generator, 1, 1, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("american ace")); - + corrections = suggester.getCorrections(wrapper, new BytesRef("american cae"), forward, 1, 1, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections.length, equalTo(0)); // only use forward with constant prefix - + corrections = suggester.getCorrections(wrapper, new BytesRef("america cae"), generator, 2, 1, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("american ace")); - + corrections = suggester.getCorrections(wrapper, new BytesRef("Zorr the Got-Jewel"), generator, 0.5f, 4, ir, "body", wordScorer, 0, 2).corrections; assertThat(corrections.length, equalTo(4)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the god jewel")); @@ -273,21 +273,18 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { corrections = suggester.getCorrections(wrapper, new BytesRef("Zorr the Got-Jewel"), generator, 0.5f, 1, ir, "body", wordScorer, 1.5f, 2).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the god jewel")); - + corrections = suggester.getCorrections(wrapper, new BytesRef("Xor the Got-Jewel"), generator, 0.5f, 1, ir, "body", wordScorer, 1.5f, 2).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the god jewel")); - // Test a special case where one of the suggest term is unchanged by the postFilter, 'II' here is unchanged by the reverse analyzer. + // Test a special case where one of the suggest term is unchanged by the postFilter, 'II' here is unchanged by the reverse analyzer. corrections = suggester.getCorrections(wrapper, new BytesRef("Quazar II"), generator, 1, 1, ir, "body", wordScorer, 1, 2).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("quasar ii")); } - @Test public void testMarvelHerosTrigram() throws IOException { - - RAMDirectory dir = new RAMDirectory(); Map mapping = new HashMap<>(); mapping.put("body_ngram", new Analyzer() { @@ -334,11 +331,11 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { Correction[] corrections = suggester.getCorrections(wrapper, new BytesRef("american ame"), generator, 1, 1, ir, "body", wordScorer, 1, 3).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("american ace")); - + corrections = suggester.getCorrections(wrapper, new BytesRef("american ame"), generator, 1, 1, ir, "body", wordScorer, 1, 1).corrections; assertThat(corrections.length, equalTo(0)); // assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("american ape")); - + wordScorer = new LinearInterpoatingScorer(ir, MultiFields.getTerms(ir, "body_ngram"), "body_ngram", 0.85d, new BytesRef(" "), 0.5, 0.4, 0.1); corrections = suggester.getCorrections(wrapper, new BytesRef("Xor the Got-Jewel"), generator, 0.5f, 4, ir, "body", wordScorer, 0, 3).corrections; assertThat(corrections.length, equalTo(4)); @@ -346,25 +343,25 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { assertThat(corrections[1].join(new BytesRef(" ")).utf8ToString(), equalTo("xor the god jewel")); assertThat(corrections[2].join(new BytesRef(" ")).utf8ToString(), equalTo("xorn the god jewel")); assertThat(corrections[3].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the got jewel")); - - - + + + corrections = suggester.getCorrections(wrapper, new BytesRef("Xor the Got-Jewel"), generator, 0.5f, 4, ir, "body", wordScorer, 1, 3).corrections; assertThat(corrections.length, equalTo(4)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the god jewel")); assertThat(corrections[1].join(new BytesRef(" ")).utf8ToString(), equalTo("xor the god jewel")); assertThat(corrections[2].join(new BytesRef(" ")).utf8ToString(), equalTo("xorn the god jewel")); assertThat(corrections[3].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the got jewel")); - + corrections = suggester.getCorrections(wrapper, new BytesRef("Xor the Got-Jewel"), generator, 0.5f, 1, ir, "body", wordScorer, 100, 3).corrections; assertThat(corrections.length, equalTo(1)); assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("xorr the god jewel")); - + // test synonyms - + Analyzer analyzer = new Analyzer() { @Override @@ -373,7 +370,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { TokenFilter filter = new LowerCaseFilter(t); try { SolrSynonymParser parser = new SolrSynonymParser(true, false, new WhitespaceAnalyzer()); - ((SolrSynonymParser) parser).parse(new StringReader("usa => usa, america, american\nursa => usa, america, american")); + parser.parse(new StringReader("usa => usa, america, american\nursa => usa, america, american")); filter = new SynonymFilter(filter, parser.build(), true); } catch (Exception e) { throw new RuntimeException(e); @@ -381,7 +378,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { return new TokenStreamComponents(t, filter); } }; - + spellchecker.setAccuracy(0.0f); spellchecker.setMinPrefix(1); spellchecker.setMinQueryLength(1); @@ -389,12 +386,12 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase { wordScorer = new LinearInterpoatingScorer(ir, MultiFields.getTerms(ir, "body_ngram"), "body_ngram", 0.95d, new BytesRef(" "), 0.5, 0.4, 0.1); corrections = suggester.getCorrections(analyzer, new BytesRef("captian usa"), generator, 2, 4, ir, "body", wordScorer, 1, 3).corrections; assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("captain america")); - + generator = new DirectCandidateGenerator(spellchecker, "body", SuggestMode.SUGGEST_MORE_POPULAR, ir, 0.95, 10, null, analyzer, MultiFields.getTerms(ir, "body")); corrections = suggester.getCorrections(analyzer, new BytesRef("captian usw"), generator, 2, 4, ir, "body", wordScorer, 1, 3).corrections; assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("captain america")); - - + + wordScorer = new StupidBackoffScorer(ir, MultiFields.getTerms(ir, "body_ngram"), "body_ngram", 0.85d, new BytesRef(" "), 0.4); corrections = suggester.getCorrections(wrapper, new BytesRef("Xor the Got-Jewel"), generator, 0.5f, 2, ir, "body", wordScorer, 0, 3).corrections; assertThat(corrections.length, equalTo(2)); diff --git a/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java b/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java index d486cdba220..ab6a10f3cd2 100644 --- a/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java +++ b/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.similarity; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -30,8 +29,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; public class SimilarityIT extends ESIntegTestCase { - - @Test public void testCustomBM25Similarity() throws Exception { try { client().admin().indices().prepareDelete("test").execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index ffddcfc1619..51ae038ca0d 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -20,12 +20,7 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.service.PendingClusterTask; @@ -208,7 +203,7 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { private void addBlock() { // We should block after this task - add blocking cluster state update task - clusterService.submitStateUpdateTask("test_block", passThroughPriority, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("test_block", new ClusterStateUpdateTask(passThroughPriority) { @Override public ClusterState execute(ClusterState currentState) throws Exception { while(System.currentTimeMillis() < stopWaitingAt) { diff --git a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java index 9047c710c4d..666ef9dfe39 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java @@ -34,17 +34,26 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.FromXContentBuilder; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.translog.BufferedChecksumStreamOutput; import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.elasticsearch.repositories.blobstore.LegacyBlobStoreFormat; -import org.junit.Test; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThan; @@ -99,6 +108,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase { return new BlobObj(text); } + @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.field("text", getText()); return builder; @@ -147,7 +157,6 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase { } } - @Test public void testBlobStoreOperations() throws IOException { BlobStore blobStore = createTestBlobStore(); BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath()); @@ -183,8 +192,6 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase { assertEquals(legacySMILE.read(blobContainer, "legacy-smile-comp").getText(), "legacy smile compressed"); } - - @Test public void testCompressionIsApplied() throws IOException { BlobStore blobStore = createTestBlobStore(); BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath()); @@ -202,7 +209,6 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase { assertThat(blobs.get("blob-not-comp").length(), greaterThan(blobs.get("blob-comp").length())); } - @Test public void testBlobCorruption() throws IOException { BlobStore blobStore = createTestBlobStore(); BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath()); diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 60aaa64311b..f9392836d8b 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.snapshots; import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.hppc.IntSet; + import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; @@ -64,9 +65,10 @@ import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepos import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction; import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.rest.FakeRestRequest; -import org.junit.Test; import java.io.IOException; import java.nio.file.Path; @@ -80,24 +82,30 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0) @ESIntegTestCase.SuppressLocalMode // TODO only restorePersistentSettingsTest needs this maybe factor out? public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(MockRepository.Plugin.class); } - @Test - public void restorePersistentSettingsTest() throws Exception { + public void testRestorePersistentSettings() throws Exception { logger.info("--> start 2 nodes"); Settings nodeSettings = settingsBuilder() .put("discovery.type", "zen") @@ -157,8 +165,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest .getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, -1), not(equalTo(2))); } - @Test - public void restoreCustomMetadata() throws Exception { + public void testRestoreCustomMetadata() throws Exception { Path tempDir = randomRepoPath(); logger.info("--> start node"); @@ -286,8 +293,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest public ClusterState execute(ClusterState currentState) throws Exception; } - @Test - public void snapshotDuringNodeShutdownTest() throws Exception { + public void testSnapshotDuringNodeShutdown() throws Exception { logger.info("--> start 2 nodes"); Client client = client(); @@ -299,7 +305,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> create repository"); logger.info("--> creating repository"); @@ -332,8 +338,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest logger.info("--> done"); } - @Test - public void snapshotWithStuckNodeTest() throws Exception { + public void testSnapshotWithStuckNode() throws Exception { logger.info("--> start 2 nodes"); ArrayList nodes = new ArrayList<>(); nodes.add(internalCluster().startNode()); @@ -348,7 +353,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> creating repository"); Path repo = randomRepoPath(); @@ -397,8 +402,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest logger.info("--> done"); } - @Test - public void restoreIndexWithMissingShards() throws Exception { + public void testRestoreIndexWithMissingShards() throws Exception { logger.info("--> start 2 nodes"); internalCluster().startNode(); internalCluster().startNode(); @@ -414,7 +418,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest index("test-idx-some", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client().prepareCount("test-idx-some").get().getCount(), equalTo(100L)); + assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> shutdown one of the nodes"); internalCluster().stopRandomDataNode(); @@ -435,7 +439,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest index("test-idx-closed", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client().prepareCount("test-idx-all").get().getCount(), equalTo(100L)); + assertThat(client().prepareSearch("test-idx-all").setSize(0).get().getHits().totalHits(), equalTo(100L)); assertAcked(client().admin().indices().prepareClose("test-idx-closed")); logger.info("--> create an index that will have no allocated shards"); @@ -522,7 +526,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(6)); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - assertThat(client().prepareCount("test-idx-all").get().getCount(), equalTo(100L)); + assertThat(client().prepareSearch("test-idx-all").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> restore snapshot for the partial index"); cluster().wipeIndices("test-idx-some"); @@ -533,7 +537,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), allOf(greaterThan(0), lessThan(6))); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), greaterThan(0)); - assertThat(client().prepareCount("test-idx-some").get().getCount(), allOf(greaterThan(0L), lessThan(100L))); + assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().totalHits(), allOf(greaterThan(0L), lessThan(100L))); logger.info("--> restore snapshot for the index that didn't have any shards snapshotted successfully"); cluster().wipeIndices("test-idx-none"); @@ -544,11 +548,10 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(0)); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(6)); - assertThat(client().prepareCount("test-idx-some").get().getCount(), allOf(greaterThan(0L), lessThan(100L))); + assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().totalHits(), allOf(greaterThan(0L), lessThan(100L))); } - @Test - public void restoreIndexWithShardsMissingInLocalGateway() throws Exception { + public void testRestoreIndexWithShardsMissingInLocalGateway() throws Exception { logger.info("--> start 2 nodes"); Settings nodeSettings = settingsBuilder() .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, EnableAllocationDecider.Rebalance.NONE) @@ -573,7 +576,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client().prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client().prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> start snapshot"); assertThat(client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setIndices("test-idx").setWaitForCompletion(true).get().getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); @@ -595,7 +598,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-1").setRestoreGlobalState(false).setWaitForCompletion(true).get().getRestoreInfo().successfulShards(), equalTo(6)); ensureGreen("test-idx"); - assertThat(client().prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client().prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); IntSet reusedShards = new IntHashSet(); for (RecoveryState recoveryState : client().admin().indices().prepareRecoveries("test-idx").get().shardRecoveryStates().get("test-idx")) { @@ -607,9 +610,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(reusedShards.size(), greaterThanOrEqualTo(numberOfShards / 2)); } - - @Test - public void registrationFailureTest() { + public void testRegistrationFailure() { logger.info("--> start first node"); internalCluster().startNode(); logger.info("--> start second node"); @@ -628,7 +629,6 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest } - @Test public void testThatSensitiveRepositorySettingsAreNotExposed() throws Exception { Settings nodeSettings = settingsBuilder().put().build(); logger.info("--> start two nodes"); @@ -683,12 +683,11 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest if (clusterStateError.get() != null) { throw clusterStateError.get(); } - + } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12621") - public void chaosSnapshotTest() throws Exception { + public void testChaosSnapshot() throws Exception { final List indices = new CopyOnWriteArrayList<>(); Settings settings = settingsBuilder().put("action.write_consistency", "one").build(); int initialNodes = between(1, 3); @@ -790,9 +789,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest logger.info("--> done"); } - @Test - public void masterShutdownDuringSnapshotTest() throws Exception { - + public void testMasterShutdownDuringSnapshot() throws Exception { Settings masterSettings = settingsBuilder().put("node.data", false).build(); Settings dataSettings = settingsBuilder().put("node.master", false).build(); diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java index c5221d12f3b..b0de06138a8 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryVerificationException; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.nio.file.Path; import java.util.List; @@ -48,8 +47,6 @@ import static org.hamcrest.Matchers.notNullValue; */ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class RepositoriesIT extends AbstractSnapshotIntegTestCase { - - @Test public void testRepositoryCreation() throws Exception { Client client = client(); @@ -97,7 +94,8 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { assertThat(repositoriesMetaData.repository("test-repo-2").type(), equalTo("fs")); logger.info("--> check that both repositories can be retrieved by getRepositories query"); - GetRepositoriesResponse repositoriesResponse = client.admin().cluster().prepareGetRepositories().get(); + GetRepositoriesResponse repositoriesResponse = client.admin().cluster() + .prepareGetRepositories(randomFrom("_all", "*", "test-repo-*")).get(); assertThat(repositoriesResponse.repositories().size(), equalTo(2)); assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-1"), notNullValue()); assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-2"), notNullValue()); @@ -123,7 +121,6 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { return null; } - @Test public void testMisconfiguredRepository() throws Exception { Client client = client(); @@ -170,8 +167,7 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { } } - @Test - public void repositoryAckTimeoutTest() throws Exception { + public void testRepositoryAckTimeout() throws Exception { logger.info("--> creating repository test-repo-1 with 0s timeout - shouldn't ack"); PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo-1") .setType("fs").setSettings(Settings.settingsBuilder() @@ -201,8 +197,7 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { assertThat(deleteRepositoryResponse.isAcknowledged(), equalTo(true)); } - @Test - public void repositoryVerificationTest() throws Exception { + public void testRepositoryVerification() throws Exception { Client client = client(); Settings settings = Settings.settingsBuilder() @@ -236,8 +231,7 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { } } - @Test - public void repositoryVerificationTimeoutTest() throws Exception { + public void testRepositoryVerificationTimeout() throws Exception { Client client = client(); Settings settings = Settings.settingsBuilder() diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 0da8905eb83..57a22c0dd15 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStage; @@ -37,7 +38,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; @@ -54,6 +55,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -64,7 +66,6 @@ import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.junit.Test; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; @@ -78,6 +79,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.client.Requests.getSnapshotsRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -104,9 +106,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase { - - @Test - public void basicWorkFlowTest() throws Exception { + public void testBasicWorkFlow() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -126,9 +126,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); - assertHitCount(client.prepareCount("test-idx-1").get(), 100L); - assertHitCount(client.prepareCount("test-idx-2").get(), 100L); - assertHitCount(client.prepareCount("test-idx-3").get(), 100L); + assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 100L); + assertHitCount(client.prepareSearch("test-idx-2").setSize(0).get(), 100L); + assertHitCount(client.prepareSearch("test-idx-3").setSize(0).get(), 100L); ListenableActionFuture flushResponseFuture = null; if (randomBoolean()) { @@ -149,7 +149,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - SnapshotInfo snapshotInfo = client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0); + List snapshotInfos = client.admin().cluster().prepareGetSnapshots("test-repo") + .setSnapshots(randomFrom("test-snap", "_all", "*", "*-snap", "test*")).get().getSnapshots(); + assertThat(snapshotInfos.size(), equalTo(1)); + SnapshotInfo snapshotInfo = snapshotInfos.get(0); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfo.version(), equalTo(Version.CURRENT)); @@ -164,9 +167,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } assertAllSuccessful(refresh()); - assertHitCount(client.prepareCount("test-idx-1").get(), 50L); - assertHitCount(client.prepareCount("test-idx-2").get(), 50L); - assertHitCount(client.prepareCount("test-idx-3").get(), 50L); + assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 50L); + assertHitCount(client.prepareSearch("test-idx-2").setSize(0).get(), 50L); + assertHitCount(client.prepareSearch("test-idx-3").setSize(0).get(), 50L); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); @@ -177,9 +180,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas ensureGreen(); for (int i=0; i<5; i++) { - assertHitCount(client.prepareCount("test-idx-1").get(), 100L); - assertHitCount(client.prepareCount("test-idx-2").get(), 100L); - assertHitCount(client.prepareCount("test-idx-3").get(), 50L); + assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 100L); + assertHitCount(client.prepareSearch("test-idx-2").setSize(0).get(), 100L); + assertHitCount(client.prepareSearch("test-idx-3").setSize(0).get(), 50L); } // Test restore after index deletion @@ -191,7 +194,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas ensureGreen(); for (int i=0; i<5; i++) { - assertHitCount(client.prepareCount("test-idx-1").get(), 100L); + assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 100L); } ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); @@ -203,9 +206,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } - - @Test - public void singleGetAfterRestoreTest() throws Exception { + public void testSingleGetAfterRestore() throws Exception { String indexName = "testindex"; String repoName = "test-restore-snapshot-repo"; String snapshotName = "test-restore-snapshot"; @@ -245,7 +246,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(client.prepareGet(restoredIndexName, typeName, docId).get().isExists(), equalTo(true)); } - @Test public void testFreshIndexUUID() { Client client = client(); @@ -294,8 +294,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertFalse("UUID has been reused on restore: " + copyRestoreUUID + " vs. " + originalIndexUUID, copyRestoreUUID.equals(originalIndexUUID)); } - @Test - public void restoreWithDifferentMappingsAndSettingsTest() throws Exception { + public void testRestoreWithDifferentMappingsAndSettings() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -343,8 +342,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(getSettingsResponse.getSetting("test-idx", "index.refresh_interval"), equalTo("10000ms")); } - @Test - public void emptySnapshotTest() throws Exception { + public void testEmptySnapshot() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -360,8 +358,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); } - @Test - public void restoreAliasesTest() throws Exception { + public void testRestoreAliases() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -416,8 +413,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } - @Test - public void restoreTemplatesTest() throws Exception { + public void testRestoreTemplates() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -449,8 +445,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } - @Test - public void includeGlobalStateTest() throws Exception { + public void testIncludeGlobalState() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -502,7 +497,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot without global state but with indices"); createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-no-global-state-with-index").setIndices("test-idx").setIncludeGlobalState(false).setWaitForCompletion(true).get(); @@ -524,12 +519,11 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas logger.info("--> check that template wasn't restored but index was"); getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get(); assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template"); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); } - @Test - public void snapshotFileFailureDuringSnapshotTest() throws Exception { + public void testSnapshotFileFailureDuringSnapshot() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -549,7 +543,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); try { @@ -579,8 +573,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } - @Test - public void dataFileFailureDuringSnapshotTest() throws Exception { + public void testDataFileFailureDuringSnapshot() throws Exception { Client client = client(); logger.info("--> creating repository"); assertAcked(client.admin().cluster().preparePutRepository("test-repo") @@ -598,7 +591,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); @@ -643,11 +636,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } assertThat(indexStatus.getShardsStats().getFailedShards(), equalTo(numberOfFailures)); } - } - @Test - public void dataFileFailureDuringRestoreTest() throws Exception { + public void testDataFileFailureDuringRestore() throws Exception { Path repositoryLocation = randomRepoPath(); Client client = client(); logger.info("--> creating repository"); @@ -662,7 +653,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); @@ -683,14 +674,12 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas logger.info("--> restore index after deletion"); RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - CountResponse countResponse = client.prepareCount("test-idx").get(); - assertThat(countResponse.getCount(), equalTo(100L)); + SearchResponse countResponse = client.prepareSearch("test-idx").setSize(0).get(); + assertThat(countResponse.getHits().totalHits(), equalTo(100L)); logger.info("--> total number of simulated failures during restore: [{}]", getFailureCount("test-repo")); } - - @Test - public void deletionOfFailingToRecoverIndexShouldStopRestore() throws Exception { + public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Exception { Path repositoryLocation = randomRepoPath(); Client client = client(); logger.info("--> creating repository"); @@ -705,7 +694,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); @@ -748,13 +737,12 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - CountResponse countResponse = client.prepareCount("test-idx").get(); - assertThat(countResponse.getCount(), equalTo(100L)); + SearchResponse countResponse = client.prepareSearch("test-idx").setSize(0).get(); + assertThat(countResponse.getHits().totalHits(), equalTo(100L)); } - @Test - public void unallocatedShardsTest() throws Exception { + public void testUnallocatedShards() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -773,8 +761,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(createSnapshotResponse.getSnapshotInfo().reason(), startsWith("Indices don't have primary shards")); } - @Test - public void deleteSnapshotTest() throws Exception { + public void testDeleteSnapshot() throws Exception { final int numberOfSnapshots = between(5, 15); Client client = client(); @@ -803,7 +790,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // Store number of files after each snapshot numberOfFiles[i] = numberOfFiles(repo); } - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(10L * numberOfSnapshots)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(10L * numberOfSnapshots)); int numberOfFilesBeforeDeletion = numberOfFiles(repo); logger.info("--> delete all snapshots except the first one and last one"); @@ -823,7 +810,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", lastSnapshot).setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(10L * numberOfSnapshots)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(10L * numberOfSnapshots)); logger.info("--> delete the last snapshot"); client.admin().cluster().prepareDeleteSnapshot("test-repo", lastSnapshot).get(); @@ -831,8 +818,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0])); } - @Test - public void deleteSnapshotWithMissingIndexAndShardMetadataTest() throws Exception { + public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exception { Client client = client(); Path repo = randomRepoPath(); @@ -870,8 +856,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class); } - @Test - public void deleteSnapshotWithMissingMetadataTest() throws Exception { + public void testDeleteSnapshotWithMissingMetadata() throws Exception { Client client = client(); Path repo = randomRepoPath(); @@ -905,8 +890,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class); } - @Test - public void deleteSnapshotWithCorruptedSnapshotFileTest() throws Exception { + public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception { Client client = client(); Path repo = randomRepoPath(); @@ -946,9 +930,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); } - - @Test - public void snapshotClosedIndexTest() throws Exception { + public void testSnapshotClosedIndex() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -961,7 +943,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas logger.info("--> closing index test-idx-closed"); assertAcked(client.admin().indices().prepareClose("test-idx-closed")); ClusterStateResponse stateResponse = client.admin().cluster().prepareState().get(); - assertThat(stateResponse.getState().metaData().index("test-idx-closed").state(), equalTo(IndexMetaData.State.CLOSE)); + assertThat(stateResponse.getState().metaData().index("test-idx-closed").getState(), equalTo(IndexMetaData.State.CLOSE)); assertThat(stateResponse.getState().routingTable().index("test-idx-closed"), nullValue()); logger.info("--> snapshot"); @@ -976,8 +958,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertBlocked(client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx", "test-idx-closed"), MetaDataIndexStateService.INDEX_CLOSED_BLOCK); } - @Test - public void snapshotSingleClosedIndexTest() throws Exception { + public void testSnapshotSingleClosedIndex() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -995,8 +976,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setWaitForCompletion(true).setIndices("test-idx"), MetaDataIndexStateService.INDEX_CLOSED_BLOCK); } - @Test - public void renameOnRestoreTest() throws Exception { + public void testRenameOnRestore() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -1019,8 +999,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx-2", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1", "test-idx-2").get(); @@ -1032,8 +1012,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setRenamePattern("(.+)").setRenameReplacement("$1-copy").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx-1-copy").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2-copy").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1-copy").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2-copy").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> close just restored indices"); client.admin().indices().prepareClose("test-idx-1-copy", "test-idx-2-copy").get(); @@ -1043,8 +1023,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setRenamePattern("(.+)").setRenameReplacement("$1-copy").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx-1-copy").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2-copy").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1-copy").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2-copy").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> close indices"); @@ -1115,8 +1095,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } - @Test - public void moveShardWhileSnapshottingTest() throws Exception { + public void testMoveShardWhileSnapshotting() throws Exception { Client client = client(); Path repositoryLocation = randomRepoPath(); logger.info("--> creating repository"); @@ -1135,7 +1114,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); // Pick one node and block it String blockedNode = blockNodeWithIndex("test-idx"); @@ -1174,11 +1153,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); } - @Test - public void deleteRepositoryWhileSnapshottingTest() throws Exception { + public void testDeleteRepositoryWhileSnapshotting() throws Exception { Client client = client(); Path repositoryLocation = randomRepoPath(); logger.info("--> creating repository"); @@ -1199,7 +1177,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); // Pick one node and block it String blockedNode = blockNodeWithIndex("test-idx"); @@ -1257,11 +1235,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); } - @Test - public void urlRepositoryTest() throws Exception { + public void testUrlRepository() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -1280,7 +1257,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); @@ -1301,7 +1278,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> list available shapshots"); GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); @@ -1318,9 +1295,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); } - - @Test - public void readonlyRepositoryTest() throws Exception { + public void testReadonlyRepository() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -1361,7 +1336,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("readonly-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> list available shapshots"); GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("readonly-repo").get(); @@ -1375,8 +1350,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThrows(client.admin().cluster().prepareCreateSnapshot("readonly-repo", "test-snap-2").setWaitForCompletion(true).setIndices("test-idx"), RepositoryException.class, "cannot create snapshot in a readonly repository"); } - @Test - public void throttlingTest() throws Exception { + public void testThrottling() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -1399,7 +1373,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); @@ -1412,7 +1386,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas logger.info("--> restore index"); RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); long snapshotPause = 0L; long restorePause = 0L; @@ -1434,9 +1408,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } - - @Test - public void snapshotStatusTest() throws Exception { + public void testSnapshotStatus() throws Exception { Client client = client(); Path repositoryLocation = randomRepoPath(); logger.info("--> creating repository"); @@ -1457,7 +1429,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); // Pick one node and block it String blockedNode = blockNodeWithIndex("test-idx"); @@ -1534,9 +1506,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } - - @Test - public void snapshotRelocatingPrimary() throws Exception { + public void testSnapshotRelocatingPrimary() throws Exception { Client client = client(); logger.info("--> creating repository"); assertAcked(client.admin().cluster().preparePutRepository("test-repo") @@ -1553,7 +1523,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); // Update settings to make sure that relocation is slow so we can start snapshot before relocation is finished assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(Settings.builder() @@ -1605,7 +1575,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } indexRandom(true, builders); flushAndRefresh(); - assertNoFailures(client().admin().indices().prepareOptimize("test").setFlush(true).setMaxNumSegments(1).get()); + assertNoFailures(client().admin().indices().prepareForceMerge("test").setFlush(true).setMaxNumSegments(1).get()); CreateSnapshotResponse createSnapshotResponseFirst = client.admin().cluster().prepareCreateSnapshot("test-repo", "test").setWaitForCompletion(true).setIndices("test").get(); assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), greaterThan(0)); @@ -1645,8 +1615,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } - @Test - public void changeSettingsOnRestoreTest() throws Exception { + public void testChangeSettingsOnRestore() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -1680,8 +1649,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas indexRandom(true, builders); flushAndRefresh(); - assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), numdocs); - assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs); + assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "foo")).get(), numdocs); + assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")).get(), numdocs); logger.info("--> snapshot it"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); @@ -1735,8 +1704,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(getSettingsResponse.getSetting("test-idx", "index.analysis.analyzer.my_analyzer.type"), equalTo("standard")); assertThat(getSettingsResponse.getSetting("test-idx", "index.analysis.filter.my_synonym.type"), nullValue()); - assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), 0); - assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs); + assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "foo")).get(), 0); + assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")).get(), numdocs); logger.info("--> delete the index and recreate it while deleting all index settings"); cluster().wipeIndices("test-idx"); @@ -1755,13 +1724,12 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // Make sure that number of shards didn't change assertThat(getSettingsResponse.getSetting("test-idx", SETTING_NUMBER_OF_SHARDS), equalTo("" + numberOfShards)); - assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "foo")).get(), 0); - assertHitCount(client.prepareCount("test-idx").setQuery(matchQuery("field1", "bar")).get(), numdocs); + assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "foo")).get(), 0); + assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")).get(), numdocs); } - @Test - public void recreateBlocksOnRestoreTest() throws Exception { + public void testRecreateBlocksOnRestore() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -1850,8 +1818,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } - @Test - public void deleteIndexDuringSnapshotTest() throws Exception { + public void testDeleteIndexDuringSnapshot() throws Exception { Client client = client(); boolean allowPartial = randomBoolean(); @@ -1875,9 +1842,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot allow partial {}", allowPartial); ListenableActionFuture future = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -1903,9 +1870,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } } - - @Test - public void deleteOrphanSnapshotTest() throws Exception { + public void testDeleteOrphanSnapshot() throws Exception { Client client = client(); logger.info("--> creating repository"); @@ -1970,10 +1935,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas return awaitBusy(() -> client().admin().cluster().prepareHealth(index).execute().actionGet().getRelocatingShards() > 0, timeout.millis(), TimeUnit.MILLISECONDS); } - @Test @TestLogging("cluster:DEBUG") - public void batchingShardUpdateTaskTest() throws Exception { - + public void testBatchingShardUpdateTask() throws Exception { final Client client = client(); logger.info("--> creating repository"); @@ -2052,9 +2015,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertEquals(1, restoreListener.count()); } - @Test - public void snapshotNameTest() throws Exception { - + public void testSnapshotName() throws Exception { final Client client = client(); logger.info("--> creating repository"); @@ -2092,4 +2053,53 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(ex.getMessage(), containsString("Invalid snapshot name")); } } -} \ No newline at end of file + + public void testListCorruptedSnapshot() throws Exception { + Client client = client(); + Path repo = randomRepoPath(); + logger.info("--> creating repository at " + repo.toAbsolutePath()); + assertAcked(client.admin().cluster().preparePutRepository("test-repo") + .setType("fs").setSettings(Settings.settingsBuilder() + .put("location", repo) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + + createIndex("test-idx-1", "test-idx-2", "test-idx-3"); + ensureYellow(); + logger.info("--> indexing some data"); + indexRandom(true, + client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"), + client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"), + client().prepareIndex("test-idx-3", "doc").setSource("foo", "bar")); + + logger.info("--> creating 2 snapshots"); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2").setWaitForCompletion(true).setIndices("test-idx-*").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + logger.info("--> truncate snapshot file to make it unreadable"); + Path snapshotPath = repo.resolve("snap-test-snap-2.dat"); + try(SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) { + outChan.truncate(randomInt(10)); + } + + logger.info("--> get snapshots request should return both snapshots"); + List snapshotInfos = client.admin().cluster() + .prepareGetSnapshots("test-repo") + .setIgnoreUnavailable(true).get().getSnapshots(); + + assertThat(snapshotInfos.size(), equalTo(1)); + assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfos.get(0).name(), equalTo("test-snap-1")); + + try { + client.admin().cluster().prepareGetSnapshots("test-repo").setIgnoreUnavailable(false).get().getSnapshots(); + } catch (SnapshotException ex) { + assertThat(ex.snapshot().getRepository(), equalTo("test-repo")); + assertThat(ex.snapshot().getSnapshot(), equalTo("test-snap-2")); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java index 21401d091f0..360f549ecbd 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java @@ -24,7 +24,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.Client; @@ -33,7 +33,6 @@ import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.test.ESBackcompatTestCase; -import org.junit.Test; import java.io.IOException; import java.nio.file.Files; @@ -50,8 +49,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { - - @Test public void testSnapshotAndRestore() throws ExecutionException, InterruptedException, IOException { logger.info("--> creating repository"); assertAcked(client().admin().cluster().preparePutRepository("test-repo") @@ -84,10 +81,10 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { } indexRandom(true, buildersBefore); indexRandom(true, buildersAfter); - assertThat(client().prepareCount(indices).get().getCount(), equalTo((long) (buildersBefore.length + buildersAfter.length))); + assertThat(client().prepareSearch(indices).setSize(0).get().getHits().totalHits(), equalTo((long) (buildersBefore.length + buildersAfter.length))); long[] counts = new long[indices.length]; for (int i = 0; i < indices.length; i++) { - counts[i] = client().prepareCount(indices[i]).get().getCount(); + counts[i] = client().prepareSearch(indices[i]).setSize(0).get().getHits().totalHits(); } logger.info("--> snapshot subset of indices before upgrage"); @@ -106,8 +103,8 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { client().prepareDelete(request.index(), request.type(), request.id()).get(); } refresh(); - final long numDocs = client().prepareCount(indices).get().getCount(); - assertThat(client().prepareCount(indices).get().getCount(), lessThan((long) (buildersBefore.length + buildersAfter.length))); + final long numDocs = client().prepareSearch(indices).setSize(0).get().getHits().totalHits(); + assertThat(client().prepareSearch(indices).setSize(0).get().getHits().totalHits(), lessThan((long) (buildersBefore.length + buildersAfter.length))); disableAllocation(indices); @@ -116,11 +113,11 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { boolean upgraded; do { logClusterState(); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); upgraded = backwardsCluster().upgradeOneNode(); ensureYellow(); - countResponse = client().prepareCount().get(); + countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); } while (upgraded); enableAllocation(indices); @@ -136,9 +133,9 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureYellow(); - assertThat(client().prepareCount(indices).get().getCount(), equalTo((long) (buildersBefore.length + buildersAfter.length))); + assertThat(client().prepareSearch(indices).setSize(0).get().getHits().totalHits(), equalTo((long) (buildersBefore.length + buildersAfter.length))); for (int i = 0; i < indices.length; i++) { - assertThat(counts[i], equalTo(client().prepareCount(indices[i]).get().getCount())); + assertThat(counts[i], equalTo(client().prepareSearch(indices[i]).setSize(0).get().getHits().totalHits())); } logger.info("--> snapshot subset of indices after upgrade"); @@ -154,9 +151,9 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setWaitForCompletion(true).setIndices(index).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureYellow(); - assertThat(client().prepareCount(indices).get().getCount(), equalTo((long) (buildersBefore.length + buildersAfter.length))); + assertThat(client().prepareSearch(indices).setSize(0).get().getHits().totalHits(), equalTo((long) (buildersBefore.length + buildersAfter.length))); for (int i = 0; i < indices.length; i++) { - assertThat(counts[i], equalTo(client().prepareCount(indices[i]).get().getCount())); + assertThat(counts[i], equalTo(client().prepareSearch(indices[i]).setSize(0).get().getHits().totalHits())); } } @@ -185,7 +182,7 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { } indexRandom(true, builders); flushAndRefresh(); - assertNoFailures(client().admin().indices().prepareOptimize("test").setFlush(true).setMaxNumSegments(1).get()); + assertNoFailures(client().admin().indices().prepareForceMerge("test").setFlush(true).setMaxNumSegments(1).get()); CreateSnapshotResponse createSnapshotResponseFirst = client.admin().cluster().prepareCreateSnapshot("test-repo", "test").setWaitForCompletion(true).setIndices("test").get(); assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), greaterThan(0)); @@ -206,11 +203,11 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { boolean upgraded; do { logClusterState(); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); upgraded = backwardsCluster().upgradeOneNode(); ensureYellow(); - countResponse = client().prepareCount().get(); + countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); } while (upgraded); enableAllocation("test"); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java index 7cf56bfc7d3..38d858c49aa 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java @@ -24,16 +24,13 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; public class SnapshotRequestsTests extends ESTestCase { - @Test public void testRestoreSnapshotRequestParsing() throws IOException { - RestoreSnapshotRequest request = new RestoreSnapshotRequest("test-repo", "test-snap"); XContentBuilder builder = jsonBuilder().startObject(); @@ -94,9 +91,7 @@ public class SnapshotRequestsTests extends ESTestCase { } - @Test public void testCreateSnapshotRequestParsing() throws IOException { - CreateSnapshotRequest request = new CreateSnapshotRequest("test-repo", "test-snap"); XContentBuilder builder = jsonBuilder().startObject(); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotUtilsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotUtilsTests.java index 8e9d7cb8428..a121427b447 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotUtilsTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.Arrays; import java.util.List; @@ -30,7 +29,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder; /** */ public class SnapshotUtilsTests extends ESTestCase { - @Test public void testIndexNameFiltering() { assertIndexNameFiltering(new String[]{"foo", "bar", "baz"}, new String[]{}, new String[]{"foo", "bar", "baz"}); assertIndexNameFiltering(new String[]{"foo", "bar", "baz"}, new String[]{"*"}, new String[]{"foo", "bar", "baz"}); diff --git a/core/src/test/java/org/elasticsearch/test/engine/MockEngineSupportModule.java b/core/src/test/java/org/elasticsearch/test/engine/MockEngineSupportModule.java deleted file mode 100644 index 4f353ee42b7..00000000000 --- a/core/src/test/java/org/elasticsearch/test/engine/MockEngineSupportModule.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.engine; - -import org.apache.lucene.index.AssertingDirectoryReader; -import org.apache.lucene.index.FilterDirectoryReader; -import org.elasticsearch.common.inject.AbstractModule; - -public class MockEngineSupportModule extends AbstractModule { - public Class wrapperImpl = AssertingDirectoryReader.class; - - @Override - protected void configure() { - bind(Class.class).annotatedWith(MockEngineFactory.MockReaderType.class).toInstance(wrapperImpl); - } -} diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomGeoGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomGeoGenerator.java new file mode 100644 index 00000000000..ad94c4e5ab4 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/test/geo/RandomGeoGenerator.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.geo; + +import org.apache.lucene.util.GeoUtils; +import org.elasticsearch.common.geo.GeoPoint; + +import java.util.Random; + +/** + * Random geo generation utilities for randomized {@code geo_point} type testing + * does not depend on jts or spatial4j. Use {@link org.elasticsearch.test.geo.RandomShapeGenerator} + * to create random OGC compliant shapes. + */ +public class RandomGeoGenerator { + + public static void randomPoint(Random r, double[] pt) { + final double[] min = {-180, -90}; + final double[] max = {180, 90}; + randomPointIn(r, min[0], min[1], max[0], max[1], pt); + } + + public static void randomPointIn(Random r, final double minLon, final double minLat, + final double maxLon, final double maxLat, double[] pt) { + assert pt != null && pt.length == 2; + + // normalize min and max + double[] min = {GeoUtils.normalizeLon(minLon), GeoUtils.normalizeLat(minLat)}; + double[] max = {GeoUtils.normalizeLon(maxLon), GeoUtils.normalizeLat(maxLat)}; + final double[] tMin = new double[2]; + final double[] tMax = new double[2]; + tMin[0] = Math.min(min[0], max[0]); + tMax[0] = Math.max(min[0], max[0]); + tMin[1] = Math.min(min[1], max[1]); + tMax[1] = Math.max(min[1], max[1]); + + pt[0] = tMin[0] + r.nextDouble() * (tMax[0] - tMin[0]); + pt[1] = tMin[1] + r.nextDouble() * (tMax[1] - tMin[1]); + } + + public static GeoPoint randomPoint(Random r) { + return randomPointIn(r, -180, -90, 180, 90); + } + + public static GeoPoint randomPointIn(Random r, final double minLon, final double minLat, + final double maxLon, final double maxLat) { + double[] pt = new double[2]; + randomPointIn(r, minLon, minLat, maxLon, maxLat, pt); + return new GeoPoint(pt[1], pt[0]); + } +} diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java index 2bf231e203f..b1e89cdf268 100644 --- a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java +++ b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java @@ -19,7 +19,6 @@ package org.elasticsearch.test.geo; -import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.generators.RandomInts; import com.spatial4j.core.context.jts.JtsSpatialContext; import com.spatial4j.core.distance.DistanceUtils; @@ -30,8 +29,8 @@ import com.spatial4j.core.shape.impl.Range; import com.vividsolutions.jts.algorithm.ConvexHull; import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Geometry; + import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.geo.builders.BaseLineStringBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; import org.elasticsearch.common.geo.builders.LineStringBuilder; import org.elasticsearch.common.geo.builders.MultiLineStringBuilder; @@ -40,15 +39,18 @@ import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.geo.builders.PointCollection; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.search.geo.GeoShapeQueryTests; +import org.junit.Assert; import java.util.Random; import static com.spatial4j.core.shape.SpatialRelation.CONTAINS; /** - * Random geoshape generation utilities for randomized Geospatial testing + * Random geoshape generation utilities for randomized {@code geo_shape} type testing + * depends on jts and spatial4j */ -public class RandomShapeGenerator { +public class RandomShapeGenerator extends RandomGeoGenerator { protected static JtsSpatialContext ctx = ShapeBuilder.SPATIAL_CONTEXT; protected static final double xDIVISIBLE = 2; @@ -153,7 +155,7 @@ public class RandomShapeGenerator { /** * Creates a random shape useful for randomized testing, NOTE: exercise caution when using this to build random GeometryCollections * as creating a large random number of random shapes can result in massive resource consumption - * see: {@link org.elasticsearch.search.geo.GeoShapeIntegrationIT#testShapeFilterWithRandomGeoCollection} + * see: {@link GeoShapeQueryTests#testShapeFilterWithRandomGeoCollection} * * The following options are included * @param nearPoint Create a shape near a provided point @@ -195,7 +197,7 @@ public class RandomShapeGenerator { case MULTILINESTRING: MultiLineStringBuilder mlsb = new MultiLineStringBuilder(); for (int i=0; i parameters() throws IOException, RestTestParseException { - return createParameters(4, 8); - } -} diff --git a/core/src/test/java/org/elasticsearch/test/rest/Rest5IT.java b/core/src/test/java/org/elasticsearch/test/rest/Rest5IT.java deleted file mode 100644 index 748b06c2c2a..00000000000 --- a/core/src/test/java/org/elasticsearch/test/rest/Rest5IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 5 */ -public class Rest5IT extends ESRestTestCase { - public Rest5IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(5, 8); - } -} diff --git a/core/src/test/java/org/elasticsearch/test/rest/Rest6IT.java b/core/src/test/java/org/elasticsearch/test/rest/Rest6IT.java deleted file mode 100644 index e8fbcd4826c..00000000000 --- a/core/src/test/java/org/elasticsearch/test/rest/Rest6IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 6 */ -public class Rest6IT extends ESRestTestCase { - public Rest6IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(6, 8); - } -} diff --git a/core/src/test/java/org/elasticsearch/test/rest/Rest7IT.java b/core/src/test/java/org/elasticsearch/test/rest/Rest7IT.java deleted file mode 100644 index cf68bdb5606..00000000000 --- a/core/src/test/java/org/elasticsearch/test/rest/Rest7IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 7 */ -public class Rest7IT extends ESRestTestCase { - public Rest7IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(7, 8); - } -} diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java new file mode 100644 index 00000000000..1df965968a2 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.search.aggregations.bucket; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; +import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESTestCase; +import org.junit.Assert; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.test.ESIntegTestCase.client; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.equalTo; + +public class SharedSignificantTermsTestMethods { + public static final String INDEX_NAME = "testidx"; + public static final String DOC_TYPE = "doc"; + public static final String TEXT_FIELD = "text"; + public static final String CLASS_FIELD = "class"; + + public static void aggregateAndCheckFromSeveralShards(ESIntegTestCase testCase) throws ExecutionException, InterruptedException { + String type = ESTestCase.randomBoolean() ? "string" : "long"; + String settings = "{\"index.number_of_shards\": 5, \"index.number_of_replicas\": 0}"; + index01Docs(type, settings, testCase); + testCase.ensureGreen(); + testCase.logClusterState(); + checkSignificantTermsAggregationCorrect(testCase); + } + + private static void checkSignificantTermsAggregationCorrect(ESIntegTestCase testCase) { + + SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE) + .addAggregation(new TermsBuilder("class").field(CLASS_FIELD).subAggregation( + new SignificantTermsBuilder("sig_terms") + .field(TEXT_FIELD))) + .execute() + .actionGet(); + assertSearchResponse(response); + StringTerms classes = response.getAggregations().get("class"); + Assert.assertThat(classes.getBuckets().size(), equalTo(2)); + for (Terms.Bucket classBucket : classes.getBuckets()) { + Map aggs = classBucket.getAggregations().asMap(); + Assert.assertTrue(aggs.containsKey("sig_terms")); + SignificantTerms agg = (SignificantTerms) aggs.get("sig_terms"); + Assert.assertThat(agg.getBuckets().size(), equalTo(1)); + SignificantTerms.Bucket sigBucket = agg.iterator().next(); + String term = sigBucket.getKeyAsString(); + String classTerm = classBucket.getKeyAsString(); + Assert.assertTrue(term.equals(classTerm)); + } + } + + public static void index01Docs(String type, String settings, ESIntegTestCase testCase) throws ExecutionException, InterruptedException { + String mappings = "{\"doc\": {\"properties\":{\"text\": {\"type\":\"" + type + "\"}}}}"; + assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings).addMapping("doc", mappings)); + String[] gb = {"0", "1"}; + List indexRequestBuilderList = new ArrayList<>(); + indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "1") + .setSource(TEXT_FIELD, "1", CLASS_FIELD, "1")); + indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "2") + .setSource(TEXT_FIELD, "1", CLASS_FIELD, "1")); + indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "3") + .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); + indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "4") + .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); + indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "5") + .setSource(TEXT_FIELD, gb, CLASS_FIELD, "1")); + indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "6") + .setSource(TEXT_FIELD, gb, CLASS_FIELD, "0")); + indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "7") + .setSource(TEXT_FIELD, "0", CLASS_FIELD, "0")); + testCase.indexRandom(true, false, indexRequestBuilderList); + } +} diff --git a/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java b/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java deleted file mode 100644 index 11a791c04f3..00000000000 --- a/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.store; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; -import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.index.store.DirectoryService; -import org.elasticsearch.index.store.FsDirectoryService; -import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.index.store.IndexStoreModule; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.store.IndicesStore; -import org.elasticsearch.plugins.Plugin; - -public class MockFSIndexStore extends IndexStore { - - private final IndicesService indicesService; - - public static class TestPlugin extends Plugin { - @Override - public String name() { - return "mock-index-store"; - } - @Override - public String description() { - return "a mock index store for testing"; - } - public void onModule(IndexStoreModule indexStoreModule) { - indexStoreModule.addIndexStore("mock", MockFSIndexStore.class); - } - @Override - public Settings additionalSettings() { - return Settings.builder().put(IndexStoreModule.STORE_TYPE, "mock").build(); - } - } - - @Inject - public MockFSIndexStore(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService, - IndicesStore indicesStore, IndicesService indicesService) { - super(index, indexSettings, indexSettingsService, indicesStore); - this.indicesService = indicesService; - } - - public DirectoryService newDirectoryService(ShardPath path) { - return new MockFSDirectoryService(indexSettings, this, indicesService, path); - } - -} diff --git a/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStoreModule.java b/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStoreModule.java deleted file mode 100644 index c4f9d2046ad..00000000000 --- a/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStoreModule.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.store; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.store.IndexStore; - -public class MockFSIndexStoreModule extends AbstractModule { - - @Override - protected void configure() { - bind(IndexStore.class).to(MockFSIndexStore.class).asEagerSingleton(); - } - -} diff --git a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index 0bf04918a02..60f1bad6089 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -29,15 +29,15 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.hamcrest.RegexMatcher; import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.tribe.TribeIT; -import org.junit.Test; import java.io.IOException; import java.lang.management.ManagementFactory; @@ -55,7 +55,6 @@ import java.util.regex.Pattern; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -66,15 +65,12 @@ import static org.hamcrest.Matchers.sameInstance; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0) public class SimpleThreadPoolIT extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put("threadpool.search.type", "cached").build(); + return Settings.settingsBuilder().build(); } - @Test - public void verifyThreadNames() throws Exception { - + public void testThreadNames() throws Exception { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); Set preNodeStartThreadNames = new HashSet<>(); for (long l : threadBean.getAllThreadIds()) { @@ -130,53 +126,46 @@ public class SimpleThreadPoolIT extends ESIntegTestCase { } } - @Test(timeout = 20000) public void testUpdatingThreadPoolSettings() throws Exception { internalCluster().startNodesAsync(2).get(); ThreadPool threadPool = internalCluster().getDataNodeInstance(ThreadPool.class); // Check that settings are changed - assertThat(((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(5L)); - client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder().put("threadpool.search.keep_alive", "10m").build()).execute().actionGet(); - assertThat(((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(10L)); + assertThat(((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getQueue().remainingCapacity(), equalTo(1000)); + client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder().put("threadpool.search.queue_size", 2000).build()).execute().actionGet(); + assertThat(((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getQueue().remainingCapacity(), equalTo(2000)); // Make sure that threads continue executing when executor is replaced final CyclicBarrier barrier = new CyclicBarrier(2); Executor oldExecutor = threadPool.executor(Names.SEARCH); - threadPool.executor(Names.SEARCH).execute(new Runnable() { - @Override - public void run() { - try { - barrier.await(); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } catch (BrokenBarrierException ex) { - // - } - } - }); - client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder().put("threadpool.search.type", "fixed").build()).execute().actionGet(); + threadPool.executor(Names.SEARCH).execute(() -> { + try { + barrier.await(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } catch (BrokenBarrierException ex) { + // + } + }); + client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder().put("threadpool.search.queue_size", 1000).build()).execute().actionGet(); assertThat(threadPool.executor(Names.SEARCH), not(sameInstance(oldExecutor))); assertThat(((ThreadPoolExecutor) oldExecutor).isShutdown(), equalTo(true)); assertThat(((ThreadPoolExecutor) oldExecutor).isTerminating(), equalTo(true)); assertThat(((ThreadPoolExecutor) oldExecutor).isTerminated(), equalTo(false)); - barrier.await(); + barrier.await(10, TimeUnit.SECONDS); // Make sure that new thread executor is functional - threadPool.executor(Names.SEARCH).execute(new Runnable() { - @Override - public void run() { - try { - barrier.await(); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } catch (BrokenBarrierException ex) { - // + threadPool.executor(Names.SEARCH).execute(() -> { + try { + barrier.await(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } catch (BrokenBarrierException ex) { + // + } } - } - }); - client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder().put("threadpool.search.type", "fixed").build()).execute().actionGet(); - barrier.await(); - Thread.sleep(200); + ); + client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder().put("threadpool.search.queue_size", 500)).execute().actionGet(); + barrier.await(10, TimeUnit.SECONDS); // Check that node info is correct NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().all().execute().actionGet(); @@ -185,18 +174,15 @@ public class SimpleThreadPoolIT extends ESIntegTestCase { boolean found = false; for (ThreadPool.Info info : nodeInfo.getThreadPool()) { if (info.getName().equals(Names.SEARCH)) { - assertThat(info.getType(), equalTo("fixed")); + assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED); found = true; break; } } assertThat(found, equalTo(true)); - - Map poolMap = getPoolSettingsThroughJson(nodeInfo.getThreadPool(), Names.SEARCH); } } - @Test public void testThreadPoolLeakingThreadsWithTribeNode() { Settings settings = Settings.builder() .put("node.name", "thread_pool_leaking_threads_tribe_node") @@ -206,7 +192,7 @@ public class SimpleThreadPoolIT extends ESIntegTestCase { .put("tribe.t1.plugin.mandatory", "non_existing").build(); try { - NodeBuilder.nodeBuilder().settings(settings).build(); + new Node(settings); fail("The node startup is supposed to fail"); } catch(Throwable t) { //all good diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java index be33df33d36..3d57c1d5206 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.threadpool; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -30,8 +31,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.junit.Before; +import java.io.IOException; import java.util.Map; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -44,12 +46,17 @@ import static org.hamcrest.Matchers.nullValue; * */ public class ThreadPoolSerializationTests extends ESTestCase { - BytesStreamOutput output = new BytesStreamOutput(); + private ThreadPool.ThreadPoolType threadPoolType; + + @Before + public void setUp() throws Exception { + super.setUp(); + threadPoolType = randomFrom(ThreadPool.ThreadPoolType.values()); + } - @Test public void testThatQueueSizeSerializationWorks() throws Exception { - ThreadPool.Info info = new ThreadPool.Info("foo", "search", 1, 10, TimeValue.timeValueMillis(3000), SizeValue.parseSizeValue("10k")); + ThreadPool.Info info = new ThreadPool.Info("foo", threadPoolType, 1, 10, TimeValue.timeValueMillis(3000), SizeValue.parseSizeValue("10k")); output.setVersion(Version.CURRENT); info.writeTo(output); @@ -60,9 +67,8 @@ public class ThreadPoolSerializationTests extends ESTestCase { assertThat(newInfo.getQueueSize().singles(), is(10000l)); } - @Test public void testThatNegativeQueueSizesCanBeSerialized() throws Exception { - ThreadPool.Info info = new ThreadPool.Info("foo", "search", 1, 10, TimeValue.timeValueMillis(3000), null); + ThreadPool.Info info = new ThreadPool.Info("foo", threadPoolType, 1, 10, TimeValue.timeValueMillis(3000), null); output.setVersion(Version.CURRENT); info.writeTo(output); @@ -73,9 +79,8 @@ public class ThreadPoolSerializationTests extends ESTestCase { assertThat(newInfo.getQueueSize(), is(nullValue())); } - @Test public void testThatToXContentWritesOutUnboundedCorrectly() throws Exception { - ThreadPool.Info info = new ThreadPool.Info("foo", "search", 1, 10, TimeValue.timeValueMillis(3000), null); + ThreadPool.Info info = new ThreadPool.Info("foo", threadPoolType, 1, 10, TimeValue.timeValueMillis(3000), null); XContentBuilder builder = jsonBuilder(); builder.startObject(); info.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -92,7 +97,6 @@ public class ThreadPoolSerializationTests extends ESTestCase { assertThat(map.get("queue_size").toString(), is("-1")); } - @Test public void testThatNegativeSettingAllowsToStart() throws InterruptedException { Settings settings = settingsBuilder().put("name", "index").put("threadpool.index.queue_size", "-1").build(); ThreadPool threadPool = new ThreadPool(settings); @@ -100,9 +104,8 @@ public class ThreadPoolSerializationTests extends ESTestCase { terminate(threadPool); } - @Test public void testThatToXContentWritesInteger() throws Exception { - ThreadPool.Info info = new ThreadPool.Info("foo", "search", 1, 10, TimeValue.timeValueMillis(3000), SizeValue.parseSizeValue("1k")); + ThreadPool.Info info = new ThreadPool.Info("foo", threadPoolType, 1, 10, TimeValue.timeValueMillis(3000), SizeValue.parseSizeValue("1k")); XContentBuilder builder = jsonBuilder(); builder.startObject(); info.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -118,4 +121,16 @@ public class ThreadPoolSerializationTests extends ESTestCase { assertThat(map, hasKey("queue_size")); assertThat(map.get("queue_size").toString(), is("1000")); } + + public void testThatThreadPoolTypeIsSerializedCorrectly() throws IOException { + ThreadPool.Info info = new ThreadPool.Info("foo", threadPoolType); + output.setVersion(Version.CURRENT); + info.writeTo(output); + + StreamInput input = StreamInput.wrap(output.bytes()); + ThreadPool.Info newInfo = new ThreadPool.Info(); + newInfo.readFrom(input); + + assertThat(newInfo.getThreadPoolType(), is(threadPoolType)); + } } diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java index 0fc4f4c7a7f..b18be91f575 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -36,8 +35,6 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; public class ThreadPoolStatsTests extends ESTestCase { - - @Test public void testThreadPoolStatsSort() throws IOException { List stats = new ArrayList<>(); stats.add(new ThreadPoolStats.Stats("z", -1, 0, 0, 0, 0, 0L)); @@ -64,7 +61,6 @@ public class ThreadPoolStatsTests extends ESTestCase { assertThat(threads, contains(-1, -1, 1, 2, 3,-1,-1)); } - @Test public void testThreadPoolStatsToXContent() throws IOException { try (BytesStreamOutput os = new BytesStreamOutput()) { @@ -73,7 +69,7 @@ public class ThreadPoolStatsTests extends ESTestCase { stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.WARMER, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.OPTIMIZE, -1, 0, 0, 0, 0, 0L)); + stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.FORCE_MERGE, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.PERCOLATE, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SAME, -1, 0, 0, 0, 0, 0L)); @@ -105,8 +101,8 @@ public class ThreadPoolStatsTests extends ESTestCase { parser.skipChildren(); token = parser.nextToken(); } - assertThat(names, contains(ThreadPool.Names.GENERIC, - ThreadPool.Names.OPTIMIZE, + assertThat(names, contains(ThreadPool.Names.FORCE_MERGE, + ThreadPool.Names.GENERIC, ThreadPool.Names.PERCOLATE, ThreadPool.Names.SAME, ThreadPool.Names.SEARCH, diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolTypeSettingsValidatorTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolTypeSettingsValidatorTests.java new file mode 100644 index 00000000000..3dfca5cb283 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolTypeSettingsValidatorTests.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed wit[√h + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.threadpool; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.settings.Validator; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.*; + +import static org.junit.Assert.*; + +public class ThreadPoolTypeSettingsValidatorTests extends ESTestCase { + private Validator validator; + + @Before + public void setUp() throws Exception { + super.setUp(); + validator = ThreadPool.THREAD_POOL_TYPE_SETTINGS_VALIDATOR; + } + + public void testValidThreadPoolTypeSettings() { + for (Map.Entry entry : ThreadPool.THREAD_POOL_TYPES.entrySet()) { + assertNull(validateSetting(validator, entry.getKey(), entry.getValue().getType())); + } + } + + public void testInvalidThreadPoolTypeSettings() { + for (Map.Entry entry : ThreadPool.THREAD_POOL_TYPES.entrySet()) { + Set set = new HashSet<>(); + set.addAll(Arrays.asList(ThreadPool.ThreadPoolType.values())); + set.remove(entry.getValue()); + ThreadPool.ThreadPoolType invalidThreadPoolType = randomFrom(set.toArray(new ThreadPool.ThreadPoolType[set.size()])); + String expectedMessage = String.format( + Locale.ROOT, + "thread pool type for [%s] can only be updated to [%s] but was [%s]", + entry.getKey(), + entry.getValue().getType(), + invalidThreadPoolType.getType()); + String message = validateSetting(validator, entry.getKey(), invalidThreadPoolType.getType()); + assertNotNull(message); + assertEquals(expectedMessage, message); + } + } + + public void testNonThreadPoolTypeSetting() { + String setting = ThreadPool.THREADPOOL_GROUP + randomAsciiOfLength(10) + "foo"; + String value = randomAsciiOfLength(10); + assertNull(validator.validate(setting, value, ClusterState.PROTO)); + } + + private String validateSetting(Validator validator, String threadPoolName, String value) { + return validator.validate(ThreadPool.THREADPOOL_GROUP + threadPoolName + ".type", value, ClusterState.PROTO); + } +} diff --git a/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java b/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java index 562adaa8a0e..95ceea1e490 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java @@ -23,13 +23,16 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool.Names; -import org.junit.Test; import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.*; @@ -37,6 +40,320 @@ import static org.hamcrest.Matchers.*; /** */ public class UpdateThreadPoolSettingsTests extends ESTestCase { + public void testCorrectThreadPoolTypePermittedInSettings() throws InterruptedException { + String threadPoolName = randomThreadPoolName(); + ThreadPool.ThreadPoolType correctThreadPoolType = ThreadPool.THREAD_POOL_TYPES.get(threadPoolName); + ThreadPool threadPool = null; + try { + threadPool = new ThreadPool(settingsBuilder() + .put("name", "testCorrectThreadPoolTypePermittedInSettings") + .put("threadpool." + threadPoolName + ".type", correctThreadPoolType.getType()) + .build()); + ThreadPool.Info info = info(threadPool, threadPoolName); + if (ThreadPool.Names.SAME.equals(threadPoolName)) { + assertNull(info); // we don't report on the "same" threadpool + } else { + // otherwise check we have the expected type + assertEquals(info.getThreadPoolType(), correctThreadPoolType); + } + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + public void testThreadPoolCanNotOverrideThreadPoolType() throws InterruptedException { + String threadPoolName = randomThreadPoolName(); + ThreadPool.ThreadPoolType incorrectThreadPoolType = randomIncorrectThreadPoolType(threadPoolName); + ThreadPool.ThreadPoolType correctThreadPoolType = ThreadPool.THREAD_POOL_TYPES.get(threadPoolName); + ThreadPool threadPool = null; + try { + threadPool = new ThreadPool( + settingsBuilder() + .put("name", "testThreadPoolCanNotOverrideThreadPoolType") + .put("threadpool." + threadPoolName + ".type", incorrectThreadPoolType.getType()) + .build()); + terminate(threadPool); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat( + e.getMessage(), + is("setting threadpool." + threadPoolName + ".type to " + incorrectThreadPoolType.getType() + " is not permitted; must be " + correctThreadPoolType.getType())); + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + public void testUpdateSettingsCanNotChangeThreadPoolType() throws InterruptedException { + String threadPoolName = randomThreadPoolName(); + ThreadPool.ThreadPoolType invalidThreadPoolType = randomIncorrectThreadPoolType(threadPoolName); + ThreadPool.ThreadPoolType validThreadPoolType = ThreadPool.THREAD_POOL_TYPES.get(threadPoolName); + ThreadPool threadPool = null; + try { + threadPool = new ThreadPool(settingsBuilder().put("name", "testUpdateSettingsCanNotChangeThreadPoolType").build()); + + + threadPool.updateSettings( + settingsBuilder() + .put("threadpool." + threadPoolName + ".type", invalidThreadPoolType.getType()) + .build() + ); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat( + e.getMessage(), + is("setting threadpool." + threadPoolName + ".type to " + invalidThreadPoolType.getType() + " is not permitted; must be " + validThreadPoolType.getType())); + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + public void testCachedExecutorType() throws InterruptedException { + String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.CACHED); + ThreadPool threadPool = null; + try { + threadPool = new ThreadPool( + Settings.settingsBuilder() + .put("name", "testCachedExecutorType").build()); + + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.CACHED); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + + threadPool.updateSettings(settingsBuilder() + .put("threadpool." + threadPoolName + ".keep_alive", "10m") + .build()); + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.CACHED); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(0)); + // Make sure keep alive value changed + assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(10L)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(10L)); + + // Make sure keep alive value reused + assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(10L)); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + + // Change keep alive + Executor oldExecutor = threadPool.executor(threadPoolName); + threadPool.updateSettings(settingsBuilder().put("threadpool." + threadPoolName + ".keep_alive", "1m").build()); + // Make sure keep alive value changed + assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(1L)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(1L)); + // Make sure executor didn't change + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.CACHED); + assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor)); + + // Set the same keep alive + threadPool.updateSettings(settingsBuilder().put("threadpool." + threadPoolName + ".keep_alive", "1m").build()); + // Make sure keep alive value didn't change + assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(1L)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(1L)); + // Make sure executor didn't change + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.CACHED); + assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor)); + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + public void testFixedExecutorType() throws InterruptedException { + String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED); + ThreadPool threadPool = null; + + try { + threadPool = new ThreadPool(settingsBuilder() + .put("name", "testCachedExecutorType").build()); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + + threadPool.updateSettings(settingsBuilder() + .put("threadpool." + threadPoolName + ".size", "15") + .build()); + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(15)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(), equalTo(15)); + assertThat(info(threadPool, threadPoolName).getMin(), equalTo(15)); + assertThat(info(threadPool, threadPoolName).getMax(), equalTo(15)); + // keep alive does not apply to fixed thread pools + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(0L)); + + // Put old type back + threadPool.updateSettings(Settings.EMPTY); + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED); + // Make sure keep alive value is not used + assertThat(info(threadPool, threadPoolName).getKeepAlive(), nullValue()); + // Make sure keep pool size value were reused + assertThat(info(threadPool, threadPoolName).getMin(), equalTo(15)); + assertThat(info(threadPool, threadPoolName).getMax(), equalTo(15)); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(15)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(), equalTo(15)); + + // Change size + Executor oldExecutor = threadPool.executor(threadPoolName); + threadPool.updateSettings(settingsBuilder().put("threadpool." + threadPoolName + ".size", "10").build()); + // Make sure size values changed + assertThat(info(threadPool, threadPoolName).getMax(), equalTo(10)); + assertThat(info(threadPool, threadPoolName).getMin(), equalTo(10)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(), equalTo(10)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(10)); + // Make sure executor didn't change + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED); + assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor)); + + // Change queue capacity + threadPool.updateSettings(settingsBuilder() + .put("threadpool." + threadPoolName + ".queue", "500") + .build()); + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + public void testScalingExecutorType() throws InterruptedException { + String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING); + ThreadPool threadPool = null; + try { + threadPool = new ThreadPool(settingsBuilder() + .put("threadpool." + threadPoolName + ".size", 10) + .put("name", "testCachedExecutorType").build()); + assertThat(info(threadPool, threadPoolName).getMin(), equalTo(1)); + assertThat(info(threadPool, threadPoolName).getMax(), equalTo(10)); + assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(5L)); + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + + // Change settings that doesn't require pool replacement + Executor oldExecutor = threadPool.executor(threadPoolName); + threadPool.updateSettings(settingsBuilder() + .put("threadpool." + threadPoolName + ".keep_alive", "10m") + .put("threadpool." + threadPoolName + ".min", "2") + .put("threadpool." + threadPoolName + ".size", "15") + .build()); + assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING); + assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(2)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(), equalTo(15)); + assertThat(info(threadPool, threadPoolName).getMin(), equalTo(2)); + assertThat(info(threadPool, threadPoolName).getMax(), equalTo(15)); + // Make sure keep alive value changed + assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(10L)); + assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(10L)); + assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor)); + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + public void testShutdownNowInterrupts() throws Exception { + String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED); + ThreadPool threadPool = null; + try { + threadPool = new ThreadPool(Settings.settingsBuilder() + .put("threadpool." + threadPoolName + ".queue_size", 1000) + .put("name", "testCachedExecutorType").build()); + assertEquals(info(threadPool, threadPoolName).getQueueSize().getSingles(), 1000L); + + final CountDownLatch latch = new CountDownLatch(1); + ThreadPoolExecutor oldExecutor = (ThreadPoolExecutor) threadPool.executor(threadPoolName); + threadPool.executor(threadPoolName).execute(() -> { + try { + new CountDownLatch(1).await(); + } catch (InterruptedException ex) { + latch.countDown(); + Thread.currentThread().interrupt(); + } + } + ); + threadPool.updateSettings(settingsBuilder().put("threadpool." + threadPoolName + ".queue_size", 2000).build()); + assertThat(threadPool.executor(threadPoolName), not(sameInstance(oldExecutor))); + assertThat(oldExecutor.isShutdown(), equalTo(true)); + assertThat(oldExecutor.isTerminating(), equalTo(true)); + assertThat(oldExecutor.isTerminated(), equalTo(false)); + threadPool.shutdownNow(); // should interrupt the thread + latch.await(3, TimeUnit.SECONDS); // If this throws then ThreadPool#shutdownNow didn't interrupt + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + public void testCustomThreadPool() throws Exception { + ThreadPool threadPool = null; + try { + threadPool = new ThreadPool(Settings.settingsBuilder() + .put("threadpool.my_pool1.type", "scaling") + .put("threadpool.my_pool2.type", "fixed") + .put("threadpool.my_pool2.size", "1") + .put("threadpool.my_pool2.queue_size", "1") + .put("name", "testCustomThreadPool").build()); + ThreadPoolInfo groups = threadPool.info(); + boolean foundPool1 = false; + boolean foundPool2 = false; + outer: + for (ThreadPool.Info info : groups) { + if ("my_pool1".equals(info.getName())) { + foundPool1 = true; + assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING); + } else if ("my_pool2".equals(info.getName())) { + foundPool2 = true; + assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED); + assertThat(info.getMin(), equalTo(1)); + assertThat(info.getMax(), equalTo(1)); + assertThat(info.getQueueSize().singles(), equalTo(1l)); + } else { + for (Field field : Names.class.getFields()) { + if (info.getName().equalsIgnoreCase(field.getName())) { + // This is ok it is a default thread pool + continue outer; + } + } + fail("Unexpected pool name: " + info.getName()); + } + } + assertThat(foundPool1, is(true)); + assertThat(foundPool2, is(true)); + + // Updating my_pool2 + Settings settings = Settings.builder() + .put("threadpool.my_pool2.size", "10") + .build(); + threadPool.updateSettings(settings); + + groups = threadPool.info(); + foundPool1 = false; + foundPool2 = false; + outer: + for (ThreadPool.Info info : groups) { + if ("my_pool1".equals(info.getName())) { + foundPool1 = true; + assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING); + } else if ("my_pool2".equals(info.getName())) { + foundPool2 = true; + assertThat(info.getMax(), equalTo(10)); + assertThat(info.getMin(), equalTo(10)); + assertThat(info.getQueueSize().singles(), equalTo(1l)); + assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED); + } else { + for (Field field : Names.class.getFields()) { + if (info.getName().equalsIgnoreCase(field.getName())) { + // This is ok it is a default thread pool + continue outer; + } + } + fail("Unexpected pool name: " + info.getName()); + } + } + assertThat(foundPool1, is(true)); + assertThat(foundPool2, is(true)); + } finally { + terminateThreadPoolIfNeeded(threadPool); + } + } + + private void terminateThreadPoolIfNeeded(ThreadPool threadPool) throws InterruptedException { + if (threadPool != null) { + terminate(threadPool); + } + } private ThreadPool.Info info(ThreadPool threadPool, String name) { for (ThreadPool.Info info : threadPool.info()) { @@ -47,253 +364,20 @@ public class UpdateThreadPoolSettingsTests extends ESTestCase { return null; } - @Test - public void testCachedExecutorType() throws InterruptedException { - ThreadPool threadPool = new ThreadPool( - Settings.settingsBuilder() - .put("threadpool.search.type", "cached") - .put("name","testCachedExecutorType").build()); - - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("cached")); - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(5L)); - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - - // Replace with different type - threadPool.updateSettings(settingsBuilder().put("threadpool.search.type", "same").build()); - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("same")); - assertThat(threadPool.executor(Names.SEARCH), is(ThreadPool.DIRECT_EXECUTOR)); - - // Replace with different type again - threadPool.updateSettings(settingsBuilder() - .put("threadpool.search.type", "scaling") - .put("threadpool.search.keep_alive", "10m") - .build()); - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("scaling")); - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getCorePoolSize(), equalTo(1)); - // Make sure keep alive value changed - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(10L)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(10L)); - - // Put old type back - threadPool.updateSettings(settingsBuilder().put("threadpool.search.type", "cached").build()); - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("cached")); - // Make sure keep alive value reused - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(10L)); - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - - // Change keep alive - Executor oldExecutor = threadPool.executor(Names.SEARCH); - threadPool.updateSettings(settingsBuilder().put("threadpool.search.keep_alive", "1m").build()); - // Make sure keep alive value changed - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(1L)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(1L)); - // Make sure executor didn't change - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("cached")); - assertThat(threadPool.executor(Names.SEARCH), sameInstance(oldExecutor)); - - // Set the same keep alive - threadPool.updateSettings(settingsBuilder().put("threadpool.search.keep_alive", "1m").build()); - // Make sure keep alive value didn't change - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(1L)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(1L)); - // Make sure executor didn't change - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("cached")); - assertThat(threadPool.executor(Names.SEARCH), sameInstance(oldExecutor)); - terminate(threadPool); + private String randomThreadPoolName() { + Set threadPoolNames = ThreadPool.THREAD_POOL_TYPES.keySet(); + return randomFrom(threadPoolNames.toArray(new String[threadPoolNames.size()])); } - @Test - public void testFixedExecutorType() throws InterruptedException { - ThreadPool threadPool = new ThreadPool(settingsBuilder() - .put("threadpool.search.type", "fixed") - .put("name","testCachedExecutorType").build()); - - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - - // Replace with different type - threadPool.updateSettings(settingsBuilder() - .put("threadpool.search.type", "scaling") - .put("threadpool.search.keep_alive", "10m") - .put("threadpool.search.min", "2") - .put("threadpool.search.size", "15") - .build()); - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("scaling")); - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getCorePoolSize(), equalTo(2)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getMaximumPoolSize(), equalTo(15)); - assertThat(info(threadPool, Names.SEARCH).getMin(), equalTo(2)); - assertThat(info(threadPool, Names.SEARCH).getMax(), equalTo(15)); - // Make sure keep alive value changed - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(10L)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(10L)); - - // Put old type back - threadPool.updateSettings(settingsBuilder() - .put("threadpool.search.type", "fixed") - .build()); - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("fixed")); - // Make sure keep alive value is not used - assertThat(info(threadPool, Names.SEARCH).getKeepAlive(), nullValue()); - // Make sure keep pool size value were reused - assertThat(info(threadPool, Names.SEARCH).getMin(), equalTo(15)); - assertThat(info(threadPool, Names.SEARCH).getMax(), equalTo(15)); - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getCorePoolSize(), equalTo(15)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getMaximumPoolSize(), equalTo(15)); - - // Change size - Executor oldExecutor = threadPool.executor(Names.SEARCH); - threadPool.updateSettings(settingsBuilder().put("threadpool.search.size", "10").build()); - // Make sure size values changed - assertThat(info(threadPool, Names.SEARCH).getMax(), equalTo(10)); - assertThat(info(threadPool, Names.SEARCH).getMin(), equalTo(10)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getMaximumPoolSize(), equalTo(10)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getCorePoolSize(), equalTo(10)); - // Make sure executor didn't change - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("fixed")); - assertThat(threadPool.executor(Names.SEARCH), sameInstance(oldExecutor)); - - // Change queue capacity - threadPool.updateSettings(settingsBuilder() - .put("threadpool.search.queue", "500") - .build()); - - terminate(threadPool); + private ThreadPool.ThreadPoolType randomIncorrectThreadPoolType(String threadPoolName) { + Set set = new HashSet<>(); + set.addAll(Arrays.asList(ThreadPool.ThreadPoolType.values())); + set.remove(ThreadPool.THREAD_POOL_TYPES.get(threadPoolName)); + ThreadPool.ThreadPoolType invalidThreadPoolType = randomFrom(set.toArray(new ThreadPool.ThreadPoolType[set.size()])); + return invalidThreadPoolType; } - - @Test - public void testScalingExecutorType() throws InterruptedException { - ThreadPool threadPool = new ThreadPool(settingsBuilder() - .put("threadpool.search.type", "scaling") - .put("threadpool.search.size", 10) - .put("name","testCachedExecutorType").build()); - - assertThat(info(threadPool, Names.SEARCH).getMin(), equalTo(1)); - assertThat(info(threadPool, Names.SEARCH).getMax(), equalTo(10)); - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(5L)); - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("scaling")); - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - - // Change settings that doesn't require pool replacement - Executor oldExecutor = threadPool.executor(Names.SEARCH); - threadPool.updateSettings(settingsBuilder() - .put("threadpool.search.type", "scaling") - .put("threadpool.search.keep_alive", "10m") - .put("threadpool.search.min", "2") - .put("threadpool.search.size", "15") - .build()); - assertThat(info(threadPool, Names.SEARCH).getType(), equalTo("scaling")); - assertThat(threadPool.executor(Names.SEARCH), instanceOf(EsThreadPoolExecutor.class)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getCorePoolSize(), equalTo(2)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getMaximumPoolSize(), equalTo(15)); - assertThat(info(threadPool, Names.SEARCH).getMin(), equalTo(2)); - assertThat(info(threadPool, Names.SEARCH).getMax(), equalTo(15)); - // Make sure keep alive value changed - assertThat(info(threadPool, Names.SEARCH).getKeepAlive().minutes(), equalTo(10L)); - assertThat(((EsThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(10L)); - assertThat(threadPool.executor(Names.SEARCH), sameInstance(oldExecutor)); - - terminate(threadPool); + private String randomThreadPool(ThreadPool.ThreadPoolType type) { + return randomFrom(ThreadPool.THREAD_POOL_TYPES.entrySet().stream().filter(t -> t.getValue().equals(type)).map(t -> t.getKey()).collect(Collectors.toList())); } - - @Test(timeout = 10000) - public void testShutdownDownNowDoesntBlock() throws Exception { - ThreadPool threadPool = new ThreadPool(Settings.settingsBuilder() - .put("threadpool.search.type", "cached") - .put("name","testCachedExecutorType").build()); - - final CountDownLatch latch = new CountDownLatch(1); - Executor oldExecutor = threadPool.executor(Names.SEARCH); - threadPool.executor(Names.SEARCH).execute(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(20000); - } catch (InterruptedException ex) { - latch.countDown(); - Thread.currentThread().interrupt(); - } - } - }); - threadPool.updateSettings(settingsBuilder().put("threadpool.search.type", "fixed").build()); - assertThat(threadPool.executor(Names.SEARCH), not(sameInstance(oldExecutor))); - assertThat(((ThreadPoolExecutor) oldExecutor).isShutdown(), equalTo(true)); - assertThat(((ThreadPoolExecutor) oldExecutor).isTerminating(), equalTo(true)); - assertThat(((ThreadPoolExecutor) oldExecutor).isTerminated(), equalTo(false)); - threadPool.shutdownNow(); // interrupt the thread - latch.await(); - terminate(threadPool); - } - - @Test - public void testCustomThreadPool() throws Exception { - ThreadPool threadPool = new ThreadPool(Settings.settingsBuilder() - .put("threadpool.my_pool1.type", "cached") - .put("threadpool.my_pool2.type", "fixed") - .put("threadpool.my_pool2.size", "1") - .put("threadpool.my_pool2.queue_size", "1") - .put("name", "testCustomThreadPool").build()); - - ThreadPoolInfo groups = threadPool.info(); - boolean foundPool1 = false; - boolean foundPool2 = false; - outer: for (ThreadPool.Info info : groups) { - if ("my_pool1".equals(info.getName())) { - foundPool1 = true; - assertThat(info.getType(), equalTo("cached")); - } else if ("my_pool2".equals(info.getName())) { - foundPool2 = true; - assertThat(info.getType(), equalTo("fixed")); - assertThat(info.getMin(), equalTo(1)); - assertThat(info.getMax(), equalTo(1)); - assertThat(info.getQueueSize().singles(), equalTo(1l)); - } else { - for (Field field : Names.class.getFields()) { - if (info.getName().equalsIgnoreCase(field.getName())) { - // This is ok it is a default thread pool - continue outer; - } - } - fail("Unexpected pool name: " + info.getName()); - } - } - assertThat(foundPool1, is(true)); - assertThat(foundPool2, is(true)); - - // Updating my_pool2 - Settings settings = Settings.builder() - .put("threadpool.my_pool2.size", "10") - .build(); - threadPool.updateSettings(settings); - - groups = threadPool.info(); - foundPool1 = false; - foundPool2 = false; - outer: for (ThreadPool.Info info : groups) { - if ("my_pool1".equals(info.getName())) { - foundPool1 = true; - assertThat(info.getType(), equalTo("cached")); - } else if ("my_pool2".equals(info.getName())) { - foundPool2 = true; - assertThat(info.getMax(), equalTo(10)); - assertThat(info.getMin(), equalTo(10)); - assertThat(info.getQueueSize().singles(), equalTo(1l)); - assertThat(info.getType(), equalTo("fixed")); - } else { - for (Field field : Names.class.getFields()) { - if (info.getName().equalsIgnoreCase(field.getName())) { - // This is ok it is a default thread pool - continue outer; - } - } - fail("Unexpected pool name: " + info.getName()); - } - } - assertThat(foundPool1, is(true)); - assertThat(foundPool2, is(true)); - terminate(threadPool); - } - } diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java index 3cfbd42e2e8..30ed8fe25ca 100644 --- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java +++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java @@ -26,7 +26,6 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Priority; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Locale; @@ -40,11 +39,8 @@ import static org.hamcrest.Matchers.notNullValue; /** */ -public class SimpleTimestampIT extends ESIntegTestCase { - - @Test +public class SimpleTimestampIT extends ESIntegTestCase { public void testSimpleTimestamp() throws Exception { - client().admin().indices().prepareCreate("test") .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).endObject().endObject().endObject()) .execute().actionGet(); @@ -94,7 +90,7 @@ public class SimpleTimestampIT extends ESIntegTestCase { assertThat(((Number) getResponse.getField("_timestamp").getValue()).longValue(), equalTo(timestamp)); } - @Test // issue 5053 + // issue #5053 public void testThatUpdatingMappingShouldNotRemoveTimestampConfiguration() throws Exception { String index = "foo"; String type = "mytype"; @@ -114,7 +110,6 @@ public class SimpleTimestampIT extends ESIntegTestCase { assertTimestampMappingEnabled(index, type, true); } - @Test public void testThatTimestampCanBeSwitchedOnAndOff() throws Exception { String index = "foo"; String type = "mytype"; diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index c423fb7dbac..becb61666da 100644 --- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -33,7 +33,6 @@ import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.concurrent.CountDownLatch; @@ -42,7 +41,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; -import static org.elasticsearch.transport.TransportRequestOptions.options; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -126,7 +124,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { terminate(threadPool); } - @Test public void testHelloWorld() { serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override @@ -173,7 +170,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { } res = serviceB.submitRequest(nodeA, "sayHello", - new StringMessageRequest("moshe"), TransportRequestOptions.options().withCompress(true), new BaseTransportResponseHandler() { + new StringMessageRequest("moshe"), TransportRequestOptions.builder().withCompress(true).build(), new BaseTransportResponseHandler() { @Override public StringMessageResponse newInstance() { return new StringMessageResponse(); @@ -206,7 +203,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHello"); } - @Test public void testLocalNodeConnection() throws InterruptedException { assertTrue("serviceA is not connected to nodeA", serviceA.nodeConnected(nodeA)); if (((TransportService) serviceA).getLocalNode() != null) { @@ -254,13 +250,12 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { assertThat(responseString.get(), equalTo("test")); } - @Test public void testVoidMessageCompressed() { serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override public void messageReceived(TransportRequest.Empty request, TransportChannel channel) { try { - channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.options().withCompress(true)); + channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.builder().withCompress(true).build()); } catch (IOException e) { e.printStackTrace(); assertThat(e.getMessage(), false, equalTo(true)); @@ -269,7 +264,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { }); TransportFuture res = serviceB.submitRequest(nodeA, "sayHello", - TransportRequest.Empty.INSTANCE, TransportRequestOptions.options().withCompress(true), new BaseTransportResponseHandler() { + TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(true).build(), new BaseTransportResponseHandler() { @Override public TransportResponse.Empty newInstance() { return TransportResponse.Empty.INSTANCE; @@ -301,14 +296,13 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHello"); } - @Test public void testHelloWorldCompressed() { serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override public void messageReceived(StringMessageRequest request, TransportChannel channel) { assertThat("moshe", equalTo(request.message)); try { - channel.sendResponse(new StringMessageResponse("hello " + request.message), TransportResponseOptions.options().withCompress(true)); + channel.sendResponse(new StringMessageResponse("hello " + request.message), TransportResponseOptions.builder().withCompress(true).build()); } catch (IOException e) { e.printStackTrace(); assertThat(e.getMessage(), false, equalTo(true)); @@ -317,7 +311,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { }); TransportFuture res = serviceB.submitRequest(nodeA, "sayHello", - new StringMessageRequest("moshe"), TransportRequestOptions.options().withCompress(true), new BaseTransportResponseHandler() { + new StringMessageRequest("moshe"), TransportRequestOptions.builder().withCompress(true).build(), new BaseTransportResponseHandler() { @Override public StringMessageResponse newInstance() { return new StringMessageResponse(); @@ -350,7 +344,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHello"); } - @Test public void testErrorMessage() { serviceA.registerRequestHandler("sayHelloException", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override @@ -393,7 +386,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHelloException"); } - @Test public void testDisconnectListener() throws Exception { final CountDownLatch latch = new CountDownLatch(1); TransportConnectionListener disconnectListener = new TransportConnectionListener() { @@ -412,7 +404,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { assertThat(latch.await(5, TimeUnit.SECONDS), equalTo(true)); } - @Test public void testNotifyOnShutdown() throws Exception { final CountDownLatch latch2 = new CountDownLatch(1); @@ -429,7 +420,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { } }); TransportFuture foobar = serviceB.submitRequest(nodeA, "foobar", - new StringMessageRequest(""), options(), EmptyTransportResponseHandler.INSTANCE_SAME); + new StringMessageRequest(""), TransportRequestOptions.EMPTY, EmptyTransportResponseHandler.INSTANCE_SAME); latch2.countDown(); try { foobar.txGet(); @@ -440,7 +431,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHelloTimeoutDelayedResponse"); } - @Test public void testTimeoutSendExceptionWithNeverSendingBackResponse() throws Exception { serviceA.registerRequestHandler("sayHelloTimeoutNoResponse", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override @@ -457,7 +447,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { }); TransportFuture res = serviceB.submitRequest(nodeA, "sayHelloTimeoutNoResponse", - new StringMessageRequest("moshe"), options().withTimeout(100), new BaseTransportResponseHandler() { + new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(), new BaseTransportResponseHandler() { @Override public StringMessageResponse newInstance() { return new StringMessageResponse(); @@ -489,7 +479,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHelloTimeoutNoResponse"); } - @Test public void testTimeoutSendExceptionWithDelayedResponse() throws Exception { serviceA.registerRequestHandler("sayHelloTimeoutDelayedResponse", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override @@ -510,7 +499,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { }); final CountDownLatch latch = new CountDownLatch(1); TransportFuture res = serviceB.submitRequest(nodeA, "sayHelloTimeoutDelayedResponse", - new StringMessageRequest("300ms"), options().withTimeout(100), new BaseTransportResponseHandler() { + new StringMessageRequest("300ms"), TransportRequestOptions.builder().withTimeout(100).build(), new BaseTransportResponseHandler() { @Override public StringMessageResponse newInstance() { return new StringMessageResponse(); @@ -546,7 +535,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { final int counter = i; // now, try and send another request, this times, with a short timeout res = serviceB.submitRequest(nodeA, "sayHelloTimeoutDelayedResponse", - new StringMessageRequest(counter + "ms"), options().withTimeout(3000), new BaseTransportResponseHandler() { + new StringMessageRequest(counter + "ms"), TransportRequestOptions.builder().withTimeout(3000).build(), new BaseTransportResponseHandler() { @Override public StringMessageResponse newInstance() { return new StringMessageResponse(); @@ -576,8 +565,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHelloTimeoutDelayedResponse"); } - - @Test @TestLogging(value = "test. transport.tracer:TRACE") public void testTracerLog() throws InterruptedException { TransportRequestHandler handler = new TransportRequestHandler() { @@ -633,7 +620,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { tracer.reset(4); boolean timeout = randomBoolean(); - TransportRequestOptions options = timeout ? new TransportRequestOptions().withTimeout(1) : TransportRequestOptions.EMPTY; + TransportRequestOptions options = timeout ? TransportRequestOptions.builder().withTimeout(1).build(): TransportRequestOptions.EMPTY; serviceA.sendRequest(nodeB, "test", new StringMessageRequest("", 10), options, noopResponseHandler); requestCompleted.acquire(); tracer.expectedEvents.get().await(); @@ -883,8 +870,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { } } - @Test - public void testVersion_from0to1() throws Exception { + public void testVersionFrom0to1() throws Exception { serviceB.registerRequestHandler("/version", Version1Request::new, ThreadPool.Names.SAME, new TransportRequestHandler() { @Override public void messageReceived(Version1Request request, TransportChannel channel) throws Exception { @@ -925,8 +911,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { assertThat(version0Response.value1, equalTo(1)); } - @Test - public void testVersion_from1to0() throws Exception { + public void testVersionFrom1to0() throws Exception { serviceA.registerRequestHandler("/version", Version0Request::new, ThreadPool.Names.SAME, new TransportRequestHandler() { @Override public void messageReceived(Version0Request request, TransportChannel channel) throws Exception { @@ -968,8 +953,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { assertThat(version1Response.value2, equalTo(0)); } - @Test - public void testVersion_from1to1() throws Exception { + public void testVersionFrom1to1() throws Exception { serviceB.registerRequestHandler("/version", Version1Request::new, ThreadPool.Names.SAME, new TransportRequestHandler() { @Override public void messageReceived(Version1Request request, TransportChannel channel) throws Exception { @@ -1013,8 +997,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { assertThat(version1Response.value2, equalTo(2)); } - @Test - public void testVersion_from0to0() throws Exception { + public void testVersionFrom0to0() throws Exception { serviceA.registerRequestHandler("/version", Version0Request::new, ThreadPool.Names.SAME, new TransportRequestHandler() { @Override public void messageReceived(Version0Request request, TransportChannel channel) throws Exception { @@ -1053,7 +1036,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { assertThat(version0Response.value1, equalTo(1)); } - @Test public void testMockFailToSendNoConnectRule() { serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override @@ -1063,7 +1045,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { } }); - serviceB.addFailToSendNoConnectRule(nodeA); + serviceB.addFailToSendNoConnectRule(serviceA); TransportFuture res = serviceB.submitRequest(nodeA, "sayHello", new StringMessageRequest("moshe"), new BaseTransportResponseHandler() { @@ -1112,7 +1094,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceA.removeHandler("sayHello"); } - @Test public void testMockUnresponsiveRule() { serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler() { @Override @@ -1122,10 +1103,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { } }); - serviceB.addUnresponsiveRule(nodeA); + serviceB.addUnresponsiveRule(serviceA); TransportFuture res = serviceB.submitRequest(nodeA, "sayHello", - new StringMessageRequest("moshe"), TransportRequestOptions.options().withTimeout(100), new BaseTransportResponseHandler() { + new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(), new BaseTransportResponseHandler() { @Override public StringMessageResponse newInstance() { return new StringMessageResponse(); @@ -1172,7 +1153,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { } - @Test public void testHostOnMessages() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(2); final AtomicReference addressA = new AtomicReference<>(); diff --git a/core/src/test/java/org/elasticsearch/transport/ActionNamesIT.java b/core/src/test/java/org/elasticsearch/transport/ActionNamesIT.java index d3b8533d554..d790137b38f 100644 --- a/core/src/test/java/org/elasticsearch/transport/ActionNamesIT.java +++ b/core/src/test/java/org/elasticsearch/transport/ActionNamesIT.java @@ -20,7 +20,6 @@ package org.elasticsearch.transport; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.startsWith; @@ -42,9 +41,6 @@ import static org.hamcrest.CoreMatchers.startsWith; * we use the `[n]` suffix to identify node actions and the `[s]` suffix to identify shard actions. */ public class ActionNamesIT extends ESIntegTestCase { - - @Test - @SuppressWarnings("unchecked") public void testActionNamesCategories() throws NoSuchFieldException, IllegalAccessException { TransportService transportService = internalCluster().getInstance(TransportService.class); for (String action : transportService.requestHandlers.keySet()) { diff --git a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java index 44039ce157f..3f140b388fd 100644 --- a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java +++ b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java @@ -19,29 +19,27 @@ package org.elasticsearch.transport; -import java.nio.charset.StandardCharsets; - import org.elasticsearch.Version; +import org.elasticsearch.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.common.util.MockBigArrays; -import org.elasticsearch.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.netty.NettyTransport; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.BufferedReader; import java.io.InputStreamReader; import java.net.InetAddress; import java.net.Socket; +import java.nio.charset.StandardCharsets; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.is; @@ -52,7 +50,11 @@ import static org.hamcrest.Matchers.is; */ public class NettySizeHeaderFrameDecoderTests extends ESTestCase { - private final Settings settings = settingsBuilder().put("name", "foo").put("transport.host", "127.0.0.1").build(); + private final Settings settings = settingsBuilder() + .put("name", "foo") + .put("transport.host", "127.0.0.1") + .put("transport.tcp.port", "0") + .build(); private ThreadPool threadPool; private NettyTransport nettyTransport; @@ -82,7 +84,6 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase { terminate(threadPool); } - @Test public void testThatTextMessageIsReturnedOnHTTPLikeRequest() throws Exception { String randomMethod = randomFrom("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH"); String data = randomMethod + " / HTTP/1.1"; @@ -97,7 +98,6 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase { } } - @Test public void testThatNothingIsReturnedForOtherInvalidPackets() throws Exception { try (Socket socket = new Socket(host, port)) { socket.getOutputStream().write("FOOBAR".getBytes(StandardCharsets.UTF_8)); diff --git a/core/src/test/java/org/elasticsearch/transport/TransportMessageTests.java b/core/src/test/java/org/elasticsearch/transport/TransportMessageTests.java index bb907e5fb26..a94b06f6f06 100644 --- a/core/src/test/java/org/elasticsearch/transport/TransportMessageTests.java +++ b/core/src/test/java/org/elasticsearch/transport/TransportMessageTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -32,8 +31,6 @@ import static org.hamcrest.Matchers.is; * */ public class TransportMessageTests extends ESTestCase { - - @Test public void testSerialization() throws Exception { Message message = new Message(); message.putHeader("key1", "value1"); @@ -57,7 +54,6 @@ public class TransportMessageTests extends ESTestCase { assertThat(key1, is("value1")); } - @Test public void testCopyHeadersAndContext() throws Exception { Message m1 = new Message(); m1.putHeader("key1", "value1"); diff --git a/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java b/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java index 43d03729bf4..9581dfff42f 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.transport.netty; import org.elasticsearch.common.util.concurrent.KeyedLock; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.Map.Entry; import java.util.Set; @@ -30,13 +29,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; public class KeyedLockTests extends ESTestCase { - - @Test - public void checkIfMapEmptyAfterLotsOfAcquireAndReleases() throws InterruptedException { + public void testIfMapEmptyAfterLotsOfAcquireAndReleases() throws InterruptedException { ConcurrentHashMap counter = new ConcurrentHashMap<>(); ConcurrentHashMap safeCounter = new ConcurrentHashMap<>(); KeyedLock connectionLock = new KeyedLock(randomBoolean()); @@ -69,19 +68,27 @@ public class KeyedLockTests extends ESTestCase { } } - @Test(expected = IllegalStateException.class) - public void checkCannotAcquireTwoLocks() throws InterruptedException { + public void testCannotAcquireTwoLocks() throws InterruptedException { KeyedLock connectionLock = new KeyedLock(); String name = randomRealisticUnicodeOfLength(scaledRandomIntBetween(10, 50)); connectionLock.acquire(name); - connectionLock.acquire(name); + try { + connectionLock.acquire(name); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), containsString("Lock already acquired")); + } } - @Test(expected = IllegalStateException.class) - public void checkCannotReleaseUnacquiredLock() throws InterruptedException { + public void testCannotReleaseUnacquiredLock() throws InterruptedException { KeyedLock connectionLock = new KeyedLock(); String name = randomRealisticUnicodeOfLength(scaledRandomIntBetween(10, 50)); - connectionLock.release(name); + try { + connectionLock.release(name); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), is("Lock not acquired")); + } } public static class AcquireAndReleaseThread extends Thread { @@ -105,7 +112,7 @@ public class KeyedLockTests extends ESTestCase { try { startLatch.await(); } catch (InterruptedException e) { - throw new RuntimeException(); + throw new RuntimeException(e); } int numRuns = scaledRandomIntBetween(5000, 50000); for (int i = 0; i < numRuns; i++) { diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java index 4e03b8ea1d3..7a939a5a1bc 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseOptions; -import org.junit.Test; import java.io.IOException; @@ -47,14 +46,10 @@ import static org.hamcrest.Matchers.greaterThan; /** */ public class NettyScheduledPingTests extends ESTestCase { - - @Test public void testScheduledPing() throws Exception { ThreadPool threadPool = new ThreadPool(getClass().getName()); - int startPort = 11000 + randomIntBetween(0, 255); - int endPort = startPort + 10; - Settings settings = Settings.builder().put(NettyTransport.PING_SCHEDULE, "5ms").put("transport.tcp.port", startPort + "-" + endPort).build(); + Settings settings = Settings.builder().put(NettyTransport.PING_SCHEDULE, "5ms").put("transport.tcp.port", 0).build(); final NettyTransport nettyA = new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, new NamedWriteableRegistry()); MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool); @@ -84,7 +79,7 @@ public class NettyScheduledPingTests extends ESTestCase { @Override public void messageReceived(TransportRequest.Empty request, TransportChannel channel) { try { - channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.options()); + channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY); } catch (IOException e) { e.printStackTrace(); assertThat(e.getMessage(), false, equalTo(true)); @@ -96,7 +91,7 @@ public class NettyScheduledPingTests extends ESTestCase { int rounds = scaledRandomIntBetween(100, 5000); for (int i = 0; i < rounds; i++) { serviceB.submitRequest(nodeA, "sayHello", - TransportRequest.Empty.INSTANCE, TransportRequestOptions.options().withCompress(randomBoolean()), new BaseTransportResponseHandler() { + TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(randomBoolean()).build(), new BaseTransportResponseHandler() { @Override public TransportResponse.Empty newInstance() { return TransportResponse.Empty.INSTANCE; diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java index 28701248817..c84a9eb9a55 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java @@ -21,7 +21,7 @@ package org.elasticsearch.transport.netty; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.inject.Inject; @@ -35,6 +35,8 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.transport.RequestHandlerRegistry; @@ -43,15 +45,12 @@ import org.elasticsearch.transport.TransportRequest; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelPipeline; import org.jboss.netty.channel.ChannelPipelineFactory; -import org.junit.Test; import java.io.IOException; import java.net.InetSocketAddress; import java.util.Collection; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; @@ -60,7 +59,6 @@ import static org.hamcrest.Matchers.is; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 1) public class NettyTransportIT extends ESIntegTestCase { - // static so we can use it in anonymous classes private static String channelProfileName = null; @@ -76,7 +74,6 @@ public class NettyTransportIT extends ESIntegTestCase { return pluginList(ExceptionThrowingNettyTransport.TestPlugin.class); } - @Test public void testThatConnectionFailsAsIntended() throws Exception { Client transportClient = internalCluster().transportClient(); ClusterHealthResponse clusterIndexHealths = transportClient.admin().cluster().prepareHealth().get(); diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java index 9a6486134da..59ef26c42af 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java @@ -19,7 +19,7 @@ package org.elasticsearch.transport.netty; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.transport.TransportClient; @@ -28,17 +28,22 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.junit.annotations.Network; import org.elasticsearch.transport.TransportModule; -import org.junit.Test; import java.net.InetAddress; import java.util.Locale; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import static org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; @ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0) public class NettyTransportMultiPortIntegrationIT extends ESIntegTestCase { @@ -64,7 +69,6 @@ public class NettyTransportMultiPortIntegrationIT extends ESIntegTestCase { return builder.build(); } - @Test public void testThatTransportClientCanConnect() throws Exception { Settings settings = settingsBuilder() .put("cluster.name", internalCluster().getClusterName()) @@ -78,7 +82,6 @@ public class NettyTransportMultiPortIntegrationIT extends ESIntegTestCase { } } - @Test @Network public void testThatInfosAreExposed() throws Exception { NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setTransport(true).get(); diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java index 9d9c093ecc7..1c8869772e2 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.transport.netty; -import com.carrotsearch.hppc.IntHashSet; -import java.nio.charset.StandardCharsets; import org.elasticsearch.Version; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.common.component.Lifecycle; @@ -27,181 +25,115 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.network.NetworkUtils; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.common.util.MockBigArrays; -import org.elasticsearch.test.junit.rule.RepeatOnExceptionRule; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.BindTransportException; import org.elasticsearch.transport.TransportService; import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.is; public class NettyTransportMultiPortTests extends ESTestCase { - private static final int MAX_RETRIES = 10; private String host; - @Rule - public RepeatOnExceptionRule repeatOnBindExceptionRule = new RepeatOnExceptionRule(logger, MAX_RETRIES, BindTransportException.class); - @Before public void setup() { - if (randomBoolean()) { - host = "localhost"; + if (NetworkUtils.SUPPORTS_V6 && randomBoolean()) { + host = "::1"; } else { - if (NetworkUtils.SUPPORTS_V6 && randomBoolean()) { - host = "::1"; - } else { - host = "127.0.0.1"; - } + host = "127.0.0.1"; } } - @Test public void testThatNettyCanBindToMultiplePorts() throws Exception { - int[] ports = getRandomPorts(3); - Settings settings = settingsBuilder() .put("network.host", host) - .put("transport.tcp.port", ports[0]) - .put("transport.profiles.default.port", ports[1]) - .put("transport.profiles.client1.port", ports[2]) + .put("transport.tcp.port", 22) // will not actually bind to this + .put("transport.profiles.default.port", 0) + .put("transport.profiles.client1.port", 0) .build(); ThreadPool threadPool = new ThreadPool("tst"); - try (NettyTransport ignored = startNettyTransport(settings, threadPool)) { - assertConnectionRefused(ports[0]); - assertPortIsBound(ports[1]); - assertPortIsBound(ports[2]); + try (NettyTransport transport = startNettyTransport(settings, threadPool)) { + assertEquals(1, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); } finally { terminate(threadPool); } } - @Test public void testThatDefaultProfileInheritsFromStandardSettings() throws Exception { - int[] ports = getRandomPorts(2); - Settings settings = settingsBuilder() .put("network.host", host) - .put("transport.tcp.port", ports[0]) - .put("transport.profiles.client1.port", ports[1]) + .put("transport.tcp.port", 0) + .put("transport.profiles.client1.port", 0) .build(); ThreadPool threadPool = new ThreadPool("tst"); - try (NettyTransport ignored = startNettyTransport(settings, threadPool)) { - assertPortIsBound(ports[0]); - assertPortIsBound(ports[1]); + try (NettyTransport transport = startNettyTransport(settings, threadPool)) { + assertEquals(1, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); } finally { terminate(threadPool); } } - @Test public void testThatProfileWithoutPortSettingsFails() throws Exception { - int[] ports = getRandomPorts(1); Settings settings = settingsBuilder() .put("network.host", host) - .put("transport.tcp.port", ports[0]) + .put("transport.tcp.port", 0) .put("transport.profiles.client1.whatever", "foo") .build(); ThreadPool threadPool = new ThreadPool("tst"); - try (NettyTransport ignored = startNettyTransport(settings, threadPool)) { - assertPortIsBound(ports[0]); + try (NettyTransport transport = startNettyTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); } finally { terminate(threadPool); } } - @Test public void testThatDefaultProfilePortOverridesGeneralConfiguration() throws Exception { - int[] ports = getRandomPorts(3); - Settings settings = settingsBuilder() .put("network.host", host) - .put("transport.tcp.port", ports[0]) - .put("transport.netty.port", ports[1]) - .put("transport.profiles.default.port", ports[2]) + .put("transport.tcp.port", 22) // will not actually bind to this + .put("transport.netty.port", 23) // will not actually bind to this + .put("transport.profiles.default.port", 0) .build(); ThreadPool threadPool = new ThreadPool("tst"); - try (NettyTransport ignored = startNettyTransport(settings, threadPool)) { - assertConnectionRefused(ports[0]); - assertConnectionRefused(ports[1]); - assertPortIsBound(ports[2]); + try (NettyTransport transport = startNettyTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); } finally { terminate(threadPool); } } - @Test public void testThatProfileWithoutValidNameIsIgnored() throws Exception { - int[] ports = getRandomPorts(3); - Settings settings = settingsBuilder() .put("network.host", host) - .put("transport.tcp.port", ports[0]) + .put("transport.tcp.port", 0) // mimics someone trying to define a profile for .local which is the profile for a node request to itself - .put("transport.profiles." + TransportService.DIRECT_RESPONSE_PROFILE + ".port", ports[1]) - .put("transport.profiles..port", ports[2]) + .put("transport.profiles." + TransportService.DIRECT_RESPONSE_PROFILE + ".port", 22) // will not actually bind to this + .put("transport.profiles..port", 23) // will not actually bind to this .build(); ThreadPool threadPool = new ThreadPool("tst"); - try (NettyTransport ignored = startNettyTransport(settings, threadPool)) { - assertPortIsBound(ports[0]); - assertConnectionRefused(ports[1]); - assertConnectionRefused(ports[2]); + try (NettyTransport transport = startNettyTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); } finally { terminate(threadPool); } } - private int[] getRandomPorts(int numberOfPorts) { - IntHashSet ports = new IntHashSet(); - - int nextPort = randomIntBetween(49152, 65535); - for (int i = 0; i < numberOfPorts; i++) { - boolean foundPortInRange = false; - while (!foundPortInRange) { - if (!ports.contains(nextPort)) { - logger.debug("looking to see if port [{}]is available", nextPort); - try (ServerSocket serverSocket = new ServerSocket()) { - // Set SO_REUSEADDR as we may bind here and not be able - // to reuse the address immediately without it. - serverSocket.setReuseAddress(NetworkUtils.defaultReuseAddress()); - serverSocket.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), nextPort)); - - // bind was a success - logger.debug("port [{}] available.", nextPort); - foundPortInRange = true; - ports.add(nextPort); - } catch (IOException e) { - // Do nothing - logger.debug("port [{}] not available.", e, nextPort); - } - } - nextPort = randomIntBetween(49152, 65535); - } - } - return ports.toArray(); - } - private NettyTransport startNettyTransport(Settings settings, ThreadPool threadPool) { BigArrays bigArrays = new MockBigArrays(new PageCacheRecycler(settings, threadPool), new NoneCircuitBreakerService()); @@ -211,36 +143,4 @@ public class NettyTransportMultiPortTests extends ESTestCase { assertThat(nettyTransport.lifecycleState(), is(Lifecycle.State.STARTED)); return nettyTransport; } - - private void assertConnectionRefused(int port) throws Exception { - try { - trySocketConnection(new InetSocketTransportAddress(InetAddress.getByName(host), port).address()); - fail("Expected to get exception when connecting to port " + port); - } catch (IOException e) { - // expected - logger.info("Got expected connection message {}", e.getMessage()); - } - } - - private void assertPortIsBound(int port) throws Exception { - assertPortIsBound(host, port); - } - - private void assertPortIsBound(String host, int port) throws Exception { - logger.info("Trying to connect to [{}]:[{}]", host, port); - trySocketConnection(new InetSocketTransportAddress(InetAddress.getByName(host), port).address()); - } - - private void trySocketConnection(InetSocketAddress address) throws Exception { - try (Socket socket = new Socket()) { - logger.info("Connecting to {}", address); - socket.connect(address, 500); - - assertThat(socket.isConnected(), is(true)); - try (OutputStream os = socket.getOutputStream()) { - os.write("foo".getBytes(StandardCharsets.UTF_8)); - os.flush(); - } - } - } } diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportPublishAddressIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportPublishAddressIT.java new file mode 100644 index 00000000000..3437701f6c9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportPublishAddressIT.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty; + +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.TransportModule; + +import java.net.Inet4Address; +import java.net.Inet6Address; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; + +/** + * Checks that Elasticsearch produces a sane publish_address when it binds to + * different ports on ipv4 and ipv6. + */ +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class NettyTransportPublishAddressIT extends ESIntegTestCase { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(TransportModule.TRANSPORT_TYPE_KEY, "netty") + .put("node.mode", "network").build(); + } + + public void testDifferentPorts() throws Exception { + if (!NetworkUtils.SUPPORTS_V6) { + return; + } + logger.info("--> starting a node on ipv4 only"); + Settings ipv4Settings = Settings.builder().put("network.host", "127.0.0.1").build(); + String ipv4OnlyNode = internalCluster().startNode(ipv4Settings); // should bind 127.0.0.1:XYZ + + logger.info("--> starting a node on ipv4 and ipv6"); + Settings bothSettings = Settings.builder().put("network.host", "_local_").build(); + internalCluster().startNode(bothSettings); // should bind [::1]:XYZ and 127.0.0.1:XYZ+1 + + logger.info("--> waiting for the cluster to declare itself stable"); + ensureStableCluster(2); // fails if port of publish address does not match corresponding bound address + + logger.info("--> checking if boundAddress matching publishAddress has same port"); + NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get(); + for (NodeInfo nodeInfo : nodesInfoResponse) { + BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getAddress(); + if (nodeInfo.getNode().getName().equals(ipv4OnlyNode)) { + assertThat(boundTransportAddress.boundAddresses().length, equalTo(1)); + assertThat(boundTransportAddress.boundAddresses()[0].getPort(), equalTo(boundTransportAddress.publishAddress().getPort())); + } else { + assertThat(boundTransportAddress.boundAddresses().length, greaterThan(1)); + for (TransportAddress boundAddress : boundTransportAddress.boundAddresses()) { + assertThat(boundAddress, instanceOf(InetSocketTransportAddress.class)); + InetSocketTransportAddress inetBoundAddress = (InetSocketTransportAddress) boundAddress; + if (inetBoundAddress.address().getAddress() instanceof Inet4Address) { + // IPv4 address is preferred publish address for _local_ + assertThat(inetBoundAddress.getPort(), equalTo(boundTransportAddress.publishAddress().getPort())); + } + } + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java index 923ed63aead..89702118b49 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java @@ -29,25 +29,28 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.AbstractSimpleTransportTestCase; import org.elasticsearch.transport.ConnectTransportException; -import org.junit.Test; import java.net.InetAddress; import java.net.UnknownHostException; -public class SimpleNettyTransportTests extends AbstractSimpleTransportTestCase { +import static org.hamcrest.Matchers.containsString; +public class SimpleNettyTransportTests extends AbstractSimpleTransportTestCase { @Override protected MockTransportService build(Settings settings, Version version, NamedWriteableRegistry namedWriteableRegistry) { - int startPort = 11000 + randomIntBetween(0, 255); - int endPort = startPort + 10; - settings = Settings.builder().put(settings).put("transport.tcp.port", startPort + "-" + endPort).build(); + settings = Settings.builder().put(settings).put("transport.tcp.port", "0").build(); MockTransportService transportService = new MockTransportService(settings, new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE, version, namedWriteableRegistry), threadPool); transportService.start(); return transportService; } - @Test(expected = ConnectTransportException.class) public void testConnectException() throws UnknownHostException { - serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9876), Version.CURRENT)); + try { + serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9876), Version.CURRENT)); + fail("Expected ConnectTransportException"); + } catch (ConnectTransportException e) { + assertThat(e.getMessage(), containsString("connect_timeout")); + assertThat(e.getMessage(), containsString("[localhost/127.0.0.1:9876]")); + } } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java index 25befab1ff6..28a3dea118e 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java +++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.tribe; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Priority; @@ -37,7 +37,6 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.NodeConfigurationSource; @@ -45,14 +44,15 @@ import org.elasticsearch.test.TestCluster; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Map; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -73,8 +73,20 @@ public class TribeIT extends ESIntegTestCase { @BeforeClass public static void setupSecondCluster() throws Exception { ESIntegTestCase.beforeClass(); + NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() { + @Override + public Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(Node.HTTP_ENABLED, false).build(); + } + + @Override + public Settings transportClientSettings() { + return null; + } + + }; cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2, - Strings.randomBase64UUID(getRandom()), NodeConfigurationSource.EMPTY, 0, false, SECOND_CLUSTER_NODE_PREFIX, true); + Strings.randomBase64UUID(getRandom()), nodeConfigurationSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, true); cluster2.beforeTest(getRandom(), 0.1); cluster2.ensureAtLeastNumDataNodes(2); @@ -130,13 +142,10 @@ public class TribeIT extends ESIntegTestCase { .put("node.name", "tribe_node") // make sure we can identify threads from this node .build(); - tribeNode = NodeBuilder.nodeBuilder() - .settings(merged) - .node(); + tribeNode = new Node(merged).start(); tribeClient = tribeNode.client(); } - @Test public void testGlobalReadWriteBlocks() throws Exception { logger.info("create 2 indices, test1 on t1, and test2 on t2"); internalCluster().client().admin().indices().prepareCreate("test1").get(); @@ -161,20 +170,19 @@ public class TribeIT extends ESIntegTestCase { // all is well! } try { - tribeClient.admin().indices().prepareOptimize("test1").execute().actionGet(); + tribeClient.admin().indices().prepareForceMerge("test1").execute().actionGet(); fail("cluster block should be thrown"); } catch (ClusterBlockException e) { // all is well! } try { - tribeClient.admin().indices().prepareOptimize("test2").execute().actionGet(); + tribeClient.admin().indices().prepareForceMerge("test2").execute().actionGet(); fail("cluster block should be thrown"); } catch (ClusterBlockException e) { // all is well! } } - @Test public void testIndexWriteBlocks() throws Exception { logger.info("create 2 indices, test1 on t1, and test2 on t2"); assertAcked(internalCluster().client().admin().indices().prepareCreate("test1")); @@ -208,7 +216,6 @@ public class TribeIT extends ESIntegTestCase { } } - @Test public void testOnConflictDrop() throws Exception { logger.info("create 2 indices, test1 on t1, and test2 on t2"); assertAcked(cluster().client().admin().indices().prepareCreate("conflict")); @@ -232,7 +239,6 @@ public class TribeIT extends ESIntegTestCase { assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().hasIndex("conflict"), equalTo(false)); } - @Test public void testOnConflictPrefer() throws Exception { testOnConflictPrefer(randomBoolean() ? "t1" : "t2"); } @@ -260,7 +266,6 @@ public class TribeIT extends ESIntegTestCase { assertThat(tribeClient.admin().cluster().prepareState().get().getState().getMetaData().index("conflict").getSettings().get(TribeService.TRIBE_NAME), equalTo(tribe)); } - @Test public void testTribeOnOneCluster() throws Exception { setupTribeNode(Settings.EMPTY); logger.info("create 2 indices, test1 on t1, and test2 on t2"); @@ -283,7 +288,7 @@ public class TribeIT extends ESIntegTestCase { tribeClient.admin().indices().prepareRefresh().get(); logger.info("verify they are there"); - assertHitCount(tribeClient.prepareCount().get(), 2l); + assertHitCount(tribeClient.prepareSearch().setSize(0).get(), 2l); assertHitCount(tribeClient.prepareSearch().get(), 2l); assertBusy(new Runnable() { @Override @@ -302,7 +307,7 @@ public class TribeIT extends ESIntegTestCase { logger.info("verify they are there"); - assertHitCount(tribeClient.prepareCount().get(), 4l); + assertHitCount(tribeClient.prepareSearch().setSize(0).get(), 4l); assertHitCount(tribeClient.prepareSearch().get(), 4l); assertBusy(new Runnable() { @Override @@ -337,7 +342,6 @@ public class TribeIT extends ESIntegTestCase { } } - @Test public void testCloseAndOpenIndex() throws Exception { //create an index and close it even before starting the tribe node assertAcked(internalCluster().client().admin().indices().prepareCreate("test1")); diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java index 468bca25e9c..f871995cc20 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java @@ -170,7 +170,7 @@ public class SimpleTTLIT extends ESIntegTestCase { if (rarely()) { client().admin().indices().prepareFlush("test").get(); } else if (rarely()) { - client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get(); + client().admin().indices().prepareForceMerge("test").setMaxNumSegments(1).get(); } IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats("test").clear().setIndexing(true).get(); // TTL deletes two docs, but it is indexed in the primary shard and replica shard. diff --git a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java index 1faee8811b1..dc10c9751cf 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.util.Collection; import java.util.HashMap; @@ -50,7 +49,6 @@ public class UpdateByNativeScriptIT extends ESIntegTestCase { return pluginList(CustomNativeScriptFactory.TestPlugin.class); } - @Test public void testThatUpdateUsingNativeScriptWorks() throws Exception { createIndex("test"); ensureYellow(); diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java index 8c62d97349a..a789bb48774 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -47,10 +47,15 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Semaphore; @@ -59,7 +64,14 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class UpdateIT extends ESIntegTestCase { @@ -457,7 +469,6 @@ public class UpdateIT extends ESIntegTestCase { .endObject())); } - @Test public void testUpsert() throws Exception { createTestIndex(); ensureGreen(); @@ -487,7 +498,6 @@ public class UpdateIT extends ESIntegTestCase { } } - @Test public void testScriptedUpsert() throws Exception { createTestIndex(); ensureGreen(); @@ -531,7 +541,6 @@ public class UpdateIT extends ESIntegTestCase { } } - @Test public void testUpsertDoc() throws Exception { createTestIndex(); ensureGreen(); @@ -547,8 +556,7 @@ public class UpdateIT extends ESIntegTestCase { assertThat(updateResponse.getGetResult().sourceAsMap().get("bar").toString(), equalTo("baz")); } - @Test - // See: https://github.com/elasticsearch/elasticsearch/issues/3265 + // Issue #3265 public void testNotUpsertDoc() throws Exception { createTestIndex(); ensureGreen(); @@ -560,7 +568,6 @@ public class UpdateIT extends ESIntegTestCase { .execute(), DocumentMissingException.class); } - @Test public void testUpsertFields() throws Exception { createTestIndex(); ensureGreen(); @@ -590,7 +597,6 @@ public class UpdateIT extends ESIntegTestCase { assertThat(updateResponse.getGetResult().sourceAsMap().get("extra").toString(), equalTo("foo")); } - @Test public void testVersionedUpdate() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); @@ -651,7 +657,6 @@ public class UpdateIT extends ESIntegTestCase { assertThrows(client().prepareUpdate(indexOrAlias(), "type", "1").setVersion(10).setRetryOnConflict(5), ActionRequestValidationException.class); } - @Test public void testIndexAutoCreation() throws Exception { UpdateResponse updateResponse = client().prepareUpdate("test", "type1", "1") .setUpsert(XContentFactory.jsonBuilder().startObject().field("bar", "baz").endObject()) @@ -666,7 +671,6 @@ public class UpdateIT extends ESIntegTestCase { assertThat(updateResponse.getGetResult().sourceAsMap().get("extra"), nullValue()); } - @Test public void testUpdate() throws Exception { createTestIndex(); ensureGreen(); @@ -813,7 +817,6 @@ public class UpdateIT extends ESIntegTestCase { } } - @Test public void testUpdateRequestWithBothScriptAndDoc() throws Exception { createTestIndex(); ensureGreen(); @@ -831,7 +834,6 @@ public class UpdateIT extends ESIntegTestCase { } } - @Test public void testUpdateRequestWithScriptAndShouldUpsertDoc() throws Exception { createTestIndex(); ensureGreen(); @@ -848,7 +850,6 @@ public class UpdateIT extends ESIntegTestCase { } } - @Test public void testContextVariables() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .addMapping("type1", XContentFactory.jsonBuilder() @@ -927,7 +928,6 @@ public class UpdateIT extends ESIntegTestCase { assertNull(updateContext.get("_ttl")); } - @Test public void testConcurrentUpdateWithRetryOnConflict() throws Exception { final boolean useBulkApi = randomBoolean(); createTestIndex(); @@ -936,16 +936,19 @@ public class UpdateIT extends ESIntegTestCase { int numberOfThreads = scaledRandomIntBetween(2,5); final CountDownLatch latch = new CountDownLatch(numberOfThreads); final CountDownLatch startLatch = new CountDownLatch(1); - final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 10000); + final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 500); final List failures = new CopyOnWriteArrayList<>(); + for (int i = 0; i < numberOfThreads; i++) { Runnable r = new Runnable() { - @Override public void run() { try { startLatch.await(); for (int i = 0; i < numberOfUpdatesPerThread; i++) { + if (i % 100 == 0) { + logger.debug("Client [{}] issued [{}] of [{}] requests", Thread.currentThread().getName(), i, numberOfUpdatesPerThread); + } if (useBulkApi) { UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "type1", Integer.toString(i)) .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)) @@ -960,6 +963,12 @@ public class UpdateIT extends ESIntegTestCase { .execute().actionGet(); } } + logger.info("Client [{}] issued all [{}] requests.", Thread.currentThread().getName(), numberOfUpdatesPerThread); + } catch (InterruptedException e) { + // test infrastructure kills long-running tests by interrupting them, thus we handle this case separately + logger.warn("Test was forcefully stopped. Client [{}] may still have outstanding requests.", Thread.currentThread().getName()); + failures.add(e); + Thread.currentThread().interrupt(); } catch (Throwable e) { failures.add(e); } finally { @@ -968,7 +977,9 @@ public class UpdateIT extends ESIntegTestCase { } }; - new Thread(r).start(); + Thread updater = new Thread(r); + updater.setName("UpdateIT-Client-" + i); + updater.start(); } startLatch.countDown(); latch.await(); @@ -985,8 +996,7 @@ public class UpdateIT extends ESIntegTestCase { } } - @Test - public void stressUpdateDeleteConcurrency() throws Exception { + public void testStressUpdateDeleteConcurrency() throws Exception { //We create an index with merging disabled so that deletes don't get merged away assertAcked(prepareCreate("test") .addMapping("type1", XContentFactory.jsonBuilder() diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java index 499fb52a98b..b4b5eefc832 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; -import org.junit.Test; import java.io.IOException; @@ -35,8 +34,7 @@ import static org.hamcrest.Matchers.notNullValue; * Tests for noop updates. */ public class UpdateNoopIT extends ESIntegTestCase { - @Test - public void singleField() throws Exception { + public void testSingleField() throws Exception { updateAndCheckSource(1, fields("bar", "baz")); updateAndCheckSource(1, fields("bar", "baz")); updateAndCheckSource(2, fields("bar", "bir")); @@ -51,8 +49,7 @@ public class UpdateNoopIT extends ESIntegTestCase { assertEquals(4, totalNoopUpdates()); } - @Test - public void twoFields() throws Exception { + public void testTwoFields() throws Exception { // Use random keys so we get random iteration order. String key1 = 1 + randomAsciiOfLength(3); String key2 = 2 + randomAsciiOfLength(3); @@ -74,8 +71,7 @@ public class UpdateNoopIT extends ESIntegTestCase { assertEquals(5, totalNoopUpdates()); } - @Test - public void arrayField() throws Exception { + public void testArrayField() throws Exception { updateAndCheckSource(1, fields("bar", "baz")); updateAndCheckSource(2, fields("bar", new String[] {"baz", "bort"})); updateAndCheckSource(2, fields("bar", new String[] {"baz", "bort"})); @@ -92,8 +88,7 @@ public class UpdateNoopIT extends ESIntegTestCase { assertEquals(5, totalNoopUpdates()); } - @Test - public void map() throws Exception { + public void testMap() throws Exception { // Use random keys so we get variable iteration order. String key1 = 1 + randomAsciiOfLength(3); String key2 = 2 + randomAsciiOfLength(3); @@ -143,8 +138,7 @@ public class UpdateNoopIT extends ESIntegTestCase { assertEquals(3, totalNoopUpdates()); } - @Test - public void mapAndField() throws Exception { + public void testMapAndField() throws Exception { updateAndCheckSource(1, XContentFactory.jsonBuilder().startObject() .field("f", "foo") .startObject("m") @@ -216,8 +210,7 @@ public class UpdateNoopIT extends ESIntegTestCase { * Totally empty requests are noop if and only if detect noops is true and * its true by default. */ - @Test - public void totallyEmpty() throws Exception { + public void testTotallyEmpty() throws Exception { updateAndCheckSource(1, XContentFactory.jsonBuilder().startObject() .field("f", "foo") .startObject("m") diff --git a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index 22c959f5cd0..d3df8d133e0 100644 --- a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.validate; -import java.nio.charset.StandardCharsets; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.client.Client; @@ -37,9 +36,9 @@ import org.hamcrest.Matcher; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.ISODateTimeFormat; -import org.junit.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; @@ -47,6 +46,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; /** @@ -54,9 +54,7 @@ import static org.hamcrest.Matchers.nullValue; */ @ClusterScope(randomDynamicTemplates = false, scope = Scope.SUITE) public class SimpleValidateQueryIT extends ESIntegTestCase { - - @Test - public void simpleValidateQuery() throws Exception { + public void testSimpleValidateQuery() throws Exception { createIndex("test"); ensureGreen(); client().admin().indices().preparePutMapping("test").setType("type1") @@ -68,7 +66,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { refresh(); - assertThat(client().admin().indices().prepareValidateQuery("test").setSource("foo".getBytes(StandardCharsets.UTF_8)).execute().actionGet().isValid(), equalTo(false)); + assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.wrapperQuery("foo".getBytes(StandardCharsets.UTF_8))).execute().actionGet().isValid(), equalTo(false)); assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("_id:1")).execute().actionGet().isValid(), equalTo(true)); assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("_i:d:1")).execute().actionGet().isValid(), equalTo(false)); @@ -80,8 +78,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("foo:1 AND")).execute().actionGet().isValid(), equalTo(false)); } - @Test - public void explainValidateQueryTwoNodes() throws IOException { + public void testExplainValidateQueryTwoNodes() throws IOException { createIndex("test"); ensureGreen(); client().admin().indices().preparePutMapping("test").setType("type1") @@ -97,12 +94,12 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { for (Client client : internalCluster()) { ValidateQueryResponse response = client.admin().indices().prepareValidateQuery("test") - .setSource("foo".getBytes(StandardCharsets.UTF_8)) + .setQuery(QueryBuilders.wrapperQuery("foo".getBytes(StandardCharsets.UTF_8))) .setExplain(true) .execute().actionGet(); assertThat(response.isValid(), equalTo(false)); assertThat(response.getQueryExplanation().size(), equalTo(1)); - assertThat(response.getQueryExplanation().get(0).getError(), containsString("Failed to parse")); + assertThat(response.getQueryExplanation().get(0).getError(), containsString("Failed to derive xcontent")); assertThat(response.getQueryExplanation().get(0).getExplanation(), nullValue()); } @@ -119,8 +116,8 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { } } - @Test //https://github.com/elasticsearch/elasticsearch/issues/3629 - public void explainDateRangeInQueryString() { + // Issue #3629 + public void testExplainDateRangeInQueryString() { assertAcked(prepareCreate("test").setSettings(Settings.settingsBuilder() .put(indexSettings()) .put("index.number_of_shards", 1))); @@ -145,13 +142,16 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { assertThat(response.isValid(), equalTo(true)); } - @Test(expected = IndexNotFoundException.class) - public void validateEmptyCluster() { - client().admin().indices().prepareValidateQuery().get(); + public void testValidateEmptyCluster() { + try { + client().admin().indices().prepareValidateQuery().get(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); + } } - @Test - public void explainNoQuery() { + public void testExplainNoQuery() { createIndex("test"); ensureGreen(); @@ -162,8 +162,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { assertThat(validateQueryResponse.getQueryExplanation().get(0).getExplanation(), equalTo("*:*")); } - @Test - public void explainFilteredAlias() { + public void testExplainFilteredAlias() { assertAcked(prepareCreate("test") .addMapping("test", "field", "type=string") .addAlias(new Alias("alias").filter(QueryBuilders.termQuery("field", "value1")))); @@ -177,8 +176,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { assertThat(validateQueryResponse.getQueryExplanation().get(0).getExplanation(), containsString("field:value1")); } - @Test - public void explainMatchPhrasePrefix() { + public void testExplainMatchPhrasePrefix() { assertAcked(prepareCreate("test").setSettings( Settings.settingsBuilder().put(indexSettings()) .put("index.analysis.filter.syns.type", "synonym") @@ -214,8 +212,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { assertThat(validateQueryResponse.getQueryExplanation().get(0).getExplanation(), containsString("field:\"foo (one* two*)\"")); } - @Test - public void explainWithRewriteValidateQuery() throws Exception { + public void testExplainWithRewriteValidateQuery() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("type1", "field", "type=string,analyzer=whitespace") .setSettings(SETTING_NUMBER_OF_SHARDS, 1).get(); @@ -258,22 +255,20 @@ public class SimpleValidateQueryIT extends ESIntegTestCase { containsString("field:huge field:pidgin"), true); } - @Test - public void irrelevantPropertiesBeforeQuery() throws IOException { + public void testIrrelevantPropertiesBeforeQuery() throws IOException { createIndex("test"); ensureGreen(); refresh(); - assertThat(client().admin().indices().prepareValidateQuery("test").setSource(new BytesArray("{\"foo\": \"bar\", \"query\": {\"term\" : { \"user\" : \"kimchy\" }}}")).get().isValid(), equalTo(false)); + assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.wrapperQuery(new BytesArray("{\"foo\": \"bar\", \"query\": {\"term\" : { \"user\" : \"kimchy\" }}}"))).get().isValid(), equalTo(false)); } - @Test - public void irrelevantPropertiesAfterQuery() throws IOException { + public void testIrrelevantPropertiesAfterQuery() throws IOException { createIndex("test"); ensureGreen(); refresh(); - assertThat(client().admin().indices().prepareValidateQuery("test").setSource(new BytesArray("{\"query\": {\"term\" : { \"user\" : \"kimchy\" }}, \"foo\": \"bar\"}")).get().isValid(), equalTo(false)); + assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.wrapperQuery(new BytesArray("{\"query\": {\"term\" : { \"user\" : \"kimchy\" }}, \"foo\": \"bar\"}"))).get().isValid(), equalTo(false)); } private static void assertExplanation(QueryBuilder queryBuilder, Matcher matcher, boolean withRewrite) { diff --git a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java index 402ec0d8dc6..bb8636d36e9 100644 --- a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java +++ b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.versioning; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -37,10 +36,7 @@ import static org.hamcrest.Matchers.nullValue; * */ public class ConcurrentDocumentOperationIT extends ESIntegTestCase { - - @Test - public void concurrentOperationOnSameDocTest() throws Exception { - + public void testConcurrentOperationOnSameDoc() throws Exception { logger.info("--> create an index with 1 shard and max replicas based on nodes"); assertAcked(prepareCreate("test") .setSettings(settingsBuilder().put(indexSettings()).put("index.number_of_shards", 1))); diff --git a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java index 93c29e0c922..edbbebbbc45 100644 --- a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java +++ b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java @@ -31,9 +31,13 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.FlushNotAllowedEngineException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Random; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -47,8 +51,6 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; * */ public class SimpleVersioningIT extends ESIntegTestCase { - - @Test public void testExternalVersioningInitialDelete() throws Exception { createIndex("test"); ensureGreen(); @@ -69,7 +71,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(indexResponse.getVersion(), equalTo(18L)); } - @Test public void testForce() throws Exception { createIndex("test"); ensureGreen("test"); // we are testing force here which doesn't work if we are recovering at the same time - zzzzz... @@ -100,7 +101,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(deleteResponse.getVersion(), equalTo(v)); } - @Test public void testExternalGTE() throws Exception { createIndex("test"); @@ -147,7 +147,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(deleteResponse.getVersion(), equalTo(18l)); } - @Test public void testExternalVersioning() throws Exception { createIndex("test"); ensureGreen(); @@ -211,7 +210,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(indexResponse.getVersion(), equalTo(20l)); } - @Test public void testRequireUnitsOnUpdateSettings() throws Exception { createIndex("test"); ensureGreen(); @@ -226,7 +224,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { } } - @Test public void testInternalVersioningInitialDelete() throws Exception { createIndex("test"); ensureGreen(); @@ -239,8 +236,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(indexResponse.getVersion(), equalTo(1l)); } - - @Test public void testInternalVersioning() throws Exception { createIndex("test"); ensureGreen(); @@ -298,7 +293,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(deleteResponse.getVersion(), equalTo(4l)); } - @Test public void testSimpleVersioningWithFlush() throws Exception { createIndex("test"); ensureGreen(); @@ -336,7 +330,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { } } - @Test public void testVersioningWithBulk() { createIndex("test"); ensureGreen(); @@ -521,8 +514,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { } } - - @Test public void testRandomIDsAndVersions() throws Exception { createIndex("test"); ensureGreen(); @@ -719,7 +710,6 @@ public class SimpleVersioningIT extends ESIntegTestCase { } } - @Test public void testDeleteNotLost() throws Exception { // We require only one shard for this test, so that the 2nd delete provokes pruning the deletes map: @@ -799,9 +789,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { equalTo(-1L)); } - @Test public void testGCDeletesZero() throws Exception { - createIndex("test"); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/watcher/FileWatcherTests.java b/core/src/test/java/org/elasticsearch/watcher/FileWatcherTests.java index 14f7eca6832..5b5a16c6043 100644 --- a/core/src/test/java/org/elasticsearch/watcher/FileWatcherTests.java +++ b/core/src/test/java/org/elasticsearch/watcher/FileWatcherTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.watcher; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.BufferedWriter; import java.io.IOException; @@ -32,13 +31,13 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; @LuceneTestCase.SuppressFileSystems("ExtrasFS") public class FileWatcherTests extends ESTestCase { - private class RecordingChangeListener extends FileChangesListener { - private Path rootDir; private RecordingChangeListener(Path rootDir) { @@ -91,7 +90,6 @@ public class FileWatcherTests extends ESTestCase { } } - @Test public void testSimpleFileOperations() throws IOException { Path tempDir = createTempDir(); RecordingChangeListener changes = new RecordingChangeListener(tempDir); @@ -120,7 +118,6 @@ public class FileWatcherTests extends ESTestCase { } - @Test public void testSimpleDirectoryOperations() throws IOException { Path tempDir = createTempDir(); RecordingChangeListener changes = new RecordingChangeListener(tempDir); @@ -210,7 +207,6 @@ public class FileWatcherTests extends ESTestCase { } - @Test public void testNestedDirectoryOperations() throws IOException { Path tempDir = createTempDir(); RecordingChangeListener changes = new RecordingChangeListener(tempDir); @@ -276,7 +272,6 @@ public class FileWatcherTests extends ESTestCase { )); } - @Test public void testFileReplacingDirectory() throws IOException { Path tempDir = createTempDir(); RecordingChangeListener changes = new RecordingChangeListener(tempDir); @@ -323,7 +318,6 @@ public class FileWatcherTests extends ESTestCase { )); } - @Test public void testEmptyDirectory() throws IOException { Path tempDir = createTempDir(); RecordingChangeListener changes = new RecordingChangeListener(tempDir); @@ -346,7 +340,6 @@ public class FileWatcherTests extends ESTestCase { )); } - @Test public void testNoDirectoryOnInit() throws IOException { Path tempDir = createTempDir(); RecordingChangeListener changes = new RecordingChangeListener(tempDir); @@ -370,7 +363,6 @@ public class FileWatcherTests extends ESTestCase { )); } - @Test public void testNoFileOnInit() throws IOException { Path tempDir = createTempDir(); RecordingChangeListener changes = new RecordingChangeListener(tempDir); @@ -389,11 +381,11 @@ public class FileWatcherTests extends ESTestCase { equalTo("onFileCreated: testfile.txt") )); } - + static void touch(Path path) throws IOException { Files.newOutputStream(path).close(); } - + static void append(String string, Path path, Charset cs) throws IOException { try (BufferedWriter writer = Files.newBufferedWriter(path, cs, StandardOpenOption.APPEND)) { writer.append(string); diff --git a/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java b/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java index b503436addd..6c6c45e9cfd 100644 --- a/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java +++ b/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java @@ -22,17 +22,16 @@ package org.elasticsearch.watcher; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.junit.Test; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; /** * */ public class ResourceWatcherServiceTests extends ESTestCase { - - @Test public void testSettings() throws Exception { ThreadPool threadPool = new ThreadPool("test"); @@ -65,8 +64,6 @@ public class ResourceWatcherServiceTests extends ESTestCase { terminate(threadPool); } - - @Test public void testHandle() throws Exception { ThreadPool threadPool = new ThreadPool("test"); Settings settings = Settings.builder().build(); diff --git a/core/src/test/resources/indices/bwc/index-2.0.0.zip b/core/src/test/resources/indices/bwc/index-2.0.0.zip new file mode 100644 index 00000000000..7110fb424a8 Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.0.0.zip differ diff --git a/core/src/test/resources/indices/bwc/index-2.0.1.zip b/core/src/test/resources/indices/bwc/index-2.0.1.zip new file mode 100644 index 00000000000..dccb7774fa6 Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.0.1.zip differ diff --git a/core/src/test/resources/indices/bwc/index-2.1.0.zip b/core/src/test/resources/indices/bwc/index-2.1.0.zip new file mode 100644 index 00000000000..8c07e922260 Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.1.0.zip differ diff --git a/core/src/test/resources/indices/bwc/repo-2.0.0.zip b/core/src/test/resources/indices/bwc/repo-2.0.0.zip new file mode 100644 index 00000000000..9605830a12c Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.0.0.zip differ diff --git a/core/src/test/resources/indices/bwc/repo-2.0.1.zip b/core/src/test/resources/indices/bwc/repo-2.0.1.zip new file mode 100644 index 00000000000..305820877bb Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.0.1.zip differ diff --git a/core/src/test/resources/indices/bwc/repo-2.1.0.zip b/core/src/test/resources/indices/bwc/repo-2.1.0.zip new file mode 100644 index 00000000000..2f287ea3481 Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.1.0.zip differ diff --git a/core/src/test/resources/indices/bwc/unsupported-1.7.3.zip b/core/src/test/resources/indices/bwc/unsupported-1.7.3.zip new file mode 100644 index 00000000000..9fcc1788ea9 Binary files /dev/null and b/core/src/test/resources/indices/bwc/unsupported-1.7.3.zip differ diff --git a/core/src/test/resources/indices/bwc/unsupportedrepo-1.7.3.zip b/core/src/test/resources/indices/bwc/unsupportedrepo-1.7.3.zip new file mode 100644 index 00000000000..12e13dafc59 Binary files /dev/null and b/core/src/test/resources/indices/bwc/unsupportedrepo-1.7.3.zip differ diff --git a/core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json b/core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json index 525a5692122..8f3cc9c5044 100644 --- a/core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json +++ b/core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json @@ -28,6 +28,16 @@ "max_value" : { "lt": 9 } + }, + "field1": { + "min_value" : { + "gte": "2014-01-01", + "format" : "date_optional_time" + }, + "max_value" : { + "lt": "2015-01-01", + "format" : "date_optional_time" + } } } } \ No newline at end of file diff --git a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json index 3d98f375153..eefec530e1f 100644 --- a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json +++ b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json @@ -1,16 +1,16 @@ {"index":"test", "ignore_unavailable" : true, "expand_wildcards" : "open,closed"}} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} {"index" : "test", "type" : "type1", "expand_wildcards" : ["open", "closed"]} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} {"index":"test", "ignore_unavailable" : false, "expand_wildcards" : ["open"]}} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} {"index":"test", "ignore_unavailable" : true, "allow_no_indices": true, "expand_wildcards" : ["open", "closed"]}} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} {"index":"test", "ignore_unavailable" : true, "allow_no_indices": false, "expand_wildcards" : ["closed"]}} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} {} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} {"search_type" : "dfs_query_then_fetch"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} -{"query" : {"match_all" {}}} +{"query" : {"match_all" :{}}} diff --git a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json index e2e06d9f951..79330d80f72 100644 --- a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json +++ b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json @@ -1,10 +1,10 @@ {"index":"test"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {"index" : "test", "type" : "type1"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {"search_type" : "dfs_query_then_fetch"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} diff --git a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json index 6416720a92f..a6b52fd3bf9 100644 --- a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json +++ b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json @@ -1,8 +1,8 @@ {"index":["test0", "test1"]} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {"index" : "test2,test3", "type" : "type1"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {"index" : ["test4", "test1"], "type" : [ "type2", "type1" ]} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {"search_type" : "dfs_query_then_fetch"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} diff --git a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json index b98e24b8660..844d8bea1f8 100644 --- a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json +++ b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json @@ -1,6 +1,6 @@ {"index":["test0", "test1"], "request_cache": true} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {"index" : "test2,test3", "type" : "type1", "preference": "_local"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} {"index" : ["test4", "test1"], "type" : [ "type2", "type1" ], "routing": "123"} -{"query" : {"match_all" {}}} +{"query" : {"match_all" : {}}} diff --git a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json index 5f08919481f..b337eae0654 100644 --- a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json +++ b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json @@ -1,6 +1,6 @@ {"index":["test0", "test1"], "request_cache": true} -{"template": {"query" : {"match_{{template}}" {}}}, "params": {"template": "all" } } } +{"template": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } } } {"index" : "test2,test3", "type" : "type1", "preference": "_local"} -{"template": {"query" : {"match_{{template}}" {}}}, "params": {"template": "all" } } } +{"template": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } } } {"index" : ["test4", "test1"], "type" : [ "type2", "type1" ], "routing": "123"} -{"template": {"query" : {"match_{{template}}" {}}}, "params": {"template": "all" } } } +{"template": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } } } diff --git a/core/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml b/core/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml deleted file mode 100644 index 89f4922a6af..00000000000 --- a/core/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml +++ /dev/null @@ -1,3 +0,0 @@ -cluster.name: tribe_node_cluster -tribe.t1.cluster.name: tribe1 -tribe.t2.cluster.name: tribe2 \ No newline at end of file diff --git a/core/src/test/resources/org/elasticsearch/validate/config/scripts/file_template_1.mustache b/core/src/test/resources/org/elasticsearch/validate/config/scripts/file_template_1.mustache deleted file mode 100644 index 969dc8d5987..00000000000 --- a/core/src/test/resources/org/elasticsearch/validate/config/scripts/file_template_1.mustache +++ /dev/null @@ -1 +0,0 @@ -{"size":"{{size}}","query":{"match":{"foo":"{{value}}"}},"aggs":{"objects":{"terms":{"field":"{{value}}","size":"{{size}}"}}}} \ No newline at end of file diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py index 5d663ca69f3..83a35941577 100644 --- a/dev-tools/create_bwc_index.py +++ b/dev-tools/create_bwc_index.py @@ -149,6 +149,16 @@ def start_node(version, release_dir, data_dir, repo_dir, tcp_port=DEFAULT_TRANSP cmd.append('-f') # version before 1.0 start in background automatically return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +def install_plugin(version, release_dir, plugin_name): + run_plugin(version, release_dir, 'install', [plugin_name]) + +def remove_plugin(version, release_dir, plugin_name): + run_plugin(version, release_dir, 'remove', [plugin_name]) + +def run_plugin(version, release_dir, plugin_cmd, args): + cmd = [os.path.join(release_dir, 'bin/plugin'), plugin_cmd] + args + subprocess.check_call(cmd) + def create_client(http_port=DEFAULT_HTTP_TCP_PORT, timeout=30): logging.info('Waiting for node to startup') for _ in range(0, timeout): diff --git a/dev-tools/create_bwc_index_with_plugin_mappings.py b/dev-tools/create_bwc_index_with_plugin_mappings.py new file mode 100644 index 00000000000..c30de412d1d --- /dev/null +++ b/dev-tools/create_bwc_index_with_plugin_mappings.py @@ -0,0 +1,124 @@ +import create_bwc_index +import logging +import os +import random +import shutil +import subprocess +import sys +import tempfile + +def fetch_version(version): + logging.info('fetching ES version %s' % version) + if subprocess.call([sys.executable, os.path.join(os.path.split(sys.argv[0])[0], 'get-bwc-version.py'), version]) != 0: + raise RuntimeError('failed to download ES version %s' % version) + +def create_index(plugin, mapping, docs): + ''' + Creates a static back compat index (.zip) with mappings using fields defined in plugins. + ''' + + logging.basicConfig(format='[%(levelname)s] [%(asctime)s] %(message)s', level=logging.INFO, + datefmt='%Y-%m-%d %I:%M:%S %p') + logging.getLogger('elasticsearch').setLevel(logging.ERROR) + logging.getLogger('urllib3').setLevel(logging.WARN) + + tmp_dir = tempfile.mkdtemp() + plugin_installed = False + node = None + try: + data_dir = os.path.join(tmp_dir, 'data') + repo_dir = os.path.join(tmp_dir, 'repo') + logging.info('Temp data dir: %s' % data_dir) + logging.info('Temp repo dir: %s' % repo_dir) + + version = '2.0.0' + classifier = '%s-%s' %(plugin, version) + index_name = 'index-%s' % classifier + + # Download old ES releases if necessary: + release_dir = os.path.join('backwards', 'elasticsearch-%s' % version) + if not os.path.exists(release_dir): + fetch_version(version) + + create_bwc_index.install_plugin(version, release_dir, plugin) + plugin_installed = True + node = create_bwc_index.start_node(version, release_dir, data_dir, repo_dir, cluster_name=index_name) + client = create_bwc_index.create_client() + put_plugin_mappings(client, index_name, mapping, docs) + create_bwc_index.shutdown_node(node) + + print('%s server output:\n%s' % (version, node.stdout.read().decode('utf-8'))) + node = None + create_bwc_index.compress_index(classifier, tmp_dir, 'plugins/%s/src/test/resources/indices/bwc' %plugin) + finally: + if node is not None: + create_bwc_index.shutdown_node(node) + if plugin_installed: + create_bwc_index.remove_plugin(version, release_dir, plugin) + shutil.rmtree(tmp_dir) + +def put_plugin_mappings(client, index_name, mapping, docs): + client.indices.delete(index=index_name, ignore=404) + logging.info('Create single shard test index') + + client.indices.create(index=index_name, body={ + 'settings': { + 'number_of_shards': 1, + 'number_of_replicas': 0 + }, + 'mappings': { + 'type': mapping + } + }) + health = client.cluster.health(wait_for_status='green', wait_for_relocating_shards=0) + assert health['timed_out'] == False, 'cluster health timed out %s' % health + + logging.info('Indexing documents') + for i in range(len(docs)): + client.index(index=index_name, doc_type="type", id=str(i), body=docs[i]) + logging.info('Flushing index') + client.indices.flush(index=index_name) + + logging.info('Running basic checks') + count = client.count(index=index_name)['count'] + assert count == len(docs), "expected %d docs, got %d" %(len(docs), count) + +def main(): + docs = [ + { + "foo": "abc" + }, + { + "foo": "abcdef" + }, + { + "foo": "a" + } + ] + + murmur3_mapping = { + 'properties': { + 'foo': { + 'type': 'string', + 'fields': { + 'hash': { + 'type': 'murmur3' + } + } + } + } + } + + create_index("mapper-murmur3", murmur3_mapping, docs) + + size_mapping = { + '_size': { + 'enabled': True + } + } + + create_index("mapper-size", size_mapping, docs) + +if __name__ == '__main__': + main() + diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl old mode 100644 new mode 100755 index c96645c1d6e..c3e93f91076 --- a/dev-tools/es_release_notes.pl +++ b/dev-tools/es_release_notes.pl @@ -35,7 +35,7 @@ my %Group_Labels = ( breaking => 'Breaking changes', build => 'Build', deprecation => 'Deprecations', - doc => 'Docs', + docs => 'Docs', feature => 'New features', enhancement => 'Enhancements', bug => 'Bug fixes', @@ -72,27 +72,27 @@ sub dump_issues { $month++; $year += 1900; - print <<"HTML"; - - - - - -HTML + print <<"ASCIIDOC"; +:issue: https://github.com/${User_Repo}issues/ +:pull: https://github.com/${User_Repo}pull/ + +[[release-notes-$version]] +== $version Release Notes + +ASCIIDOC for my $group ( @Groups, 'other' ) { my $group_issues = $issues->{$group} or next; - print "

    $Group_Labels{$group}

    \n\n
      \n"; + print "[[$group-$version]]\n" + . "[float]\n" + . "=== $Group_Labels{$group}\n\n"; for my $header ( sort keys %$group_issues ) { my $header_issues = $group_issues->{$header}; - my $prefix = "
    • "; - if ($header) { - print "
    • $header:
        "; - } + print( $header || 'HEADER MISSING', "::\n" ); + for my $issue (@$header_issues) { my $title = $issue->{title}; - $title =~ s{`([^`]+)`}{$1}g; if ( $issue->{state} eq 'open' ) { $title .= " [OPEN]"; @@ -102,30 +102,23 @@ HTML } my $number = $issue->{number}; - print encode_utf8( $prefix - . $title - . qq[ #${number}] ); + print encode_utf8("* $title {pull}${number}[#${number}]"); if ( my $related = $issue->{related_issues} ) { my %uniq = map { $_ => 1 } @$related; print keys %uniq > 1 ? " (issues: " : " (issue: "; - print join ", ", - map {qq[#${_}]} + print join ", ", map {"{issue}${_}[#${_}]"} sort keys %uniq; print ")"; } - print "\n"; - } - if ($header) { - print "
      • \n"; + print "\n"; } + print "\n"; } - print "
    "; print "\n\n"; } - print "\n"; } #=================================== diff --git a/dev-tools/pom.xml b/dev-tools/pom.xml deleted file mode 100644 index f02d6a8762a..00000000000 --- a/dev-tools/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - 4.0.0 - org.elasticsearch - dev-tools - 3.0.0-SNAPSHOT - Build Tools and Resources - Tools to assist in building and developing in the Elasticsearch project - - org.sonatype.oss - oss-parent - 7 - - - - UTF-8 - s3://download.elasticsearch.org/elasticsearch/staging/ - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - 1.5 - - - - bundle - - - - - - **/* - - - - - - - - release - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - ${gpg.keyname} - ${gpg.passphrase} - ${gpg.keyring} - - - - - - - - - diff --git a/dev-tools/prepare_release_candidate.py b/dev-tools/prepare_release_candidate.py index 24450a63559..31b07043389 100644 --- a/dev-tools/prepare_release_candidate.py +++ b/dev-tools/prepare_release_candidate.py @@ -42,20 +42,26 @@ POM_FILE = 'pom.xml' MAIL_TEMPLATE = """ Hi all -The new release candidate for %(version)s based on this commit[1] is now available, including the x-plugins, and RPM/deb repos: +The new release candidate for %(version)s is now available, including the x-plugins and RPM/deb repos. This release is based on: - - ZIP [2] - - tar.gz [3] - - RPM [4] - - deb [5] + * Elasticsearch commit: %(hash)s - https://github.com/elastic/elasticsearch/commit/%(hash)s + * X-Plugins commit: FILL_IN_X-PLUGINS_HASH - https://github.com/elastic/x-plugins/commit/FILL_IN_X-PLUGINS_HASH -Plugins can be installed as follows, +The packages may be downloaded from the following URLs: + + * ZIP - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/zip/elasticsearch/%(version)s/elasticsearch-%(version)s.zip + * tar.gz - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/tar/elasticsearch/%(version)s/elasticsearch-%(version)s.tar.gz + * RPM - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/rpm/elasticsearch/%(version)s/elasticsearch-%(version)s.rpm + * deb - http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/deb/elasticsearch/%(version)s/elasticsearch-%(version)s.deb + +Plugins can be installed as follows: bin/plugin -Des.plugins.staging=true install cloud-aws The same goes for the x-plugins: bin/plugin -Des.plugins.staging=true install license + bin/plugin -Des.plugins.staging=true install marvel-agent bin/plugin -Des.plugins.staging=true install shield bin/plugin -Des.plugins.staging=true install watcher @@ -82,13 +88,8 @@ To smoke-test the release please run: python3 -B ./dev-tools/smoke_test_rc.py --version %(version)s --hash %(hash)s --plugins license,shield,watcher -NOTE: this script requires JAVA_HOME to point to a Java 7 Runtime +NOTE: this script requires JAVA_HOME to point to a Java 7 Runtime -[1] https://github.com/elastic/elasticsearch/commit/%(hash)s -[2] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/zip/elasticsearch/%(version)s/elasticsearch-%(version)s.zip -[3] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/tar/elasticsearch/%(version)s/elasticsearch-%(version)s.tar.gz -[4] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/rpm/elasticsearch/%(version)s/elasticsearch-%(version)s.rpm -[5] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/deb/elasticsearch/%(version)s/elasticsearch-%(version)s.deb """ # console colors @@ -355,7 +356,7 @@ if __name__ == "__main__": debs3_list_cmd = 'deb-s3 list -b %s --prefix %s' % (bucket, debs3_prefix) debs3_verify_cmd = 'deb-s3 verify -b %s --prefix %s' % (bucket, debs3_prefix) rpms3_prefix = 'elasticsearch/staging/%s-%s/repos/%s/centos' % (release_version, shortHash, package_repo_version) - rpms3_upload_cmd = 'rpm-s3 -v -b %s -p %s --sign --visibility public-read -k 0 %s' % (bucket, rpms3_prefix, rpm) + rpms3_upload_cmd = 'rpm-s3 -v -b %s -p %s --sign --visibility public-read -k 100 %s' % (bucket, rpms3_prefix, rpm) if deploy_s3: run(s3cmd_sync_to_staging_bucket_cmd) diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py index b7bc00df0ab..3fa61c4361f 100644 --- a/dev-tools/smoke_test_rc.py +++ b/dev-tools/smoke_test_rc.py @@ -70,6 +70,7 @@ DEFAULT_PLUGINS = ["analysis-icu", "lang-expression", "lang-groovy", "lang-javascript", + "lang-plan-a", "lang-python", "mapper-murmur3", "mapper-size", diff --git a/dev-tools/src/main/resources/ant/integration-tests.xml b/dev-tools/src/main/resources/ant/integration-tests.xml deleted file mode 100644 index f64f4403157..00000000000 --- a/dev-tools/src/main/resources/ant/integration-tests.xml +++ /dev/null @@ -1,406 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Installing plugin @{name}... - - - - - - - - - - - - - - - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - Waiting for elasticsearch to form a cluster of two... - - - - - - - - - - - - - - - - - - - - - - - - - - - Starting up external cluster... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - External node started PID ${integ.pid} - - - - - - - - - - - - - - - - - - - - - - - - - Shutting down external node PID ${integ.pid} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dev-tools/src/main/resources/forbidden/third-party-signatures.txt b/dev-tools/src/main/resources/forbidden/third-party-signatures.txt deleted file mode 100644 index ac1ce33ac92..00000000000 --- a/dev-tools/src/main/resources/forbidden/third-party-signatures.txt +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -@defaultMessage unsafe encoders/decoders have problems in the lzf compress library. Use variants of encode/decode functions which take Encoder/Decoder. -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createEncoder(int) -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createNonAllocatingEncoder(int) -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createEncoder(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createNonAllocatingEncoder(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.impl.UnsafeChunkDecoder#() -com.ning.compress.lzf.parallel.CompressTask -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance() -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(int) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalNonAllocatingInstance(int) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(com.ning.compress.BufferRecycler) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalNonAllocatingInstance(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.util.ChunkDecoderFactory#optimalInstance() -com.ning.compress.lzf.util.LZFFileInputStream#(java.io.File) -com.ning.compress.lzf.util.LZFFileInputStream#(java.io.FileDescriptor) -com.ning.compress.lzf.util.LZFFileInputStream#(java.lang.String) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.File) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.File, boolean) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.FileDescriptor) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.lang.String) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.lang.String, boolean) -com.ning.compress.lzf.LZFEncoder#encode(byte[]) -com.ning.compress.lzf.LZFEncoder#encode(byte[], int, int) -com.ning.compress.lzf.LZFEncoder#encode(byte[], int, int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFEncoder#appendEncoded(byte[], int, int, byte[], int) -com.ning.compress.lzf.LZFEncoder#appendEncoded(byte[], int, int, byte[], int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFCompressingInputStream#(java.io.InputStream) -com.ning.compress.lzf.LZFDecoder#fastDecoder() -com.ning.compress.lzf.LZFDecoder#decode(byte[]) -com.ning.compress.lzf.LZFDecoder#decode(byte[], int, int) -com.ning.compress.lzf.LZFDecoder#decode(byte[], byte[]) -com.ning.compress.lzf.LZFDecoder#decode(byte[], int, int, byte[]) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, boolean) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, com.ning.compress.BufferRecycler, boolean) -com.ning.compress.lzf.LZFOutputStream#(java.io.OutputStream) -com.ning.compress.lzf.LZFOutputStream#(java.io.OutputStream, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFUncompressor#(com.ning.compress.DataHandler) -com.ning.compress.lzf.LZFUncompressor#(com.ning.compress.DataHandler, com.ning.compress.BufferRecycler) - -@defaultMessage Constructing a DateTime without a time zone is dangerous -org.joda.time.DateTime#() -org.joda.time.DateTime#(long) -org.joda.time.DateTime#(int, int, int, int, int) -org.joda.time.DateTime#(int, int, int, int, int, int) -org.joda.time.DateTime#(int, int, int, int, int, int, int) -org.joda.time.DateTime#now() -org.joda.time.DateTimeZone#getDefault() diff --git a/dev-tools/src/main/resources/license-check/check_license_and_sha.pl b/dev-tools/src/main/resources/license-check/check_license_and_sha.pl deleted file mode 100755 index c6b0f04b6db..00000000000 --- a/dev-tools/src/main/resources/license-check/check_license_and_sha.pl +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use 5.010_000; - -use FindBin qw($RealBin); -use lib "$RealBin/lib"; -use File::Spec(); -use File::Temp 0.2304 (); -use File::Find(); -use File::Basename qw(basename); -use Archive::Extract(); -use Digest::SHA(); -$Archive::Extract::PREFER_BIN = 1; - -my $mode = shift(@ARGV) || ""; -die usage() unless $mode =~ /^--(check|update)$/; - -my $License_Dir = shift(@ARGV) || die usage(); -my $Source = shift(@ARGV) || die usage(); -my $Ignore = shift(@ARGV) || ''; -my $ignore - = $Ignore - ? qr/${Ignore}[^\/]*$/ - : qr/elasticsearch[^\/]*$/; - -$License_Dir = File::Spec->rel2abs($License_Dir) . '/'; -$Source = File::Spec->rel2abs($Source); - -say "LICENSE DIR: $License_Dir"; -say "SOURCE: $Source"; -say "IGNORE: $Ignore"; - -die "License dir is not a directory: $License_Dir\n" . usage() - unless -d $License_Dir; - -my %shas - = -f $Source ? jars_from_zip( $Source, $ignore ) - : -d $Source ? jars_from_dir( $Source, $ignore ) - : die "Source is neither a directory nor a zip file: $Source" . usage(); - -$mode eq '--check' - ? exit check_shas_and_licenses(%shas) - : exit write_shas(%shas); - -#=================================== -sub check_shas_and_licenses { -#=================================== - my %new = @_; - - my %old = get_sha_files(); - my %licenses = get_files_with('LICENSE'); - my %notices = get_files_with('NOTICE'); - - my $error = 0; - my $sha_error = 0; - - for my $jar ( sort keys %new ) { - my $old_sha = delete $old{$jar}; - unless ($old_sha) { - say STDERR "$jar: SHA is missing"; - $error++; - $sha_error++; - next; - } - - unless ( $old_sha eq $new{$jar} ) { - say STDERR - "$jar: SHA has changed, expected $old_sha but found $new{$jar}"; - $error++; - $sha_error++; - next; - } - - my $license_found; - my $notice_found; - my $prefix = $jar; - $prefix =~ s/\.sha1//; - - while ( $prefix =~ s/-[^\-]+$// ) { - if ( exists $licenses{$prefix} ) { - $license_found = 1; - - # mark all licenses with the same prefix as used - for ( keys %licenses ) { - $licenses{$_}++ if index( $prefix, $_ ) == 0; - } - - if ( exists $notices{$prefix} ) { - $notices{$prefix}++; - $notice_found = 1; - } - last; - } - } - unless ($license_found) { - say STDERR "$jar: LICENSE is missing"; - $error++; - $sha_error++; - } - unless ($notice_found) { - say STDERR "$jar: NOTICE is missing"; - $error++; - } - } - - if ( keys %old ) { - say STDERR "Extra SHA files present for: " . join ", ", sort keys %old; - $error++; - } - - my @unused_licenses = grep { !$licenses{$_} } keys %licenses; - if (@unused_licenses) { - $error++; - say STDERR "Extra LICENCE file present: " . join ", ", - sort @unused_licenses; - } - - my @unused_notices = grep { !$notices{$_} } keys %notices; - if (@unused_notices) { - $error++; - say STDERR "Extra NOTICE file present: " . join ", ", - sort @unused_notices; - } - - if ($sha_error) { - say STDERR <<"SHAS" - -You can update the SHA files by running: - -$0 --update $License_Dir $Source $Ignore - -SHAS - } - say("All SHAs and licenses OK") unless $error; - return $error; -} - -#=================================== -sub write_shas { -#=================================== - my %new = @_; - my %old = get_sha_files(); - - for my $jar ( sort keys %new ) { - if ( $old{$jar} ) { - next if $old{$jar} eq $new{$jar}; - say "Updating $jar"; - } - else { - say "Adding $jar"; - } - open my $fh, '>', $License_Dir . $jar or die $!; - say $fh $new{$jar} or die $!; - close $fh or die $!; - } - continue { - delete $old{$jar}; - } - - for my $jar ( sort keys %old ) { - say "Deleting $jar"; - unlink $License_Dir . $jar or die $!; - } - say "SHAs updated"; - return 0; -} - -#=================================== -sub get_files_with { -#=================================== - my $pattern = shift; - my %files; - for my $path ( grep {-f} glob("$License_Dir/*$pattern*") ) { - my ($file) = ( $path =~ m{([^/]+)-${pattern}.*$} ); - $files{$file} = 0; - } - return %files; -} - -#=================================== -sub get_sha_files { -#=================================== - my %shas; - - die "Missing directory: $License_Dir\n" - unless -d $License_Dir; - - for my $file ( grep {-f} glob("$License_Dir/*.sha1") ) { - my ($jar) = ( $file =~ m{([^/]+)$} ); - open my $fh, '<', $file or die $!; - my $sha = <$fh>; - $sha ||= ''; - chomp $sha; - $shas{$jar} = $sha; - } - return %shas; -} - -#=================================== -sub jars_from_zip { -#=================================== - my ( $source, $ignore ) = @_; - my $temp_dir = File::Temp->newdir; - my $dir_name = $temp_dir->dirname; - my $archive = Archive::Extract->new( archive => $source, type => 'zip' ); - $archive->extract( to => $dir_name ) || die $archive->error; - my @jars = map { File::Spec->rel2abs( $_, $dir_name ) } - grep { /\.jar$/ && !/$ignore/ } @{ $archive->files }; - return calculate_shas(@jars); -} - -#=================================== -sub jars_from_dir { -#=================================== - my ( $source, $ignore ) = @_; - my @jars; - File::Find::find( - { wanted => sub { - push @jars, File::Spec->rel2abs( $_, $source ) - if /\.jar$/ && !/$ignore/; - }, - no_chdir => 1 - }, - $source - ); - return calculate_shas(@jars); -} - -#=================================== -sub calculate_shas { -#=================================== - my %shas; - while ( my $file = shift() ) { - my $digest = eval { Digest::SHA->new(1)->addfile($file) } - or die "Error calculating SHA1 for <$file>: $!\n"; - $shas{ basename($file) . ".sha1" } = $digest->hexdigest; - } - return %shas; -} - -#=================================== -sub usage { -#=================================== - return <<"USAGE"; - -USAGE: - - # check the sha1 and LICENSE files for each jar in the zip or directory - $0 --check path/to/licenses/ path/to/package.zip [prefix_to_ignore] - $0 --check path/to/licenses/ path/to/dir/ [prefix_to_ignore] - - # updates the sha1s for each jar in the zip or directory - $0 --update path/to/licenses/ path/to/package.zip [prefix_to_ignore] - $0 --update path/to/licenses/ path/to/dir/ [prefix_to_ignore] - -The optional prefix_to_ignore parameter defaults to "elasticsearch". - -USAGE - -} - diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Extract.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Extract.pm deleted file mode 100644 index e88cf11f037..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Extract.pm +++ /dev/null @@ -1,1694 +0,0 @@ -package Archive::Extract; -use if $] > 5.017, 'deprecate'; - -use strict; - -use Cwd qw[cwd chdir]; -use Carp qw[carp]; -use IPC::Cmd qw[run can_run]; -use FileHandle; -use File::Path qw[mkpath]; -use File::Spec; -use File::Basename qw[dirname basename]; -use Params::Check qw[check]; -use Module::Load::Conditional qw[can_load check_install]; -use Locale::Maketext::Simple Style => 'gettext'; - -### solaris has silly /bin/tar output ### -use constant ON_SOLARIS => $^O eq 'solaris' ? 1 : 0; -use constant ON_NETBSD => $^O eq 'netbsd' ? 1 : 0; -use constant ON_OPENBSD => $^O =~ m!^(openbsd|bitrig)$! ? 1 : 0; -use constant ON_FREEBSD => $^O =~ m!^(free|midnight|dragonfly)(bsd)?$! ? 1 : 0; -use constant ON_LINUX => $^O eq 'linux' ? 1 : 0; -use constant FILE_EXISTS => sub { -e $_[0] ? 1 : 0 }; - -### VMS may require quoting upper case command options -use constant ON_VMS => $^O eq 'VMS' ? 1 : 0; - -### Windows needs special treatment of Tar options -use constant ON_WIN32 => $^O eq 'MSWin32' ? 1 : 0; - -### we can't use this extraction method, because of missing -### modules/binaries: -use constant METHOD_NA => []; - -### If these are changed, update @TYPES and the new() POD -use constant TGZ => 'tgz'; -use constant TAR => 'tar'; -use constant GZ => 'gz'; -use constant ZIP => 'zip'; -use constant BZ2 => 'bz2'; -use constant TBZ => 'tbz'; -use constant Z => 'Z'; -use constant LZMA => 'lzma'; -use constant XZ => 'xz'; -use constant TXZ => 'txz'; - -use vars qw[$VERSION $PREFER_BIN $PROGRAMS $WARN $DEBUG - $_ALLOW_BIN $_ALLOW_PURE_PERL $_ALLOW_TAR_ITER - ]; - -$VERSION = '0.76'; -$PREFER_BIN = 0; -$WARN = 1; -$DEBUG = 0; -$_ALLOW_PURE_PERL = 1; # allow pure perl extractors -$_ALLOW_BIN = 1; # allow binary extractors -$_ALLOW_TAR_ITER = 1; # try to use Archive::Tar->iter if available - -# same as all constants -my @Types = ( TGZ, TAR, GZ, ZIP, BZ2, TBZ, Z, LZMA, XZ, TXZ ); - -local $Params::Check::VERBOSE = $Params::Check::VERBOSE = 1; - -=pod - -=head1 NAME - -Archive::Extract - A generic archive extracting mechanism - -=head1 SYNOPSIS - - use Archive::Extract; - - ### build an Archive::Extract object ### - my $ae = Archive::Extract->new( archive => 'foo.tgz' ); - - ### extract to cwd() ### - my $ok = $ae->extract; - - ### extract to /tmp ### - my $ok = $ae->extract( to => '/tmp' ); - - ### what if something went wrong? - my $ok = $ae->extract or die $ae->error; - - ### files from the archive ### - my $files = $ae->files; - - ### dir that was extracted to ### - my $outdir = $ae->extract_path; - - - ### quick check methods ### - $ae->is_tar # is it a .tar file? - $ae->is_tgz # is it a .tar.gz or .tgz file? - $ae->is_gz; # is it a .gz file? - $ae->is_zip; # is it a .zip file? - $ae->is_bz2; # is it a .bz2 file? - $ae->is_tbz; # is it a .tar.bz2 or .tbz file? - $ae->is_lzma; # is it a .lzma file? - $ae->is_xz; # is it a .xz file? - $ae->is_txz; # is it a .tar.xz or .txz file? - - ### absolute path to the archive you provided ### - $ae->archive; - - ### commandline tools, if found ### - $ae->bin_tar # path to /bin/tar, if found - $ae->bin_gzip # path to /bin/gzip, if found - $ae->bin_unzip # path to /bin/unzip, if found - $ae->bin_bunzip2 # path to /bin/bunzip2 if found - $ae->bin_unlzma # path to /bin/unlzma if found - $ae->bin_unxz # path to /bin/unxz if found - -=head1 DESCRIPTION - -Archive::Extract is a generic archive extraction mechanism. - -It allows you to extract any archive file of the type .tar, .tar.gz, -.gz, .Z, tar.bz2, .tbz, .bz2, .zip, .xz,, .txz, .tar.xz or .lzma -without having to worry how it -does so, or use different interfaces for each type by using either -perl modules, or commandline tools on your system. - -See the C section further down for details. - -=cut - - -### see what /bin/programs are available ### -$PROGRAMS = {}; -CMD: for my $pgm (qw[tar unzip gzip bunzip2 uncompress unlzma unxz]) { - if ( $pgm eq 'unzip' and ON_FREEBSD and my $unzip = can_run('info-unzip') ) { - $PROGRAMS->{$pgm} = $unzip; - next CMD; - } - if ( $pgm eq 'unzip' and ( ON_NETBSD or ON_FREEBSD ) ) { - local $IPC::Cmd::INSTANCES = 1; - ($PROGRAMS->{$pgm}) = grep { ON_NETBSD ? m!/usr/pkg/! : m!/usr/local! } can_run($pgm); - next CMD; - } - if ( $pgm eq 'unzip' and ON_LINUX ) { - # Check if 'unzip' is busybox masquerading - local $IPC::Cmd::INSTANCES = 1; - my $opt = ON_VMS ? '"-Z"' : '-Z'; - ($PROGRAMS->{$pgm}) = grep { scalar run(command=> [ $_, $opt, '-1' ]) } can_run($pgm); - next CMD; - } - if ( $pgm eq 'tar' and ( ON_OPENBSD || ON_SOLARIS || ON_NETBSD ) ) { - # try gtar first - next CMD if $PROGRAMS->{$pgm} = can_run('gtar'); - } - $PROGRAMS->{$pgm} = can_run($pgm); -} - -### mapping from types to extractor methods ### -my $Mapping = { # binary program # pure perl module - is_tgz => { bin => '_untar_bin', pp => '_untar_at' }, - is_tar => { bin => '_untar_bin', pp => '_untar_at' }, - is_gz => { bin => '_gunzip_bin', pp => '_gunzip_cz' }, - is_zip => { bin => '_unzip_bin', pp => '_unzip_az' }, - is_tbz => { bin => '_untar_bin', pp => '_untar_at' }, - is_bz2 => { bin => '_bunzip2_bin', pp => '_bunzip2_bz2'}, - is_Z => { bin => '_uncompress_bin', pp => '_gunzip_cz' }, - is_lzma => { bin => '_unlzma_bin', pp => '_unlzma_cz' }, - is_xz => { bin => '_unxz_bin', pp => '_unxz_cz' }, - is_txz => { bin => '_untar_bin', pp => '_untar_at' }, -}; - -{ ### use subs so we re-generate array refs etc for the no-override flags - ### if we don't, then we reuse the same arrayref, meaning objects store - ### previous errors - my $tmpl = { - archive => sub { { required => 1, allow => FILE_EXISTS } }, - type => sub { { default => '', allow => [ @Types ] } }, - _error_msg => sub { { no_override => 1, default => [] } }, - _error_msg_long => sub { { no_override => 1, default => [] } }, - }; - - ### build accessors ### - for my $method( keys %$tmpl, - qw[_extractor _gunzip_to files extract_path], - ) { - no strict 'refs'; - *$method = sub { - my $self = shift; - $self->{$method} = $_[0] if @_; - return $self->{$method}; - } - } - -=head1 METHODS - -=head2 $ae = Archive::Extract->new(archive => '/path/to/archive',[type => TYPE]) - -Creates a new C object based on the archive file you -passed it. Automatically determines the type of archive based on the -extension, but you can override that by explicitly providing the -C argument. - -Valid values for C are: - -=over 4 - -=item tar - -Standard tar files, as produced by, for example, C. -Corresponds to a C<.tar> suffix. - -=item tgz - -Gzip compressed tar files, as produced by, for example C. -Corresponds to a C<.tgz> or C<.tar.gz> suffix. - -=item gz - -Gzip compressed file, as produced by, for example C. -Corresponds to a C<.gz> suffix. - -=item Z - -Lempel-Ziv compressed file, as produced by, for example C. -Corresponds to a C<.Z> suffix. - -=item zip - -Zip compressed file, as produced by, for example C. -Corresponds to a C<.zip>, C<.jar> or C<.par> suffix. - -=item bz2 - -Bzip2 compressed file, as produced by, for example, C. -Corresponds to a C<.bz2> suffix. - -=item tbz - -Bzip2 compressed tar file, as produced by, for example C. -Corresponds to a C<.tbz> or C<.tar.bz2> suffix. - -=item lzma - -Lzma compressed file, as produced by C. -Corresponds to a C<.lzma> suffix. - -=item xz - -Xz compressed file, as produced by C. -Corresponds to a C<.xz> suffix. - -=item txz - -Xz compressed tar file, as produced by, for example C. -Corresponds to a C<.txz> or C<.tar.xz> suffix. - -=back - -Returns a C object on success, or false on failure. - -=cut - - ### constructor ### - sub new { - my $class = shift; - my %hash = @_; - - ### see above why we use subs here and generate the template; - ### it's basically to not re-use arrayrefs - my %utmpl = map { $_ => $tmpl->{$_}->() } keys %$tmpl; - - my $parsed = check( \%utmpl, \%hash ) or return; - - ### make sure we have an absolute path ### - my $ar = $parsed->{archive} = File::Spec->rel2abs( $parsed->{archive} ); - - ### figure out the type, if it wasn't already specified ### - unless ( $parsed->{type} ) { - $parsed->{type} = - $ar =~ /.+?\.(?:tar\.gz|tgz)$/i ? TGZ : - $ar =~ /.+?\.gz$/i ? GZ : - $ar =~ /.+?\.tar$/i ? TAR : - $ar =~ /.+?\.(zip|jar|ear|war|par)$/i ? ZIP : - $ar =~ /.+?\.(?:tbz2?|tar\.bz2?)$/i ? TBZ : - $ar =~ /.+?\.bz2$/i ? BZ2 : - $ar =~ /.+?\.Z$/ ? Z : - $ar =~ /.+?\.lzma$/ ? LZMA : - $ar =~ /.+?\.(?:txz|tar\.xz)$/i ? TXZ : - $ar =~ /.+?\.xz$/ ? XZ : - ''; - - } - - bless $parsed, $class; - - ### don't know what type of file it is - ### XXX this *has* to be an object call, not a package call - return $parsed->_error(loc("Cannot determine file type for '%1'", - $parsed->{archive} )) unless $parsed->{type}; - return $parsed; - } -} - -=head2 $ae->extract( [to => '/output/path'] ) - -Extracts the archive represented by the C object to -the path of your choice as specified by the C argument. Defaults to -C. - -Since C<.gz> files never hold a directory, but only a single file; if -the C argument is an existing directory, the file is extracted -there, with its C<.gz> suffix stripped. -If the C argument is not an existing directory, the C argument -is understood to be a filename, if the archive type is C. -In the case that you did not specify a C argument, the output -file will be the name of the archive file, stripped from its C<.gz> -suffix, in the current working directory. - -C will try a pure perl solution first, and then fall back to -commandline tools if they are available. See the C -section below on how to alter this behaviour. - -It will return true on success, and false on failure. - -On success, it will also set the follow attributes in the object: - -=over 4 - -=item $ae->extract_path - -This is the directory that the files where extracted to. - -=item $ae->files - -This is an array ref with the paths of all the files in the archive, -relative to the C argument you specified. -To get the full path to an extracted file, you would use: - - File::Spec->catfile( $to, $ae->files->[0] ); - -Note that all files from a tar archive will be in unix format, as per -the tar specification. - -=back - -=cut - -sub extract { - my $self = shift; - my %hash = @_; - - ### reset error messages - $self->_error_msg( [] ); - $self->_error_msg_long( [] ); - - my $to; - my $tmpl = { - to => { default => '.', store => \$to } - }; - - check( $tmpl, \%hash ) or return; - - ### so 'to' could be a file or a dir, depending on whether it's a .gz - ### file, or basically anything else. - ### so, check that, then act accordingly. - ### set an accessor specifically so _gunzip can know what file to extract - ### to. - my $dir; - { ### a foo.gz file - if( $self->is_gz or $self->is_bz2 or $self->is_Z or $self->is_lzma or $self->is_xz ) { - - my $cp = $self->archive; $cp =~ s/\.(?:gz|bz2?|Z|lzma|xz)$//i; - - ### to is a dir? - if ( -d $to ) { - $dir = $to; - $self->_gunzip_to( basename($cp) ); - - ### then it's a filename - } else { - $dir = dirname($to); - $self->_gunzip_to( basename($to) ); - } - - ### not a foo.gz file - } else { - $dir = $to; - } - } - - ### make the dir if it doesn't exist ### - unless( -d $dir ) { - eval { mkpath( $dir ) }; - - return $self->_error(loc("Could not create path '%1': %2", $dir, $@)) - if $@; - } - - ### get the current dir, to restore later ### - my $cwd = cwd(); - - my $ok = 1; - EXTRACT: { - - ### chdir to the target dir ### - unless( chdir $dir ) { - $self->_error(loc("Could not chdir to '%1': %2", $dir, $!)); - $ok = 0; last EXTRACT; - } - - ### set files to an empty array ref, so there's always an array - ### ref IN the accessor, to avoid errors like: - ### Can't use an undefined value as an ARRAY reference at - ### ../lib/Archive/Extract.pm line 742. (rt #19815) - $self->files( [] ); - - ### find out the dispatch methods needed for this type of - ### archive. Do a $self->is_XXX to figure out the type, then - ### get the hashref with bin + pure perl dispatchers. - my ($map) = map { $Mapping->{$_} } grep { $self->$_ } keys %$Mapping; - - ### add pure perl extractor if allowed & add bin extractor if allowed - my @methods; - push @methods, $map->{'pp'} if $_ALLOW_PURE_PERL; - push @methods, $map->{'bin'} if $_ALLOW_BIN; - - ### reverse it if we prefer bin extractors - @methods = reverse @methods if $PREFER_BIN; - - my($na, $fail); - for my $method (@methods) { - $self->debug( "# Extracting with ->$method\n" ); - - my $rv = $self->$method; - - ### a positive extraction - if( $rv and $rv ne METHOD_NA ) { - $self->debug( "# Extraction succeeded\n" ); - $self->_extractor($method); - last; - - ### method is not available - } elsif ( $rv and $rv eq METHOD_NA ) { - $self->debug( "# Extraction method not available\n" ); - $na++; - } else { - $self->debug( "# Extraction method failed\n" ); - $fail++; - } - } - - ### warn something went wrong if we didn't get an extractor - unless( $self->_extractor ) { - my $diag = $fail ? loc("Extract failed due to errors") : - $na ? loc("Extract failed; no extractors available") : - ''; - - $self->_error($diag); - $ok = 0; - } - } - - ### and chdir back ### - unless( chdir $cwd ) { - $self->_error(loc("Could not chdir back to start dir '%1': %2'", - $cwd, $!)); - } - - return $ok; -} - -=pod - -=head1 ACCESSORS - -=head2 $ae->error([BOOL]) - -Returns the last encountered error as string. -Pass it a true value to get the C output instead. - -=head2 $ae->extract_path - -This is the directory the archive got extracted to. -See C for details. - -=head2 $ae->files - -This is an array ref holding all the paths from the archive. -See C for details. - -=head2 $ae->archive - -This is the full path to the archive file represented by this -C object. - -=head2 $ae->type - -This is the type of archive represented by this C -object. See accessors below for an easier way to use this. -See the C method for details. - -=head2 $ae->types - -Returns a list of all known C for C's -C method. - -=cut - -sub types { return @Types } - -=head2 $ae->is_tgz - -Returns true if the file is of type C<.tar.gz>. -See the C method for details. - -=head2 $ae->is_tar - -Returns true if the file is of type C<.tar>. -See the C method for details. - -=head2 $ae->is_gz - -Returns true if the file is of type C<.gz>. -See the C method for details. - -=head2 $ae->is_Z - -Returns true if the file is of type C<.Z>. -See the C method for details. - -=head2 $ae->is_zip - -Returns true if the file is of type C<.zip>. -See the C method for details. - -=head2 $ae->is_lzma - -Returns true if the file is of type C<.lzma>. -See the C method for details. - -=head2 $ae->is_xz - -Returns true if the file is of type C<.xz>. -See the C method for details. - -=cut - -### quick check methods ### -sub is_tgz { return $_[0]->type eq TGZ } -sub is_tar { return $_[0]->type eq TAR } -sub is_gz { return $_[0]->type eq GZ } -sub is_zip { return $_[0]->type eq ZIP } -sub is_tbz { return $_[0]->type eq TBZ } -sub is_bz2 { return $_[0]->type eq BZ2 } -sub is_Z { return $_[0]->type eq Z } -sub is_lzma { return $_[0]->type eq LZMA } -sub is_xz { return $_[0]->type eq XZ } -sub is_txz { return $_[0]->type eq TXZ } - -=pod - -=head2 $ae->bin_tar - -Returns the full path to your tar binary, if found. - -=head2 $ae->bin_gzip - -Returns the full path to your gzip binary, if found - -=head2 $ae->bin_unzip - -Returns the full path to your unzip binary, if found - -=head2 $ae->bin_unlzma - -Returns the full path to your unlzma binary, if found - -=head2 $ae->bin_unxz - -Returns the full path to your unxz binary, if found - -=cut - -### paths to commandline tools ### -sub bin_gzip { return $PROGRAMS->{'gzip'} if $PROGRAMS->{'gzip'} } -sub bin_unzip { return $PROGRAMS->{'unzip'} if $PROGRAMS->{'unzip'} } -sub bin_tar { return $PROGRAMS->{'tar'} if $PROGRAMS->{'tar'} } -sub bin_bunzip2 { return $PROGRAMS->{'bunzip2'} if $PROGRAMS->{'bunzip2'} } -sub bin_uncompress { return $PROGRAMS->{'uncompress'} - if $PROGRAMS->{'uncompress'} } -sub bin_unlzma { return $PROGRAMS->{'unlzma'} if $PROGRAMS->{'unlzma'} } -sub bin_unxz { return $PROGRAMS->{'unxz'} if $PROGRAMS->{'unxz'} } - -=head2 $bool = $ae->have_old_bunzip2 - -Older versions of C, from before the C release, -require all archive names to end in C<.bz2> or it will not extract -them. This method checks if you have a recent version of C -that allows any extension, or an older one that doesn't. - -=cut - -sub have_old_bunzip2 { - my $self = shift; - - ### no bunzip2? no old bunzip2 either :) - return unless $self->bin_bunzip2; - - ### if we can't run this, we can't be sure if it's too old or not - ### XXX stupid stupid stupid bunzip2 doesn't understand --version - ### is not a request to extract data: - ### $ bunzip2 --version - ### bzip2, a block-sorting file compressor. Version 1.0.2, 30-Dec-2001. - ### [...] - ### bunzip2: I won't read compressed data from a terminal. - ### bunzip2: For help, type: `bunzip2 --help'. - ### $ echo $? - ### 1 - ### HATEFUL! - - ### double hateful: bunzip2 --version also hangs if input is a pipe - ### See #32370: Archive::Extract will hang if stdin is a pipe [+PATCH] - ### So, we have to provide *another* argument which is a fake filename, - ### just so it wont try to read from stdin to print its version.. - ### *sigh* - ### Even if the file exists, it won't clobber or change it. - my $buffer; - scalar run( - command => [$self->bin_bunzip2, '--version', 'NoSuchFile'], - verbose => 0, - buffer => \$buffer - ); - - ### no output - return unless $buffer; - - my ($version) = $buffer =~ /version \s+ (\d+)/ix; - - return 1 if $version < 1; - return; -} - -################################# -# -# Untar code -# -################################# - -### annoying issue with (gnu) tar on win32, as illustrated by this -### bug: https://rt.cpan.org/Ticket/Display.html?id=40138 -### which shows that (gnu) tar will interpret a file name with a : -### in it as a remote file name, so C:\tmp\foo.txt is interpreted -### as a remote shell, and the extract fails. -{ my @ExtraTarFlags; - if( ON_WIN32 and my $cmd = __PACKAGE__->bin_tar ) { - - ### if this is gnu tar we are running, we need to use --force-local - push @ExtraTarFlags, '--force-local' if `$cmd --version` =~ /gnu tar/i; - } - - - ### use /bin/tar to extract ### - sub _untar_bin { - my $self = shift; - - ### check for /bin/tar ### - ### check for /bin/gzip if we need it ### - ### if any of the binaries are not available, return NA - { my $diag = !$self->bin_tar ? - loc("No '%1' program found", '/bin/tar') : - $self->is_tgz && !$self->bin_gzip ? - loc("No '%1' program found", '/bin/gzip') : - $self->is_tbz && !$self->bin_bunzip2 ? - loc("No '%1' program found", '/bin/bunzip2') : - $self->is_txz && !$self->bin_unxz ? - loc("No '%1' program found", '/bin/unxz') : - ''; - - if( $diag ) { - $self->_error( $diag ); - return METHOD_NA; - } - } - - ### XXX figure out how to make IPC::Run do this in one call -- - ### currently i don't know how to get output of a command after a pipe - ### trapped in a scalar. Mailed barries about this 5th of june 2004. - - ### see what command we should run, based on whether - ### it's a .tgz or .tar - - ### GNU tar can't handled VMS filespecs, but VMSTAR can handle Unix filespecs. - my $archive = $self->archive; - $archive = VMS::Filespec::unixify($archive) if ON_VMS; - - ### XXX solaris tar and bsdtar are having different outputs - ### depending whether you run with -x or -t - ### compensate for this insanity by running -t first, then -x - { my $cmd = - $self->is_tgz ? [$self->bin_gzip, '-c', '-d', '-f', $archive, '|', - $self->bin_tar, '-tf', '-'] : - $self->is_tbz ? [$self->bin_bunzip2, '-cd', $archive, '|', - $self->bin_tar, '-tf', '-'] : - $self->is_txz ? [$self->bin_unxz, '-cd', $archive, '|', - $self->bin_tar, '-tf', '-'] : - [$self->bin_tar, @ExtraTarFlags, '-tf', $archive]; - - ### run the command - ### newer versions of 'tar' (1.21 and up) now print record size - ### to STDERR as well if v OR t is given (used to be both). This - ### is a 'feature' according to the changelog, so we must now only - ### inspect STDOUT, otherwise, failures like these occur: - ### http://www.cpantesters.org/cpan/report/3230366 - my $buffer = ''; - my @out = run( command => $cmd, - buffer => \$buffer, - verbose => $DEBUG ); - - ### command was unsuccessful - unless( $out[0] ) { - return $self->_error(loc( - "Error listing contents of archive '%1': %2", - $archive, $buffer )); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_files( $archive ) ); - - } else { - ### if we're on solaris we /might/ be using /bin/tar, which has - ### a weird output format... we might also be using - ### /usr/local/bin/tar, which is gnu tar, which is perfectly - ### fine... so we have to do some guessing here =/ - my @files = map { chomp; - !ON_SOLARIS ? $_ - : (m|^ x \s+ # 'xtract' -- sigh - (.+?), # the actual file name - \s+ [\d,.]+ \s bytes, - \s+ [\d,.]+ \s tape \s blocks - |x ? $1 : $_); - - ### only STDOUT, see above. Sometimes, extra whitespace - ### is present, so make sure we only pick lines with - ### a length - } grep { length } map { split $/, $_ } join '', @{$out[3]}; - - ### store the files that are in the archive ### - $self->files(\@files); - } - } - - ### now actually extract it ### - { my $cmd = - $self->is_tgz ? [$self->bin_gzip, '-c', '-d', '-f', $archive, '|', - $self->bin_tar, '-xf', '-'] : - $self->is_tbz ? [$self->bin_bunzip2, '-cd', $archive, '|', - $self->bin_tar, '-xf', '-'] : - $self->is_txz ? [$self->bin_unxz, '-cd', $archive, '|', - $self->bin_tar, '-xf', '-'] : - [$self->bin_tar, @ExtraTarFlags, '-xf', $archive]; - - my $buffer = ''; - unless( scalar run( command => $cmd, - buffer => \$buffer, - verbose => $DEBUG ) - ) { - return $self->_error(loc("Error extracting archive '%1': %2", - $archive, $buffer )); - } - - ### we might not have them, due to lack of buffers - if( $self->files ) { - ### now that we've extracted, figure out where we extracted to - my $dir = $self->__get_extract_dir( $self->files ); - - ### store the extraction dir ### - $self->extract_path( $dir ); - } - } - - ### we got here, no error happened - return 1; - } -} - - -### use archive::tar to extract ### -sub _untar_at { - my $self = shift; - - ### Loading Archive::Tar is going to set it to 1, so make it local - ### within this block, starting with its initial value. Whatever - ### Achive::Tar does will be undone when we return. - ### - ### Also, later, set $Archive::Tar::WARN to $Archive::Extract::WARN - ### so users don't have to even think about this variable. If they - ### do, they still get their set value outside of this call. - local $Archive::Tar::WARN = $Archive::Tar::WARN; - - ### we definitely need Archive::Tar, so load that first - { my $use_list = { 'Archive::Tar' => '0.0' }; - - unless( can_load( modules => $use_list ) ) { - - $self->_error(loc("You do not have '%1' installed - " . - "Please install it as soon as possible.", - 'Archive::Tar')); - - return METHOD_NA; - } - } - - ### we might pass it a filehandle if it's a .tbz file.. - my $fh_to_read = $self->archive; - - ### we will need Compress::Zlib too, if it's a tgz... and IO::Zlib - ### if A::T's version is 0.99 or higher - if( $self->is_tgz ) { - my $use_list = { 'Compress::Zlib' => '0.0' }; - $use_list->{ 'IO::Zlib' } = '0.0' - if $Archive::Tar::VERSION >= '0.99'; - - unless( can_load( modules => $use_list ) ) { - my $which = join '/', sort keys %$use_list; - - $self->_error(loc( - "You do not have '%1' installed - Please ". - "install it as soon as possible.", $which) - ); - - return METHOD_NA; - } - - } elsif ( $self->is_tbz ) { - my $use_list = { 'IO::Uncompress::Bunzip2' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc( - "You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::Bunzip2') - ); - - return METHOD_NA; - } - - my $bz = IO::Uncompress::Bunzip2->new( $self->archive ) or - return $self->_error(loc("Unable to open '%1': %2", - $self->archive, - $IO::Uncompress::Bunzip2::Bunzip2Error)); - - $fh_to_read = $bz; - } elsif ( $self->is_txz ) { - my $use_list = { 'IO::Uncompress::UnXz' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc( - "You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::UnXz') - ); - - return METHOD_NA; - } - - my $xz = IO::Uncompress::UnXz->new( $self->archive ) or - return $self->_error(loc("Unable to open '%1': %2", - $self->archive, - $IO::Uncompress::UnXz::UnXzError)); - - $fh_to_read = $xz; - } - - my @files; - { - ### $Archive::Tar::WARN is 1 by default in Archive::Tar, but we've - ### localized $Archive::Tar::WARN already. - $Archive::Tar::WARN = $Archive::Extract::WARN; - - ### only tell it it's compressed if it's a .tgz, as we give it a file - ### handle if it's a .tbz - my @read = ( $fh_to_read, ( $self->is_tgz ? 1 : 0 ) ); - - ### for version of Archive::Tar > 1.04 - local $Archive::Tar::CHOWN = 0; - - ### use the iterator if we can. it's a feature of A::T 1.40 and up - if ( $_ALLOW_TAR_ITER && Archive::Tar->can( 'iter' ) ) { - - my $next; - unless ( $next = Archive::Tar->iter( @read ) ) { - return $self->_error(loc( - "Unable to read '%1': %2", $self->archive, - $Archive::Tar::error)); - } - - while ( my $file = $next->() ) { - push @files, $file->full_path; - - $file->extract or return $self->_error(loc( - "Unable to read '%1': %2", - $self->archive, - $Archive::Tar::error)); - } - - ### older version, read the archive into memory - } else { - - my $tar = Archive::Tar->new(); - - unless( $tar->read( @read ) ) { - return $self->_error(loc("Unable to read '%1': %2", - $self->archive, $Archive::Tar::error)); - } - - ### workaround to prevent Archive::Tar from setting uid, which - ### is a potential security hole. -autrijus - ### have to do it here, since A::T needs to be /loaded/ first ### - { no strict 'refs'; local $^W; - - ### older versions of archive::tar <= 0.23 - *Archive::Tar::chown = sub {}; - } - - { local $^W; # quell 'splice() offset past end of array' warnings - # on older versions of A::T - - ### older archive::tar always returns $self, return value - ### slightly fux0r3d because of it. - $tar->extract or return $self->_error(loc( - "Unable to extract '%1': %2", - $self->archive, $Archive::Tar::error )); - } - - @files = $tar->list_files; - } - } - - my $dir = $self->__get_extract_dir( \@files ); - - ### store the files that are in the archive ### - $self->files(\@files); - - ### store the extraction dir ### - $self->extract_path( $dir ); - - ### check if the dir actually appeared ### - return 1 if -d $self->extract_path; - - ### no dir, we failed ### - return $self->_error(loc("Unable to extract '%1': %2", - $self->archive, $Archive::Tar::error )); -} - -################################# -# -# Gunzip code -# -################################# - -sub _gunzip_bin { - my $self = shift; - - ### check for /bin/gzip -- we need it ### - unless( $self->bin_gzip ) { - $self->_error(loc("No '%1' program found", '/bin/gzip')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_gzip, '-c', '-d', '-f', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to gunzip '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -sub _gunzip_cz { - my $self = shift; - - my $use_list = { 'Compress::Zlib' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", 'Compress::Zlib')); - return METHOD_NA; - } - - my $gz = Compress::Zlib::gzopen( $self->archive, "rb" ) or - return $self->_error(loc("Unable to open '%1': %2", - $self->archive, $Compress::Zlib::gzerrno)); - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $buffer; - $self->_print($fh, $buffer) while $gz->gzread($buffer) > 0; - $fh->close; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -################################# -# -# Uncompress code -# -################################# - -sub _uncompress_bin { - my $self = shift; - - ### check for /bin/gzip -- we need it ### - unless( $self->bin_uncompress ) { - $self->_error(loc("No '%1' program found", '/bin/uncompress')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_uncompress, '-c', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - - -################################# -# -# Unzip code -# -################################# - - -sub _unzip_bin { - my $self = shift; - - ### check for /bin/gzip if we need it ### - unless( $self->bin_unzip ) { - $self->_error(loc("No '%1' program found", '/bin/unzip')); - return METHOD_NA; - } - - ### first, get the files.. it must be 2 different commands with 'unzip' :( - { ### on VMS, capital letter options have to be quoted. This is - ### reported by John Malmberg on P5P Tue 21 Aug 2007 05:05:11 - ### Subject: [patch@31735]Archive Extract fix on VMS. - my $opt = ON_VMS ? '"-Z"' : '-Z'; - my $cmd = [ $self->bin_unzip, $opt, '-1', $self->archive ]; - - my $buffer = ''; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unzip '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_files( $self->archive ) ); - - } else { - ### Annoyingly, pesky MSWin32 can either have 'native' tools - ### which have \r\n line endings or Cygwin-based tools which - ### have \n line endings. Jan Dubois suggested using this fix - my $split = ON_WIN32 ? qr/\r?\n/ : "\n"; - $self->files( [split $split, $buffer] ); - } - } - - ### now, extract the archive ### - { my $cmd = [ $self->bin_unzip, '-qq', '-o', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unzip '%1': %2", - $self->archive, $buffer)); - } - - if( scalar @{$self->files} ) { - my $files = $self->files; - my $dir = $self->__get_extract_dir( $files ); - - $self->extract_path( $dir ); - } - } - - return 1; -} - -sub _unzip_az { - my $self = shift; - - my $use_list = { 'Archive::Zip' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", 'Archive::Zip')); - return METHOD_NA; - } - - my $zip = Archive::Zip->new(); - - unless( $zip->read( $self->archive ) == &Archive::Zip::AZ_OK ) { - return $self->_error(loc("Unable to read '%1'", $self->archive)); - } - - my @files; - - - ### Address: #43278: Explicitly tell Archive::Zip where to put the files: - ### "In my BackPAN indexing, Archive::Zip was extracting things - ### in my script's directory instead of the current working directory. - ### I traced this back through Archive::Zip::_asLocalName which - ### eventually calls File::Spec::Win32::rel2abs which on Windows might - ### call Cwd::getdcwd. getdcwd returns the wrong directory in my - ### case, even though I think I'm on the same drive. - ### - ### To fix this, I pass the optional second argument to - ### extractMember using the cwd from Archive::Extract." --bdfoy - - ## store cwd() before looping; calls to cwd() can be expensive, and - ### it won't change during the loop - my $extract_dir = cwd(); - - ### have to extract every member individually ### - for my $member ($zip->members) { - push @files, $member->{fileName}; - - ### file to extract to, to avoid the above problem - my $to = File::Spec->catfile( $extract_dir, $member->{fileName} ); - - unless( $zip->extractMember($member, $to) == &Archive::Zip::AZ_OK ) { - return $self->_error(loc("Extraction of '%1' from '%2' failed", - $member->{fileName}, $self->archive )); - } - } - - my $dir = $self->__get_extract_dir( \@files ); - - ### set what files where extract, and where they went ### - $self->files( \@files ); - $self->extract_path( File::Spec->rel2abs($dir) ); - - return 1; -} - -sub __get_extract_dir { - my $self = shift; - my $files = shift || []; - - return unless scalar @$files; - - my($dir1, $dir2); - for my $aref ( [ \$dir1, 0 ], [ \$dir2, -1 ] ) { - my($dir,$pos) = @$aref; - - ### add a catdir(), so that any trailing slashes get - ### take care of (removed) - ### also, a catdir() normalises './dir/foo' to 'dir/foo'; - ### which was the problem in bug #23999 - my $res = -d $files->[$pos] - ? File::Spec->catdir( $files->[$pos], '' ) - : File::Spec->catdir( dirname( $files->[$pos] ) ); - - $$dir = $res; - } - - ### if the first and last dir don't match, make sure the - ### dirname is not set wrongly - my $dir; - - ### dirs are the same, so we know for sure what the extract dir is - if( $dir1 eq $dir2 ) { - $dir = $dir1; - - ### dirs are different.. do they share the base dir? - ### if so, use that, if not, fall back to '.' - } else { - my $base1 = [ File::Spec->splitdir( $dir1 ) ]->[0]; - my $base2 = [ File::Spec->splitdir( $dir2 ) ]->[0]; - - $dir = File::Spec->rel2abs( $base1 eq $base2 ? $base1 : '.' ); - } - - return File::Spec->rel2abs( $dir ); -} - -################################# -# -# Bunzip2 code -# -################################# - -sub _bunzip2_bin { - my $self = shift; - - ### check for /bin/gzip -- we need it ### - unless( $self->bin_bunzip2 ) { - $self->_error(loc("No '%1' program found", '/bin/bunzip2')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - ### guard against broken bunzip2. See ->have_old_bunzip2() - ### for details - if( $self->have_old_bunzip2 and $self->archive !~ /\.bz2$/i ) { - return $self->_error(loc("Your bunzip2 version is too old and ". - "can only extract files ending in '%1'", - '.bz2')); - } - - my $cmd = [ $self->bin_bunzip2, '-cd', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to bunzip2 '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -### using cz2, the compact versions... this we use mainly in archive::tar -### extractor.. -# sub _bunzip2_cz1 { -# my $self = shift; -# -# my $use_list = { 'IO::Uncompress::Bunzip2' => '0.0' }; -# unless( can_load( modules => $use_list ) ) { -# return $self->_error(loc("You do not have '%1' installed - Please " . -# "install it as soon as possible.", -# 'IO::Uncompress::Bunzip2')); -# } -# -# my $bz = IO::Uncompress::Bunzip2->new( $self->archive ) or -# return $self->_error(loc("Unable to open '%1': %2", -# $self->archive, -# $IO::Uncompress::Bunzip2::Bunzip2Error)); -# -# my $fh = FileHandle->new('>'. $self->_gunzip_to) or -# return $self->_error(loc("Could not open '%1' for writing: %2", -# $self->_gunzip_to, $! )); -# -# my $buffer; -# $fh->print($buffer) while $bz->read($buffer) > 0; -# $fh->close; -# -# ### set what files where extract, and where they went ### -# $self->files( [$self->_gunzip_to] ); -# $self->extract_path( File::Spec->rel2abs(cwd()) ); -# -# return 1; -# } - -sub _bunzip2_bz2 { - my $self = shift; - - my $use_list = { 'IO::Uncompress::Bunzip2' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::Bunzip2')); - return METHOD_NA; - } - - IO::Uncompress::Bunzip2::bunzip2($self->archive => $self->_gunzip_to) - or return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, - $IO::Uncompress::Bunzip2::Bunzip2Error)); - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -################################# -# -# UnXz code -# -################################# - -sub _unxz_bin { - my $self = shift; - - ### check for /bin/unxz -- we need it ### - unless( $self->bin_unxz ) { - $self->_error(loc("No '%1' program found", '/bin/unxz')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_unxz, '-c', '-d', '-f', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unxz '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -sub _unxz_cz { - my $self = shift; - - my $use_list = { 'IO::Uncompress::UnXz' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::UnXz')); - return METHOD_NA; - } - - IO::Uncompress::UnXz::unxz($self->archive => $self->_gunzip_to) - or return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, - $IO::Uncompress::UnXz::UnXzError)); - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - - -################################# -# -# unlzma code -# -################################# - -sub _unlzma_bin { - my $self = shift; - - ### check for /bin/unlzma -- we need it ### - unless( $self->bin_unlzma ) { - $self->_error(loc("No '%1' program found", '/bin/unlzma')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_unlzma, '-c', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unlzma '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -sub _unlzma_cz { - my $self = shift; - - my $use_list1 = { 'IO::Uncompress::UnLzma' => '0.0' }; - my $use_list2 = { 'Compress::unLZMA' => '0.0' }; - - if (can_load( modules => $use_list1 ) ) { - IO::Uncompress::UnLzma::unlzma($self->archive => $self->_gunzip_to) - or return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, - $IO::Uncompress::UnLzma::UnLzmaError)); - } - elsif (can_load( modules => $use_list2 ) ) { - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $buffer; - $buffer = Compress::unLZMA::uncompressfile( $self->archive ); - unless ( defined $buffer ) { - return $self->_error(loc("Could not unlzma '%1': %2", - $self->archive, $@)); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - } - else { - $self->_error(loc("You do not have '%1' or '%2' installed - Please " . - "install it as soon as possible.", 'Compress::unLZMA', 'IO::Uncompress::UnLzma')); - return METHOD_NA; - } - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -################################# -# -# Error code -# -################################# - -# For printing binaries that avoids interfering globals -sub _print { - my $self = shift; - my $fh = shift; - - local( $\, $", $, ) = ( undef, ' ', '' ); - return print $fh @_; -} - -sub _error { - my $self = shift; - my $error = shift; - my $lerror = Carp::longmess($error); - - push @{$self->_error_msg}, $error; - push @{$self->_error_msg_long}, $lerror; - - ### set $Archive::Extract::WARN to 0 to disable printing - ### of errors - if( $WARN ) { - carp $DEBUG ? $lerror : $error; - } - - return; -} - -sub error { - my $self = shift; - - ### make sure we have a fallback aref - my $aref = do { - shift() - ? $self->_error_msg_long - : $self->_error_msg - } || []; - - return join $/, @$aref; -} - -=head2 debug( MESSAGE ) - -This method outputs MESSAGE to the default filehandle if C<$DEBUG> is -true. It's a small method, but it's here if you'd like to subclass it -so you can so something else with any debugging output. - -=cut - -### this is really a stub for subclassing -sub debug { - return unless $DEBUG; - - print $_[1]; -} - -sub _no_buffer_files { - my $self = shift; - my $file = shift or return; - return loc("No buffer captured, unable to tell ". - "extracted files or extraction dir for '%1'", $file); -} - -sub _no_buffer_content { - my $self = shift; - my $file = shift or return; - return loc("No buffer captured, unable to get content for '%1'", $file); -} -1; - -=pod - -=head1 HOW IT WORKS - -C tries first to determine what type of archive you -are passing it, by inspecting its suffix. It does not do this by using -Mime magic, or something related. See C below. - -Once it has determined the file type, it knows which extraction methods -it can use on the archive. It will try a perl solution first, then fall -back to a commandline tool if that fails. If that also fails, it will -return false, indicating it was unable to extract the archive. -See the section on C to see how to alter this order. - -=head1 CAVEATS - -=head2 File Extensions - -C trusts on the extension of the archive to determine -what type it is, and what extractor methods therefore can be used. If -your archives do not have any of the extensions as described in the -C method, you will have to specify the type explicitly, or -C will not be able to extract the archive for you. - -=head2 Supporting Very Large Files - -C can use either pure perl modules or command line -programs under the hood. Some of the pure perl modules (like -C and Compress::unLZMA) take the entire contents of the archive into memory, -which may not be feasible on your system. Consider setting the global -variable C<$Archive::Extract::PREFER_BIN> to C<1>, which will prefer -the use of command line programs and won't consume so much memory. - -See the C section below for details. - -=head2 Bunzip2 support of arbitrary extensions. - -Older versions of C do not support arbitrary file -extensions and insist on a C<.bz2> suffix. Although we do our best -to guard against this, if you experience a bunzip2 error, it may -be related to this. For details, please see the C -method. - -=head1 GLOBAL VARIABLES - -=head2 $Archive::Extract::DEBUG - -Set this variable to C to have all calls to command line tools -be printed out, including all their output. -This also enables C errors, instead of the regular -C errors. - -Good for tracking down why things don't work with your particular -setup. - -Defaults to C. - -=head2 $Archive::Extract::WARN - -This variable controls whether errors encountered internally by -C should be C'd or not. - -Set to false to silence warnings. Inspect the output of the C -method manually to see what went wrong. - -Defaults to C. - -=head2 $Archive::Extract::PREFER_BIN - -This variables controls whether C should prefer the -use of perl modules, or commandline tools to extract archives. - -Set to C to have C prefer commandline tools. - -Defaults to C. - -=head1 TODO / CAVEATS - -=over 4 - -=item Mime magic support - -Maybe this module should use something like C to determine -the type, rather than blindly trust the suffix. - -=item Thread safety - -Currently, C does a C to the extraction dir before -extraction, and a C back again after. This is not necessarily -thread safe. See C bug C<#45671> for details. - -=back - -=head1 BUG REPORTS - -Please report bugs or other issues to Ebug-archive-extract@rt.cpan.orgE. - -=head1 AUTHOR - -This module by Jos Boumans Ekane@cpan.orgE. - -=head1 COPYRIGHT - -This library is free software; you may redistribute and/or modify it -under the same terms as Perl itself. - -=cut - -# Local variables: -# c-indentation-style: bsd -# c-basic-offset: 4 -# indent-tabs-mode: nil -# End: -# vim: expandtab shiftwidth=4: - diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip.pm deleted file mode 100644 index 0fdbf17bb39..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip.pm +++ /dev/null @@ -1,2136 +0,0 @@ -package Archive::Zip; - -use 5.006; -use strict; -use Carp (); -use Cwd (); -use IO::File (); -use IO::Seekable (); -use Compress::Raw::Zlib (); -use File::Spec (); -use File::Temp (); -use FileHandle (); - -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - - require Exporter; - @ISA = qw( Exporter ); -} - -use vars qw( $ChunkSize $ErrorHandler ); - -BEGIN { - # This is the size we'll try to read, write, and (de)compress. - # You could set it to something different if you had lots of memory - # and needed more speed. - $ChunkSize ||= 32768; - - $ErrorHandler = \&Carp::carp; -} - -# BEGIN block is necessary here so that other modules can use the constants. -use vars qw( @EXPORT_OK %EXPORT_TAGS ); - -BEGIN { - @EXPORT_OK = ('computeCRC32'); - %EXPORT_TAGS = ( - CONSTANTS => [ - qw( - FA_MSDOS - FA_UNIX - GPBF_ENCRYPTED_MASK - GPBF_DEFLATING_COMPRESSION_MASK - GPBF_HAS_DATA_DESCRIPTOR_MASK - COMPRESSION_STORED - COMPRESSION_DEFLATED - COMPRESSION_LEVEL_NONE - COMPRESSION_LEVEL_DEFAULT - COMPRESSION_LEVEL_FASTEST - COMPRESSION_LEVEL_BEST_COMPRESSION - IFA_TEXT_FILE_MASK - IFA_TEXT_FILE - IFA_BINARY_FILE - ) - ], - - MISC_CONSTANTS => [ - qw( - FA_AMIGA - FA_VAX_VMS - FA_VM_CMS - FA_ATARI_ST - FA_OS2_HPFS - FA_MACINTOSH - FA_Z_SYSTEM - FA_CPM - FA_TOPS20 - FA_WINDOWS_NTFS - FA_QDOS - FA_ACORN - FA_VFAT - FA_MVS - FA_BEOS - FA_TANDEM - FA_THEOS - GPBF_IMPLODING_8K_SLIDING_DICTIONARY_MASK - GPBF_IMPLODING_3_SHANNON_FANO_TREES_MASK - GPBF_IS_COMPRESSED_PATCHED_DATA_MASK - COMPRESSION_SHRUNK - DEFLATING_COMPRESSION_NORMAL - DEFLATING_COMPRESSION_MAXIMUM - DEFLATING_COMPRESSION_FAST - DEFLATING_COMPRESSION_SUPER_FAST - COMPRESSION_REDUCED_1 - COMPRESSION_REDUCED_2 - COMPRESSION_REDUCED_3 - COMPRESSION_REDUCED_4 - COMPRESSION_IMPLODED - COMPRESSION_TOKENIZED - COMPRESSION_DEFLATED_ENHANCED - COMPRESSION_PKWARE_DATA_COMPRESSION_LIBRARY_IMPLODED - ) - ], - - ERROR_CODES => [ - qw( - AZ_OK - AZ_STREAM_END - AZ_ERROR - AZ_FORMAT_ERROR - AZ_IO_ERROR - ) - ], - - # For Internal Use Only - PKZIP_CONSTANTS => [ - qw( - SIGNATURE_FORMAT - SIGNATURE_LENGTH - LOCAL_FILE_HEADER_SIGNATURE - LOCAL_FILE_HEADER_FORMAT - LOCAL_FILE_HEADER_LENGTH - CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE - DATA_DESCRIPTOR_FORMAT - DATA_DESCRIPTOR_LENGTH - DATA_DESCRIPTOR_SIGNATURE - DATA_DESCRIPTOR_FORMAT_NO_SIG - DATA_DESCRIPTOR_LENGTH_NO_SIG - CENTRAL_DIRECTORY_FILE_HEADER_FORMAT - CENTRAL_DIRECTORY_FILE_HEADER_LENGTH - END_OF_CENTRAL_DIRECTORY_SIGNATURE - END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING - END_OF_CENTRAL_DIRECTORY_FORMAT - END_OF_CENTRAL_DIRECTORY_LENGTH - ) - ], - - # For Internal Use Only - UTILITY_METHODS => [ - qw( - _error - _printError - _ioError - _formatError - _subclassResponsibility - _binmode - _isSeekable - _newFileHandle - _readSignature - _asZipDirName - ) - ], - ); - - # Add all the constant names and error code names to @EXPORT_OK - Exporter::export_ok_tags( - qw( - CONSTANTS - ERROR_CODES - PKZIP_CONSTANTS - UTILITY_METHODS - MISC_CONSTANTS - )); - -} - -# Error codes -use constant AZ_OK => 0; -use constant AZ_STREAM_END => 1; -use constant AZ_ERROR => 2; -use constant AZ_FORMAT_ERROR => 3; -use constant AZ_IO_ERROR => 4; - -# File types -# Values of Archive::Zip::Member->fileAttributeFormat() - -use constant FA_MSDOS => 0; -use constant FA_AMIGA => 1; -use constant FA_VAX_VMS => 2; -use constant FA_UNIX => 3; -use constant FA_VM_CMS => 4; -use constant FA_ATARI_ST => 5; -use constant FA_OS2_HPFS => 6; -use constant FA_MACINTOSH => 7; -use constant FA_Z_SYSTEM => 8; -use constant FA_CPM => 9; -use constant FA_TOPS20 => 10; -use constant FA_WINDOWS_NTFS => 11; -use constant FA_QDOS => 12; -use constant FA_ACORN => 13; -use constant FA_VFAT => 14; -use constant FA_MVS => 15; -use constant FA_BEOS => 16; -use constant FA_TANDEM => 17; -use constant FA_THEOS => 18; - -# general-purpose bit flag masks -# Found in Archive::Zip::Member->bitFlag() - -use constant GPBF_ENCRYPTED_MASK => 1 << 0; -use constant GPBF_DEFLATING_COMPRESSION_MASK => 3 << 1; -use constant GPBF_HAS_DATA_DESCRIPTOR_MASK => 1 << 3; - -# deflating compression types, if compressionMethod == COMPRESSION_DEFLATED -# ( Archive::Zip::Member->bitFlag() & GPBF_DEFLATING_COMPRESSION_MASK ) - -use constant DEFLATING_COMPRESSION_NORMAL => 0 << 1; -use constant DEFLATING_COMPRESSION_MAXIMUM => 1 << 1; -use constant DEFLATING_COMPRESSION_FAST => 2 << 1; -use constant DEFLATING_COMPRESSION_SUPER_FAST => 3 << 1; - -# compression method - -# these two are the only ones supported in this module -use constant COMPRESSION_STORED => 0; # file is stored (no compression) -use constant COMPRESSION_DEFLATED => 8; # file is Deflated -use constant COMPRESSION_LEVEL_NONE => 0; -use constant COMPRESSION_LEVEL_DEFAULT => -1; -use constant COMPRESSION_LEVEL_FASTEST => 1; -use constant COMPRESSION_LEVEL_BEST_COMPRESSION => 9; - -# internal file attribute bits -# Found in Archive::Zip::Member::internalFileAttributes() - -use constant IFA_TEXT_FILE_MASK => 1; -use constant IFA_TEXT_FILE => 1; -use constant IFA_BINARY_FILE => 0; - -# PKZIP file format miscellaneous constants (for internal use only) -use constant SIGNATURE_FORMAT => "V"; -use constant SIGNATURE_LENGTH => 4; - -# these lengths are without the signature. -use constant LOCAL_FILE_HEADER_SIGNATURE => 0x04034b50; -use constant LOCAL_FILE_HEADER_FORMAT => "v3 V4 v2"; -use constant LOCAL_FILE_HEADER_LENGTH => 26; - -# PKZIP docs don't mention the signature, but Info-Zip writes it. -use constant DATA_DESCRIPTOR_SIGNATURE => 0x08074b50; -use constant DATA_DESCRIPTOR_FORMAT => "V3"; -use constant DATA_DESCRIPTOR_LENGTH => 12; - -# but the signature is apparently optional. -use constant DATA_DESCRIPTOR_FORMAT_NO_SIG => "V2"; -use constant DATA_DESCRIPTOR_LENGTH_NO_SIG => 8; - -use constant CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE => 0x02014b50; -use constant CENTRAL_DIRECTORY_FILE_HEADER_FORMAT => "C2 v3 V4 v5 V2"; -use constant CENTRAL_DIRECTORY_FILE_HEADER_LENGTH => 42; - -use constant END_OF_CENTRAL_DIRECTORY_SIGNATURE => 0x06054b50; -use constant END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING => - pack("V", END_OF_CENTRAL_DIRECTORY_SIGNATURE); -use constant END_OF_CENTRAL_DIRECTORY_FORMAT => "v4 V2 v"; -use constant END_OF_CENTRAL_DIRECTORY_LENGTH => 18; - -use constant GPBF_IMPLODING_8K_SLIDING_DICTIONARY_MASK => 1 << 1; -use constant GPBF_IMPLODING_3_SHANNON_FANO_TREES_MASK => 1 << 2; -use constant GPBF_IS_COMPRESSED_PATCHED_DATA_MASK => 1 << 5; - -# the rest of these are not supported in this module -use constant COMPRESSION_SHRUNK => 1; # file is Shrunk -use constant COMPRESSION_REDUCED_1 => 2; # file is Reduced CF=1 -use constant COMPRESSION_REDUCED_2 => 3; # file is Reduced CF=2 -use constant COMPRESSION_REDUCED_3 => 4; # file is Reduced CF=3 -use constant COMPRESSION_REDUCED_4 => 5; # file is Reduced CF=4 -use constant COMPRESSION_IMPLODED => 6; # file is Imploded -use constant COMPRESSION_TOKENIZED => 7; # reserved for Tokenizing compr. -use constant COMPRESSION_DEFLATED_ENHANCED => 9; # reserved for enh. Deflating -use constant COMPRESSION_PKWARE_DATA_COMPRESSION_LIBRARY_IMPLODED => 10; - -# Load the various required classes -require Archive::Zip::Archive; -require Archive::Zip::Member; -require Archive::Zip::FileMember; -require Archive::Zip::DirectoryMember; -require Archive::Zip::ZipFileMember; -require Archive::Zip::NewFileMember; -require Archive::Zip::StringMember; - -# Convenience functions - -sub _ISA ($$) { - - # Can't rely on Scalar::Util, so use the next best way - local $@; - !!eval { ref $_[0] and $_[0]->isa($_[1]) }; -} - -sub _CAN ($$) { - local $@; - !!eval { ref $_[0] and $_[0]->can($_[1]) }; -} - -##################################################################### -# Methods - -sub new { - my $class = shift; - return Archive::Zip::Archive->new(@_); -} - -sub computeCRC32 { - my ($data, $crc); - - if (ref($_[0]) eq 'HASH') { - $data = $_[0]->{string}; - $crc = $_[0]->{checksum}; - } else { - $data = shift; - $data = shift if ref($data); - $crc = shift; - } - - return Compress::Raw::Zlib::crc32($data, $crc); -} - -# Report or change chunk size used for reading and writing. -# Also sets Zlib's default buffer size (eventually). -sub setChunkSize { - shift if ref($_[0]) eq 'Archive::Zip::Archive'; - my $chunkSize = (ref($_[0]) eq 'HASH') ? shift->{chunkSize} : shift; - my $oldChunkSize = $Archive::Zip::ChunkSize; - $Archive::Zip::ChunkSize = $chunkSize if ($chunkSize); - return $oldChunkSize; -} - -sub chunkSize { - return $Archive::Zip::ChunkSize; -} - -sub setErrorHandler { - my $errorHandler = (ref($_[0]) eq 'HASH') ? shift->{subroutine} : shift; - $errorHandler = \&Carp::carp unless defined($errorHandler); - my $oldErrorHandler = $Archive::Zip::ErrorHandler; - $Archive::Zip::ErrorHandler = $errorHandler; - return $oldErrorHandler; -} - -###################################################################### -# Private utility functions (not methods). - -sub _printError { - my $string = join(' ', @_, "\n"); - my $oldCarpLevel = $Carp::CarpLevel; - $Carp::CarpLevel += 2; - &{$ErrorHandler}($string); - $Carp::CarpLevel = $oldCarpLevel; -} - -# This is called on format errors. -sub _formatError { - shift if ref($_[0]); - _printError('format error:', @_); - return AZ_FORMAT_ERROR; -} - -# This is called on IO errors. -sub _ioError { - shift if ref($_[0]); - _printError('IO error:', @_, ':', $!); - return AZ_IO_ERROR; -} - -# This is called on generic errors. -sub _error { - shift if ref($_[0]); - _printError('error:', @_); - return AZ_ERROR; -} - -# Called when a subclass should have implemented -# something but didn't -sub _subclassResponsibility { - Carp::croak("subclass Responsibility\n"); -} - -# Try to set the given file handle or object into binary mode. -sub _binmode { - my $fh = shift; - return _CAN($fh, 'binmode') ? $fh->binmode() : binmode($fh); -} - -# Attempt to guess whether file handle is seekable. -# Because of problems with Windows, this only returns true when -# the file handle is a real file. -sub _isSeekable { - my $fh = shift; - return 0 unless ref $fh; - _ISA($fh, "IO::Scalar") # IO::Scalar objects are brokenly-seekable - and return 0; - _ISA($fh, "IO::String") - and return 1; - if (_ISA($fh, "IO::Seekable")) { - - # Unfortunately, some things like FileHandle objects - # return true for Seekable, but AREN'T!!!!! - _ISA($fh, "FileHandle") - and return 0; - return 1; - } - - # open my $fh, "+<", \$data; - ref $fh eq "GLOB" && eval { seek $fh, 0, 1 } and return 1; - _CAN($fh, "stat") - and return -f $fh; - return (_CAN($fh, "seek") and _CAN($fh, "tell")) ? 1 : 0; -} - -# Print to the filehandle, while making sure the pesky Perl special global -# variables don't interfere. -sub _print { - my ($self, $fh, @data) = @_; - - local $\; - - return $fh->print(@data); -} - -# Return an opened IO::Handle -# my ( $status, fh ) = _newFileHandle( 'fileName', 'w' ); -# Can take a filename, file handle, or ref to GLOB -# Or, if given something that is a ref but not an IO::Handle, -# passes back the same thing. -sub _newFileHandle { - my $fd = shift; - my $status = 1; - my $handle; - - if (ref($fd)) { - if (_ISA($fd, 'IO::Scalar') or _ISA($fd, 'IO::String')) { - $handle = $fd; - } elsif (_ISA($fd, 'IO::Handle') or ref($fd) eq 'GLOB') { - $handle = IO::File->new; - $status = $handle->fdopen($fd, @_); - } else { - $handle = $fd; - } - } else { - $handle = IO::File->new; - $status = $handle->open($fd, @_); - } - - return ($status, $handle); -} - -# Returns next signature from given file handle, leaves -# file handle positioned afterwards. -# In list context, returns ($status, $signature) -# ( $status, $signature) = _readSignature( $fh, $fileName ); - -sub _readSignature { - my $fh = shift; - my $fileName = shift; - my $expectedSignature = shift; # optional - - my $signatureData; - my $bytesRead = $fh->read($signatureData, SIGNATURE_LENGTH); - if ($bytesRead != SIGNATURE_LENGTH) { - return _ioError("reading header signature"); - } - my $signature = unpack(SIGNATURE_FORMAT, $signatureData); - my $status = AZ_OK; - - # compare with expected signature, if any, or any known signature. - if ( - (defined($expectedSignature) && $signature != $expectedSignature) - || ( !defined($expectedSignature) - && $signature != CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE - && $signature != LOCAL_FILE_HEADER_SIGNATURE - && $signature != END_OF_CENTRAL_DIRECTORY_SIGNATURE - && $signature != DATA_DESCRIPTOR_SIGNATURE) - ) { - my $errmsg = sprintf("bad signature: 0x%08x", $signature); - if (_isSeekable($fh)) { - $errmsg .= sprintf(" at offset %d", $fh->tell() - SIGNATURE_LENGTH); - } - - $status = _formatError("$errmsg in file $fileName"); - } - - return ($status, $signature); -} - -# Utility method to make and open a temp file. -# Will create $temp_dir if it does not exist. -# Returns file handle and name: -# -# my ($fh, $name) = Archive::Zip::tempFile(); -# my ($fh, $name) = Archive::Zip::tempFile('mytempdir'); -# - -sub tempFile { - my $dir = (ref($_[0]) eq 'HASH') ? shift->{tempDir} : shift; - my ($fh, $filename) = File::Temp::tempfile( - SUFFIX => '.zip', - UNLINK => 1, - $dir ? (DIR => $dir) : ()); - return (undef, undef) unless $fh; - my ($status, $newfh) = _newFileHandle($fh, 'w+'); - return ($newfh, $filename); -} - -# Return the normalized directory name as used in a zip file (path -# separators become slashes, etc.). -# Will translate internal slashes in path components (i.e. on Macs) to -# underscores. Discards volume names. -# When $forceDir is set, returns paths with trailing slashes (or arrays -# with trailing blank members). -# -# If third argument is a reference, returns volume information there. -# -# input output -# . ('.') '.' -# ./a ('a') a -# ./a/b ('a','b') a/b -# ./a/b/ ('a','b') a/b -# a/b/ ('a','b') a/b -# /a/b/ ('','a','b') a/b -# c:\a\b\c.doc ('','a','b','c.doc') a/b/c.doc # on Windows -# "i/o maps:whatever" ('i_o maps', 'whatever') "i_o maps/whatever" # on Macs -sub _asZipDirName { - my $name = shift; - my $forceDir = shift; - my $volReturn = shift; - my ($volume, $directories, $file) = - File::Spec->splitpath(File::Spec->canonpath($name), $forceDir); - $$volReturn = $volume if (ref($volReturn)); - my @dirs = map { $_ =~ s{/}{_}g; $_ } File::Spec->splitdir($directories); - if (@dirs > 0) { pop(@dirs) unless $dirs[-1] } # remove empty component - push(@dirs, defined($file) ? $file : ''); - - #return wantarray ? @dirs : join ( '/', @dirs ); - - my $normalised_path = join '/', @dirs; - - # Leading directory separators should not be stored in zip archives. - # Example: - # C:\a\b\c\ a/b/c - # C:\a\b\c.txt a/b/c.txt - # /a/b/c/ a/b/c - # /a/b/c.txt a/b/c.txt - $normalised_path =~ s{^/}{}; # remove leading separator - - return $normalised_path; -} - -# Return an absolute local name for a zip name. -# Assume a directory if zip name has trailing slash. -# Takes an optional volume name in FS format (like 'a:'). -# -sub _asLocalName { - my $name = shift; # zip format - my $volume = shift; - $volume = '' unless defined($volume); # local FS format - - my @paths = split(/\//, $name); - my $filename = pop(@paths); - $filename = '' unless defined($filename); - my $localDirs = @paths ? File::Spec->catdir(@paths) : ''; - my $localName = File::Spec->catpath($volume, $localDirs, $filename); - unless ($volume) { - $localName = File::Spec->rel2abs($localName, Cwd::getcwd()); - } - return $localName; -} - -1; - -__END__ - -=pod - -=encoding utf8 - -=head1 NAME - -Archive::Zip - Provide an interface to ZIP archive files. - -=head1 SYNOPSIS - - # Create a Zip file - use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); - my $zip = Archive::Zip->new(); - - # Add a directory - my $dir_member = $zip->addDirectory( 'dirname/' ); - - # Add a file from a string with compression - my $string_member = $zip->addString( 'This is a test', 'stringMember.txt' ); - $string_member->desiredCompressionMethod( COMPRESSION_DEFLATED ); - - # Add a file from disk - my $file_member = $zip->addFile( 'xyz.pl', 'AnotherName.pl' ); - - # Save the Zip file - unless ( $zip->writeToFileNamed('someZip.zip') == AZ_OK ) { - die 'write error'; - } - - # Read a Zip file - my $somezip = Archive::Zip->new(); - unless ( $somezip->read( 'someZip.zip' ) == AZ_OK ) { - die 'read error'; - } - - # Change the compression type for a file in the Zip - my $member = $somezip->memberNamed( 'stringMember.txt' ); - $member->desiredCompressionMethod( COMPRESSION_STORED ); - unless ( $zip->writeToFileNamed( 'someOtherZip.zip' ) == AZ_OK ) { - die 'write error'; - } - -=head1 DESCRIPTION - -The Archive::Zip module allows a Perl program to create, manipulate, read, -and write Zip archive files. - -Zip archives can be created, or you can read from existing zip files. - -Once created, they can be written to files, streams, or strings. Members -can be added, removed, extracted, replaced, rearranged, and enumerated. -They can also be renamed or have their dates, comments, or other attributes -queried or modified. Their data can be compressed or uncompressed as needed. - -Members can be created from members in existing Zip files, or from existing -directories, files, or strings. - -This module uses the L library to read and write the -compressed streams inside the files. - -One can use L to read the zip file archive members -as if they were files. - -=head2 File Naming - -Regardless of what your local file system uses for file naming, names in a -Zip file are in Unix format (I slashes (/) separating directory -names, etc.). - -C tries to be consistent with file naming conventions, and will -translate back and forth between native and Zip file names. - -However, it can't guess which format names are in. So two rules control what -kind of file name you must pass various routines: - -=over 4 - -=item Names of files are in local format. - -C and C are used for various file -operations. When you're referring to a file on your system, use its -file naming conventions. - -=item Names of archive members are in Unix format. - -This applies to every method that refers to an archive member, or -provides a name for new archive members. The C methods -that can take one or two names will convert from local to zip names -if you call them with a single name. - -=back - -=head2 Archive::Zip Object Model - -=head3 Overview - -Archive::Zip::Archive objects are what you ordinarily deal with. -These maintain the structure of a zip file, without necessarily -holding data. When a zip is read from a disk file, the (possibly -compressed) data still lives in the file, not in memory. Archive -members hold information about the individual members, but not -(usually) the actual member data. When the zip is written to a -(different) file, the member data is compressed or copied as needed. -It is possible to make archive members whose data is held in a string -in memory, but this is not done when a zip file is read. Directory -members don't have any data. - -=head2 Inheritance - - Exporter - Archive::Zip Common base class, has defs. - Archive::Zip::Archive A Zip archive. - Archive::Zip::Member Abstract superclass for all members. - Archive::Zip::StringMember Member made from a string - Archive::Zip::FileMember Member made from an external file - Archive::Zip::ZipFileMember Member that lives in a zip file - Archive::Zip::NewFileMember Member whose data is in a file - Archive::Zip::DirectoryMember Member that is a directory - -=head1 EXPORTS - -=over 4 - -=item :CONSTANTS - -Exports the following constants: - -FA_MSDOS FA_UNIX GPBF_ENCRYPTED_MASK -GPBF_DEFLATING_COMPRESSION_MASK GPBF_HAS_DATA_DESCRIPTOR_MASK -COMPRESSION_STORED COMPRESSION_DEFLATED IFA_TEXT_FILE_MASK -IFA_TEXT_FILE IFA_BINARY_FILE COMPRESSION_LEVEL_NONE -COMPRESSION_LEVEL_DEFAULT COMPRESSION_LEVEL_FASTEST -COMPRESSION_LEVEL_BEST_COMPRESSION - -=item :MISC_CONSTANTS - -Exports the following constants (only necessary for extending the -module): - -FA_AMIGA FA_VAX_VMS FA_VM_CMS FA_ATARI_ST FA_OS2_HPFS -FA_MACINTOSH FA_Z_SYSTEM FA_CPM FA_WINDOWS_NTFS -GPBF_IMPLODING_8K_SLIDING_DICTIONARY_MASK -GPBF_IMPLODING_3_SHANNON_FANO_TREES_MASK -GPBF_IS_COMPRESSED_PATCHED_DATA_MASK COMPRESSION_SHRUNK -DEFLATING_COMPRESSION_NORMAL DEFLATING_COMPRESSION_MAXIMUM -DEFLATING_COMPRESSION_FAST DEFLATING_COMPRESSION_SUPER_FAST -COMPRESSION_REDUCED_1 COMPRESSION_REDUCED_2 COMPRESSION_REDUCED_3 -COMPRESSION_REDUCED_4 COMPRESSION_IMPLODED COMPRESSION_TOKENIZED -COMPRESSION_DEFLATED_ENHANCED -COMPRESSION_PKWARE_DATA_COMPRESSION_LIBRARY_IMPLODED - -=item :ERROR_CODES - -Explained below. Returned from most methods. - -AZ_OK AZ_STREAM_END AZ_ERROR AZ_FORMAT_ERROR AZ_IO_ERROR - -=back - -=head1 ERROR CODES - -Many of the methods in Archive::Zip return error codes. These are implemented -as inline subroutines, using the C pragma. They can be imported -into your namespace using the C<:ERROR_CODES> tag: - - use Archive::Zip qw( :ERROR_CODES ); - - ... - - unless ( $zip->read( 'myfile.zip' ) == AZ_OK ) { - die "whoops!"; - } - -=over 4 - -=item AZ_OK (0) - -Everything is fine. - -=item AZ_STREAM_END (1) - -The read stream (or central directory) ended normally. - -=item AZ_ERROR (2) - -There was some generic kind of error. - -=item AZ_FORMAT_ERROR (3) - -There is a format error in a ZIP file being read. - -=item AZ_IO_ERROR (4) - -There was an IO error. - -=back - -=head2 Compression - -Archive::Zip allows each member of a ZIP file to be compressed (using the -Deflate algorithm) or uncompressed. - -Other compression algorithms that some versions of ZIP have been able to -produce are not supported. Each member has two compression methods: the -one it's stored as (this is always COMPRESSION_STORED for string and external -file members), and the one you desire for the member in the zip file. - -These can be different, of course, so you can make a zip member that is not -compressed out of one that is, and vice versa. - -You can inquire about the current compression and set the desired -compression method: - - my $member = $zip->memberNamed( 'xyz.txt' ); - $member->compressionMethod(); # return current compression - - # set to read uncompressed - $member->desiredCompressionMethod( COMPRESSION_STORED ); - - # set to read compressed - $member->desiredCompressionMethod( COMPRESSION_DEFLATED ); - -There are two different compression methods: - -=over 4 - -=item COMPRESSION_STORED - -File is stored (no compression) - -=item COMPRESSION_DEFLATED - -File is Deflated - -=back - -=head2 Compression Levels - -If a member's desiredCompressionMethod is COMPRESSION_DEFLATED, you -can choose different compression levels. This choice may affect the -speed of compression and decompression, as well as the size of the -compressed member data. - - $member->desiredCompressionLevel( 9 ); - -The levels given can be: - -=over 4 - -=item * 0 or COMPRESSION_LEVEL_NONE - -This is the same as saying - - $member->desiredCompressionMethod( COMPRESSION_STORED ); - -=item * 1 .. 9 - -1 gives the best speed and worst compression, and 9 gives the -best compression and worst speed. - -=item * COMPRESSION_LEVEL_FASTEST - -This is a synonym for level 1. - -=item * COMPRESSION_LEVEL_BEST_COMPRESSION - -This is a synonym for level 9. - -=item * COMPRESSION_LEVEL_DEFAULT - -This gives a good compromise between speed and compression, -and is currently equivalent to 6 (this is in the zlib code). -This is the level that will be used if not specified. - -=back - -=head1 Archive::Zip Methods - -The Archive::Zip class (and its invisible subclass Archive::Zip::Archive) -implement generic zip file functionality. Creating a new Archive::Zip object -actually makes an Archive::Zip::Archive object, but you don't have to worry -about this unless you're subclassing. - -=head2 Constructor - -=over 4 - -=item new( [$fileName] ) - -=item new( { filename => $fileName } ) - -Make a new, empty zip archive. - - my $zip = Archive::Zip->new(); - -If an additional argument is passed, new() will call read() -to read the contents of an archive: - - my $zip = Archive::Zip->new( 'xyz.zip' ); - -If a filename argument is passed and the read fails for any -reason, new will return undef. For this reason, it may be -better to call read separately. - -=back - -=head2 Zip Archive Utility Methods - -These Archive::Zip methods may be called as functions or as object -methods. Do not call them as class methods: - - $zip = Archive::Zip->new(); - $crc = Archive::Zip::computeCRC32( 'ghijkl' ); # OK - $crc = $zip->computeCRC32( 'ghijkl' ); # also OK - $crc = Archive::Zip->computeCRC32( 'ghijkl' ); # NOT OK - -=over 4 - -=item Archive::Zip::computeCRC32( $string [, $crc] ) - -=item Archive::Zip::computeCRC32( { string => $string [, checksum => $crc ] } ) - -This is a utility function that uses the Compress::Raw::Zlib CRC -routine to compute a CRC-32. You can get the CRC of a string: - - $crc = Archive::Zip::computeCRC32( $string ); - -Or you can compute the running CRC: - - $crc = 0; - $crc = Archive::Zip::computeCRC32( 'abcdef', $crc ); - $crc = Archive::Zip::computeCRC32( 'ghijkl', $crc ); - -=item Archive::Zip::setChunkSize( $number ) - -=item Archive::Zip::setChunkSize( { chunkSize => $number } ) - -Report or change chunk size used for reading and writing. -This can make big differences in dealing with large files. -Currently, this defaults to 32K. This also changes the chunk -size used for Compress::Raw::Zlib. You must call setChunkSize() -before reading or writing. This is not exportable, so you -must call it like: - - Archive::Zip::setChunkSize( 4096 ); - -or as a method on a zip (though this is a global setting). -Returns old chunk size. - -=item Archive::Zip::chunkSize() - -Returns the current chunk size: - - my $chunkSize = Archive::Zip::chunkSize(); - -=item Archive::Zip::setErrorHandler( \&subroutine ) - -=item Archive::Zip::setErrorHandler( { subroutine => \&subroutine } ) - -Change the subroutine called with error strings. This -defaults to \&Carp::carp, but you may want to change it to -get the error strings. This is not exportable, so you must -call it like: - - Archive::Zip::setErrorHandler( \&myErrorHandler ); - -If myErrorHandler is undef, resets handler to default. -Returns old error handler. Note that if you call Carp::carp -or a similar routine or if you're chaining to the default -error handler from your error handler, you may want to -increment the number of caller levels that are skipped (do -not just set it to a number): - - $Carp::CarpLevel++; - -=item Archive::Zip::tempFile( [ $tmpdir ] ) - -=item Archive::Zip::tempFile( { tempDir => $tmpdir } ) - -Create a uniquely named temp file. It will be returned open -for read/write. If C<$tmpdir> is given, it is used as the -name of a directory to create the file in. If not given, -creates the file using C. Generally, you can -override this choice using the - - $ENV{TMPDIR} - -environment variable. But see the L -documentation for your system. Note that on many systems, if you're -running in taint mode, then you must make sure that C<$ENV{TMPDIR}> is -untainted for it to be used. -Will I create C<$tmpdir> if it does not exist (this is a change -from prior versions!). Returns file handle and name: - - my ($fh, $name) = Archive::Zip::tempFile(); - my ($fh, $name) = Archive::Zip::tempFile('myTempDir'); - my $fh = Archive::Zip::tempFile(); # if you don't need the name - -=back - -=head2 Zip Archive Accessors - -=over 4 - -=item members() - -Return a copy of the members array - - my @members = $zip->members(); - -=item numberOfMembers() - -Return the number of members I have - -=item memberNames() - -Return a list of the (internal) file names of the zip members - -=item memberNamed( $string ) - -=item memberNamed( { zipName => $string } ) - -Return ref to member whose filename equals given filename or -undef. C<$string> must be in Zip (Unix) filename format. - -=item membersMatching( $regex ) - -=item membersMatching( { regex => $regex } ) - -Return array of members whose filenames match given regular -expression in list context. Returns number of matching -members in scalar context. - - my @textFileMembers = $zip->membersMatching( '.*\.txt' ); - # or - my $numberOfTextFiles = $zip->membersMatching( '.*\.txt' ); - -=item diskNumber() - -Return the disk that I start on. Not used for writing zips, -but might be interesting if you read a zip in. This should be -0, as Archive::Zip does not handle multi-volume archives. - -=item diskNumberWithStartOfCentralDirectory() - -Return the disk number that holds the beginning of the -central directory. Not used for writing zips, but might be -interesting if you read a zip in. This should be 0, as -Archive::Zip does not handle multi-volume archives. - -=item numberOfCentralDirectoriesOnThisDisk() - -Return the number of CD structures in the zipfile last read in. -Not used for writing zips, but might be interesting if you read a zip -in. - -=item numberOfCentralDirectories() - -Return the number of CD structures in the zipfile last read in. -Not used for writing zips, but might be interesting if you read a zip -in. - -=item centralDirectorySize() - -Returns central directory size, as read from an external zip -file. Not used for writing zips, but might be interesting if -you read a zip in. - -=item centralDirectoryOffsetWRTStartingDiskNumber() - -Returns the offset into the zip file where the CD begins. Not -used for writing zips, but might be interesting if you read a -zip in. - -=item zipfileComment( [ $string ] ) - -=item zipfileComment( [ { comment => $string } ] ) - -Get or set the zipfile comment. Returns the old comment. - - print $zip->zipfileComment(); - $zip->zipfileComment( 'New Comment' ); - -=item eocdOffset() - -Returns the (unexpected) number of bytes between where the -EOCD was found and where it expected to be. This is normally -0, but would be positive if something (a virus, perhaps) had -added bytes somewhere before the EOCD. Not used for writing -zips, but might be interesting if you read a zip in. Here is -an example of how you can diagnose this: - - my $zip = Archive::Zip->new('somefile.zip'); - if ($zip->eocdOffset()) - { - warn "A virus has added ", $zip->eocdOffset, " bytes of garbage\n"; - } - -The C is used to adjust the starting position of member -headers, if necessary. - -=item fileName() - -Returns the name of the file last read from. If nothing has -been read yet, returns an empty string; if read from a file -handle, returns the handle in string form. - -=back - -=head2 Zip Archive Member Operations - -Various operations on a zip file modify members. When a member is -passed as an argument, you can either use a reference to the member -itself, or the name of a member. Of course, using the name requires -that names be unique within a zip (this is not enforced). - -=over 4 - -=item removeMember( $memberOrName ) - -=item removeMember( { memberOrZipName => $memberOrName } ) - -Remove and return the given member, or match its name and -remove it. Returns undef if member or name does not exist in this -Zip. No-op if member does not belong to this zip. - -=item replaceMember( $memberOrName, $newMember ) - -=item replaceMember( { memberOrZipName => $memberOrName, - newMember => $newMember } ) - -Remove and return the given member, or match its name and -remove it. Replace with new member. Returns undef if member or -name does not exist in this Zip, or if C<$newMember> is undefined. - -It is an (undiagnosed) error to provide a C<$newMember> that is a -member of the zip being modified. - - my $member1 = $zip->removeMember( 'xyz' ); - my $member2 = $zip->replaceMember( 'abc', $member1 ); - # now, $member2 (named 'abc') is not in $zip, - # and $member1 (named 'xyz') is, having taken $member2's place. - -=item extractMember( $memberOrName [, $extractedName ] ) - -=item extractMember( { memberOrZipName => $memberOrName - [, name => $extractedName ] } ) - -Extract the given member, or match its name and extract it. -Returns undef if member does not exist in this Zip. If -optional second arg is given, use it as the name of the -extracted member. Otherwise, the internal filename of the -member is used as the name of the extracted file or -directory. -If you pass C<$extractedName>, it should be in the local file -system's format. -All necessary directories will be created. Returns C -on success. - -=item extractMemberWithoutPaths( $memberOrName [, $extractedName ] ) - -=item extractMemberWithoutPaths( { memberOrZipName => $memberOrName - [, name => $extractedName ] } ) - -Extract the given member, or match its name and extract it. -Does not use path information (extracts into the current -directory). Returns undef if member does not exist in this -Zip. -If optional second arg is given, use it as the name of the -extracted member (its paths will be deleted too). Otherwise, -the internal filename of the member (minus paths) is used as -the name of the extracted file or directory. Returns C -on success. - -=item addMember( $member ) - -=item addMember( { member => $member } ) - -Append a member (possibly from another zip file) to the zip -file. Returns the new member. Generally, you will use -addFile(), addDirectory(), addFileOrDirectory(), addString(), -or read() to add members. - - # Move member named 'abc' to end of zip: - my $member = $zip->removeMember( 'abc' ); - $zip->addMember( $member ); - -=item updateMember( $memberOrName, $fileName ) - -=item updateMember( { memberOrZipName => $memberOrName, name => $fileName } ) - -Update a single member from the file or directory named C<$fileName>. -Returns the (possibly added or updated) member, if any; C on -errors. -The comparison is based on C and (in the case of a -non-directory) the size of the file. - -=item addFile( $fileName [, $newName, $compressionLevel ] ) - -=item addFile( { filename => $fileName - [, zipName => $newName, compressionLevel => $compressionLevel } ] ) - -Append a member whose data comes from an external file, -returning the member or undef. The member will have its file -name set to the name of the external file, and its -desiredCompressionMethod set to COMPRESSION_DEFLATED. The -file attributes and last modification time will be set from -the file. -If the name given does not represent a readable plain file or -symbolic link, undef will be returned. C<$fileName> must be -in the format required for the local file system. -The optional C<$newName> argument sets the internal file name -to something different than the given $fileName. C<$newName>, -if given, must be in Zip name format (i.e. Unix). -The text mode bit will be set if the contents appears to be -text (as returned by the C<-T> perl operator). - - -I that you should not (generally) use absolute path names -in zip member names, as this will cause problems with some zip -tools as well as introduce a security hole and make the zip -harder to use. - -=item addDirectory( $directoryName [, $fileName ] ) - -=item addDirectory( { directoryName => $directoryName - [, zipName => $fileName ] } ) - - -Append a member created from the given directory name. The -directory name does not have to name an existing directory. -If the named directory exists, the file modification time and -permissions are set from the existing directory, otherwise -they are set to now and permissive default permissions. -C<$directoryName> must be in local file system format. -The optional second argument sets the name of the archive -member (which defaults to C<$directoryName>). If given, it -must be in Zip (Unix) format. -Returns the new member. - -=item addFileOrDirectory( $name [, $newName, $compressionLevel ] ) - -=item addFileOrDirectory( { name => $name [, zipName => $newName, - compressionLevel => $compressionLevel ] } ) - - -Append a member from the file or directory named $name. If -$newName is given, use it for the name of the new member. -Will add or remove trailing slashes from $newName as needed. -C<$name> must be in local file system format. -The optional second argument sets the name of the archive -member (which defaults to C<$name>). If given, it must be in -Zip (Unix) format. - -=item addString( $stringOrStringRef, $name, [$compressionLevel] ) - -=item addString( { string => $stringOrStringRef [, zipName => $name, - compressionLevel => $compressionLevel ] } ) - -Append a member created from the given string or string -reference. The name is given by the second argument. -Returns the new member. The last modification time will be -set to now, and the file attributes will be set to permissive -defaults. - - my $member = $zip->addString( 'This is a test', 'test.txt' ); - -=item contents( $memberOrMemberName [, $newContents ] ) - -=item contents( { memberOrZipName => $memberOrMemberName - [, contents => $newContents ] } ) - - -Returns the uncompressed data for a particular member, or -undef. - - print "xyz.txt contains " . $zip->contents( 'xyz.txt' ); - -Also can change the contents of a member: - - $zip->contents( 'xyz.txt', 'This is the new contents' ); - -If called expecting an array as the return value, it will include -the status as the second value in the array. - - ($content, $status) = $zip->contents( 'xyz.txt'); - -=back - -=head2 Zip Archive I/O operations - - -A Zip archive can be written to a file or file handle, or read from -one. - -=over 4 - -=item writeToFileNamed( $fileName ) - -=item writeToFileNamed( { fileName => $fileName } ) - -Write a zip archive to named file. Returns C on -success. - - my $status = $zip->writeToFileNamed( 'xx.zip' ); - die "error somewhere" if $status != AZ_OK; - -Note that if you use the same name as an existing zip file -that you read in, you will clobber ZipFileMembers. So -instead, write to a different file name, then delete the -original. -If you use the C or C methods, you can -re-write the original zip in this way. -C<$fileName> should be a valid file name on your system. - -=item writeToFileHandle( $fileHandle [, $seekable] ) - -Write a zip archive to a file handle. Return AZ_OK on -success. The optional second arg tells whether or not to try -to seek backwards to re-write headers. If not provided, it is -set if the Perl C<-f> test returns true. This could fail on -some operating systems, though. - - my $fh = IO::File->new( 'someFile.zip', 'w' ); - unless ( $zip->writeToFileHandle( $fh ) == AZ_OK ) { - # error handling - } - -If you pass a file handle that is not seekable (like if -you're writing to a pipe or a socket), pass a false second -argument: - - my $fh = IO::File->new( '| cat > somefile.zip', 'w' ); - $zip->writeToFileHandle( $fh, 0 ); # fh is not seekable - -If this method fails during the write of a member, that -member and all following it will return false from -C. See writeCentralDirectory() for a way to -deal with this. -If you want, you can write data to the file handle before -passing it to writeToFileHandle(); this could be used (for -instance) for making self-extracting archives. However, this -only works reliably when writing to a real file (as opposed -to STDOUT or some other possible non-file). - -See examples/selfex.pl for how to write a self-extracting -archive. - -=item writeCentralDirectory( $fileHandle [, $offset ] ) - -=item writeCentralDirectory( { fileHandle => $fileHandle - [, offset => $offset ] } ) - -Writes the central directory structure to the given file -handle. - -Returns AZ_OK on success. If given an $offset, will -seek to that point before writing. This can be used for -recovery in cases where writeToFileHandle or writeToFileNamed -returns an IO error because of running out of space on the -destination file. - -You can truncate the zip by seeking backwards and then writing the -directory: - - my $fh = IO::File->new( 'someFile.zip', 'w' ); - my $retval = $zip->writeToFileHandle( $fh ); - if ( $retval == AZ_IO_ERROR ) { - my @unwritten = grep { not $_->wasWritten() } $zip->members(); - if (@unwritten) { - $zip->removeMember( $member ) foreach my $member ( @unwritten ); - $zip->writeCentralDirectory( $fh, - $unwritten[0]->writeLocalHeaderRelativeOffset()); - } - } - -=item overwriteAs( $newName ) - -=item overwriteAs( { filename => $newName } ) - -Write the zip to the specified file, as safely as possible. -This is done by first writing to a temp file, then renaming -the original if it exists, then renaming the temp file, then -deleting the renamed original if it exists. Returns AZ_OK if -successful. - -=item overwrite() - -Write back to the original zip file. See overwriteAs() above. -If the zip was not ever read from a file, this generates an -error. - -=item read( $fileName ) - -=item read( { filename => $fileName } ) - -Read zipfile headers from a zip file, appending new members. -Returns C or error code. - - my $zipFile = Archive::Zip->new(); - my $status = $zipFile->read( '/some/FileName.zip' ); - -=item readFromFileHandle( $fileHandle, $filename ) - -=item readFromFileHandle( { fileHandle => $fileHandle, filename => $filename } ) - -Read zipfile headers from an already-opened file handle, -appending new members. Does not close the file handle. -Returns C or error code. Note that this requires a -seekable file handle; reading from a stream is not yet -supported, but using in-memory data is. - - my $fh = IO::File->new( '/some/FileName.zip', 'r' ); - my $zip1 = Archive::Zip->new(); - my $status = $zip1->readFromFileHandle( $fh ); - my $zip2 = Archive::Zip->new(); - $status = $zip2->readFromFileHandle( $fh ); - -Read zip using in-memory data (recursable): - - open my $fh, "<", "archive.zip" or die $!; - my $zip_data = do { local $.; <$fh> }; - my $zip = Archive::Zip->new; - open my $dh, "+<", \$zip_data; - $zip->readFromFileHandle ($dh); - -=back - -=head2 Zip Archive Tree operations - -These used to be in Archive::Zip::Tree but got moved into -Archive::Zip. They enable operation on an entire tree of members or -files. -A usage example: - - use Archive::Zip; - my $zip = Archive::Zip->new(); - - # add all readable files and directories below . as xyz/* - $zip->addTree( '.', 'xyz' ); - - # add all readable plain files below /abc as def/* - $zip->addTree( '/abc', 'def', sub { -f && -r } ); - - # add all .c files below /tmp as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.c$' ); - - # add all .o files below /tmp as stuff/* if they aren't writable - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { ! -w } ); - - # add all .so files below /tmp that are smaller than 200 bytes as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { -s < 200 } ); - - # and write them into a file - $zip->writeToFileNamed('xxx.zip'); - - # now extract the same files into /tmpx - $zip->extractTree( 'stuff', '/tmpx' ); - -=over 4 - -=item $zip->addTree( $root, $dest [, $pred, $compressionLevel ] ) -- Add tree of files to a zip - -=item $zip->addTree( { root => $root, zipName => $dest [, select => $pred, - compressionLevel => $compressionLevel ] ) - -C<$root> is the root of the tree of files and directories to be -added. It is a valid directory name on your system. C<$dest> is -the name for the root in the zip file (undef or blank means -to use relative pathnames). It is a valid ZIP directory name -(that is, it uses forward slashes (/) for separating -directory components). C<$pred> is an optional subroutine -reference to select files: it is passed the name of the -prospective file or directory using C<$_>, and if it returns -true, the file or directory will be included. The default is -to add all readable files and directories. For instance, -using - - my $pred = sub { /\.txt/ }; - $zip->addTree( '.', '', $pred ); - -will add all the .txt files in and below the current -directory, using relative names, and making the names -identical in the zipfile: - - original name zip member name - ./xyz xyz - ./a/ a/ - ./a/b a/b - -To translate absolute to relative pathnames, just pass them -in: $zip->addTree( '/c/d', 'a' ); - - original name zip member name - /c/d/xyz a/xyz - /c/d/a/ a/a/ - /c/d/a/b a/a/b - -Returns AZ_OK on success. Note that this will not follow -symbolic links to directories. Note also that this does not -check for the validity of filenames. - -Note that you generally I want to make zip archive member names -absolute. - -=item $zip->addTreeMatching( $root, $dest, $pattern [, $pred, $compressionLevel ] ) - -=item $zip->addTreeMatching( { root => $root, zipName => $dest, pattern => - $pattern [, select => $pred, compressionLevel => $compressionLevel ] } ) - -$root is the root of the tree of files and directories to be -added $dest is the name for the root in the zip file (undef -means to use relative pathnames) $pattern is a (non-anchored) -regular expression for filenames to match $pred is an -optional subroutine reference to select files: it is passed -the name of the prospective file or directory in C<$_>, and -if it returns true, the file or directory will be included. -The default is to add all readable files and directories. To -add all files in and below the current directory whose names -end in C<.pl>, and make them extract into a subdirectory -named C, do this: - - $zip->addTreeMatching( '.', 'xyz', '\.pl$' ) - -To add all I files in and below the directory named -C whose names end in C<.pl>, and make them extract into -a subdirectory named C, do this: - - $zip->addTreeMatching( '/abc', 'xyz', '\.pl$', sub { -w } ) - -Returns AZ_OK on success. Note that this will not follow -symbolic links to directories. - -=item $zip->updateTree( $root [, $dest , $pred , $mirror, $compressionLevel ] ); - -=item $zip->updateTree( { root => $root [, zipName => $dest, select => $pred, - mirror => $mirror, compressionLevel => $compressionLevel ] } ); - -Update a zip file from a directory tree. - -C takes the same arguments as C, but first -checks to see whether the file or directory already exists in the zip -file, and whether it has been changed. - -If the fourth argument C<$mirror> is true, then delete all my members -if corresponding files were not found. - -Returns an error code or AZ_OK if all is well. - -=item $zip->extractTree( [ $root, $dest, $volume } ] ) - -=item $zip->extractTree( [ { root => $root, zipName => $dest, volume => $volume } ] ) - -If you don't give any arguments at all, will extract all the -files in the zip with their original names. - -If you supply one argument for C<$root>, C will extract -all the members whose names start with C<$root> into the current -directory, stripping off C<$root> first. -C<$root> is in Zip (Unix) format. -For instance, - - $zip->extractTree( 'a' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x as ./x - -a/b/c as ./b/c - -If you give two arguments, C extracts all the members -whose names start with C<$root>. It will translate C<$root> into -C<$dest> to construct the destination file name. -C<$root> and C<$dest> are in Zip (Unix) format. -For instance, - - $zip->extractTree( 'a', 'd/e' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x to d/e/x - -a/b/c to d/e/b/c and ignore ax/d/e and d/e - -If you give three arguments, C extracts all the members -whose names start with C<$root>. It will translate C<$root> into -C<$dest> to construct the destination file name, and then it will -convert to local file system format, using C<$volume> as the name of -the destination volume. - -C<$root> and C<$dest> are in Zip (Unix) format. - -C<$volume> is in local file system format. - -For instance, under Windows, - - $zip->extractTree( 'a', 'd/e', 'f:' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x to f:d/e/x - -a/b/c to f:d/e/b/c and ignore ax/d/e and d/e - -If you want absolute paths (the prior example used paths relative to -the current directory on the destination volume, you can specify these -in C<$dest>: - - $zip->extractTree( 'a', '/d/e', 'f:' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x to f:\d\e\x - -a/b/c to f:\d\e\b\c and ignore ax/d/e and d/e - -Returns an error code or AZ_OK if everything worked OK. - -=back - -=head1 Archive::Zip Global Variables - -=over 4 - -=item $Archive::Zip::UNICODE - -This variable governs how Unicode file and directory names are added -to or extracted from an archive. If set, file and directory names are considered -to be UTF-8 encoded. This is I. Please report problems. - - { - local $Archive::Zip::UNICODE = 1; - $zip->addFile('Déjà vu.txt'); - } - -=back - -=head1 MEMBER OPERATIONS - -=head2 Member Class Methods - -Several constructors allow you to construct members without adding -them to a zip archive. These work the same as the addFile(), -addDirectory(), and addString() zip instance methods described above, -but they don't add the new members to a zip. - -=over 4 - -=item Archive::Zip::Member->newFromString( $stringOrStringRef [, $fileName ] ) - -=item Archive::Zip::Member->newFromString( { string => $stringOrStringRef - [, zipName => $fileName ] ) - -Construct a new member from the given string. Returns undef -on error. - - my $member = Archive::Zip::Member->newFromString( 'This is a test', - -=item newFromFile( $fileName [, $zipName ] ) - -=item newFromFile( { filename => $fileName [, zipName => $zipName ] } ) - -Construct a new member from the given file. Returns undef on -error. - - my $member = Archive::Zip::Member->newFromFile( 'xyz.txt' ); - -=item newDirectoryNamed( $directoryName [, $zipname ] ) - -=item newDirectoryNamed( { directoryName => $directoryName - [, zipName => $zipname ] } ) - -Construct a new member from the given directory. -C<$directoryName> must be a valid name on your file system; it does not -have to exist. - -If given, C<$zipname> will be the name of the zip member; it must be a -valid Zip (Unix) name. If not given, it will be converted from -C<$directoryName>. - -Returns undef on error. - - my $member = Archive::Zip::Member->newDirectoryNamed( 'CVS/' ); - -=back - -=head2 Member Simple accessors - -These methods get (and/or set) member attribute values. - -=over 4 - -=item versionMadeBy() - -Gets the field from the member header. - -=item fileAttributeFormat( [ $format ] ) - -=item fileAttributeFormat( [ { format => $format ] } ) - -Gets or sets the field from the member header. These are -C values. - -=item versionNeededToExtract() - -Gets the field from the member header. - -=item bitFlag() - -Gets the general purpose bit field from the member header. -This is where the C bits live. - -=item compressionMethod() - -Returns the member compression method. This is the method -that is currently being used to compress the member data. -This will be COMPRESSION_STORED for added string or file -members, or any of the C values for members -from a zip file. However, this module can only handle members -whose data is in COMPRESSION_STORED or COMPRESSION_DEFLATED -format. - -=item desiredCompressionMethod( [ $method ] ) - -=item desiredCompressionMethod( [ { compressionMethod => $method } ] ) - -Get or set the member's C. This is -the compression method that will be used when the member is -written. Returns prior desiredCompressionMethod. Only -COMPRESSION_DEFLATED or COMPRESSION_STORED are valid -arguments. Changing to COMPRESSION_STORED will change the -member desiredCompressionLevel to 0; changing to -COMPRESSION_DEFLATED will change the member -desiredCompressionLevel to COMPRESSION_LEVEL_DEFAULT. - -=item desiredCompressionLevel( [ $level ] ) - -=item desiredCompressionLevel( [ { compressionLevel => $level } ] ) - -Get or set the member's desiredCompressionLevel This is the -method that will be used to write. Returns prior -desiredCompressionLevel. Valid arguments are 0 through 9, -COMPRESSION_LEVEL_NONE, COMPRESSION_LEVEL_DEFAULT, -COMPRESSION_LEVEL_BEST_COMPRESSION, and -COMPRESSION_LEVEL_FASTEST. 0 or COMPRESSION_LEVEL_NONE will -change the desiredCompressionMethod to COMPRESSION_STORED. -All other arguments will change the desiredCompressionMethod -to COMPRESSION_DEFLATED. - -=item externalFileName() - -Return the member's external file name, if any, or undef. - -=item fileName() - -Get or set the member's internal filename. Returns the -(possibly new) filename. Names will have backslashes -converted to forward slashes, and will have multiple -consecutive slashes converted to single ones. - -=item lastModFileDateTime() - -Return the member's last modification date/time stamp in -MS-DOS format. - -=item lastModTime() - -Return the member's last modification date/time stamp, -converted to unix localtime format. - - print "Mod Time: " . scalar( localtime( $member->lastModTime() ) ); - -=item setLastModFileDateTimeFromUnix() - -Set the member's lastModFileDateTime from the given unix -time. - - $member->setLastModFileDateTimeFromUnix( time() ); - -=item internalFileAttributes() - -Return the internal file attributes field from the zip -header. This is only set for members read from a zip file. - -=item externalFileAttributes() - -Return member attributes as read from the ZIP file. Note that -these are NOT UNIX! - -=item unixFileAttributes( [ $newAttributes ] ) - -=item unixFileAttributes( [ { attributes => $newAttributes } ] ) - -Get or set the member's file attributes using UNIX file -attributes. Returns old attributes. - - my $oldAttribs = $member->unixFileAttributes( 0666 ); - -Note that the return value has more than just the file -permissions, so you will have to mask off the lowest bits for -comparisons. - -=item localExtraField( [ $newField ] ) - -=item localExtraField( [ { field => $newField } ] ) - -Gets or sets the extra field that was read from the local -header. This is not set for a member from a zip file until -after the member has been written out. The extra field must -be in the proper format. - -=item cdExtraField( [ $newField ] ) - -=item cdExtraField( [ { field => $newField } ] ) - -Gets or sets the extra field that was read from the central -directory header. The extra field must be in the proper -format. - -=item extraFields() - -Return both local and CD extra fields, concatenated. - -=item fileComment( [ $newComment ] ) - -=item fileComment( [ { comment => $newComment } ] ) - -Get or set the member's file comment. - -=item hasDataDescriptor() - -Get or set the data descriptor flag. If this is set, the -local header will not necessarily have the correct data -sizes. Instead, a small structure will be stored at the end -of the member data with these values. This should be -transparent in normal operation. - -=item crc32() - -Return the CRC-32 value for this member. This will not be set -for members that were constructed from strings or external -files until after the member has been written. - -=item crc32String() - -Return the CRC-32 value for this member as an 8 character -printable hex string. This will not be set for members that -were constructed from strings or external files until after -the member has been written. - -=item compressedSize() - -Return the compressed size for this member. This will not be -set for members that were constructed from strings or -external files until after the member has been written. - -=item uncompressedSize() - -Return the uncompressed size for this member. - -=item password( [ $password ] ) - -Returns the password for this member to be used on decryption. -If $password is given, it will set the password for the decryption. - -=item isEncrypted() - -Return true if this member is encrypted. The Archive::Zip -module does not currently support creation of encrypted -members. Decryption works more or less like this: - - my $zip = Archive::Zip->new; - $zip->read ("encrypted.zip"); - for my $m (map { $zip->memberNamed ($_) } $zip->memberNames) { - $m->password ("secret"); - $m->contents; # is "" when password was wrong - -That shows that the password has to be set per member, and not per -archive. This might change in the future. - -=item isTextFile( [ $flag ] ) - -=item isTextFile( [ { flag => $flag } ] ) - -Returns true if I am a text file. Also can set the status if -given an argument (then returns old state). Note that this -module does not currently do anything with this flag upon -extraction or storage. That is, bytes are stored in native -format whether or not they came from a text file. - -=item isBinaryFile() - -Returns true if I am a binary file. Also can set the status -if given an argument (then returns old state). Note that this -module does not currently do anything with this flag upon -extraction or storage. That is, bytes are stored in native -format whether or not they came from a text file. - -=item extractToFileNamed( $fileName ) - -=item extractToFileNamed( { name => $fileName } ) - -Extract me to a file with the given name. The file will be -created with default modes. Directories will be created as -needed. -The C<$fileName> argument should be a valid file name on your -file system. -Returns AZ_OK on success. - -=item isDirectory() - -Returns true if I am a directory. - -=item writeLocalHeaderRelativeOffset() - -Returns the file offset in bytes the last time I was written. - -=item wasWritten() - -Returns true if I was successfully written. Reset at the -beginning of a write attempt. - -=back - -=head2 Low-level member data reading - -It is possible to use lower-level routines to access member data -streams, rather than the extract* methods and contents(). For -instance, here is how to print the uncompressed contents of a member -in chunks using these methods: - - my ( $member, $status, $bufferRef ); - $member = $zip->memberNamed( 'xyz.txt' ); - $member->desiredCompressionMethod( COMPRESSION_STORED ); - $status = $member->rewindData(); - die "error $status" unless $status == AZ_OK; - while ( ! $member->readIsDone() ) - { - ( $bufferRef, $status ) = $member->readChunk(); - die "error $status" - if $status != AZ_OK && $status != AZ_STREAM_END; - # do something with $bufferRef: - print $$bufferRef; - } - $member->endRead(); - -=over 4 - -=item readChunk( [ $chunkSize ] ) - -=item readChunk( [ { chunkSize => $chunkSize } ] ) - -This reads the next chunk of given size from the member's -data stream and compresses or uncompresses it as necessary, -returning a reference to the bytes read and a status. If size -argument is not given, defaults to global set by -Archive::Zip::setChunkSize. Status is AZ_OK on success until -the last chunk, where it returns AZ_STREAM_END. Returns C<( -\$bytes, $status)>. - - my ( $outRef, $status ) = $self->readChunk(); - print $$outRef if $status != AZ_OK && $status != AZ_STREAM_END; - -=item rewindData() - -Rewind data and set up for reading data streams or writing -zip files. Can take options for C or -C, but this is not likely to be necessary. -Subclass overrides should call this method. Returns C -on success. - -=item endRead() - -Reset the read variables and free the inflater or deflater. -Must be called to close files, etc. Returns AZ_OK on success. - -=item readIsDone() - -Return true if the read has run out of data or encountered an error. - -=item contents() - -Return the entire uncompressed member data or undef in scalar -context. When called in array context, returns C<( $string, -$status )>; status will be AZ_OK on success: - - my $string = $member->contents(); - # or - my ( $string, $status ) = $member->contents(); - die "error $status" unless $status == AZ_OK; - -Can also be used to set the contents of a member (this may -change the class of the member): - - $member->contents( "this is my new contents" ); - -=item extractToFileHandle( $fh ) - -=item extractToFileHandle( { fileHandle => $fh } ) - -Extract (and uncompress, if necessary) the member's contents -to the given file handle. Return AZ_OK on success. - -=back - -=head1 Archive::Zip::FileMember methods - -The Archive::Zip::FileMember class extends Archive::Zip::Member. It is the -base class for both ZipFileMember and NewFileMember classes. This class adds -an C and an C member to keep track of the external -file. - -=over 4 - -=item externalFileName() - -Return the member's external filename. - -=item fh() - -Return the member's read file handle. Automatically opens file if -necessary. - -=back - -=head1 Archive::Zip::ZipFileMember methods - -The Archive::Zip::ZipFileMember class represents members that have been read -from external zip files. - -=over 4 - -=item diskNumberStart() - -Returns the disk number that the member's local header resides in. -Should be 0. - -=item localHeaderRelativeOffset() - -Returns the offset into the zip file where the member's local header -is. - -=item dataOffset() - -Returns the offset from the beginning of the zip file to the member's -data. - -=back - -=head1 REQUIRED MODULES - -L requires several other modules: - -L - -L - -L - -L - -L - -L - -L - -L - -L - -L - -L - -=head1 BUGS AND CAVEATS - -=head2 When not to use Archive::Zip - -If you are just going to be extracting zips (and/or other archives) you -are recommended to look at using L instead, as it is much -easier to use and factors out archive-specific functionality. - -=head2 Try to avoid IO::Scalar - -One of the most common ways to use Archive::Zip is to generate Zip files -in-memory. Most people use L for this purpose. - -Unfortunately, as of 1.11 this module no longer works with L -as it incorrectly implements seeking. - -Anybody using L should consider porting to L, -which is smaller, lighter, and is implemented to be perfectly compatible -with regular seekable filehandles. - -Support for L most likely will B be restored in the -future, as L itself cannot change the way it is implemented -due to back-compatibility issues. - -=head2 Wrong password for encrypted members - -When an encrypted member is read using the wrong password, you currently -have to re-read the entire archive to try again with the correct password. - -=head1 TO DO - -* auto-choosing storing vs compression - -* extra field hooks (see notes.txt) - -* check for duplicates on addition/renaming? - -* Text file extraction (line end translation) - -* Reading zip files from non-seekable inputs - (Perhaps by proxying through IO::String?) - -* separate unused constants into separate module - -* cookbook style docs - -* Handle tainted paths correctly - -* Work on better compatibility with other IO:: modules - -* Support encryption - -* More user-friendly decryption - -=head1 SUPPORT - -Bugs should be reported via the CPAN bug tracker - -L - -For other issues contact the maintainer - -=head1 AUTHOR - -Currently maintained by Fred Moyer - -Previously maintained by Adam Kennedy - -Previously maintained by Steve Peters Esteve@fisharerojo.orgE. - -File attributes code by Maurice Aubrey Emaurice@lovelyfilth.comE. - -Originally by Ned Konz Enedkonz@cpan.orgE. - -=head1 COPYRIGHT - -Some parts copyright 2006 - 2012 Adam Kennedy. - -Some parts copyright 2005 Steve Peters. - -Original work copyright 2000 - 2004 Ned Konz. - -This program is free software; you can redistribute it and/or modify -it under the same terms as Perl itself. - -=head1 SEE ALSO - -Look at L which is a wrapper that allows one to -read Zip archive members as if they were files. - -L, L, L - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Archive.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Archive.pm deleted file mode 100644 index c185612390e..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Archive.pm +++ /dev/null @@ -1,1020 +0,0 @@ -package Archive::Zip::Archive; - -# Represents a generic ZIP archive - -use strict; -use File::Path; -use File::Find (); -use File::Spec (); -use File::Copy (); -use File::Basename; -use Cwd; - -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip ); - - if ($^O eq 'MSWin32') { - require Win32; - require Encode; - Encode->import(qw{ encode_utf8 decode_utf8 }); - } -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES - :PKZIP_CONSTANTS - :UTILITY_METHODS -); - -our $UNICODE; - -# Note that this returns undef on read errors, else new zip object. - -sub new { - my $class = shift; - my $self = bless( - { - 'diskNumber' => 0, - 'diskNumberWithStartOfCentralDirectory' => 0, - 'numberOfCentralDirectoriesOnThisDisk' => - 0, # should be # of members - 'numberOfCentralDirectories' => 0, # should be # of members - 'centralDirectorySize' => 0, # must re-compute on write - 'centralDirectoryOffsetWRTStartingDiskNumber' => - 0, # must re-compute - 'writeEOCDOffset' => 0, - 'writeCentralDirectoryOffset' => 0, - 'zipfileComment' => '', - 'eocdOffset' => 0, - 'fileName' => '' - }, - $class - ); - $self->{'members'} = []; - my $fileName = (ref($_[0]) eq 'HASH') ? shift->{filename} : shift; - if ($fileName) { - my $status = $self->read($fileName); - return $status == AZ_OK ? $self : undef; - } - return $self; -} - -sub storeSymbolicLink { - my $self = shift; - $self->{'storeSymbolicLink'} = shift; -} - -sub members { - @{shift->{'members'}}; -} - -sub numberOfMembers { - scalar(shift->members()); -} - -sub memberNames { - my $self = shift; - return map { $_->fileName() } $self->members(); -} - -# return ref to member with given name or undef -sub memberNamed { - my $self = shift; - my $fileName = (ref($_[0]) eq 'HASH') ? shift->{zipName} : shift; - foreach my $member ($self->members()) { - return $member if $member->fileName() eq $fileName; - } - return undef; -} - -sub membersMatching { - my $self = shift; - my $pattern = (ref($_[0]) eq 'HASH') ? shift->{regex} : shift; - return grep { $_->fileName() =~ /$pattern/ } $self->members(); -} - -sub diskNumber { - shift->{'diskNumber'}; -} - -sub diskNumberWithStartOfCentralDirectory { - shift->{'diskNumberWithStartOfCentralDirectory'}; -} - -sub numberOfCentralDirectoriesOnThisDisk { - shift->{'numberOfCentralDirectoriesOnThisDisk'}; -} - -sub numberOfCentralDirectories { - shift->{'numberOfCentralDirectories'}; -} - -sub centralDirectorySize { - shift->{'centralDirectorySize'}; -} - -sub centralDirectoryOffsetWRTStartingDiskNumber { - shift->{'centralDirectoryOffsetWRTStartingDiskNumber'}; -} - -sub zipfileComment { - my $self = shift; - my $comment = $self->{'zipfileComment'}; - if (@_) { - my $new_comment = (ref($_[0]) eq 'HASH') ? shift->{comment} : shift; - $self->{'zipfileComment'} = pack('C0a*', $new_comment); # avoid Unicode - } - return $comment; -} - -sub eocdOffset { - shift->{'eocdOffset'}; -} - -# Return the name of the file last read. -sub fileName { - shift->{'fileName'}; -} - -sub removeMember { - my $self = shift; - my $member = (ref($_[0]) eq 'HASH') ? shift->{memberOrZipName} : shift; - $member = $self->memberNamed($member) unless ref($member); - return undef unless $member; - my @newMembers = grep { $_ != $member } $self->members(); - $self->{'members'} = \@newMembers; - return $member; -} - -sub replaceMember { - my $self = shift; - - my ($oldMember, $newMember); - if (ref($_[0]) eq 'HASH') { - $oldMember = $_[0]->{memberOrZipName}; - $newMember = $_[0]->{newMember}; - } else { - ($oldMember, $newMember) = @_; - } - - $oldMember = $self->memberNamed($oldMember) unless ref($oldMember); - return undef unless $oldMember; - return undef unless $newMember; - my @newMembers = - map { ($_ == $oldMember) ? $newMember : $_ } $self->members(); - $self->{'members'} = \@newMembers; - return $oldMember; -} - -sub extractMember { - my $self = shift; - - my ($member, $name); - if (ref($_[0]) eq 'HASH') { - $member = $_[0]->{memberOrZipName}; - $name = $_[0]->{name}; - } else { - ($member, $name) = @_; - } - - $member = $self->memberNamed($member) unless ref($member); - return _error('member not found') unless $member; - my $originalSize = $member->compressedSize(); - my ($volumeName, $dirName, $fileName); - if (defined($name)) { - ($volumeName, $dirName, $fileName) = File::Spec->splitpath($name); - $dirName = File::Spec->catpath($volumeName, $dirName, ''); - } else { - $name = $member->fileName(); - ($dirName = $name) =~ s{[^/]*$}{}; - $dirName = Archive::Zip::_asLocalName($dirName); - $name = Archive::Zip::_asLocalName($name); - } - if ($dirName && !-d $dirName) { - mkpath($dirName); - return _ioError("can't create dir $dirName") if (!-d $dirName); - } - my $rc = $member->extractToFileNamed($name, @_); - - # TODO refactor this fix into extractToFileNamed() - $member->{'compressedSize'} = $originalSize; - return $rc; -} - -sub extractMemberWithoutPaths { - my $self = shift; - - my ($member, $name); - if (ref($_[0]) eq 'HASH') { - $member = $_[0]->{memberOrZipName}; - $name = $_[0]->{name}; - } else { - ($member, $name) = @_; - } - - $member = $self->memberNamed($member) unless ref($member); - return _error('member not found') unless $member; - my $originalSize = $member->compressedSize(); - return AZ_OK if $member->isDirectory(); - unless ($name) { - $name = $member->fileName(); - $name =~ s{.*/}{}; # strip off directories, if any - $name = Archive::Zip::_asLocalName($name); - } - my $rc = $member->extractToFileNamed($name, @_); - $member->{'compressedSize'} = $originalSize; - return $rc; -} - -sub addMember { - my $self = shift; - my $newMember = (ref($_[0]) eq 'HASH') ? shift->{member} : shift; - push(@{$self->{'members'}}, $newMember) if $newMember; - return $newMember; -} - -sub addFile { - my $self = shift; - - my ($fileName, $newName, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $fileName = $_[0]->{filename}; - $newName = $_[0]->{zipName}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($fileName, $newName, $compressionLevel) = @_; - } - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $fileName = Win32::GetANSIPathName($fileName); - } - - my $newMember = Archive::Zip::Member->newFromFile($fileName, $newName); - $newMember->desiredCompressionLevel($compressionLevel); - if ($self->{'storeSymbolicLink'} && -l $fileName) { - my $newMember = - Archive::Zip::Member->newFromString(readlink $fileName, $newName); - - # For symbolic links, External File Attribute is set to 0xA1FF0000 by Info-ZIP - $newMember->{'externalFileAttributes'} = 0xA1FF0000; - $self->addMember($newMember); - } else { - $self->addMember($newMember); - } - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $newMember->{'fileName'} = - encode_utf8(Win32::GetLongPathName($fileName)); - } - return $newMember; -} - -sub addString { - my $self = shift; - - my ($stringOrStringRef, $name, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $stringOrStringRef = $_[0]->{string}; - $name = $_[0]->{zipName}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($stringOrStringRef, $name, $compressionLevel) = @_; - } - - my $newMember = - Archive::Zip::Member->newFromString($stringOrStringRef, $name); - $newMember->desiredCompressionLevel($compressionLevel); - return $self->addMember($newMember); -} - -sub addDirectory { - my $self = shift; - - my ($name, $newName); - if (ref($_[0]) eq 'HASH') { - $name = $_[0]->{directoryName}; - $newName = $_[0]->{zipName}; - } else { - ($name, $newName) = @_; - } - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $name = Win32::GetANSIPathName($name); - } - - my $newMember = Archive::Zip::Member->newDirectoryNamed($name, $newName); - if ($self->{'storeSymbolicLink'} && -l $name) { - my $link = readlink $name; - ($newName =~ s{/$}{}) if $newName; # Strip trailing / - my $newMember = Archive::Zip::Member->newFromString($link, $newName); - - # For symbolic links, External File Attribute is set to 0xA1FF0000 by Info-ZIP - $newMember->{'externalFileAttributes'} = 0xA1FF0000; - $self->addMember($newMember); - } else { - $self->addMember($newMember); - } - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $newMember->{'fileName'} = encode_utf8(Win32::GetLongPathName($name)); - } - return $newMember; -} - -# add either a file or a directory. - -sub addFileOrDirectory { - my $self = shift; - - my ($name, $newName, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $name = $_[0]->{name}; - $newName = $_[0]->{zipName}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($name, $newName, $compressionLevel) = @_; - } - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $name = Win32::GetANSIPathName($name); - } - - $name =~ s{/$}{}; - if ($newName) { - $newName =~ s{/$}{}; - } else { - $newName = $name; - } - if (-f $name) { - return $self->addFile($name, $newName, $compressionLevel); - } elsif (-d $name) { - return $self->addDirectory($name, $newName); - } else { - return _error("$name is neither a file nor a directory"); - } -} - -sub contents { - my $self = shift; - - my ($member, $newContents); - if (ref($_[0]) eq 'HASH') { - $member = $_[0]->{memberOrZipName}; - $newContents = $_[0]->{contents}; - } else { - ($member, $newContents) = @_; - } - - return _error('No member name given') unless $member; - $member = $self->memberNamed($member) unless ref($member); - return undef unless $member; - return $member->contents($newContents); -} - -sub writeToFileNamed { - my $self = shift; - my $fileName = - (ref($_[0]) eq 'HASH') ? shift->{filename} : shift; # local FS format - foreach my $member ($self->members()) { - if ($member->_usesFileNamed($fileName)) { - return _error("$fileName is needed by member " - . $member->fileName() - . "; consider using overwrite() or overwriteAs() instead."); - } - } - my ($status, $fh) = _newFileHandle($fileName, 'w'); - return _ioError("Can't open $fileName for write") unless $status; - my $retval = $self->writeToFileHandle($fh, 1); - $fh->close(); - $fh = undef; - - return $retval; -} - -# It is possible to write data to the FH before calling this, -# perhaps to make a self-extracting archive. -sub writeToFileHandle { - my $self = shift; - - my ($fh, $fhIsSeekable); - if (ref($_[0]) eq 'HASH') { - $fh = $_[0]->{fileHandle}; - $fhIsSeekable = - exists($_[0]->{seek}) ? $_[0]->{seek} : _isSeekable($fh); - } else { - $fh = shift; - $fhIsSeekable = @_ ? shift : _isSeekable($fh); - } - - return _error('No filehandle given') unless $fh; - return _ioError('filehandle not open') unless $fh->opened(); - _binmode($fh); - - # Find out where the current position is. - my $offset = $fhIsSeekable ? $fh->tell() : 0; - $offset = 0 if $offset < 0; - - foreach my $member ($self->members()) { - my $retval = $member->_writeToFileHandle($fh, $fhIsSeekable, $offset); - $member->endRead(); - return $retval if $retval != AZ_OK; - $offset += $member->_localHeaderSize() + $member->_writeOffset(); - $offset += - $member->hasDataDescriptor() - ? DATA_DESCRIPTOR_LENGTH + SIGNATURE_LENGTH - : 0; - - # changed this so it reflects the last successful position - $self->{'writeCentralDirectoryOffset'} = $offset; - } - return $self->writeCentralDirectory($fh); -} - -# Write zip back to the original file, -# as safely as possible. -# Returns AZ_OK if successful. -sub overwrite { - my $self = shift; - return $self->overwriteAs($self->{'fileName'}); -} - -# Write zip to the specified file, -# as safely as possible. -# Returns AZ_OK if successful. -sub overwriteAs { - my $self = shift; - my $zipName = (ref($_[0]) eq 'HASH') ? $_[0]->{filename} : shift; - return _error("no filename in overwriteAs()") unless defined($zipName); - - my ($fh, $tempName) = Archive::Zip::tempFile(); - return _error("Can't open temp file", $!) unless $fh; - - (my $backupName = $zipName) =~ s{(\.[^.]*)?$}{.zbk}; - - my $status = $self->writeToFileHandle($fh); - $fh->close(); - $fh = undef; - - if ($status != AZ_OK) { - unlink($tempName); - _printError("Can't write to $tempName"); - return $status; - } - - my $err; - - # rename the zip - if (-f $zipName && !rename($zipName, $backupName)) { - $err = $!; - unlink($tempName); - return _error("Can't rename $zipName as $backupName", $err); - } - - # move the temp to the original name (possibly copying) - unless (File::Copy::move($tempName, $zipName) - || File::Copy::copy($tempName, $zipName)) { - $err = $!; - rename($backupName, $zipName); - unlink($tempName); - return _error("Can't move $tempName to $zipName", $err); - } - - # unlink the backup - if (-f $backupName && !unlink($backupName)) { - $err = $!; - return _error("Can't unlink $backupName", $err); - } - - return AZ_OK; -} - -# Used only during writing -sub _writeCentralDirectoryOffset { - shift->{'writeCentralDirectoryOffset'}; -} - -sub _writeEOCDOffset { - shift->{'writeEOCDOffset'}; -} - -# Expects to have _writeEOCDOffset() set -sub _writeEndOfCentralDirectory { - my ($self, $fh) = @_; - - $self->_print($fh, END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING) - or return _ioError('writing EOCD Signature'); - my $zipfileCommentLength = length($self->zipfileComment()); - - my $header = pack( - END_OF_CENTRAL_DIRECTORY_FORMAT, - 0, # {'diskNumber'}, - 0, # {'diskNumberWithStartOfCentralDirectory'}, - $self->numberOfMembers(), # {'numberOfCentralDirectoriesOnThisDisk'}, - $self->numberOfMembers(), # {'numberOfCentralDirectories'}, - $self->_writeEOCDOffset() - $self->_writeCentralDirectoryOffset(), - $self->_writeCentralDirectoryOffset(), - $zipfileCommentLength - ); - $self->_print($fh, $header) - or return _ioError('writing EOCD header'); - if ($zipfileCommentLength) { - $self->_print($fh, $self->zipfileComment()) - or return _ioError('writing zipfile comment'); - } - return AZ_OK; -} - -# $offset can be specified to truncate a zip file. -sub writeCentralDirectory { - my $self = shift; - - my ($fh, $offset); - if (ref($_[0]) eq 'HASH') { - $fh = $_[0]->{fileHandle}; - $offset = $_[0]->{offset}; - } else { - ($fh, $offset) = @_; - } - - if (defined($offset)) { - $self->{'writeCentralDirectoryOffset'} = $offset; - $fh->seek($offset, IO::Seekable::SEEK_SET) - or return _ioError('seeking to write central directory'); - } else { - $offset = $self->_writeCentralDirectoryOffset(); - } - - foreach my $member ($self->members()) { - my $status = $member->_writeCentralDirectoryFileHeader($fh); - return $status if $status != AZ_OK; - $offset += $member->_centralDirectoryHeaderSize(); - $self->{'writeEOCDOffset'} = $offset; - } - return $self->_writeEndOfCentralDirectory($fh); -} - -sub read { - my $self = shift; - my $fileName = (ref($_[0]) eq 'HASH') ? shift->{filename} : shift; - return _error('No filename given') unless $fileName; - my ($status, $fh) = _newFileHandle($fileName, 'r'); - return _ioError("opening $fileName for read") unless $status; - - $status = $self->readFromFileHandle($fh, $fileName); - return $status if $status != AZ_OK; - - $fh->close(); - $self->{'fileName'} = $fileName; - return AZ_OK; -} - -sub readFromFileHandle { - my $self = shift; - - my ($fh, $fileName); - if (ref($_[0]) eq 'HASH') { - $fh = $_[0]->{fileHandle}; - $fileName = $_[0]->{filename}; - } else { - ($fh, $fileName) = @_; - } - - $fileName = $fh unless defined($fileName); - return _error('No filehandle given') unless $fh; - return _ioError('filehandle not open') unless $fh->opened(); - - _binmode($fh); - $self->{'fileName'} = "$fh"; - - # TODO: how to support non-seekable zips? - return _error('file not seekable') - unless _isSeekable($fh); - - $fh->seek(0, 0); # rewind the file - - my $status = $self->_findEndOfCentralDirectory($fh); - return $status if $status != AZ_OK; - - my $eocdPosition = $fh->tell(); - - $status = $self->_readEndOfCentralDirectory($fh); - return $status if $status != AZ_OK; - - $fh->seek($eocdPosition - $self->centralDirectorySize(), - IO::Seekable::SEEK_SET) - or return _ioError("Can't seek $fileName"); - - # Try to detect garbage at beginning of archives - # This should be 0 - $self->{'eocdOffset'} = $eocdPosition - $self->centralDirectorySize() # here - - $self->centralDirectoryOffsetWRTStartingDiskNumber(); - - for (; ;) { - my $newMember = - Archive::Zip::Member->_newFromZipFile($fh, $fileName, - $self->eocdOffset()); - my $signature; - ($status, $signature) = _readSignature($fh, $fileName); - return $status if $status != AZ_OK; - last if $signature == END_OF_CENTRAL_DIRECTORY_SIGNATURE; - $status = $newMember->_readCentralDirectoryFileHeader(); - return $status if $status != AZ_OK; - $status = $newMember->endRead(); - return $status if $status != AZ_OK; - $newMember->_becomeDirectoryIfNecessary(); - push(@{$self->{'members'}}, $newMember); - } - - return AZ_OK; -} - -# Read EOCD, starting from position before signature. -# Return AZ_OK on success. -sub _readEndOfCentralDirectory { - my $self = shift; - my $fh = shift; - - # Skip past signature - $fh->seek(SIGNATURE_LENGTH, IO::Seekable::SEEK_CUR) - or return _ioError("Can't seek past EOCD signature"); - - my $header = ''; - my $bytesRead = $fh->read($header, END_OF_CENTRAL_DIRECTORY_LENGTH); - if ($bytesRead != END_OF_CENTRAL_DIRECTORY_LENGTH) { - return _ioError("reading end of central directory"); - } - - my $zipfileCommentLength; - ( - $self->{'diskNumber'}, - $self->{'diskNumberWithStartOfCentralDirectory'}, - $self->{'numberOfCentralDirectoriesOnThisDisk'}, - $self->{'numberOfCentralDirectories'}, - $self->{'centralDirectorySize'}, - $self->{'centralDirectoryOffsetWRTStartingDiskNumber'}, - $zipfileCommentLength - ) = unpack(END_OF_CENTRAL_DIRECTORY_FORMAT, $header); - - if ($self->{'diskNumber'} == 0xFFFF || - $self->{'diskNumberWithStartOfCentralDirectory'} == 0xFFFF || - $self->{'numberOfCentralDirectoriesOnThisDisk'} == 0xFFFF || - $self->{'numberOfCentralDirectories'} == 0xFFFF || - $self->{'centralDirectorySize'} == 0xFFFFFFFF || - $self->{'centralDirectoryOffsetWRTStartingDiskNumber'} == 0xFFFFFFFF) { - return _formatError("zip64 not supported"); - } - - if ($zipfileCommentLength) { - my $zipfileComment = ''; - $bytesRead = $fh->read($zipfileComment, $zipfileCommentLength); - if ($bytesRead != $zipfileCommentLength) { - return _ioError("reading zipfile comment"); - } - $self->{'zipfileComment'} = $zipfileComment; - } - - return AZ_OK; -} - -# Seek in my file to the end, then read backwards until we find the -# signature of the central directory record. Leave the file positioned right -# before the signature. Returns AZ_OK if success. -sub _findEndOfCentralDirectory { - my $self = shift; - my $fh = shift; - my $data = ''; - $fh->seek(0, IO::Seekable::SEEK_END) - or return _ioError("seeking to end"); - - my $fileLength = $fh->tell(); - if ($fileLength < END_OF_CENTRAL_DIRECTORY_LENGTH + 4) { - return _formatError("file is too short"); - } - - my $seekOffset = 0; - my $pos = -1; - for (; ;) { - $seekOffset += 512; - $seekOffset = $fileLength if ($seekOffset > $fileLength); - $fh->seek(-$seekOffset, IO::Seekable::SEEK_END) - or return _ioError("seek failed"); - my $bytesRead = $fh->read($data, $seekOffset); - if ($bytesRead != $seekOffset) { - return _ioError("read failed"); - } - $pos = rindex($data, END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING); - last - if ( $pos >= 0 - or $seekOffset == $fileLength - or $seekOffset >= $Archive::Zip::ChunkSize); - } - - if ($pos >= 0) { - $fh->seek($pos - $seekOffset, IO::Seekable::SEEK_CUR) - or return _ioError("seeking to EOCD"); - return AZ_OK; - } else { - return _formatError("can't find EOCD signature"); - } -} - -# Used to avoid taint problems when chdir'ing. -# Not intended to increase security in any way; just intended to shut up the -T -# complaints. If your Cwd module is giving you unreliable returns from cwd() -# you have bigger problems than this. -sub _untaintDir { - my $dir = shift; - $dir =~ m/\A(.+)\z/s; - return $1; -} - -sub addTree { - my $self = shift; - - my ($root, $dest, $pred, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $pred = $_[0]->{select}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($root, $dest, $pred, $compressionLevel) = @_; - } - - return _error("root arg missing in call to addTree()") - unless defined($root); - $dest = '' unless defined($dest); - $pred = sub { -r } - unless defined($pred); - - my @files; - my $startDir = _untaintDir(cwd()); - - return _error('undef returned by _untaintDir on cwd ', cwd()) - unless $startDir; - - # This avoids chdir'ing in Find, in a way compatible with older - # versions of File::Find. - my $wanted = sub { - local $main::_ = $File::Find::name; - my $dir = _untaintDir($File::Find::dir); - chdir($startDir); - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - push(@files, Win32::GetANSIPathName($File::Find::name)) if (&$pred); - $dir = Win32::GetANSIPathName($dir); - } else { - push(@files, $File::Find::name) if (&$pred); - } - chdir($dir); - }; - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $root = Win32::GetANSIPathName($root); - } - File::Find::find($wanted, $root); - - my $rootZipName = _asZipDirName($root, 1); # with trailing slash - my $pattern = $rootZipName eq './' ? '^' : "^\Q$rootZipName\E"; - - $dest = _asZipDirName($dest, 1); # with trailing slash - - foreach my $fileName (@files) { - my $isDir; - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $isDir = -d Win32::GetANSIPathName($fileName); - } else { - $isDir = -d $fileName; - } - - # normalize, remove leading ./ - my $archiveName = _asZipDirName($fileName, $isDir); - if ($archiveName eq $rootZipName) { $archiveName = $dest } - else { $archiveName =~ s{$pattern}{$dest} } - next if $archiveName =~ m{^\.?/?$}; # skip current dir - my $member = - $isDir - ? $self->addDirectory($fileName, $archiveName) - : $self->addFile($fileName, $archiveName); - $member->desiredCompressionLevel($compressionLevel); - - return _error("add $fileName failed in addTree()") if !$member; - } - return AZ_OK; -} - -sub addTreeMatching { - my $self = shift; - - my ($root, $dest, $pattern, $pred, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $pattern = $_[0]->{pattern}; - $pred = $_[0]->{select}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($root, $dest, $pattern, $pred, $compressionLevel) = @_; - } - - return _error("root arg missing in call to addTreeMatching()") - unless defined($root); - $dest = '' unless defined($dest); - return _error("pattern missing in call to addTreeMatching()") - unless defined($pattern); - my $matcher = - $pred ? sub { m{$pattern} && &$pred } : sub { m{$pattern} && -r }; - return $self->addTree($root, $dest, $matcher, $compressionLevel); -} - -# $zip->extractTree( $root, $dest [, $volume] ); -# -# $root and $dest are Unix-style. -# $volume is in local FS format. -# -sub extractTree { - my $self = shift; - - my ($root, $dest, $volume); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $volume = $_[0]->{volume}; - } else { - ($root, $dest, $volume) = @_; - } - - $root = '' unless defined($root); - if (defined $dest) { - if ($dest !~ m{/$}) { - $dest .= '/'; - } - } else { - $dest = './'; - } - - my $pattern = "^\Q$root"; - my @members = $self->membersMatching($pattern); - - foreach my $member (@members) { - my $fileName = $member->fileName(); # in Unix format - $fileName =~ s{$pattern}{$dest}; # in Unix format - # convert to platform format: - $fileName = Archive::Zip::_asLocalName($fileName, $volume); - my $status = $member->extractToFileNamed($fileName); - return $status if $status != AZ_OK; - } - return AZ_OK; -} - -# $zip->updateMember( $memberOrName, $fileName ); -# Returns (possibly updated) member, if any; undef on errors. - -sub updateMember { - my $self = shift; - - my ($oldMember, $fileName); - if (ref($_[0]) eq 'HASH') { - $oldMember = $_[0]->{memberOrZipName}; - $fileName = $_[0]->{name}; - } else { - ($oldMember, $fileName) = @_; - } - - if (!defined($fileName)) { - _error("updateMember(): missing fileName argument"); - return undef; - } - - my @newStat = stat($fileName); - if (!@newStat) { - _ioError("Can't stat $fileName"); - return undef; - } - - my $isDir = -d _; - - my $memberName; - - if (ref($oldMember)) { - $memberName = $oldMember->fileName(); - } else { - $oldMember = $self->memberNamed($memberName = $oldMember) - || $self->memberNamed($memberName = - _asZipDirName($oldMember, $isDir)); - } - - unless (defined($oldMember) - && $oldMember->lastModTime() == $newStat[9] - && $oldMember->isDirectory() == $isDir - && ($isDir || ($oldMember->uncompressedSize() == $newStat[7]))) { - - # create the new member - my $newMember = - $isDir - ? Archive::Zip::Member->newDirectoryNamed($fileName, $memberName) - : Archive::Zip::Member->newFromFile($fileName, $memberName); - - unless (defined($newMember)) { - _error("creation of member $fileName failed in updateMember()"); - return undef; - } - - # replace old member or append new one - if (defined($oldMember)) { - $self->replaceMember($oldMember, $newMember); - } else { - $self->addMember($newMember); - } - - return $newMember; - } - - return $oldMember; -} - -# $zip->updateTree( $root, [ $dest, [ $pred [, $mirror]]] ); -# -# This takes the same arguments as addTree, but first checks to see -# whether the file or directory already exists in the zip file. -# -# If the fourth argument $mirror is true, then delete all my members -# if corresponding files were not found. - -sub updateTree { - my $self = shift; - - my ($root, $dest, $pred, $mirror, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $pred = $_[0]->{select}; - $mirror = $_[0]->{mirror}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($root, $dest, $pred, $mirror, $compressionLevel) = @_; - } - - return _error("root arg missing in call to updateTree()") - unless defined($root); - $dest = '' unless defined($dest); - $pred = sub { -r } - unless defined($pred); - - $dest = _asZipDirName($dest, 1); - my $rootZipName = _asZipDirName($root, 1); # with trailing slash - my $pattern = $rootZipName eq './' ? '^' : "^\Q$rootZipName\E"; - - my @files; - my $startDir = _untaintDir(cwd()); - - return _error('undef returned by _untaintDir on cwd ', cwd()) - unless $startDir; - - # This avoids chdir'ing in Find, in a way compatible with older - # versions of File::Find. - my $wanted = sub { - local $main::_ = $File::Find::name; - my $dir = _untaintDir($File::Find::dir); - chdir($startDir); - push(@files, $File::Find::name) if (&$pred); - chdir($dir); - }; - - File::Find::find($wanted, $root); - - # Now @files has all the files that I could potentially be adding to - # the zip. Only add the ones that are necessary. - # For each file (updated or not), add its member name to @done. - my %done; - foreach my $fileName (@files) { - my @newStat = stat($fileName); - my $isDir = -d _; - - # normalize, remove leading ./ - my $memberName = _asZipDirName($fileName, $isDir); - if ($memberName eq $rootZipName) { $memberName = $dest } - else { $memberName =~ s{$pattern}{$dest} } - next if $memberName =~ m{^\.?/?$}; # skip current dir - - $done{$memberName} = 1; - my $changedMember = $self->updateMember($memberName, $fileName); - $changedMember->desiredCompressionLevel($compressionLevel); - return _error("updateTree failed to update $fileName") - unless ref($changedMember); - } - - # @done now has the archive names corresponding to all the found files. - # If we're mirroring, delete all those members that aren't in @done. - if ($mirror) { - foreach my $member ($self->members()) { - $self->removeMember($member) - unless $done{$member->fileName()}; - } - } - - return AZ_OK; -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/BufferedFileHandle.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/BufferedFileHandle.pm deleted file mode 100644 index 2c770c7fb4f..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/BufferedFileHandle.pm +++ /dev/null @@ -1,131 +0,0 @@ -package Archive::Zip::BufferedFileHandle; - -# File handle that uses a string internally and can seek -# This is given as a demo for getting a zip file written -# to a string. -# I probably should just use IO::Scalar instead. -# Ned Konz, March 2000 - -use strict; -use IO::File; -use Carp; - -use vars qw{$VERSION}; - -BEGIN { - $VERSION = '1.48'; - $VERSION = eval $VERSION; -} - -sub new { - my $class = shift || __PACKAGE__; - $class = ref($class) || $class; - my $self = bless( - { - content => '', - position => 0, - size => 0 - }, - $class - ); - return $self; -} - -# Utility method to read entire file -sub readFromFile { - my $self = shift; - my $fileName = shift; - my $fh = IO::File->new($fileName, "r"); - CORE::binmode($fh); - if (!$fh) { - Carp::carp("Can't open $fileName: $!\n"); - return undef; - } - local $/ = undef; - $self->{content} = <$fh>; - $self->{size} = length($self->{content}); - return $self; -} - -sub contents { - my $self = shift; - if (@_) { - $self->{content} = shift; - $self->{size} = length($self->{content}); - } - return $self->{content}; -} - -sub binmode { 1 } - -sub close { 1 } - -sub opened { 1 } - -sub eof { - my $self = shift; - return $self->{position} >= $self->{size}; -} - -sub seek { - my $self = shift; - my $pos = shift; - my $whence = shift; - - # SEEK_SET - if ($whence == 0) { $self->{position} = $pos; } - - # SEEK_CUR - elsif ($whence == 1) { $self->{position} += $pos; } - - # SEEK_END - elsif ($whence == 2) { $self->{position} = $self->{size} + $pos; } - else { return 0; } - - return 1; -} - -sub tell { return shift->{position}; } - -# Copy my data to given buffer -sub read { - my $self = shift; - my $buf = \($_[0]); - shift; - my $len = shift; - my $offset = shift || 0; - - $$buf = '' if not defined($$buf); - my $bytesRead = - ($self->{position} + $len > $self->{size}) - ? ($self->{size} - $self->{position}) - : $len; - substr($$buf, $offset, $bytesRead) = - substr($self->{content}, $self->{position}, $bytesRead); - $self->{position} += $bytesRead; - return $bytesRead; -} - -# Copy given buffer to me -sub write { - my $self = shift; - my $buf = \($_[0]); - shift; - my $len = shift; - my $offset = shift || 0; - - $$buf = '' if not defined($$buf); - my $bufLen = length($$buf); - my $bytesWritten = - ($offset + $len > $bufLen) - ? $bufLen - $offset - : $len; - substr($self->{content}, $self->{position}, $bytesWritten) = - substr($$buf, $offset, $bytesWritten); - $self->{size} = length($self->{content}); - return $bytesWritten; -} - -sub clearerr() { 1 } - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/DirectoryMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/DirectoryMember.pm deleted file mode 100644 index fa686343a53..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/DirectoryMember.pm +++ /dev/null @@ -1,80 +0,0 @@ -package Archive::Zip::DirectoryMember; - -use strict; -use File::Path; - -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip::Member ); -} - -use Archive::Zip qw( - :ERROR_CODES - :UTILITY_METHODS -); - -sub _newNamed { - my $class = shift; - my $fileName = shift; # FS name - my $newName = shift; # Zip name - $newName = _asZipDirName($fileName) unless $newName; - my $self = $class->new(@_); - $self->{'externalFileName'} = $fileName; - $self->fileName($newName); - - if (-e $fileName) { - - # -e does NOT do a full stat, so we need to do one now - if (-d _ ) { - my @stat = stat(_); - $self->unixFileAttributes($stat[2]); - my $mod_t = $stat[9]; - if ($^O eq 'MSWin32' and !$mod_t) { - $mod_t = time(); - } - $self->setLastModFileDateTimeFromUnix($mod_t); - - } else { # hmm.. trying to add a non-directory? - _error($fileName, ' exists but is not a directory'); - return undef; - } - } else { - $self->unixFileAttributes($self->DEFAULT_DIRECTORY_PERMISSIONS); - $self->setLastModFileDateTimeFromUnix(time()); - } - return $self; -} - -sub externalFileName { - shift->{'externalFileName'}; -} - -sub isDirectory { - return 1; -} - -sub extractToFileNamed { - my $self = shift; - my $name = shift; # local FS name - my $attribs = $self->unixFileAttributes() & 07777; - mkpath($name, 0, $attribs); # croaks on error - utime($self->lastModTime(), $self->lastModTime(), $name); - return AZ_OK; -} - -sub fileName { - my $self = shift; - my $newName = shift; - $newName =~ s{/?$}{/} if defined($newName); - return $self->SUPER::fileName($newName); -} - -# So people don't get too confused. This way it looks like the problem -# is in their code... -sub contents { - return wantarray ? (undef, AZ_OK) : undef; -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FAQ.pod b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FAQ.pod deleted file mode 100644 index d03f883c869..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FAQ.pod +++ /dev/null @@ -1,344 +0,0 @@ -=head1 NAME - -Archive::Zip::FAQ - Answers to a few frequently asked questions about Archive::Zip - -=head1 DESCRIPTION - -It seems that I keep answering the same questions over and over again. I -assume that this is because my documentation is deficient, rather than that -people don't read the documentation. - -So this FAQ is an attempt to cut down on the number of personal answers I have -to give. At least I can now say "You I read the FAQ, right?". - -The questions are not in any particular order. The answers assume the current -version of Archive::Zip; some of the answers depend on newly added/fixed -functionality. - -=head1 Install problems on RedHat 8 or 9 with Perl 5.8.0 - -B Archive::Zip won't install on my RedHat 9 system! It's broke! - -B This has become something of a FAQ. -Basically, RedHat broke some versions of Perl by setting LANG to UTF8. -They apparently have a fixed version out as an update. - -You might try running CPAN or creating your Makefile after exporting the LANG -environment variable as - -C - -L - -=head1 Why is my zip file so big? - -B My zip file is actually bigger than what I stored in it! Why? - -B Some things to make sure of: - -=over 4 - -=item Make sure that you are requesting COMPRESSION_DEFLATED if you are storing strings. - -$member->desiredCompressionMethod( COMPRESSION_DEFLATED ); - -=item Don't make lots of little files if you can help it. - -Since zip computes the compression tables for each member, small -members without much entropy won't compress well. Instead, if you've -got lots of repeated strings in your data, try to combine them into -one big member. - -=item Make sure that you are requesting COMPRESSION_STORED if you are storing things that are already compressed. - -If you're storing a .zip, .jpg, .mp3, or other compressed file in a zip, -then don't compress them again. They'll get bigger. - -=back - -=head1 Sample code? - -B Can you send me code to do (whatever)? - -B Have you looked in the C directory yet? It contains: - -=over 4 - -=item examples/calcSizes.pl -- How to find out how big a Zip file will be before writing it - -=item examples/copy.pl -- Copies one Zip file to another - -=item examples/extract.pl -- extract file(s) from a Zip - -=item examples/mailZip.pl -- make and mail a zip file - -=item examples/mfh.pl -- demo for use of MockFileHandle - -=item examples/readScalar.pl -- shows how to use IO::Scalar as the source of a Zip read - -=item examples/selfex.pl -- a brief example of a self-extracting Zip - -=item examples/unzipAll.pl -- uses Archive::Zip::Tree to unzip an entire Zip - -=item examples/updateZip.pl -- shows how to read/modify/write a Zip - -=item examples/updateTree.pl -- shows how to update a Zip in place - -=item examples/writeScalar.pl -- shows how to use IO::Scalar as the destination of a Zip write - -=item examples/writeScalar2.pl -- shows how to use IO::String as the destination of a Zip write - -=item examples/zip.pl -- Constructs a Zip file - -=item examples/zipcheck.pl -- One way to check a Zip file for validity - -=item examples/zipinfo.pl -- Prints out information about a Zip archive file - -=item examples/zipGrep.pl -- Searches for text in Zip files - -=item examples/ziptest.pl -- Lists a Zip file and checks member CRCs - -=item examples/ziprecent.pl -- Puts recent files into a zipfile - -=item examples/ziptest.pl -- Another way to check a Zip file for validity - -=back - -=head1 Can't Read/modify/write same Zip file - -B Why can't I open a Zip file, add a member, and write it back? I get an -error message when I try. - -B Because Archive::Zip doesn't (and can't, generally) read file contents into memory, -the original Zip file is required to stay around until the writing of the new -file is completed. - -The best way to do this is to write the Zip to a temporary file and then -rename the temporary file to have the old name (possibly after deleting the -old one). - -Archive::Zip v1.02 added the archive methods C and -C to do this simply and carefully. - -See C for an example of this technique. - -=head1 File creation time not set - -B Upon extracting files, I see that their modification (and access) times are -set to the time in the Zip archive. However, their creation time is not set to -the same time. Why? - -B Mostly because Perl doesn't give cross-platform access to I. -Indeed, many systems (like Unix) don't support such a concept. -However, if yours does, you can easily set it. Get the modification time from -the member using C. - -=head1 Can't use Archive::Zip on gzip files - -B Can I use Archive::Zip to extract Unix gzip files? - -B No. - -There is a distinction between Unix gzip files, and Zip archives that -also can use the gzip compression. - -Depending on the format of the gzip file, you can use L, or -L to decompress it (and de-archive it in the case of Tar files). - -You can unzip PKZIP/WinZip/etc/ archives using Archive::Zip (that's what -it's for) as long as any compressed members are compressed using -Deflate compression. - -=head1 Add a directory/tree to a Zip - -B How can I add a directory (or tree) full of files to a Zip? - -B You can use the Archive::Zip::addTree*() methods: - - use Archive::Zip; - my $zip = Archive::Zip->new(); - # add all readable files and directories below . as xyz/* - $zip->addTree( '.', 'xyz' ); - # add all readable plain files below /abc as def/* - $zip->addTree( '/abc', 'def', sub { -f && -r } ); - # add all .c files below /tmp as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.c$' ); - # add all .o files below /tmp as stuff/* if they aren't writable - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { ! -w } ); - # add all .so files below /tmp that are smaller than 200 bytes as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { -s < 200 } ); - # and write them into a file - $zip->writeToFileNamed('xxx.zip'); - -=head1 Extract a directory/tree - -B How can I extract some (or all) files from a Zip into a different -directory? - -B You can use the Archive::Zip::extractTree() method: -??? || - - # now extract the same files into /tmpx - $zip->extractTree( 'stuff', '/tmpx' ); - -=head1 Update a directory/tree - -B How can I update a Zip from a directory tree, adding or replacing only -the newer files? - -B You can use the Archive::Zip::updateTree() method that was added in version 1.09. - -=head1 Zip times might be off by 1 second - -B It bothers me greatly that my file times are wrong by one second about half -the time. Why don't you do something about it? - -B Get over it. This is a result of the Zip format storing times in DOS -format, which has a resolution of only two seconds. - -=head1 Zip times don't include time zone information - -B My file times don't respect time zones. What gives? - -B If this is important to you, please submit patches to read the various -Extra Fields that encode times with time zones. I'm just using the DOS -Date/Time, which doesn't have a time zone. - -=head1 How do I make a self-extracting Zip - -B I want to make a self-extracting Zip file. Can I do this? - -B Yes. You can write a self-extracting archive stub (that is, a version of -unzip) to the output filehandle that you pass to writeToFileHandle(). See -examples/selfex.pl for how to write a self-extracting archive. - -However, you should understand that this will only work on one kind of -platform (the one for which the stub was compiled). - -=head1 How can I deal with Zips with prepended garbage (i.e. from Sircam) - -B How can I tell if a Zip has been damaged by adding garbage to the -beginning or inside the file? - -B I added code for this for the Amavis virus scanner. You can query archives -for their 'eocdOffset' property, which should be 0: - - if ($zip->eocdOffset > 0) - { warn($zip->eocdOffset . " bytes of garbage at beginning or within Zip") } - -When members are extracted, this offset will be used to adjust the start of -the member if necessary. - -=head1 Can't extract Shrunk files - -B I'm trying to extract a file out of a Zip produced by PKZIP, and keep -getting this error message: - - error: Unsupported compression combination: read 6, write 0 - -B You can't uncompress this archive member. Archive::Zip only supports uncompressed -members, and compressed members that are compressed using the compression -supported by Compress::Raw::Zlib. That means only Deflated and Stored members. - -Your file is compressed using the Shrink format, which is not supported by -Compress::Raw::Zlib. - -You could, perhaps, use a command-line UnZip program (like the Info-Zip -one) to extract this. - -=head1 Can't do decryption - -B How do I decrypt encrypted Zip members? - -B With some other program or library. Archive::Zip doesn't support decryption, -and probably never will (unless I write it). - -=head1 How to test file integrity? - -B How can Archive::Zip can test the validity of a Zip file? - -B If you try to decompress the file, the gzip streams will report errors -if you have garbage. Most of the time. - -If you try to open the file and a central directory structure can't be -found, an error will be reported. - -When a file is being read, if we can't find a proper PK.. signature in -the right places we report a format error. - -If there is added garbage at the beginning of a Zip file (as inserted -by some viruses), you can find out about it, but Archive::Zip will ignore it, -and you can still use the archive. When it gets written back out the -added stuff will be gone. - -There are two ready-to-use utilities in the examples directory that can -be used to test file integrity, or that you can use as examples -for your own code: - -=over 4 - -=item examples/zipcheck.pl shows how to use an attempted extraction to test a file. - -=item examples/ziptest.pl shows how to test CRCs in a file. - -=back - -=head1 Duplicate files in Zip? - -B Archive::Zip let me put the same file in my Zip twice! Why don't you prevent this? - -B As far as I can tell, this is not disallowed by the Zip spec. If you -think it's a bad idea, check for it yourself: - - $zip->addFile($someFile, $someName) unless $zip->memberNamed($someName); - -I can even imagine cases where this might be useful (for instance, multiple -versions of files). - -=head1 File ownership/permissions/ACLS/etc - -B Why doesn't Archive::Zip deal with file ownership, ACLs, etc.? - -B There is no standard way to represent these in the Zip file format. If -you want to send me code to properly handle the various extra fields that -have been used to represent these through the years, I'll look at it. - -=head1 I can't compile but ActiveState only has an old version of Archive::Zip - -B I've only installed modules using ActiveState's PPM program and -repository. But they have a much older version of Archive::Zip than is in CPAN. Will -you send me a newer PPM? - -B Probably not, unless I get lots of extra time. But there's no reason you -can't install the version from CPAN. Archive::Zip is pure Perl, so all you need is -NMAKE, which you can get for free from Microsoft (see the FAQ in the -ActiveState documentation for details on how to install CPAN modules). - -=head1 My JPEGs (or MP3's) don't compress when I put them into Zips! - -B How come my JPEGs and MP3's don't compress much when I put them into Zips? - -B Because they're already compressed. - -=head1 Under Windows, things lock up/get damaged - -B I'm using Windows. When I try to use Archive::Zip, my machine locks up/makes -funny sounds/displays a BSOD/corrupts data. How can I fix this? - -B First, try the newest version of Compress::Raw::Zlib. I know of -Windows-related problems prior to v1.14 of that library. - -=head1 Zip contents in a scalar - -B I want to read a Zip file from (or write one to) a scalar variable instead -of a file. How can I do this? - -B Use C and the C and -C methods. -See C and C. - -=head1 Reading from streams - -B How do I read from a stream (like for the Info-Zip C program)? - -B This is not currently supported, though writing to a stream is. diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FileMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FileMember.pm deleted file mode 100644 index 64e7c9ae06f..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FileMember.pm +++ /dev/null @@ -1,64 +0,0 @@ -package Archive::Zip::FileMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw ( Archive::Zip::Member ); -} - -use Archive::Zip qw( - :UTILITY_METHODS -); - -sub externalFileName { - shift->{'externalFileName'}; -} - -# Return true if I depend on the named file -sub _usesFileNamed { - my $self = shift; - my $fileName = shift; - my $xfn = $self->externalFileName(); - return undef if ref($xfn); - return $xfn eq $fileName; -} - -sub fh { - my $self = shift; - $self->_openFile() - if !defined($self->{'fh'}) || !$self->{'fh'}->opened(); - return $self->{'fh'}; -} - -# opens my file handle from my file name -sub _openFile { - my $self = shift; - my ($status, $fh) = _newFileHandle($self->externalFileName(), 'r'); - if (!$status) { - _ioError("Can't open", $self->externalFileName()); - return undef; - } - $self->{'fh'} = $fh; - _binmode($fh); - return $fh; -} - -# Make sure I close my file handle -sub endRead { - my $self = shift; - undef $self->{'fh'}; # _closeFile(); - return $self->SUPER::endRead(@_); -} - -sub _become { - my $self = shift; - my $newClass = shift; - return $self if ref($self) eq $newClass; - delete($self->{'externalFileName'}); - delete($self->{'fh'}); - return $self->SUPER::_become($newClass); -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Member.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Member.pm deleted file mode 100644 index 94f9d38a8bd..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Member.pm +++ /dev/null @@ -1,1247 +0,0 @@ -package Archive::Zip::Member; - -# A generic member of an archive - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip ); - - if ($^O eq 'MSWin32') { - require Win32; - require Encode; - Encode->import(qw{ decode_utf8 }); - } -} - -use Archive::Zip qw( - :CONSTANTS - :MISC_CONSTANTS - :ERROR_CODES - :PKZIP_CONSTANTS - :UTILITY_METHODS -); - -use Time::Local (); -use Compress::Raw::Zlib qw( Z_OK Z_STREAM_END MAX_WBITS ); -use File::Path; -use File::Basename; - -# Unix perms for default creation of files/dirs. -use constant DEFAULT_DIRECTORY_PERMISSIONS => 040755; -use constant DEFAULT_FILE_PERMISSIONS => 0100666; -use constant DIRECTORY_ATTRIB => 040000; -use constant FILE_ATTRIB => 0100000; - -# Returns self if successful, else undef -# Assumes that fh is positioned at beginning of central directory file header. -# Leaves fh positioned immediately after file header or EOCD signature. -sub _newFromZipFile { - my $class = shift; - my $self = Archive::Zip::ZipFileMember->_newFromZipFile(@_); - return $self; -} - -sub newFromString { - my $class = shift; - - my ($stringOrStringRef, $fileName); - if (ref($_[0]) eq 'HASH') { - $stringOrStringRef = $_[0]->{string}; - $fileName = $_[0]->{zipName}; - } else { - ($stringOrStringRef, $fileName) = @_; - } - - my $self = - Archive::Zip::StringMember->_newFromString($stringOrStringRef, $fileName); - return $self; -} - -sub newFromFile { - my $class = shift; - - my ($fileName, $zipName); - if (ref($_[0]) eq 'HASH') { - $fileName = $_[0]->{fileName}; - $zipName = $_[0]->{zipName}; - } else { - ($fileName, $zipName) = @_; - } - - my $self = - Archive::Zip::NewFileMember->_newFromFileNamed($fileName, $zipName); - return $self; -} - -sub newDirectoryNamed { - my $class = shift; - - my ($directoryName, $newName); - if (ref($_[0]) eq 'HASH') { - $directoryName = $_[0]->{directoryName}; - $newName = $_[0]->{zipName}; - } else { - ($directoryName, $newName) = @_; - } - - my $self = - Archive::Zip::DirectoryMember->_newNamed($directoryName, $newName); - return $self; -} - -sub new { - my $class = shift; - my $self = { - 'lastModFileDateTime' => 0, - 'fileAttributeFormat' => FA_UNIX, - 'versionMadeBy' => 20, - 'versionNeededToExtract' => 20, - 'bitFlag' => ($Archive::Zip::UNICODE ? 0x0800 : 0), - 'compressionMethod' => COMPRESSION_STORED, - 'desiredCompressionMethod' => COMPRESSION_STORED, - 'desiredCompressionLevel' => COMPRESSION_LEVEL_NONE, - 'internalFileAttributes' => 0, - 'externalFileAttributes' => 0, # set later - 'fileName' => '', - 'cdExtraField' => '', - 'localExtraField' => '', - 'fileComment' => '', - 'crc32' => 0, - 'compressedSize' => 0, - 'uncompressedSize' => 0, - 'isSymbolicLink' => 0, - 'password' => undef, # password for encrypted data - 'crc32c' => -1, # crc for decrypted data - @_ - }; - bless($self, $class); - $self->unixFileAttributes($self->DEFAULT_FILE_PERMISSIONS); - return $self; -} - -sub _becomeDirectoryIfNecessary { - my $self = shift; - $self->_become('Archive::Zip::DirectoryMember') - if $self->isDirectory(); - return $self; -} - -# Morph into given class (do whatever cleanup I need to do) -sub _become { - return bless($_[0], $_[1]); -} - -sub versionMadeBy { - shift->{'versionMadeBy'}; -} - -sub fileAttributeFormat { - my $self = shift; - - if (@_) { - $self->{fileAttributeFormat} = - (ref($_[0]) eq 'HASH') ? $_[0]->{format} : $_[0]; - } else { - return $self->{fileAttributeFormat}; - } -} - -sub versionNeededToExtract { - shift->{'versionNeededToExtract'}; -} - -sub bitFlag { - my $self = shift; - -# Set General Purpose Bit Flags according to the desiredCompressionLevel setting - if ( $self->desiredCompressionLevel == 1 - || $self->desiredCompressionLevel == 2) { - $self->{'bitFlag'} = DEFLATING_COMPRESSION_FAST; - } elsif ($self->desiredCompressionLevel == 3 - || $self->desiredCompressionLevel == 4 - || $self->desiredCompressionLevel == 5 - || $self->desiredCompressionLevel == 6 - || $self->desiredCompressionLevel == 7) { - $self->{'bitFlag'} = DEFLATING_COMPRESSION_NORMAL; - } elsif ($self->desiredCompressionLevel == 8 - || $self->desiredCompressionLevel == 9) { - $self->{'bitFlag'} = DEFLATING_COMPRESSION_MAXIMUM; - } - - if ($Archive::Zip::UNICODE) { - $self->{'bitFlag'} |= 0x0800; - } - $self->{'bitFlag'}; -} - -sub password { - my $self = shift; - $self->{'password'} = shift if @_; - $self->{'password'}; -} - -sub compressionMethod { - shift->{'compressionMethod'}; -} - -sub desiredCompressionMethod { - my $self = shift; - my $newDesiredCompressionMethod = - (ref($_[0]) eq 'HASH') ? shift->{compressionMethod} : shift; - my $oldDesiredCompressionMethod = $self->{'desiredCompressionMethod'}; - if (defined($newDesiredCompressionMethod)) { - $self->{'desiredCompressionMethod'} = $newDesiredCompressionMethod; - if ($newDesiredCompressionMethod == COMPRESSION_STORED) { - $self->{'desiredCompressionLevel'} = 0; - $self->{'bitFlag'} &= ~GPBF_HAS_DATA_DESCRIPTOR_MASK - if $self->uncompressedSize() == 0; - } elsif ($oldDesiredCompressionMethod == COMPRESSION_STORED) { - $self->{'desiredCompressionLevel'} = COMPRESSION_LEVEL_DEFAULT; - } - } - return $oldDesiredCompressionMethod; -} - -sub desiredCompressionLevel { - my $self = shift; - my $newDesiredCompressionLevel = - (ref($_[0]) eq 'HASH') ? shift->{compressionLevel} : shift; - my $oldDesiredCompressionLevel = $self->{'desiredCompressionLevel'}; - if (defined($newDesiredCompressionLevel)) { - $self->{'desiredCompressionLevel'} = $newDesiredCompressionLevel; - $self->{'desiredCompressionMethod'} = ( - $newDesiredCompressionLevel - ? COMPRESSION_DEFLATED - : COMPRESSION_STORED - ); - } - return $oldDesiredCompressionLevel; -} - -sub fileName { - my $self = shift; - my $newName = shift; - if (defined $newName) { - $newName =~ s{[\\/]+}{/}g; # deal with dos/windoze problems - $self->{'fileName'} = $newName; - } - return $self->{'fileName'}; -} - -sub lastModFileDateTime { - my $modTime = shift->{'lastModFileDateTime'}; - $modTime =~ m/^(\d+)$/; # untaint - return $1; -} - -sub lastModTime { - my $self = shift; - return _dosToUnixTime($self->lastModFileDateTime()); -} - -sub setLastModFileDateTimeFromUnix { - my $self = shift; - my $time_t = shift; - $self->{'lastModFileDateTime'} = _unixToDosTime($time_t); -} - -sub internalFileAttributes { - shift->{'internalFileAttributes'}; -} - -sub externalFileAttributes { - shift->{'externalFileAttributes'}; -} - -# Convert UNIX permissions into proper value for zip file -# Usable as a function or a method -sub _mapPermissionsFromUnix { - my $self = shift; - my $mode = shift; - my $attribs = $mode << 16; - - # Microsoft Windows Explorer needs this bit set for directories - if ($mode & DIRECTORY_ATTRIB) { - $attribs |= 16; - } - - return $attribs; - - # TODO: map more MS-DOS perms -} - -# Convert ZIP permissions into Unix ones -# -# This was taken from Info-ZIP group's portable UnZip -# zipfile-extraction program, version 5.50. -# http://www.info-zip.org/pub/infozip/ -# -# See the mapattr() function in unix/unix.c -# See the attribute format constants in unzpriv.h -# -# XXX Note that there's one situation that is not implemented -# yet that depends on the "extra field." -sub _mapPermissionsToUnix { - my $self = shift; - - my $format = $self->{'fileAttributeFormat'}; - my $attribs = $self->{'externalFileAttributes'}; - - my $mode = 0; - - if ($format == FA_AMIGA) { - $attribs = $attribs >> 17 & 7; # Amiga RWE bits - $mode = $attribs << 6 | $attribs << 3 | $attribs; - return $mode; - } - - if ($format == FA_THEOS) { - $attribs &= 0xF1FFFFFF; - if (($attribs & 0xF0000000) != 0x40000000) { - $attribs &= 0x01FFFFFF; # not a dir, mask all ftype bits - } else { - $attribs &= 0x41FFFFFF; # leave directory bit as set - } - } - - if ( $format == FA_UNIX - || $format == FA_VAX_VMS - || $format == FA_ACORN - || $format == FA_ATARI_ST - || $format == FA_BEOS - || $format == FA_QDOS - || $format == FA_TANDEM) { - $mode = $attribs >> 16; - return $mode if $mode != 0 or not $self->localExtraField; - - # warn("local extra field is: ", $self->localExtraField, "\n"); - - # XXX This condition is not implemented - # I'm just including the comments from the info-zip section for now. - - # Some (non-Info-ZIP) implementations of Zip for Unix and - # VMS (and probably others ??) leave 0 in the upper 16-bit - # part of the external_file_attributes field. Instead, they - # store file permission attributes in some extra field. - # As a work-around, we search for the presence of one of - # these extra fields and fall back to the MSDOS compatible - # part of external_file_attributes if one of the known - # e.f. types has been detected. - # Later, we might implement extraction of the permission - # bits from the VMS extra field. But for now, the work-around - # should be sufficient to provide "readable" extracted files. - # (For ASI Unix e.f., an experimental remap from the e.f. - # mode value IS already provided!) - } - - # PKWARE's PKZip for Unix marks entries as FA_MSDOS, but stores the - # Unix attributes in the upper 16 bits of the external attributes - # field, just like Info-ZIP's Zip for Unix. We try to use that - # value, after a check for consistency with the MSDOS attribute - # bits (see below). - if ($format == FA_MSDOS) { - $mode = $attribs >> 16; - } - - # FA_MSDOS, FA_OS2_HPFS, FA_WINDOWS_NTFS, FA_MACINTOSH, FA_TOPS20 - $attribs = !($attribs & 1) << 1 | ($attribs & 0x10) >> 4; - - # keep previous $mode setting when its "owner" - # part appears to be consistent with DOS attribute flags! - return $mode if ($mode & 0700) == (0400 | $attribs << 6); - $mode = 0444 | $attribs << 6 | $attribs << 3 | $attribs; - return $mode; -} - -sub unixFileAttributes { - my $self = shift; - my $oldPerms = $self->_mapPermissionsToUnix; - - my $perms; - if (@_) { - $perms = (ref($_[0]) eq 'HASH') ? $_[0]->{attributes} : $_[0]; - - if ($self->isDirectory) { - $perms &= ~FILE_ATTRIB; - $perms |= DIRECTORY_ATTRIB; - } else { - $perms &= ~DIRECTORY_ATTRIB; - $perms |= FILE_ATTRIB; - } - $self->{externalFileAttributes} = - $self->_mapPermissionsFromUnix($perms); - } - - return $oldPerms; -} - -sub localExtraField { - my $self = shift; - - if (@_) { - $self->{localExtraField} = - (ref($_[0]) eq 'HASH') ? $_[0]->{field} : $_[0]; - } else { - return $self->{localExtraField}; - } -} - -sub cdExtraField { - my $self = shift; - - if (@_) { - $self->{cdExtraField} = (ref($_[0]) eq 'HASH') ? $_[0]->{field} : $_[0]; - } else { - return $self->{cdExtraField}; - } -} - -sub extraFields { - my $self = shift; - return $self->localExtraField() . $self->cdExtraField(); -} - -sub fileComment { - my $self = shift; - - if (@_) { - $self->{fileComment} = - (ref($_[0]) eq 'HASH') - ? pack('C0a*', $_[0]->{comment}) - : pack('C0a*', $_[0]); - } else { - return $self->{fileComment}; - } -} - -sub hasDataDescriptor { - my $self = shift; - if (@_) { - my $shouldHave = shift; - if ($shouldHave) { - $self->{'bitFlag'} |= GPBF_HAS_DATA_DESCRIPTOR_MASK; - } else { - $self->{'bitFlag'} &= ~GPBF_HAS_DATA_DESCRIPTOR_MASK; - } - } - return $self->{'bitFlag'} & GPBF_HAS_DATA_DESCRIPTOR_MASK; -} - -sub crc32 { - shift->{'crc32'}; -} - -sub crc32String { - sprintf("%08x", shift->{'crc32'}); -} - -sub compressedSize { - shift->{'compressedSize'}; -} - -sub uncompressedSize { - shift->{'uncompressedSize'}; -} - -sub isEncrypted { - shift->{'bitFlag'} & GPBF_ENCRYPTED_MASK; -} - -sub isTextFile { - my $self = shift; - my $bit = $self->internalFileAttributes() & IFA_TEXT_FILE_MASK; - if (@_) { - my $flag = (ref($_[0]) eq 'HASH') ? shift->{flag} : shift; - $self->{'internalFileAttributes'} &= ~IFA_TEXT_FILE_MASK; - $self->{'internalFileAttributes'} |= - ($flag ? IFA_TEXT_FILE : IFA_BINARY_FILE); - } - return $bit == IFA_TEXT_FILE; -} - -sub isBinaryFile { - my $self = shift; - my $bit = $self->internalFileAttributes() & IFA_TEXT_FILE_MASK; - if (@_) { - my $flag = shift; - $self->{'internalFileAttributes'} &= ~IFA_TEXT_FILE_MASK; - $self->{'internalFileAttributes'} |= - ($flag ? IFA_BINARY_FILE : IFA_TEXT_FILE); - } - return $bit == IFA_BINARY_FILE; -} - -sub extractToFileNamed { - my $self = shift; - - # local FS name - my $name = (ref($_[0]) eq 'HASH') ? $_[0]->{name} : $_[0]; - $self->{'isSymbolicLink'} = 0; - - # Check if the file / directory is a symbolic link or not - if ($self->{'externalFileAttributes'} == 0xA1FF0000) { - $self->{'isSymbolicLink'} = 1; - $self->{'newName'} = $name; - my ($status, $fh) = _newFileHandle($name, 'r'); - my $retval = $self->extractToFileHandle($fh); - $fh->close(); - } else { - - #return _writeSymbolicLink($self, $name) if $self->isSymbolicLink(); - - my ($status, $fh); - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $name = decode_utf8(Win32::GetFullPathName($name)); - mkpath_win32($name); - Win32::CreateFile($name); - ($status, $fh) = _newFileHandle(Win32::GetANSIPathName($name), 'w'); - } else { - mkpath(dirname($name)); # croaks on error - ($status, $fh) = _newFileHandle($name, 'w'); - } - return _ioError("Can't open file $name for write") unless $status; - my $retval = $self->extractToFileHandle($fh); - $fh->close(); - chmod($self->unixFileAttributes(), $name) - or return _error("Can't chmod() ${name}: $!"); - utime($self->lastModTime(), $self->lastModTime(), $name); - return $retval; - } -} - -sub mkpath_win32 { - my $path = shift; - use File::Spec; - - my ($volume, @path) = File::Spec->splitdir($path); - $path = File::Spec->catfile($volume, shift @path); - pop @path; - while (@path) { - $path = File::Spec->catfile($path, shift @path); - Win32::CreateDirectory($path); - } -} - -sub _writeSymbolicLink { - my $self = shift; - my $name = shift; - my $chunkSize = $Archive::Zip::ChunkSize; - - #my ( $outRef, undef ) = $self->readChunk($chunkSize); - my $fh; - my $retval = $self->extractToFileHandle($fh); - my ($outRef, undef) = $self->readChunk(100); -} - -sub isSymbolicLink { - my $self = shift; - if ($self->{'externalFileAttributes'} == 0xA1FF0000) { - $self->{'isSymbolicLink'} = 1; - } else { - return 0; - } - 1; -} - -sub isDirectory { - return 0; -} - -sub externalFileName { - return undef; -} - -# The following are used when copying data -sub _writeOffset { - shift->{'writeOffset'}; -} - -sub _readOffset { - shift->{'readOffset'}; -} - -sub writeLocalHeaderRelativeOffset { - shift->{'writeLocalHeaderRelativeOffset'}; -} - -sub wasWritten { shift->{'wasWritten'} } - -sub _dataEnded { - shift->{'dataEnded'}; -} - -sub _readDataRemaining { - shift->{'readDataRemaining'}; -} - -sub _inflater { - shift->{'inflater'}; -} - -sub _deflater { - shift->{'deflater'}; -} - -# Return the total size of my local header -sub _localHeaderSize { - my $self = shift; - { - use bytes; - return SIGNATURE_LENGTH + - LOCAL_FILE_HEADER_LENGTH + - length($self->fileName()) + - length($self->localExtraField()); - } -} - -# Return the total size of my CD header -sub _centralDirectoryHeaderSize { - my $self = shift; - { - use bytes; - return SIGNATURE_LENGTH + - CENTRAL_DIRECTORY_FILE_HEADER_LENGTH + - length($self->fileName()) + - length($self->cdExtraField()) + - length($self->fileComment()); - } -} - -# DOS date/time format -# 0-4 (5) Second divided by 2 -# 5-10 (6) Minute (0-59) -# 11-15 (5) Hour (0-23 on a 24-hour clock) -# 16-20 (5) Day of the month (1-31) -# 21-24 (4) Month (1 = January, 2 = February, etc.) -# 25-31 (7) Year offset from 1980 (add 1980 to get actual year) - -# Convert DOS date/time format to unix time_t format -# NOT AN OBJECT METHOD! -sub _dosToUnixTime { - my $dt = shift; - return time() unless defined($dt); - - my $year = (($dt >> 25) & 0x7f) + 80; - my $mon = (($dt >> 21) & 0x0f) - 1; - my $mday = (($dt >> 16) & 0x1f); - - my $hour = (($dt >> 11) & 0x1f); - my $min = (($dt >> 5) & 0x3f); - my $sec = (($dt << 1) & 0x3e); - - # catch errors - my $time_t = - eval { Time::Local::timelocal($sec, $min, $hour, $mday, $mon, $year); }; - return time() if ($@); - return $time_t; -} - -# Note, this is not exactly UTC 1980, it's 1980 + 12 hours and 1 -# minute so that nothing timezoney can muck us up. -my $safe_epoch = 315576060; - -# convert a unix time to DOS date/time -# NOT AN OBJECT METHOD! -sub _unixToDosTime { - my $time_t = shift; - unless ($time_t) { - _error("Tried to add member with zero or undef value for time"); - $time_t = $safe_epoch; - } - if ($time_t < $safe_epoch) { - _ioError("Unsupported date before 1980 encountered, moving to 1980"); - $time_t = $safe_epoch; - } - my ($sec, $min, $hour, $mday, $mon, $year) = localtime($time_t); - my $dt = 0; - $dt += ($sec >> 1); - $dt += ($min << 5); - $dt += ($hour << 11); - $dt += ($mday << 16); - $dt += (($mon + 1) << 21); - $dt += (($year - 80) << 25); - return $dt; -} - -sub head { - my ($self, $mode) = (@_, 0); - - use bytes; - return pack LOCAL_FILE_HEADER_FORMAT, - $self->versionNeededToExtract(), - $self->{'bitFlag'}, - $self->desiredCompressionMethod(), - $self->lastModFileDateTime(), - $self->hasDataDescriptor() - ? (0,0,0) # crc, compr & uncompr all zero if data descriptor present - : ( - $self->crc32(), - $mode - ? $self->_writeOffset() # compressed size - : $self->compressedSize(), # may need to be re-written later - $self->uncompressedSize(), - ), - length($self->fileName()), - length($self->localExtraField()); -} - -# Write my local header to a file handle. -# Stores the offset to the start of the header in my -# writeLocalHeaderRelativeOffset member. -# Returns AZ_OK on success. -sub _writeLocalFileHeader { - my $self = shift; - my $fh = shift; - - my $signatureData = pack(SIGNATURE_FORMAT, LOCAL_FILE_HEADER_SIGNATURE); - $self->_print($fh, $signatureData) - or return _ioError("writing local header signature"); - - my $header = $self->head(1); - - $self->_print($fh, $header) or return _ioError("writing local header"); - - # Check for a valid filename or a filename equal to a literal `0' - if ($self->fileName() || $self->fileName eq '0') { - $self->_print($fh, $self->fileName()) - or return _ioError("writing local header filename"); - } - if ($self->localExtraField()) { - $self->_print($fh, $self->localExtraField()) - or return _ioError("writing local extra field"); - } - - return AZ_OK; -} - -sub _writeCentralDirectoryFileHeader { - my $self = shift; - my $fh = shift; - - my $sigData = - pack(SIGNATURE_FORMAT, CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE); - $self->_print($fh, $sigData) - or return _ioError("writing central directory header signature"); - - my ($fileNameLength, $extraFieldLength, $fileCommentLength); - { - use bytes; - $fileNameLength = length($self->fileName()); - $extraFieldLength = length($self->cdExtraField()); - $fileCommentLength = length($self->fileComment()); - } - - my $header = pack( - CENTRAL_DIRECTORY_FILE_HEADER_FORMAT, - $self->versionMadeBy(), - $self->fileAttributeFormat(), - $self->versionNeededToExtract(), - $self->bitFlag(), - $self->desiredCompressionMethod(), - $self->lastModFileDateTime(), - $self->crc32(), # these three fields should have been updated - $self->_writeOffset(), # by writing the data stream out - $self->uncompressedSize(), # - $fileNameLength, - $extraFieldLength, - $fileCommentLength, - 0, # {'diskNumberStart'}, - $self->internalFileAttributes(), - $self->externalFileAttributes(), - $self->writeLocalHeaderRelativeOffset()); - - $self->_print($fh, $header) - or return _ioError("writing central directory header"); - if ($fileNameLength) { - $self->_print($fh, $self->fileName()) - or return _ioError("writing central directory header signature"); - } - if ($extraFieldLength) { - $self->_print($fh, $self->cdExtraField()) - or return _ioError("writing central directory extra field"); - } - if ($fileCommentLength) { - $self->_print($fh, $self->fileComment()) - or return _ioError("writing central directory file comment"); - } - - return AZ_OK; -} - -# This writes a data descriptor to the given file handle. -# Assumes that crc32, writeOffset, and uncompressedSize are -# set correctly (they should be after a write). -# Further, the local file header should have the -# GPBF_HAS_DATA_DESCRIPTOR_MASK bit set. -sub _writeDataDescriptor { - my $self = shift; - my $fh = shift; - my $header = pack( - SIGNATURE_FORMAT . DATA_DESCRIPTOR_FORMAT, - DATA_DESCRIPTOR_SIGNATURE, - $self->crc32(), - $self->_writeOffset(), # compressed size - $self->uncompressedSize()); - - $self->_print($fh, $header) - or return _ioError("writing data descriptor"); - return AZ_OK; -} - -# Re-writes the local file header with new crc32 and compressedSize fields. -# To be called after writing the data stream. -# Assumes that filename and extraField sizes didn't change since last written. -sub _refreshLocalFileHeader { - my $self = shift; - my $fh = shift; - - my $here = $fh->tell(); - $fh->seek($self->writeLocalHeaderRelativeOffset() + SIGNATURE_LENGTH, - IO::Seekable::SEEK_SET) - or return _ioError("seeking to rewrite local header"); - - my $header = $self->head(1); - - $self->_print($fh, $header) - or return _ioError("re-writing local header"); - $fh->seek($here, IO::Seekable::SEEK_SET) - or return _ioError("seeking after rewrite of local header"); - - return AZ_OK; -} - -sub readChunk { - my $self = shift; - my $chunkSize = (ref($_[0]) eq 'HASH') ? $_[0]->{chunkSize} : $_[0]; - - if ($self->readIsDone()) { - $self->endRead(); - my $dummy = ''; - return (\$dummy, AZ_STREAM_END); - } - - $chunkSize = $Archive::Zip::ChunkSize if not defined($chunkSize); - $chunkSize = $self->_readDataRemaining() - if $chunkSize > $self->_readDataRemaining(); - - my $buffer = ''; - my $outputRef; - my ($bytesRead, $status) = $self->_readRawChunk(\$buffer, $chunkSize); - return (\$buffer, $status) unless $status == AZ_OK; - - $buffer && $self->isEncrypted and $buffer = $self->_decode($buffer); - $self->{'readDataRemaining'} -= $bytesRead; - $self->{'readOffset'} += $bytesRead; - - if ($self->compressionMethod() == COMPRESSION_STORED) { - $self->{'crc32'} = $self->computeCRC32($buffer, $self->{'crc32'}); - } - - ($outputRef, $status) = &{$self->{'chunkHandler'}}($self, \$buffer); - $self->{'writeOffset'} += length($$outputRef); - - $self->endRead() - if $self->readIsDone(); - - return ($outputRef, $status); -} - -# Read the next raw chunk of my data. Subclasses MUST implement. -# my ( $bytesRead, $status) = $self->_readRawChunk( \$buffer, $chunkSize ); -sub _readRawChunk { - my $self = shift; - return $self->_subclassResponsibility(); -} - -# A place holder to catch rewindData errors if someone ignores -# the error code. -sub _noChunk { - my $self = shift; - return (\undef, _error("trying to copy chunk when init failed")); -} - -# Basically a no-op so that I can have a consistent interface. -# ( $outputRef, $status) = $self->_copyChunk( \$buffer ); -sub _copyChunk { - my ($self, $dataRef) = @_; - return ($dataRef, AZ_OK); -} - -# ( $outputRef, $status) = $self->_deflateChunk( \$buffer ); -sub _deflateChunk { - my ($self, $buffer) = @_; - my ($status) = $self->_deflater()->deflate($buffer, my $out); - - if ($self->_readDataRemaining() == 0) { - my $extraOutput; - ($status) = $self->_deflater()->flush($extraOutput); - $out .= $extraOutput; - $self->endRead(); - return (\$out, AZ_STREAM_END); - } elsif ($status == Z_OK) { - return (\$out, AZ_OK); - } else { - $self->endRead(); - my $retval = _error('deflate error', $status); - my $dummy = ''; - return (\$dummy, $retval); - } -} - -# ( $outputRef, $status) = $self->_inflateChunk( \$buffer ); -sub _inflateChunk { - my ($self, $buffer) = @_; - my ($status) = $self->_inflater()->inflate($buffer, my $out); - my $retval; - $self->endRead() unless $status == Z_OK; - if ($status == Z_OK || $status == Z_STREAM_END) { - $retval = ($status == Z_STREAM_END) ? AZ_STREAM_END : AZ_OK; - return (\$out, $retval); - } else { - $retval = _error('inflate error', $status); - my $dummy = ''; - return (\$dummy, $retval); - } -} - -sub rewindData { - my $self = shift; - my $status; - - # set to trap init errors - $self->{'chunkHandler'} = $self->can('_noChunk'); - - # Work around WinZip bug with 0-length DEFLATED files - $self->desiredCompressionMethod(COMPRESSION_STORED) - if $self->uncompressedSize() == 0; - - # assume that we're going to read the whole file, and compute the CRC anew. - $self->{'crc32'} = 0 - if ($self->compressionMethod() == COMPRESSION_STORED); - - # These are the only combinations of methods we deal with right now. - if ( $self->compressionMethod() == COMPRESSION_STORED - and $self->desiredCompressionMethod() == COMPRESSION_DEFLATED) { - ($self->{'deflater'}, $status) = Compress::Raw::Zlib::Deflate->new( - '-Level' => $self->desiredCompressionLevel(), - '-WindowBits' => -MAX_WBITS(), # necessary magic - '-Bufsize' => $Archive::Zip::ChunkSize, - @_ - ); # pass additional options - return _error('deflateInit error:', $status) - unless $status == Z_OK; - $self->{'chunkHandler'} = $self->can('_deflateChunk'); - } elsif ($self->compressionMethod() == COMPRESSION_DEFLATED - and $self->desiredCompressionMethod() == COMPRESSION_STORED) { - ($self->{'inflater'}, $status) = Compress::Raw::Zlib::Inflate->new( - '-WindowBits' => -MAX_WBITS(), # necessary magic - '-Bufsize' => $Archive::Zip::ChunkSize, - @_ - ); # pass additional options - return _error('inflateInit error:', $status) - unless $status == Z_OK; - $self->{'chunkHandler'} = $self->can('_inflateChunk'); - } elsif ($self->compressionMethod() == $self->desiredCompressionMethod()) { - $self->{'chunkHandler'} = $self->can('_copyChunk'); - } else { - return _error( - sprintf( - "Unsupported compression combination: read %d, write %d", - $self->compressionMethod(), - $self->desiredCompressionMethod())); - } - - $self->{'readDataRemaining'} = - ($self->compressionMethod() == COMPRESSION_STORED) - ? $self->uncompressedSize() - : $self->compressedSize(); - $self->{'dataEnded'} = 0; - $self->{'readOffset'} = 0; - - return AZ_OK; -} - -sub endRead { - my $self = shift; - delete $self->{'inflater'}; - delete $self->{'deflater'}; - $self->{'dataEnded'} = 1; - $self->{'readDataRemaining'} = 0; - return AZ_OK; -} - -sub readIsDone { - my $self = shift; - return ($self->_dataEnded() or !$self->_readDataRemaining()); -} - -sub contents { - my $self = shift; - my $newContents = shift; - - if (defined($newContents)) { - - # change our type and call the subclass contents method. - $self->_become('Archive::Zip::StringMember'); - return $self->contents(pack('C0a*', $newContents)); # in case of Unicode - } else { - my $oldCompression = - $self->desiredCompressionMethod(COMPRESSION_STORED); - my $status = $self->rewindData(@_); - if ($status != AZ_OK) { - $self->endRead(); - return $status; - } - my $retval = ''; - while ($status == AZ_OK) { - my $ref; - ($ref, $status) = $self->readChunk($self->_readDataRemaining()); - - # did we get it in one chunk? - if (length($$ref) == $self->uncompressedSize()) { - $retval = $$ref; - } else { - $retval .= $$ref - } - } - $self->desiredCompressionMethod($oldCompression); - $self->endRead(); - $status = AZ_OK if $status == AZ_STREAM_END; - $retval = undef unless $status == AZ_OK; - return wantarray ? ($retval, $status) : $retval; - } -} - -sub extractToFileHandle { - my $self = shift; - my $fh = (ref($_[0]) eq 'HASH') ? shift->{fileHandle} : shift; - _binmode($fh); - my $oldCompression = $self->desiredCompressionMethod(COMPRESSION_STORED); - my $status = $self->rewindData(@_); - $status = $self->_writeData($fh) if $status == AZ_OK; - $self->desiredCompressionMethod($oldCompression); - $self->endRead(); - return $status; -} - -# write local header and data stream to file handle -sub _writeToFileHandle { - my $self = shift; - my $fh = shift; - my $fhIsSeekable = shift; - my $offset = shift; - - return _error("no member name given for $self") - if $self->fileName() eq ''; - - $self->{'writeLocalHeaderRelativeOffset'} = $offset; - $self->{'wasWritten'} = 0; - - # Determine if I need to write a data descriptor - # I need to do this if I can't refresh the header - # and I don't know compressed size or crc32 fields. - my $headerFieldsUnknown = ( - ($self->uncompressedSize() > 0) - and ($self->compressionMethod() == COMPRESSION_STORED - or $self->desiredCompressionMethod() == COMPRESSION_DEFLATED)); - - my $shouldWriteDataDescriptor = - ($headerFieldsUnknown and not $fhIsSeekable); - - $self->hasDataDescriptor(1) - if ($shouldWriteDataDescriptor); - - $self->{'writeOffset'} = 0; - - my $status = $self->rewindData(); - ($status = $self->_writeLocalFileHeader($fh)) - if $status == AZ_OK; - ($status = $self->_writeData($fh)) - if $status == AZ_OK; - if ($status == AZ_OK) { - $self->{'wasWritten'} = 1; - if ($self->hasDataDescriptor()) { - $status = $self->_writeDataDescriptor($fh); - } elsif ($headerFieldsUnknown) { - $status = $self->_refreshLocalFileHeader($fh); - } - } - - return $status; -} - -# Copy my (possibly compressed) data to given file handle. -# Returns C on success -sub _writeData { - my $self = shift; - my $writeFh = shift; - -# If symbolic link, just create one if the operating system is Linux, Unix, BSD or VMS -# TODO: Add checks for other operating systems - if ($self->{'isSymbolicLink'} == 1 && $^O eq 'linux') { - my $chunkSize = $Archive::Zip::ChunkSize; - my ($outRef, $status) = $self->readChunk($chunkSize); - symlink $$outRef, $self->{'newName'}; - } else { - return AZ_OK if ($self->uncompressedSize() == 0); - my $status; - my $chunkSize = $Archive::Zip::ChunkSize; - while ($self->_readDataRemaining() > 0) { - my $outRef; - ($outRef, $status) = $self->readChunk($chunkSize); - return $status if ($status != AZ_OK and $status != AZ_STREAM_END); - - if (length($$outRef) > 0) { - $self->_print($writeFh, $$outRef) - or return _ioError("write error during copy"); - } - - last if $status == AZ_STREAM_END; - } - } - return AZ_OK; -} - -# Return true if I depend on the named file -sub _usesFileNamed { - return 0; -} - -# ############################################################################## -# -# Decrypt section -# -# H.Merijn Brand (Tux) 2011-06-28 -# -# ############################################################################## - -# This code is derived from the crypt source of unzip-6.0 dated 05 Jan 2007 -# Its license states: -# -# --8<--- -# Copyright (c) 1990-2007 Info-ZIP. All rights reserved. - -# See the accompanying file LICENSE, version 2005-Feb-10 or later -# (the contents of which are also included in (un)zip.h) for terms of use. -# If, for some reason, all these files are missing, the Info-ZIP license -# also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html -# -# crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] - -# The main encryption/decryption source code for Info-Zip software was -# originally written in Europe. To the best of our knowledge, it can -# be freely distributed in both source and object forms from any country, -# including the USA under License Exception TSU of the U.S. Export -# Administration Regulations (section 740.13(e)) of 6 June 2002. - -# NOTE on copyright history: -# Previous versions of this source package (up to version 2.8) were -# not copyrighted and put in the public domain. If you cannot comply -# with the Info-Zip LICENSE, you may want to look for one of those -# public domain versions. -# -# This encryption code is a direct transcription of the algorithm from -# Roger Schlafly, described by Phil Katz in the file appnote.txt. This -# file (appnote.txt) is distributed with the PKZIP program (even in the -# version without encryption capabilities). -# -->8--- - -# As of January 2000, US export regulations were amended to allow export -# of free encryption source code from the US. As of June 2002, these -# regulations were further relaxed to allow export of encryption binaries -# associated with free encryption source code. The Zip 2.31, UnZip 5.52 -# and Wiz 5.02 archives now include full crypto source code. As of the -# Zip 2.31 release, all official binaries include encryption support; the -# former "zcr" archives ceased to exist. -# (Note that restrictions may still exist in other countries, of course.) - -# For now, we just support the decrypt stuff -# All below methods are supposed to be private - -# use Data::Peek; - -my @keys; -my @crct = do { - my $xor = 0xedb88320; - my @crc = (0) x 1024; - - # generate a crc for every 8-bit value - foreach my $n (0 .. 255) { - my $c = $n; - $c = $c & 1 ? $xor ^ ($c >> 1) : $c >> 1 for 1 .. 8; - $crc[$n] = _revbe($c); - } - - # generate crc for each value followed by one, two, and three zeros */ - foreach my $n (0 .. 255) { - my $c = ($crc[($crc[$n] >> 24) ^ 0] ^ ($crc[$n] << 8)) & 0xffffffff; - $crc[$_ * 256 + $n] = $c for 1 .. 3; - } - map { _revbe($crc[$_]) } 0 .. 1023; -}; - -sub _crc32 { - my ($c, $b) = @_; - return ($crct[($c ^ $b) & 0xff] ^ ($c >> 8)); -} # _crc32 - -sub _revbe { - my $w = shift; - return (($w >> 24) + - (($w >> 8) & 0xff00) + - (($w & 0xff00) << 8) + - (($w & 0xff) << 24)); -} # _revbe - -sub _update_keys { - use integer; - my $c = shift; # signed int - $keys[0] = _crc32($keys[0], $c); - $keys[1] = (($keys[1] + ($keys[0] & 0xff)) * 0x08088405 + 1) & 0xffffffff; - my $keyshift = $keys[1] >> 24; - $keys[2] = _crc32($keys[2], $keyshift); -} # _update_keys - -sub _zdecode ($) { - my $c = shift; - my $t = ($keys[2] & 0xffff) | 2; - _update_keys($c ^= ((($t * ($t ^ 1)) >> 8) & 0xff)); - return $c; -} # _zdecode - -sub _decode { - my $self = shift; - my $buff = shift; - - $self->isEncrypted or return $buff; - - my $pass = $self->password; - defined $pass or return ""; - - @keys = (0x12345678, 0x23456789, 0x34567890); - _update_keys($_) for unpack "C*", $pass; - - # DDumper { uk => [ @keys ] }; - - my $head = substr $buff, 0, 12, ""; - my @head = map { _zdecode($_) } unpack "C*", $head; - my $x = - $self->{externalFileAttributes} - ? ($self->{lastModFileDateTime} >> 8) & 0xff - : $self->{crc32} >> 24; - $head[-1] == $x or return ""; # Password fail - - # Worth checking ... - $self->{crc32c} = (unpack LOCAL_FILE_HEADER_FORMAT, pack "C*", @head)[3]; - - # DHexDump ($buff); - $buff = pack "C*" => map { _zdecode($_) } unpack "C*" => $buff; - - # DHexDump ($buff); - return $buff; -} # _decode - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MemberRead.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MemberRead.pm deleted file mode 100644 index acb91ebb16a..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MemberRead.pm +++ /dev/null @@ -1,348 +0,0 @@ -package Archive::Zip::MemberRead; - -=head1 NAME - -Archive::Zip::MemberRead - A wrapper that lets you read Zip archive members as if they were files. - -=cut - -=head1 SYNOPSIS - - use Archive::Zip; - use Archive::Zip::MemberRead; - $zip = Archive::Zip->new("file.zip"); - $fh = Archive::Zip::MemberRead->new($zip, "subdir/abc.txt"); - while (defined($line = $fh->getline())) - { - print $fh->input_line_number . "#: $line\n"; - } - - $read = $fh->read($buffer, 32*1024); - print "Read $read bytes as :$buffer:\n"; - -=head1 DESCRIPTION - -The Archive::Zip::MemberRead module lets you read Zip archive member data -just like you read data from files. - -=head1 METHODS - -=over 4 - -=cut - -use strict; - -use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); - -use vars qw{$VERSION}; - -my $nl; - -BEGIN { - $VERSION = '1.48'; - $VERSION = eval $VERSION; - -# Requirement for newline conversion. Should check for e.g., DOS and OS/2 as well, but am too lazy. - $nl = $^O eq 'MSWin32' ? "\r\n" : "\n"; -} - -=item Archive::Zip::Member::readFileHandle() - -You can get a C from an archive member by -calling C: - - my $member = $zip->memberNamed('abc/def.c'); - my $fh = $member->readFileHandle(); - while (defined($line = $fh->getline())) - { - # ... - } - $fh->close(); - -=cut - -sub Archive::Zip::Member::readFileHandle { - return Archive::Zip::MemberRead->new(shift()); -} - -=item Archive::Zip::MemberRead->new($zip, $fileName) - -=item Archive::Zip::MemberRead->new($zip, $member) - -=item Archive::Zip::MemberRead->new($member) - -Construct a new Archive::Zip::MemberRead on the specified member. - - my $fh = Archive::Zip::MemberRead->new($zip, 'fred.c') - -=cut - -sub new { - my ($class, $zip, $file) = @_; - my ($self, $member); - - if ($zip && $file) # zip and filename, or zip and member - { - $member = ref($file) ? $file : $zip->memberNamed($file); - } elsif ($zip && !$file && ref($zip)) # just member - { - $member = $zip; - } else { - die( - 'Archive::Zip::MemberRead::new needs a zip and filename, zip and member, or member' - ); - } - - $self = {}; - bless($self, $class); - $self->set_member($member); - return $self; -} - -sub set_member { - my ($self, $member) = @_; - - $self->{member} = $member; - $self->set_compression(COMPRESSION_STORED); - $self->rewind(); -} - -sub set_compression { - my ($self, $compression) = @_; - $self->{member}->desiredCompressionMethod($compression) if $self->{member}; -} - -=item setLineEnd(expr) - -Set the line end character to use. This is set to \n by default -except on Windows systems where it is set to \r\n. You will -only need to set this on systems which are not Windows or Unix -based and require a line end different from \n. -This is a class method so call as C->C - -=cut - -sub setLineEnd { - shift; - $nl = shift; -} - -=item rewind() - -Rewinds an C so that you can read from it again -starting at the beginning. - -=cut - -sub rewind { - my $self = shift; - - $self->_reset_vars(); - $self->{member}->rewindData() if $self->{member}; -} - -sub _reset_vars { - my $self = shift; - - $self->{line_no} = 0; - $self->{at_end} = 0; - - delete $self->{buffer}; -} - -=item input_record_separator(expr) - -If the argument is given, input_record_separator for this -instance is set to it. The current setting (which may be -the global $/) is always returned. - -=cut - -sub input_record_separator { - my $self = shift; - if (@_) { - $self->{sep} = shift; - $self->{sep_re} = - _sep_as_re($self->{sep}); # Cache the RE as an optimization - } - return exists $self->{sep} ? $self->{sep} : $/; -} - -# Return the input_record_separator in use as an RE fragment -# Note that if we have a per-instance input_record_separator -# we can just return the already converted value. Otherwise, -# the conversion must be done on $/ every time since we cannot -# know whether it has changed or not. -sub _sep_re { - my $self = shift; - - # Important to phrase this way: sep's value may be undef. - return exists $self->{sep} ? $self->{sep_re} : _sep_as_re($/); -} - -# Convert the input record separator into an RE and return it. -sub _sep_as_re { - my $sep = shift; - if (defined $sep) { - if ($sep eq '') { - return "(?:$nl){2,}"; - } else { - $sep =~ s/\n/$nl/og; - return quotemeta $sep; - } - } else { - return undef; - } -} - -=item input_line_number() - -Returns the current line number, but only if you're using C. -Using C will not update the line number. - -=cut - -sub input_line_number { - my $self = shift; - return $self->{line_no}; -} - -=item close() - -Closes the given file handle. - -=cut - -sub close { - my $self = shift; - - $self->_reset_vars(); - $self->{member}->endRead(); -} - -=item buffer_size([ $size ]) - -Gets or sets the buffer size used for reads. -Default is the chunk size used by Archive::Zip. - -=cut - -sub buffer_size { - my ($self, $size) = @_; - - if (!$size) { - return $self->{chunkSize} || Archive::Zip::chunkSize(); - } else { - $self->{chunkSize} = $size; - } -} - -=item getline() - -Returns the next line from the currently open member. -Makes sense only for text files. -A read error is considered fatal enough to die. -Returns undef on eof. All subsequent calls would return undef, -unless a rewind() is called. -Note: The line returned has the input_record_separator (default: newline) removed. - -=item getline( { preserve_line_ending => 1 } ) - -Returns the next line including the line ending. - -=cut - -sub getline { - my ($self, $argref) = @_; - - my $size = $self->buffer_size(); - my $sep = $self->_sep_re(); - - my $preserve_line_ending; - if (ref $argref eq 'HASH') { - $preserve_line_ending = $argref->{'preserve_line_ending'}; - $sep =~ s/\\([^A-Za-z_0-9])+/$1/g; - } - - for (; ;) { - if ( $sep - && defined($self->{buffer}) - && $self->{buffer} =~ s/^(.*?)$sep//s) { - my $line = $1; - $self->{line_no}++; - if ($preserve_line_ending) { - return $line . $sep; - } else { - return $line; - } - } elsif ($self->{at_end}) { - $self->{line_no}++ if $self->{buffer}; - return delete $self->{buffer}; - } - my ($temp, $status) = $self->{member}->readChunk($size); - if ($status != AZ_OK && $status != AZ_STREAM_END) { - die "ERROR: Error reading chunk from archive - $status"; - } - $self->{at_end} = $status == AZ_STREAM_END; - $self->{buffer} .= $$temp; - } -} - -=item read($buffer, $num_bytes_to_read) - -Simulates a normal C system call. -Returns the no. of bytes read. C on error, 0 on eof, I: - - $fh = Archive::Zip::MemberRead->new($zip, "sreeji/secrets.bin"); - while (1) - { - $read = $fh->read($buffer, 1024); - die "FATAL ERROR reading my secrets !\n" if (!defined($read)); - last if (!$read); - # Do processing. - .... - } - -=cut - -# -# All these $_ are required to emulate read(). -# -sub read { - my $self = $_[0]; - my $size = $_[2]; - my ($temp, $status, $ret); - - ($temp, $status) = $self->{member}->readChunk($size); - if ($status != AZ_OK && $status != AZ_STREAM_END) { - $_[1] = undef; - $ret = undef; - } else { - $_[1] = $$temp; - $ret = length($$temp); - } - return $ret; -} - -1; - -=back - -=head1 AUTHOR - -Sreeji K. Das Esreeji_k@yahoo.comE - -See L by Ned Konz without which this module does not make -any sense! - -Minor mods by Ned Konz. - -=head1 COPYRIGHT - -Copyright 2002 Sreeji K. Das. - -This program is free software; you can redistribute it and/or modify it under -the same terms as Perl itself. - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MockFileHandle.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MockFileHandle.pm deleted file mode 100644 index 7d1d65ce682..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MockFileHandle.pm +++ /dev/null @@ -1,69 +0,0 @@ -package Archive::Zip::MockFileHandle; - -# Output file handle that calls a custom write routine -# Ned Konz, March 2000 -# This is provided to help with writing zip files -# when you have to process them a chunk at a time. - -use strict; - -use vars qw{$VERSION}; - -BEGIN { - $VERSION = '1.48'; - $VERSION = eval $VERSION; -} - -sub new { - my $class = shift || __PACKAGE__; - $class = ref($class) || $class; - my $self = bless( - { - 'position' => 0, - 'size' => 0 - }, - $class - ); - return $self; -} - -sub eof { - my $self = shift; - return $self->{'position'} >= $self->{'size'}; -} - -# Copy given buffer to me -sub print { - my $self = shift; - my $bytes = join('', @_); - my $bytesWritten = $self->writeHook($bytes); - if ($self->{'position'} + $bytesWritten > $self->{'size'}) { - $self->{'size'} = $self->{'position'} + $bytesWritten; - } - $self->{'position'} += $bytesWritten; - return $bytesWritten; -} - -# Called on each write. -# Override in subclasses. -# Return number of bytes written (0 on error). -sub writeHook { - my $self = shift; - my $bytes = shift; - return length($bytes); -} - -sub binmode { 1 } - -sub close { 1 } - -sub clearerr { 1 } - -# I'm write-only! -sub read { 0 } - -sub tell { return shift->{'position'} } - -sub opened { 1 } - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/NewFileMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/NewFileMember.pm deleted file mode 100644 index a7c69b6e1b4..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/NewFileMember.pm +++ /dev/null @@ -1,77 +0,0 @@ -package Archive::Zip::NewFileMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw ( Archive::Zip::FileMember ); -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES - :UTILITY_METHODS -); - -# Given a file name, set up for eventual writing. -sub _newFromFileNamed { - my $class = shift; - my $fileName = shift; # local FS format - my $newName = shift; - $newName = _asZipDirName($fileName) unless defined($newName); - return undef unless (stat($fileName) && -r _ && !-d _ ); - my $self = $class->new(@_); - $self->{'fileName'} = $newName; - $self->{'externalFileName'} = $fileName; - $self->{'compressionMethod'} = COMPRESSION_STORED; - my @stat = stat(_); - $self->{'compressedSize'} = $self->{'uncompressedSize'} = $stat[7]; - $self->desiredCompressionMethod( - ($self->compressedSize() > 0) - ? COMPRESSION_DEFLATED - : COMPRESSION_STORED - ); - $self->unixFileAttributes($stat[2]); - $self->setLastModFileDateTimeFromUnix($stat[9]); - $self->isTextFile(-T _ ); - return $self; -} - -sub rewindData { - my $self = shift; - - my $status = $self->SUPER::rewindData(@_); - return $status unless $status == AZ_OK; - - return AZ_IO_ERROR unless $self->fh(); - $self->fh()->clearerr(); - $self->fh()->seek(0, IO::Seekable::SEEK_SET) - or return _ioError("rewinding", $self->externalFileName()); - return AZ_OK; -} - -# Return bytes read. Note that first parameter is a ref to a buffer. -# my $data; -# my ( $bytesRead, $status) = $self->readRawChunk( \$data, $chunkSize ); -sub _readRawChunk { - my ($self, $dataRef, $chunkSize) = @_; - return (0, AZ_OK) unless $chunkSize; - my $bytesRead = $self->fh()->read($$dataRef, $chunkSize) - or return (0, _ioError("reading data")); - return ($bytesRead, AZ_OK); -} - -# If I already exist, extraction is a no-op. -sub extractToFileNamed { - my $self = shift; - my $name = shift; # local FS name - if (File::Spec->rel2abs($name) eq - File::Spec->rel2abs($self->externalFileName()) and -r $name) { - return AZ_OK; - } else { - return $self->SUPER::extractToFileNamed($name, @_); - } -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/StringMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/StringMember.pm deleted file mode 100644 index 74a0e8347db..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/StringMember.pm +++ /dev/null @@ -1,64 +0,0 @@ -package Archive::Zip::StringMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip::Member ); -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES -); - -# Create a new string member. Default is COMPRESSION_STORED. -# Can take a ref to a string as well. -sub _newFromString { - my $class = shift; - my $string = shift; - my $name = shift; - my $self = $class->new(@_); - $self->contents($string); - $self->fileName($name) if defined($name); - - # Set the file date to now - $self->setLastModFileDateTimeFromUnix(time()); - $self->unixFileAttributes($self->DEFAULT_FILE_PERMISSIONS); - return $self; -} - -sub _become { - my $self = shift; - my $newClass = shift; - return $self if ref($self) eq $newClass; - delete($self->{'contents'}); - return $self->SUPER::_become($newClass); -} - -# Get or set my contents. Note that we do not call the superclass -# version of this, because it calls us. -sub contents { - my $self = shift; - my $string = shift; - if (defined($string)) { - $self->{'contents'} = - pack('C0a*', (ref($string) eq 'SCALAR') ? $$string : $string); - $self->{'uncompressedSize'} = $self->{'compressedSize'} = - length($self->{'contents'}); - $self->{'compressionMethod'} = COMPRESSION_STORED; - } - return $self->{'contents'}; -} - -# Return bytes read. Note that first parameter is a ref to a buffer. -# my $data; -# my ( $bytesRead, $status) = $self->readRawChunk( \$data, $chunkSize ); -sub _readRawChunk { - my ($self, $dataRef, $chunkSize) = @_; - $$dataRef = substr($self->contents(), $self->_readOffset(), $chunkSize); - return (length($$dataRef), AZ_OK); -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Tree.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Tree.pm deleted file mode 100644 index 6e84011998c..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Tree.pm +++ /dev/null @@ -1,48 +0,0 @@ -package Archive::Zip::Tree; - -use strict; -use vars qw{$VERSION}; - -BEGIN { - $VERSION = '1.48'; -} - -use Archive::Zip; - -warn( - "Archive::Zip::Tree is deprecated; its methods have been moved into Archive::Zip." -) if $^W; - -1; - -__END__ - -=head1 NAME - -Archive::Zip::Tree - (DEPRECATED) methods for adding/extracting trees using Archive::Zip - -=head1 DESCRIPTION - -This module is deprecated, because all its methods were moved into the main -Archive::Zip module. - -It is included in the distribution merely to avoid breaking old code. - -See L. - -=head1 AUTHOR - -Ned Konz, perl@bike-nomad.com - -=head1 COPYRIGHT - -Copyright (c) 2000-2002 Ned Konz. All rights reserved. This program is free -software; you can redistribute it and/or modify it under the same terms -as Perl itself. - -=head1 SEE ALSO - -L - -=cut - diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/ZipFileMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/ZipFileMember.pm deleted file mode 100644 index 1716aa12420..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/ZipFileMember.pm +++ /dev/null @@ -1,416 +0,0 @@ -package Archive::Zip::ZipFileMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw ( Archive::Zip::FileMember ); -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES - :PKZIP_CONSTANTS - :UTILITY_METHODS -); - -# Create a new Archive::Zip::ZipFileMember -# given a filename and optional open file handle -# -sub _newFromZipFile { - my $class = shift; - my $fh = shift; - my $externalFileName = shift; - my $possibleEocdOffset = shift; # normally 0 - - my $self = $class->new( - 'crc32' => 0, - 'diskNumberStart' => 0, - 'localHeaderRelativeOffset' => 0, - 'dataOffset' => 0, # localHeaderRelativeOffset + header length - @_ - ); - $self->{'externalFileName'} = $externalFileName; - $self->{'fh'} = $fh; - $self->{'possibleEocdOffset'} = $possibleEocdOffset; - return $self; -} - -sub isDirectory { - my $self = shift; - return (substr($self->fileName, -1, 1) eq '/' - and $self->uncompressedSize == 0); -} - -# Seek to the beginning of the local header, just past the signature. -# Verify that the local header signature is in fact correct. -# Update the localHeaderRelativeOffset if necessary by adding the possibleEocdOffset. -# Returns status. - -sub _seekToLocalHeader { - my $self = shift; - my $where = shift; # optional - my $previousWhere = shift; # optional - - $where = $self->localHeaderRelativeOffset() unless defined($where); - - # avoid loop on certain corrupt files (from Julian Field) - return _formatError("corrupt zip file") - if defined($previousWhere) && $where == $previousWhere; - - my $status; - my $signature; - - $status = $self->fh()->seek($where, IO::Seekable::SEEK_SET); - return _ioError("seeking to local header") unless $status; - - ($status, $signature) = - _readSignature($self->fh(), $self->externalFileName(), - LOCAL_FILE_HEADER_SIGNATURE); - return $status if $status == AZ_IO_ERROR; - - # retry with EOCD offset if any was given. - if ($status == AZ_FORMAT_ERROR && $self->{'possibleEocdOffset'}) { - $status = $self->_seekToLocalHeader( - $self->localHeaderRelativeOffset() + $self->{'possibleEocdOffset'}, - $where - ); - if ($status == AZ_OK) { - $self->{'localHeaderRelativeOffset'} += - $self->{'possibleEocdOffset'}; - $self->{'possibleEocdOffset'} = 0; - } - } - - return $status; -} - -# Because I'm going to delete the file handle, read the local file -# header if the file handle is seekable. If it is not, I assume that -# I've already read the local header. -# Return ( $status, $self ) - -sub _become { - my $self = shift; - my $newClass = shift; - return $self if ref($self) eq $newClass; - - my $status = AZ_OK; - - if (_isSeekable($self->fh())) { - my $here = $self->fh()->tell(); - $status = $self->_seekToLocalHeader(); - $status = $self->_readLocalFileHeader() if $status == AZ_OK; - $self->fh()->seek($here, IO::Seekable::SEEK_SET); - return $status unless $status == AZ_OK; - } - - delete($self->{'eocdCrc32'}); - delete($self->{'diskNumberStart'}); - delete($self->{'localHeaderRelativeOffset'}); - delete($self->{'dataOffset'}); - - return $self->SUPER::_become($newClass); -} - -sub diskNumberStart { - shift->{'diskNumberStart'}; -} - -sub localHeaderRelativeOffset { - shift->{'localHeaderRelativeOffset'}; -} - -sub dataOffset { - shift->{'dataOffset'}; -} - -# Skip local file header, updating only extra field stuff. -# Assumes that fh is positioned before signature. -sub _skipLocalFileHeader { - my $self = shift; - my $header; - my $bytesRead = $self->fh()->read($header, LOCAL_FILE_HEADER_LENGTH); - if ($bytesRead != LOCAL_FILE_HEADER_LENGTH) { - return _ioError("reading local file header"); - } - my $fileNameLength; - my $extraFieldLength; - my $bitFlag; - ( - undef, # $self->{'versionNeededToExtract'}, - $bitFlag, - undef, # $self->{'compressionMethod'}, - undef, # $self->{'lastModFileDateTime'}, - undef, # $crc32, - undef, # $compressedSize, - undef, # $uncompressedSize, - $fileNameLength, - $extraFieldLength - ) = unpack(LOCAL_FILE_HEADER_FORMAT, $header); - - if ($fileNameLength) { - $self->fh()->seek($fileNameLength, IO::Seekable::SEEK_CUR) - or return _ioError("skipping local file name"); - } - - if ($extraFieldLength) { - $bytesRead = - $self->fh()->read($self->{'localExtraField'}, $extraFieldLength); - if ($bytesRead != $extraFieldLength) { - return _ioError("reading local extra field"); - } - } - - $self->{'dataOffset'} = $self->fh()->tell(); - - if ($bitFlag & GPBF_HAS_DATA_DESCRIPTOR_MASK) { - - # Read the crc32, compressedSize, and uncompressedSize from the - # extended data descriptor, which directly follows the compressed data. - # - # Skip over the compressed file data (assumes that EOCD compressedSize - # was correct) - $self->fh()->seek($self->{'compressedSize'}, IO::Seekable::SEEK_CUR) - or return _ioError("seeking to extended local header"); - - # these values should be set correctly from before. - my $oldCrc32 = $self->{'eocdCrc32'}; - my $oldCompressedSize = $self->{'compressedSize'}; - my $oldUncompressedSize = $self->{'uncompressedSize'}; - - my $status = $self->_readDataDescriptor(); - return $status unless $status == AZ_OK; - - # The buffer withe encrypted data is prefixed with a new - # encrypted 12 byte header. The size only changes when - # the buffer is also compressed - $self->isEncrypted && $oldUncompressedSize > $self->{uncompressedSize} - and $oldUncompressedSize -= DATA_DESCRIPTOR_LENGTH; - - return _formatError( - "CRC or size mismatch while skipping data descriptor") - if ( $oldCrc32 != $self->{'crc32'} - || $oldUncompressedSize != $self->{'uncompressedSize'}); - - $self->{'crc32'} = 0 - if $self->compressionMethod() == COMPRESSION_STORED ; - } - - return AZ_OK; -} - -# Read from a local file header into myself. Returns AZ_OK if successful. -# Assumes that fh is positioned after signature. -# Note that crc32, compressedSize, and uncompressedSize will be 0 if -# GPBF_HAS_DATA_DESCRIPTOR_MASK is set in the bitFlag. - -sub _readLocalFileHeader { - my $self = shift; - my $header; - my $bytesRead = $self->fh()->read($header, LOCAL_FILE_HEADER_LENGTH); - if ($bytesRead != LOCAL_FILE_HEADER_LENGTH) { - return _ioError("reading local file header"); - } - my $fileNameLength; - my $crc32; - my $compressedSize; - my $uncompressedSize; - my $extraFieldLength; - ( - $self->{'versionNeededToExtract'}, $self->{'bitFlag'}, - $self->{'compressionMethod'}, $self->{'lastModFileDateTime'}, - $crc32, $compressedSize, - $uncompressedSize, $fileNameLength, - $extraFieldLength - ) = unpack(LOCAL_FILE_HEADER_FORMAT, $header); - - if ($fileNameLength) { - my $fileName; - $bytesRead = $self->fh()->read($fileName, $fileNameLength); - if ($bytesRead != $fileNameLength) { - return _ioError("reading local file name"); - } - $self->fileName($fileName); - } - - if ($extraFieldLength) { - $bytesRead = - $self->fh()->read($self->{'localExtraField'}, $extraFieldLength); - if ($bytesRead != $extraFieldLength) { - return _ioError("reading local extra field"); - } - } - - $self->{'dataOffset'} = $self->fh()->tell(); - - if ($self->hasDataDescriptor()) { - - # Read the crc32, compressedSize, and uncompressedSize from the - # extended data descriptor. - # Skip over the compressed file data (assumes that EOCD compressedSize - # was correct) - $self->fh()->seek($self->{'compressedSize'}, IO::Seekable::SEEK_CUR) - or return _ioError("seeking to extended local header"); - - my $status = $self->_readDataDescriptor(); - return $status unless $status == AZ_OK; - } else { - return _formatError( - "CRC or size mismatch after reading data descriptor") - if ( $self->{'crc32'} != $crc32 - || $self->{'uncompressedSize'} != $uncompressedSize); - } - - return AZ_OK; -} - -# This will read the data descriptor, which is after the end of compressed file -# data in members that have GPBF_HAS_DATA_DESCRIPTOR_MASK set in their bitFlag. -# The only reliable way to find these is to rely on the EOCD compressedSize. -# Assumes that file is positioned immediately after the compressed data. -# Returns status; sets crc32, compressedSize, and uncompressedSize. -sub _readDataDescriptor { - my $self = shift; - my $signatureData; - my $header; - my $crc32; - my $compressedSize; - my $uncompressedSize; - - my $bytesRead = $self->fh()->read($signatureData, SIGNATURE_LENGTH); - return _ioError("reading header signature") - if $bytesRead != SIGNATURE_LENGTH; - my $signature = unpack(SIGNATURE_FORMAT, $signatureData); - - # unfortunately, the signature appears to be optional. - if ($signature == DATA_DESCRIPTOR_SIGNATURE - && ($signature != $self->{'crc32'})) { - $bytesRead = $self->fh()->read($header, DATA_DESCRIPTOR_LENGTH); - return _ioError("reading data descriptor") - if $bytesRead != DATA_DESCRIPTOR_LENGTH; - - ($crc32, $compressedSize, $uncompressedSize) = - unpack(DATA_DESCRIPTOR_FORMAT, $header); - } else { - $bytesRead = $self->fh()->read($header, DATA_DESCRIPTOR_LENGTH_NO_SIG); - return _ioError("reading data descriptor") - if $bytesRead != DATA_DESCRIPTOR_LENGTH_NO_SIG; - - $crc32 = $signature; - ($compressedSize, $uncompressedSize) = - unpack(DATA_DESCRIPTOR_FORMAT_NO_SIG, $header); - } - - $self->{'eocdCrc32'} = $self->{'crc32'} - unless defined($self->{'eocdCrc32'}); - $self->{'crc32'} = $crc32; - $self->{'compressedSize'} = $compressedSize; - $self->{'uncompressedSize'} = $uncompressedSize; - - return AZ_OK; -} - -# Read a Central Directory header. Return AZ_OK on success. -# Assumes that fh is positioned right after the signature. - -sub _readCentralDirectoryFileHeader { - my $self = shift; - my $fh = $self->fh(); - my $header = ''; - my $bytesRead = $fh->read($header, CENTRAL_DIRECTORY_FILE_HEADER_LENGTH); - if ($bytesRead != CENTRAL_DIRECTORY_FILE_HEADER_LENGTH) { - return _ioError("reading central dir header"); - } - my ($fileNameLength, $extraFieldLength, $fileCommentLength); - ( - $self->{'versionMadeBy'}, - $self->{'fileAttributeFormat'}, - $self->{'versionNeededToExtract'}, - $self->{'bitFlag'}, - $self->{'compressionMethod'}, - $self->{'lastModFileDateTime'}, - $self->{'crc32'}, - $self->{'compressedSize'}, - $self->{'uncompressedSize'}, - $fileNameLength, - $extraFieldLength, - $fileCommentLength, - $self->{'diskNumberStart'}, - $self->{'internalFileAttributes'}, - $self->{'externalFileAttributes'}, - $self->{'localHeaderRelativeOffset'} - ) = unpack(CENTRAL_DIRECTORY_FILE_HEADER_FORMAT, $header); - - $self->{'eocdCrc32'} = $self->{'crc32'}; - - if ($fileNameLength) { - $bytesRead = $fh->read($self->{'fileName'}, $fileNameLength); - if ($bytesRead != $fileNameLength) { - _ioError("reading central dir filename"); - } - } - if ($extraFieldLength) { - $bytesRead = $fh->read($self->{'cdExtraField'}, $extraFieldLength); - if ($bytesRead != $extraFieldLength) { - return _ioError("reading central dir extra field"); - } - } - if ($fileCommentLength) { - $bytesRead = $fh->read($self->{'fileComment'}, $fileCommentLength); - if ($bytesRead != $fileCommentLength) { - return _ioError("reading central dir file comment"); - } - } - - # NK 10/21/04: added to avoid problems with manipulated headers - if ( $self->{'uncompressedSize'} != $self->{'compressedSize'} - and $self->{'compressionMethod'} == COMPRESSION_STORED) { - $self->{'uncompressedSize'} = $self->{'compressedSize'}; - } - - $self->desiredCompressionMethod($self->compressionMethod()); - - return AZ_OK; -} - -sub rewindData { - my $self = shift; - - my $status = $self->SUPER::rewindData(@_); - return $status unless $status == AZ_OK; - - return AZ_IO_ERROR unless $self->fh(); - - $self->fh()->clearerr(); - - # Seek to local file header. - # The only reason that I'm doing this this way is that the extraField - # length seems to be different between the CD header and the LF header. - $status = $self->_seekToLocalHeader(); - return $status unless $status == AZ_OK; - - # skip local file header - $status = $self->_skipLocalFileHeader(); - return $status unless $status == AZ_OK; - - # Seek to beginning of file data - $self->fh()->seek($self->dataOffset(), IO::Seekable::SEEK_SET) - or return _ioError("seeking to beginning of file data"); - - return AZ_OK; -} - -# Return bytes read. Note that first parameter is a ref to a buffer. -# my $data; -# my ( $bytesRead, $status) = $self->readRawChunk( \$data, $chunkSize ); -sub _readRawChunk { - my ($self, $dataRef, $chunkSize) = @_; - return (0, AZ_OK) unless $chunkSize; - my $bytesRead = $self->fh()->read($$dataRef, $chunkSize) - or return (0, _ioError("reading data")); - return ($bytesRead, AZ_OK); -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/File/Path.pm b/dev-tools/src/main/resources/license-check/lib/File/Path.pm deleted file mode 100644 index 3ee17bcea29..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/File/Path.pm +++ /dev/null @@ -1,1165 +0,0 @@ -package File::Path; - -use 5.005_04; -use strict; - -use Cwd 'getcwd'; -use File::Basename (); -use File::Spec (); - -BEGIN { - if ( $] < 5.006 ) { - - # can't say 'opendir my $dh, $dirname' - # need to initialise $dh - eval 'use Symbol'; - } -} - -use Exporter (); -use vars qw($VERSION @ISA @EXPORT @EXPORT_OK); -$VERSION = '2.11'; -$VERSION = eval $VERSION; -@ISA = qw(Exporter); -@EXPORT = qw(mkpath rmtree); -@EXPORT_OK = qw(make_path remove_tree); - -BEGIN { - for (qw(VMS MacOS MSWin32 os2)) { - no strict 'refs'; - *{"_IS_\U$_"} = $^O eq $_ ? sub () { 1 } : sub () { 0 }; - } - - # These OSes complain if you want to remove a file that you have no - # write permission to: - *_FORCE_WRITABLE = ( - grep { $^O eq $_ } qw(amigaos dos epoc MSWin32 MacOS os2) - ) ? sub () { 1 } : sub () { 0 }; - - # Unix-like systems need to stat each directory in order to detect - # race condition. MS-Windows is immune to this particular attack. - *_NEED_STAT_CHECK = !(_IS_MSWIN32()) ? sub () { 1 } : sub () { 0 }; -} - -sub _carp { - require Carp; - goto &Carp::carp; -} - -sub _croak { - require Carp; - goto &Carp::croak; -} - -sub _error { - my $arg = shift; - my $message = shift; - my $object = shift; - - if ( $arg->{error} ) { - $object = '' unless defined $object; - $message .= ": $!" if $!; - push @{ ${ $arg->{error} } }, { $object => $message }; - } - else { - _carp( defined($object) ? "$message for $object: $!" : "$message: $!" ); - } -} - -sub __is_arg { - my ($arg) = @_; - - # If client code blessed an array ref to HASH, this will not work - # properly. We could have done $arg->isa() wrapped in eval, but - # that would be expensive. This implementation should suffice. - # We could have also used Scalar::Util:blessed, but we choose not - # to add this dependency - return ( ref $arg eq 'HASH' ); -} - -sub make_path { - push @_, {} unless @_ and __is_arg( $_[-1] ); - goto &mkpath; -} - -sub mkpath { - my $old_style = !( @_ and __is_arg( $_[-1] ) ); - - my $arg; - my $paths; - - if ($old_style) { - my ( $verbose, $mode ); - ( $paths, $verbose, $mode ) = @_; - $paths = [$paths] unless UNIVERSAL::isa( $paths, 'ARRAY' ); - $arg->{verbose} = $verbose; - $arg->{mode} = defined $mode ? $mode : oct '777'; - } - else { - my %args_permitted = map { $_ => 1 } ( qw| - chmod - error - group - mask - mode - owner - uid - user - verbose - | ); - my @bad_args = (); - $arg = pop @_; - for my $k (sort keys %{$arg}) { - push @bad_args, $k unless $args_permitted{$k}; - } - _carp("Unrecognized option(s) passed to make_path(): @bad_args") - if @bad_args; - $arg->{mode} = delete $arg->{mask} if exists $arg->{mask}; - $arg->{mode} = oct '777' unless exists $arg->{mode}; - ${ $arg->{error} } = [] if exists $arg->{error}; - $arg->{owner} = delete $arg->{user} if exists $arg->{user}; - $arg->{owner} = delete $arg->{uid} if exists $arg->{uid}; - if ( exists $arg->{owner} and $arg->{owner} =~ /\D/ ) { - my $uid = ( getpwnam $arg->{owner} )[2]; - if ( defined $uid ) { - $arg->{owner} = $uid; - } - else { - _error( $arg, -"unable to map $arg->{owner} to a uid, ownership not changed" - ); - delete $arg->{owner}; - } - } - if ( exists $arg->{group} and $arg->{group} =~ /\D/ ) { - my $gid = ( getgrnam $arg->{group} )[2]; - if ( defined $gid ) { - $arg->{group} = $gid; - } - else { - _error( $arg, -"unable to map $arg->{group} to a gid, group ownership not changed" - ); - delete $arg->{group}; - } - } - if ( exists $arg->{owner} and not exists $arg->{group} ) { - $arg->{group} = -1; # chown will leave group unchanged - } - if ( exists $arg->{group} and not exists $arg->{owner} ) { - $arg->{owner} = -1; # chown will leave owner unchanged - } - $paths = [@_]; - } - return _mkpath( $arg, $paths ); -} - -sub _mkpath { - my $arg = shift; - my $paths = shift; - - my ( @created ); - foreach my $path ( @{$paths} ) { - next unless defined($path) and length($path); - $path .= '/' if _IS_OS2 and $path =~ /^\w:\z/s; # feature of CRT - - # Logic wants Unix paths, so go with the flow. - if (_IS_VMS) { - next if $path eq '/'; - $path = VMS::Filespec::unixify($path); - } - next if -d $path; - my $parent = File::Basename::dirname($path); - unless ( -d $parent or $path eq $parent ) { - push( @created, _mkpath( $arg, [$parent] ) ); - } - print "mkdir $path\n" if $arg->{verbose}; - if ( mkdir( $path, $arg->{mode} ) ) { - push( @created, $path ); - if ( exists $arg->{owner} ) { - - # NB: $arg->{group} guaranteed to be set during initialisation - if ( !chown $arg->{owner}, $arg->{group}, $path ) { - _error( $arg, -"Cannot change ownership of $path to $arg->{owner}:$arg->{group}" - ); - } - } - if ( exists $arg->{chmod} ) { - if ( !chmod $arg->{chmod}, $path ) { - _error( $arg, - "Cannot change permissions of $path to $arg->{chmod}" ); - } - } - } - else { - my $save_bang = $!; - my ( $e, $e1 ) = ( $save_bang, $^E ); - $e .= "; $e1" if $e ne $e1; - - # allow for another process to have created it meanwhile - if ( ! -d $path ) { - $! = $save_bang; - if ( $arg->{error} ) { - push @{ ${ $arg->{error} } }, { $path => $e }; - } - else { - _croak("mkdir $path: $e"); - } - } - } - } - return @created; -} - -sub remove_tree { - push @_, {} unless @_ and __is_arg( $_[-1] ); - goto &rmtree; -} - -sub _is_subdir { - my ( $dir, $test ) = @_; - - my ( $dv, $dd ) = File::Spec->splitpath( $dir, 1 ); - my ( $tv, $td ) = File::Spec->splitpath( $test, 1 ); - - # not on same volume - return 0 if $dv ne $tv; - - my @d = File::Spec->splitdir($dd); - my @t = File::Spec->splitdir($td); - - # @t can't be a subdir if it's shorter than @d - return 0 if @t < @d; - - return join( '/', @d ) eq join( '/', splice @t, 0, +@d ); -} - -sub rmtree { - my $old_style = !( @_ and __is_arg( $_[-1] ) ); - - my $arg; - my $paths; - - if ($old_style) { - my ( $verbose, $safe ); - ( $paths, $verbose, $safe ) = @_; - $arg->{verbose} = $verbose; - $arg->{safe} = defined $safe ? $safe : 0; - - if ( defined($paths) and length($paths) ) { - $paths = [$paths] unless UNIVERSAL::isa( $paths, 'ARRAY' ); - } - else { - _carp("No root path(s) specified\n"); - return 0; - } - } - else { - my %args_permitted = map { $_ => 1 } ( qw| - error - keep_root - result - safe - verbose - | ); - my @bad_args = (); - $arg = pop @_; - for my $k (sort keys %{$arg}) { - push @bad_args, $k unless $args_permitted{$k}; - } - _carp("Unrecognized option(s) passed to remove_tree(): @bad_args") - if @bad_args; - ${ $arg->{error} } = [] if exists $arg->{error}; - ${ $arg->{result} } = [] if exists $arg->{result}; - $paths = [@_]; - } - - $arg->{prefix} = ''; - $arg->{depth} = 0; - - my @clean_path; - $arg->{cwd} = getcwd() or do { - _error( $arg, "cannot fetch initial working directory" ); - return 0; - }; - for ( $arg->{cwd} ) { /\A(.*)\Z/s; $_ = $1 } # untaint - - for my $p (@$paths) { - - # need to fixup case and map \ to / on Windows - my $ortho_root = _IS_MSWIN32 ? _slash_lc($p) : $p; - my $ortho_cwd = - _IS_MSWIN32 ? _slash_lc( $arg->{cwd} ) : $arg->{cwd}; - my $ortho_root_length = length($ortho_root); - $ortho_root_length-- if _IS_VMS; # don't compare '.' with ']' - if ( $ortho_root_length && _is_subdir( $ortho_root, $ortho_cwd ) ) { - local $! = 0; - _error( $arg, "cannot remove path when cwd is $arg->{cwd}", $p ); - next; - } - - if (_IS_MACOS) { - $p = ":$p" unless $p =~ /:/; - $p .= ":" unless $p =~ /:\z/; - } - elsif ( _IS_MSWIN32 ) { - $p =~ s{[/\\]\z}{}; - } - else { - $p =~ s{/\z}{}; - } - push @clean_path, $p; - } - - @{$arg}{qw(device inode perm)} = ( lstat $arg->{cwd} )[ 0, 1 ] or do { - _error( $arg, "cannot stat initial working directory", $arg->{cwd} ); - return 0; - }; - - return _rmtree( $arg, \@clean_path ); -} - -sub _rmtree { - my $arg = shift; - my $paths = shift; - - my $count = 0; - my $curdir = File::Spec->curdir(); - my $updir = File::Spec->updir(); - - my ( @files, $root ); - ROOT_DIR: - foreach my $root (@$paths) { - - # since we chdir into each directory, it may not be obvious - # to figure out where we are if we generate a message about - # a file name. We therefore construct a semi-canonical - # filename, anchored from the directory being unlinked (as - # opposed to being truly canonical, anchored from the root (/). - - my $canon = - $arg->{prefix} - ? File::Spec->catfile( $arg->{prefix}, $root ) - : $root; - - my ( $ldev, $lino, $perm ) = ( lstat $root )[ 0, 1, 2 ] - or ( _error( $arg, "$root", $root ) and next ROOT_DIR ); - - if ( -d _ ) { - $root = VMS::Filespec::vmspath( VMS::Filespec::pathify($root) ) - if _IS_VMS; - - if ( !chdir($root) ) { - - # see if we can escalate privileges to get in - # (e.g. funny protection mask such as -w- instead of rwx) - $perm &= oct '7777'; - my $nperm = $perm | oct '700'; - if ( - !( - $arg->{safe} - or $nperm == $perm - or chmod( $nperm, $root ) - ) - ) - { - _error( $arg, - "cannot make child directory read-write-exec", $canon ); - next ROOT_DIR; - } - elsif ( !chdir($root) ) { - _error( $arg, "cannot chdir to child", $canon ); - next ROOT_DIR; - } - } - - my ( $cur_dev, $cur_inode, $perm ) = ( stat $curdir )[ 0, 1, 2 ] - or do { - _error( $arg, "cannot stat current working directory", $canon ); - next ROOT_DIR; - }; - - if (_NEED_STAT_CHECK) { - ( $ldev eq $cur_dev and $lino eq $cur_inode ) - or _croak( -"directory $canon changed before chdir, expected dev=$ldev ino=$lino, actual dev=$cur_dev ino=$cur_inode, aborting." - ); - } - - $perm &= oct '7777'; # don't forget setuid, setgid, sticky bits - my $nperm = $perm | oct '700'; - - # notabene: 0700 is for making readable in the first place, - # it's also intended to change it to writable in case we have - # to recurse in which case we are better than rm -rf for - # subtrees with strange permissions - - if ( - !( - $arg->{safe} - or $nperm == $perm - or chmod( $nperm, $curdir ) - ) - ) - { - _error( $arg, "cannot make directory read+writeable", $canon ); - $nperm = $perm; - } - - my $d; - $d = gensym() if $] < 5.006; - if ( !opendir $d, $curdir ) { - _error( $arg, "cannot opendir", $canon ); - @files = (); - } - else { - if ( !defined ${^TAINT} or ${^TAINT} ) { - # Blindly untaint dir names if taint mode is active - @files = map { /\A(.*)\z/s; $1 } readdir $d; - } - else { - @files = readdir $d; - } - closedir $d; - } - - if (_IS_VMS) { - - # Deleting large numbers of files from VMS Files-11 - # filesystems is faster if done in reverse ASCIIbetical order. - # include '.' to '.;' from blead patch #31775 - @files = map { $_ eq '.' ? '.;' : $_ } reverse @files; - } - - @files = grep { $_ ne $updir and $_ ne $curdir } @files; - - if (@files) { - - # remove the contained files before the directory itself - my $narg = {%$arg}; - @{$narg}{qw(device inode cwd prefix depth)} = - ( $cur_dev, $cur_inode, $updir, $canon, $arg->{depth} + 1 ); - $count += _rmtree( $narg, \@files ); - } - - # restore directory permissions of required now (in case the rmdir - # below fails), while we are still in the directory and may do so - # without a race via '.' - if ( $nperm != $perm and not chmod( $perm, $curdir ) ) { - _error( $arg, "cannot reset chmod", $canon ); - } - - # don't leave the client code in an unexpected directory - chdir( $arg->{cwd} ) - or - _croak("cannot chdir to $arg->{cwd} from $canon: $!, aborting."); - - # ensure that a chdir upwards didn't take us somewhere other - # than we expected (see CVE-2002-0435) - ( $cur_dev, $cur_inode ) = ( stat $curdir )[ 0, 1 ] - or _croak( - "cannot stat prior working directory $arg->{cwd}: $!, aborting." - ); - - if (_NEED_STAT_CHECK) { - ( $arg->{device} eq $cur_dev and $arg->{inode} eq $cur_inode ) - or _croak( "previous directory $arg->{cwd} " - . "changed before entering $canon, " - . "expected dev=$ldev ino=$lino, " - . "actual dev=$cur_dev ino=$cur_inode, aborting." - ); - } - - if ( $arg->{depth} or !$arg->{keep_root} ) { - if ( $arg->{safe} - && ( _IS_VMS - ? !&VMS::Filespec::candelete($root) - : !-w $root ) ) - { - print "skipped $root\n" if $arg->{verbose}; - next ROOT_DIR; - } - if ( _FORCE_WRITABLE and !chmod $perm | oct '700', $root ) { - _error( $arg, "cannot make directory writeable", $canon ); - } - print "rmdir $root\n" if $arg->{verbose}; - if ( rmdir $root ) { - push @{ ${ $arg->{result} } }, $root if $arg->{result}; - ++$count; - } - else { - _error( $arg, "cannot remove directory", $canon ); - if ( - _FORCE_WRITABLE - && !chmod( $perm, - ( _IS_VMS ? VMS::Filespec::fileify($root) : $root ) - ) - ) - { - _error( - $arg, - sprintf( "cannot restore permissions to 0%o", - $perm ), - $canon - ); - } - } - } - } - else { - # not a directory - $root = VMS::Filespec::vmsify("./$root") - if _IS_VMS - && !File::Spec->file_name_is_absolute($root) - && ( $root !~ m/(?]+/ ); # not already in VMS syntax - - if ( - $arg->{safe} - && ( - _IS_VMS - ? !&VMS::Filespec::candelete($root) - : !( -l $root || -w $root ) - ) - ) - { - print "skipped $root\n" if $arg->{verbose}; - next ROOT_DIR; - } - - my $nperm = $perm & oct '7777' | oct '600'; - if ( _FORCE_WRITABLE - and $nperm != $perm - and not chmod $nperm, $root ) - { - _error( $arg, "cannot make file writeable", $canon ); - } - print "unlink $canon\n" if $arg->{verbose}; - - # delete all versions under VMS - for ( ; ; ) { - if ( unlink $root ) { - push @{ ${ $arg->{result} } }, $root if $arg->{result}; - } - else { - _error( $arg, "cannot unlink file", $canon ); - _FORCE_WRITABLE and chmod( $perm, $root ) - or _error( $arg, - sprintf( "cannot restore permissions to 0%o", $perm ), - $canon ); - last; - } - ++$count; - last unless _IS_VMS && lstat $root; - } - } - } - return $count; -} - -sub _slash_lc { - - # fix up slashes and case on MSWin32 so that we can determine that - # c:\path\to\dir is underneath C:/Path/To - my $path = shift; - $path =~ tr{\\}{/}; - return lc($path); -} - -1; - -__END__ - -=head1 NAME - -File::Path - Create or remove directory trees - -=head1 VERSION - -This document describes version 2.09 of File::Path, released -2013-01-17. - -=head1 SYNOPSIS - - use File::Path qw(make_path remove_tree); - - @created = make_path('foo/bar/baz', '/zug/zwang'); - @created = make_path('foo/bar/baz', '/zug/zwang', { - verbose => 1, - mode => 0711, - }); - make_path('foo/bar/baz', '/zug/zwang', { - chmod => 0777, - }); - - $removed_count = remove_tree('foo/bar/baz', '/zug/zwang'); - $removed_count = remove_tree('foo/bar/baz', '/zug/zwang', { - verbose => 1, - error => \my $err_list, - }); - - # legacy (interface promoted before v2.00) - @created = mkpath('/foo/bar/baz'); - @created = mkpath('/foo/bar/baz', 1, 0711); - @created = mkpath(['/foo/bar/baz', 'blurfl/quux'], 1, 0711); - $removed_count = rmtree('foo/bar/baz', 1, 1); - $removed_count = rmtree(['foo/bar/baz', 'blurfl/quux'], 1, 1); - - # legacy (interface promoted before v2.06) - @created = mkpath('foo/bar/baz', '/zug/zwang', { verbose => 1, mode => 0711 }); - $removed_count = rmtree('foo/bar/baz', '/zug/zwang', { verbose => 1, mode => 0711 }); - -=head1 DESCRIPTION - -This module provide a convenient way to create directories of -arbitrary depth and to delete an entire directory subtree from the -filesystem. - -The following functions are provided: - -=over - -=item make_path( $dir1, $dir2, .... ) - -=item make_path( $dir1, $dir2, ...., \%opts ) - -The C function creates the given directories if they don't -exists before, much like the Unix command C. - -The function accepts a list of directories to be created. Its -behaviour may be tuned by an optional hashref appearing as the last -parameter on the call. - -The function returns the list of directories actually created during -the call; in scalar context the number of directories created. - -The following keys are recognised in the option hash: - -=over - -=item mode => $num - -The numeric permissions mode to apply to each created directory -(defaults to 0777), to be modified by the current C. If the -directory already exists (and thus does not need to be created), -the permissions will not be modified. - -C is recognised as an alias for this parameter. - -=item chmod => $num - -Takes a numeric mode to apply to each created directory (not -modified by the current C). If the directory already exists -(and thus does not need to be created), the permissions will -not be modified. - -=item verbose => $bool - -If present, will cause C to print the name of each directory -as it is created. By default nothing is printed. - -=item error => \$err - -If present, it should be a reference to a scalar. -This scalar will be made to reference an array, which will -be used to store any errors that are encountered. See the L section for more information. - -If this parameter is not used, certain error conditions may raise -a fatal error that will cause the program to halt, unless trapped -in an C block. - -=item owner => $owner - -=item user => $owner - -=item uid => $owner - -If present, will cause any created directory to be owned by C<$owner>. -If the value is numeric, it will be interpreted as a uid, otherwise -as username is assumed. An error will be issued if the username cannot be -mapped to a uid, or the uid does not exist, or the process lacks the -privileges to change ownership. - -Ownership of directories that already exist will not be changed. - -C and C are aliases of C. - -=item group => $group - -If present, will cause any created directory to be owned by the group C<$group>. -If the value is numeric, it will be interpreted as a gid, otherwise -as group name is assumed. An error will be issued if the group name cannot be -mapped to a gid, or the gid does not exist, or the process lacks the -privileges to change group ownership. - -Group ownership of directories that already exist will not be changed. - - make_path '/var/tmp/webcache', {owner=>'nobody', group=>'nogroup'}; - -=back - -=item mkpath( $dir ) - -=item mkpath( $dir, $verbose, $mode ) - -=item mkpath( [$dir1, $dir2,...], $verbose, $mode ) - -=item mkpath( $dir1, $dir2,..., \%opt ) - -The mkpath() function provide the legacy interface of make_path() with -a different interpretation of the arguments passed. The behaviour and -return value of the function is otherwise identical to make_path(). - -=item remove_tree( $dir1, $dir2, .... ) - -=item remove_tree( $dir1, $dir2, ...., \%opts ) - -The C function deletes the given directories and any -files and subdirectories they might contain, much like the Unix -command C or the Windows commands C and C. - -The function accepts a list of directories to be -removed. Its behaviour may be tuned by an optional hashref -appearing as the last parameter on the call. - -The functions returns the number of files successfully deleted. - -The following keys are recognised in the option hash: - -=over - -=item verbose => $bool - -If present, will cause C to print the name of each file as -it is unlinked. By default nothing is printed. - -=item safe => $bool - -When set to a true value, will cause C to skip the files -for which the process lacks the required privileges needed to delete -files, such as delete privileges on VMS. In other words, the code -will make no attempt to alter file permissions. Thus, if the process -is interrupted, no filesystem object will be left in a more -permissive mode. - -=item keep_root => $bool - -When set to a true value, will cause all files and subdirectories -to be removed, except the initially specified directories. This comes -in handy when cleaning out an application's scratch directory. - - remove_tree( '/tmp', {keep_root => 1} ); - -=item result => \$res - -If present, it should be a reference to a scalar. -This scalar will be made to reference an array, which will -be used to store all files and directories unlinked -during the call. If nothing is unlinked, the array will be empty. - - remove_tree( '/tmp', {result => \my $list} ); - print "unlinked $_\n" for @$list; - -This is a useful alternative to the C key. - -=item error => \$err - -If present, it should be a reference to a scalar. -This scalar will be made to reference an array, which will -be used to store any errors that are encountered. See the L section for more information. - -Removing things is a much more dangerous proposition than -creating things. As such, there are certain conditions that -C may encounter that are so dangerous that the only -sane action left is to kill the program. - -Use C to trap all that is reasonable (problems with -permissions and the like), and let it die if things get out -of hand. This is the safest course of action. - -=back - -=item rmtree( $dir ) - -=item rmtree( $dir, $verbose, $safe ) - -=item rmtree( [$dir1, $dir2,...], $verbose, $safe ) - -=item rmtree( $dir1, $dir2,..., \%opt ) - -The rmtree() function provide the legacy interface of remove_tree() -with a different interpretation of the arguments passed. The behaviour -and return value of the function is otherwise identical to -remove_tree(). - -=back - -=head2 ERROR HANDLING - -=over 4 - -=item B - -The following error handling mechanism is considered -experimental and is subject to change pending feedback from -users. - -=back - -If C or C encounter an error, a diagnostic -message will be printed to C via C (for non-fatal -errors), or via C (for fatal errors). - -If this behaviour is not desirable, the C attribute may be -used to hold a reference to a variable, which will be used to store -the diagnostics. The variable is made a reference to an array of hash -references. Each hash contain a single key/value pair where the key -is the name of the file, and the value is the error message (including -the contents of C<$!> when appropriate). If a general error is -encountered the diagnostic key will be empty. - -An example usage looks like: - - remove_tree( 'foo/bar', 'bar/rat', {error => \my $err} ); - if (@$err) { - for my $diag (@$err) { - my ($file, $message) = %$diag; - if ($file eq '') { - print "general error: $message\n"; - } - else { - print "problem unlinking $file: $message\n"; - } - } - } - else { - print "No error encountered\n"; - } - -Note that if no errors are encountered, C<$err> will reference an -empty array. This means that C<$err> will always end up TRUE; so you -need to test C<@$err> to determine if errors occurred. - -=head2 NOTES - -C blindly exports C and C into the -current namespace. These days, this is considered bad style, but -to change it now would break too much code. Nonetheless, you are -invited to specify what it is you are expecting to use: - - use File::Path 'rmtree'; - -The routines C and C are B exported -by default. You must specify which ones you want to use. - - use File::Path 'remove_tree'; - -Note that a side-effect of the above is that C and C -are no longer exported at all. This is due to the way the C -module works. If you are migrating a codebase to use the new -interface, you will have to list everything explicitly. But that's -just good practice anyway. - - use File::Path qw(remove_tree rmtree); - -=head3 API CHANGES - -The API was changed in the 2.0 branch. For a time, C and -C tried, unsuccessfully, to deal with the two different -calling mechanisms. This approach was considered a failure. - -The new semantics are now only available with C and -C. The old semantics are only available through -C and C. Users are strongly encouraged to upgrade -to at least 2.08 in order to avoid surprises. - -=head3 SECURITY CONSIDERATIONS - -There were race conditions 1.x implementations of File::Path's -C function (although sometimes patched depending on the OS -distribution or platform). The 2.0 version contains code to avoid the -problem mentioned in CVE-2002-0435. - -See the following pages for more information: - - http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=286905 - http://www.nntp.perl.org/group/perl.perl5.porters/2005/01/msg97623.html - http://www.debian.org/security/2005/dsa-696 - -Additionally, unless the C parameter is set (or the -third parameter in the traditional interface is TRUE), should a -C be interrupted, files that were originally in read-only -mode may now have their permissions set to a read-write (or "delete -OK") mode. - -=head1 DIAGNOSTICS - -FATAL errors will cause the program to halt (C), since the -problem is so severe that it would be dangerous to continue. (This -can always be trapped with C, but it's not a good idea. Under -the circumstances, dying is the best thing to do). - -SEVERE errors may be trapped using the modern interface. If the -they are not trapped, or the old interface is used, such an error -will cause the program will halt. - -All other errors may be trapped using the modern interface, otherwise -they will be Ced about. Program execution will not be halted. - -=over 4 - -=item mkdir [path]: [errmsg] (SEVERE) - -C was unable to create the path. Probably some sort of -permissions error at the point of departure, or insufficient resources -(such as free inodes on Unix). - -=item No root path(s) specified - -C was not given any paths to create. This message is only -emitted if the routine is called with the traditional interface. -The modern interface will remain silent if given nothing to do. - -=item No such file or directory - -On Windows, if C gives you this warning, it may mean that -you have exceeded your filesystem's maximum path length. - -=item cannot fetch initial working directory: [errmsg] - -C attempted to determine the initial directory by calling -C, but the call failed for some reason. No attempt -will be made to delete anything. - -=item cannot stat initial working directory: [errmsg] - -C attempted to stat the initial directory (after having -successfully obtained its name via C), however, the call -failed for some reason. No attempt will be made to delete anything. - -=item cannot chdir to [dir]: [errmsg] - -C attempted to set the working directory in order to -begin deleting the objects therein, but was unsuccessful. This is -usually a permissions issue. The routine will continue to delete -other things, but this directory will be left intact. - -=item directory [dir] changed before chdir, expected dev=[n] ino=[n], actual dev=[n] ino=[n], aborting. (FATAL) - -C recorded the device and inode of a directory, and then -moved into it. It then performed a C on the current directory -and detected that the device and inode were no longer the same. As -this is at the heart of the race condition problem, the program -will die at this point. - -=item cannot make directory [dir] read+writeable: [errmsg] - -C attempted to change the permissions on the current directory -to ensure that subsequent unlinkings would not run into problems, -but was unable to do so. The permissions remain as they were, and -the program will carry on, doing the best it can. - -=item cannot read [dir]: [errmsg] - -C tried to read the contents of the directory in order -to acquire the names of the directory entries to be unlinked, but -was unsuccessful. This is usually a permissions issue. The -program will continue, but the files in this directory will remain -after the call. - -=item cannot reset chmod [dir]: [errmsg] - -C, after having deleted everything in a directory, attempted -to restore its permissions to the original state but failed. The -directory may wind up being left behind. - -=item cannot remove [dir] when cwd is [dir] - -The current working directory of the program is F -and you are attempting to remove an ancestor, such as F. -The directory tree is left untouched. - -The solution is to C out of the child directory to a place -outside the directory tree to be removed. - -=item cannot chdir to [parent-dir] from [child-dir]: [errmsg], aborting. (FATAL) - -C, after having deleted everything and restored the permissions -of a directory, was unable to chdir back to the parent. The program -halts to avoid a race condition from occurring. - -=item cannot stat prior working directory [dir]: [errmsg], aborting. (FATAL) - -C was unable to stat the parent directory after have returned -from the child. Since there is no way of knowing if we returned to -where we think we should be (by comparing device and inode) the only -way out is to C. - -=item previous directory [parent-dir] changed before entering [child-dir], expected dev=[n] ino=[n], actual dev=[n] ino=[n], aborting. (FATAL) - -When C returned from deleting files in a child directory, a -check revealed that the parent directory it returned to wasn't the one -it started out from. This is considered a sign of malicious activity. - -=item cannot make directory [dir] writeable: [errmsg] - -Just before removing a directory (after having successfully removed -everything it contained), C attempted to set the permissions -on the directory to ensure it could be removed and failed. Program -execution continues, but the directory may possibly not be deleted. - -=item cannot remove directory [dir]: [errmsg] - -C attempted to remove a directory, but failed. This may because -some objects that were unable to be removed remain in the directory, or -a permissions issue. The directory will be left behind. - -=item cannot restore permissions of [dir] to [0nnn]: [errmsg] - -After having failed to remove a directory, C was unable to -restore its permissions from a permissive state back to a possibly -more restrictive setting. (Permissions given in octal). - -=item cannot make file [file] writeable: [errmsg] - -C attempted to force the permissions of a file to ensure it -could be deleted, but failed to do so. It will, however, still attempt -to unlink the file. - -=item cannot unlink file [file]: [errmsg] - -C failed to remove a file. Probably a permissions issue. - -=item cannot restore permissions of [file] to [0nnn]: [errmsg] - -After having failed to remove a file, C was also unable -to restore the permissions on the file to a possibly less permissive -setting. (Permissions given in octal). - -=item unable to map [owner] to a uid, ownership not changed"); - -C was instructed to give the ownership of created -directories to the symbolic name [owner], but C did -not return the corresponding numeric uid. The directory will -be created, but ownership will not be changed. - -=item unable to map [group] to a gid, group ownership not changed - -C was instructed to give the group ownership of created -directories to the symbolic name [group], but C did -not return the corresponding numeric gid. The directory will -be created, but group ownership will not be changed. - -=back - -=head1 SEE ALSO - -=over 4 - -=item * - -L - -Allows files and directories to be moved to the Trashcan/Recycle -Bin (where they may later be restored if necessary) if the operating -system supports such functionality. This feature may one day be -made available directly in C. - -=item * - -L - -When removing directory trees, if you want to examine each file to -decide whether to delete it (and possibly leaving large swathes -alone), F offers a convenient and flexible approach -to examining directory trees. - -=back - -=head1 BUGS AND LIMITATIONS - -The following describes F limitations and how to report bugs. - -=head2 MULTITHREAD APPLICATIONS - -F B and B will not work with multithreaded -applications due to its use of B. At this time, no warning or error -results and you will certainly encounter unexpected results. - -The implementation that surfaces this limitation may change in a future -release. - -=head2 NFS Mount Points - -F is not responsible for triggering the automounts, mirror mounts, -and the contents of network mounted filesystems. If your NFS implementation -requires an action to be performed on the filesystem in order for -F to perform operations, it is strongly suggested you assure -filesystem availability by reading the root of the mounted filesystem. - -=head2 REPORTING BUGS - -Please report all bugs on the RT queue, either via the web interface: - -L - -or by email: - - bug-File-Path@rt.cpan.org - -In either case, please B patches to the bug report rather than -including them inline in the web post or the body of the email. - -You can also send pull requests to the Github repository: - -L - -=head1 ACKNOWLEDGEMENTS - -Paul Szabo identified the race condition originally, and Brendan -O'Dea wrote an implementation for Debian that addressed the problem. -That code was used as a basis for the current code. Their efforts -are greatly appreciated. - -Gisle Aas made a number of improvements to the documentation for -2.07 and his advice and assistance is also greatly appreciated. - -=head1 AUTHORS - -Prior authors and maintainers: Tim Bunce, Charles Bailey, and -David Landgren >. - -Current maintainers are Richard Elberger > and -James (Jim) Keenan >. - -=head1 CONTRIBUTORS - -Contributors to File::Path, in alphabetical order. - -=over 1 - -=item > - -=item Richard Elberger > - -=item Ryan Yee > - -=item Skye Shaw > - -=item Tom Lutz > - -=back - -=head1 COPYRIGHT - -This module is copyright (C) Charles Bailey, Tim Bunce, David Landgren, -James Keenan, and Richard Elberger 1995-2015. All rights reserved. - -=head1 LICENSE - -This library is free software; you can redistribute it and/or modify -it under the same terms as Perl itself. - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/File/Temp.pm b/dev-tools/src/main/resources/license-check/lib/File/Temp.pm deleted file mode 100644 index 817c6d90c6b..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/File/Temp.pm +++ /dev/null @@ -1,2594 +0,0 @@ -package File::Temp; -# ABSTRACT: return name and handle of a temporary file safely -our $VERSION = '0.2304'; # VERSION - - -# Toolchain targets v5.8.1, but we'll try to support back to v5.6 anyway. -# It might be possible to make this v5.5, but many v5.6isms are creeping -# into the code and tests. -use 5.006; -use strict; -use Carp; -use File::Spec 0.8; -use Cwd (); -use File::Path 2.06 qw/ rmtree /; -use Fcntl 1.03; -use IO::Seekable; # For SEEK_* -use Errno; -use Scalar::Util 'refaddr'; -require VMS::Stdio if $^O eq 'VMS'; - -# pre-emptively load Carp::Heavy. If we don't when we run out of file -# handles and attempt to call croak() we get an error message telling -# us that Carp::Heavy won't load rather than an error telling us we -# have run out of file handles. We either preload croak() or we -# switch the calls to croak from _gettemp() to use die. -eval { require Carp::Heavy; }; - -# Need the Symbol package if we are running older perl -require Symbol if $] < 5.006; - -### For the OO interface -use parent 0.221 qw/ IO::Handle IO::Seekable /; -use overload '""' => "STRINGIFY", '0+' => "NUMIFY", - fallback => 1; - -# use 'our' on v5.6.0 -use vars qw(@EXPORT_OK %EXPORT_TAGS $DEBUG $KEEP_ALL); - -$DEBUG = 0; -$KEEP_ALL = 0; - -# We are exporting functions - -use Exporter 5.57 'import'; # 5.57 lets us import 'import' - -# Export list - to allow fine tuning of export table - -@EXPORT_OK = qw{ - tempfile - tempdir - tmpnam - tmpfile - mktemp - mkstemp - mkstemps - mkdtemp - unlink0 - cleanup - SEEK_SET - SEEK_CUR - SEEK_END - }; - -# Groups of functions for export - -%EXPORT_TAGS = ( - 'POSIX' => [qw/ tmpnam tmpfile /], - 'mktemp' => [qw/ mktemp mkstemp mkstemps mkdtemp/], - 'seekable' => [qw/ SEEK_SET SEEK_CUR SEEK_END /], - ); - -# add contents of these tags to @EXPORT -Exporter::export_tags('POSIX','mktemp','seekable'); - -# This is a list of characters that can be used in random filenames - -my @CHARS = (qw/ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z - a b c d e f g h i j k l m n o p q r s t u v w x y z - 0 1 2 3 4 5 6 7 8 9 _ - /); - -# Maximum number of tries to make a temp file before failing - -use constant MAX_TRIES => 1000; - -# Minimum number of X characters that should be in a template -use constant MINX => 4; - -# Default template when no template supplied - -use constant TEMPXXX => 'X' x 10; - -# Constants for the security level - -use constant STANDARD => 0; -use constant MEDIUM => 1; -use constant HIGH => 2; - -# OPENFLAGS. If we defined the flag to use with Sysopen here this gives -# us an optimisation when many temporary files are requested - -my $OPENFLAGS = O_CREAT | O_EXCL | O_RDWR; -my $LOCKFLAG; - -unless ($^O eq 'MacOS') { - for my $oflag (qw/ NOFOLLOW BINARY LARGEFILE NOINHERIT /) { - my ($bit, $func) = (0, "Fcntl::O_" . $oflag); - no strict 'refs'; - $OPENFLAGS |= $bit if eval { - # Make sure that redefined die handlers do not cause problems - # e.g. CGI::Carp - local $SIG{__DIE__} = sub {}; - local $SIG{__WARN__} = sub {}; - $bit = &$func(); - 1; - }; - } - # Special case O_EXLOCK - $LOCKFLAG = eval { - local $SIG{__DIE__} = sub {}; - local $SIG{__WARN__} = sub {}; - &Fcntl::O_EXLOCK(); - }; -} - -# On some systems the O_TEMPORARY flag can be used to tell the OS -# to automatically remove the file when it is closed. This is fine -# in most cases but not if tempfile is called with UNLINK=>0 and -# the filename is requested -- in the case where the filename is to -# be passed to another routine. This happens on windows. We overcome -# this by using a second open flags variable - -my $OPENTEMPFLAGS = $OPENFLAGS; -unless ($^O eq 'MacOS') { - for my $oflag (qw/ TEMPORARY /) { - my ($bit, $func) = (0, "Fcntl::O_" . $oflag); - local($@); - no strict 'refs'; - $OPENTEMPFLAGS |= $bit if eval { - # Make sure that redefined die handlers do not cause problems - # e.g. CGI::Carp - local $SIG{__DIE__} = sub {}; - local $SIG{__WARN__} = sub {}; - $bit = &$func(); - 1; - }; - } -} - -# Private hash tracking which files have been created by each process id via the OO interface -my %FILES_CREATED_BY_OBJECT; - -# INTERNAL ROUTINES - not to be used outside of package - -# Generic routine for getting a temporary filename -# modelled on OpenBSD _gettemp() in mktemp.c - -# The template must contain X's that are to be replaced -# with the random values - -# Arguments: - -# TEMPLATE - string containing the XXXXX's that is converted -# to a random filename and opened if required - -# Optionally, a hash can also be supplied containing specific options -# "open" => if true open the temp file, else just return the name -# default is 0 -# "mkdir"=> if true, we are creating a temp directory rather than tempfile -# default is 0 -# "suffixlen" => number of characters at end of PATH to be ignored. -# default is 0. -# "unlink_on_close" => indicates that, if possible, the OS should remove -# the file as soon as it is closed. Usually indicates -# use of the O_TEMPORARY flag to sysopen. -# Usually irrelevant on unix -# "use_exlock" => Indicates that O_EXLOCK should be used. Default is true. - -# Optionally a reference to a scalar can be passed into the function -# On error this will be used to store the reason for the error -# "ErrStr" => \$errstr - -# "open" and "mkdir" can not both be true -# "unlink_on_close" is not used when "mkdir" is true. - -# The default options are equivalent to mktemp(). - -# Returns: -# filehandle - open file handle (if called with doopen=1, else undef) -# temp name - name of the temp file or directory - -# For example: -# ($fh, $name) = _gettemp($template, "open" => 1); - -# for the current version, failures are associated with -# stored in an error string and returned to give the reason whilst debugging -# This routine is not called by any external function -sub _gettemp { - - croak 'Usage: ($fh, $name) = _gettemp($template, OPTIONS);' - unless scalar(@_) >= 1; - - # the internal error string - expect it to be overridden - # Need this in case the caller decides not to supply us a value - # need an anonymous scalar - my $tempErrStr; - - # Default options - my %options = ( - "open" => 0, - "mkdir" => 0, - "suffixlen" => 0, - "unlink_on_close" => 0, - "use_exlock" => 1, - "ErrStr" => \$tempErrStr, - ); - - # Read the template - my $template = shift; - if (ref($template)) { - # Use a warning here since we have not yet merged ErrStr - carp "File::Temp::_gettemp: template must not be a reference"; - return (); - } - - # Check that the number of entries on stack are even - if (scalar(@_) % 2 != 0) { - # Use a warning here since we have not yet merged ErrStr - carp "File::Temp::_gettemp: Must have even number of options"; - return (); - } - - # Read the options and merge with defaults - %options = (%options, @_) if @_; - - # Make sure the error string is set to undef - ${$options{ErrStr}} = undef; - - # Can not open the file and make a directory in a single call - if ($options{"open"} && $options{"mkdir"}) { - ${$options{ErrStr}} = "doopen and domkdir can not both be true\n"; - return (); - } - - # Find the start of the end of the Xs (position of last X) - # Substr starts from 0 - my $start = length($template) - 1 - $options{"suffixlen"}; - - # Check that we have at least MINX x X (e.g. 'XXXX") at the end of the string - # (taking suffixlen into account). Any fewer is insecure. - - # Do it using substr - no reason to use a pattern match since - # we know where we are looking and what we are looking for - - if (substr($template, $start - MINX + 1, MINX) ne 'X' x MINX) { - ${$options{ErrStr}} = "The template must end with at least ". - MINX . " 'X' characters\n"; - return (); - } - - # Replace all the X at the end of the substring with a - # random character or just all the XX at the end of a full string. - # Do it as an if, since the suffix adjusts which section to replace - # and suffixlen=0 returns nothing if used in the substr directly - # and generate a full path from the template - - my $path = _replace_XX($template, $options{"suffixlen"}); - - - # Split the path into constituent parts - eventually we need to check - # whether the directory exists - # We need to know whether we are making a temp directory - # or a tempfile - - my ($volume, $directories, $file); - my $parent; # parent directory - if ($options{"mkdir"}) { - # There is no filename at the end - ($volume, $directories, $file) = File::Spec->splitpath( $path, 1); - - # The parent is then $directories without the last directory - # Split the directory and put it back together again - my @dirs = File::Spec->splitdir($directories); - - # If @dirs only has one entry (i.e. the directory template) that means - # we are in the current directory - if ($#dirs == 0) { - $parent = File::Spec->curdir; - } else { - - if ($^O eq 'VMS') { # need volume to avoid relative dir spec - $parent = File::Spec->catdir($volume, @dirs[0..$#dirs-1]); - $parent = 'sys$disk:[]' if $parent eq ''; - } else { - - # Put it back together without the last one - $parent = File::Spec->catdir(@dirs[0..$#dirs-1]); - - # ...and attach the volume (no filename) - $parent = File::Spec->catpath($volume, $parent, ''); - } - - } - - } else { - - # Get rid of the last filename (use File::Basename for this?) - ($volume, $directories, $file) = File::Spec->splitpath( $path ); - - # Join up without the file part - $parent = File::Spec->catpath($volume,$directories,''); - - # If $parent is empty replace with curdir - $parent = File::Spec->curdir - unless $directories ne ''; - - } - - # Check that the parent directories exist - # Do this even for the case where we are simply returning a name - # not a file -- no point returning a name that includes a directory - # that does not exist or is not writable - - unless (-e $parent) { - ${$options{ErrStr}} = "Parent directory ($parent) does not exist"; - return (); - } - unless (-d $parent) { - ${$options{ErrStr}} = "Parent directory ($parent) is not a directory"; - return (); - } - - # Check the stickiness of the directory and chown giveaway if required - # If the directory is world writable the sticky bit - # must be set - - if (File::Temp->safe_level == MEDIUM) { - my $safeerr; - unless (_is_safe($parent,\$safeerr)) { - ${$options{ErrStr}} = "Parent directory ($parent) is not safe ($safeerr)"; - return (); - } - } elsif (File::Temp->safe_level == HIGH) { - my $safeerr; - unless (_is_verysafe($parent, \$safeerr)) { - ${$options{ErrStr}} = "Parent directory ($parent) is not safe ($safeerr)"; - return (); - } - } - - - # Now try MAX_TRIES time to open the file - for (my $i = 0; $i < MAX_TRIES; $i++) { - - # Try to open the file if requested - if ($options{"open"}) { - my $fh; - - # If we are running before perl5.6.0 we can not auto-vivify - if ($] < 5.006) { - $fh = &Symbol::gensym; - } - - # Try to make sure this will be marked close-on-exec - # XXX: Win32 doesn't respect this, nor the proper fcntl, - # but may have O_NOINHERIT. This may or may not be in Fcntl. - local $^F = 2; - - # Attempt to open the file - my $open_success = undef; - if ( $^O eq 'VMS' and $options{"unlink_on_close"} && !$KEEP_ALL) { - # make it auto delete on close by setting FAB$V_DLT bit - $fh = VMS::Stdio::vmssysopen($path, $OPENFLAGS, 0600, 'fop=dlt'); - $open_success = $fh; - } else { - my $flags = ( ($options{"unlink_on_close"} && !$KEEP_ALL) ? - $OPENTEMPFLAGS : - $OPENFLAGS ); - $flags |= $LOCKFLAG if (defined $LOCKFLAG && $options{use_exlock}); - $open_success = sysopen($fh, $path, $flags, 0600); - } - if ( $open_success ) { - - # in case of odd umask force rw - chmod(0600, $path); - - # Opened successfully - return file handle and name - return ($fh, $path); - - } else { - - # Error opening file - abort with error - # if the reason was anything but EEXIST - unless ($!{EEXIST}) { - ${$options{ErrStr}} = "Could not create temp file $path: $!"; - return (); - } - - # Loop round for another try - - } - } elsif ($options{"mkdir"}) { - - # Open the temp directory - if (mkdir( $path, 0700)) { - # in case of odd umask - chmod(0700, $path); - - return undef, $path; - } else { - - # Abort with error if the reason for failure was anything - # except EEXIST - unless ($!{EEXIST}) { - ${$options{ErrStr}} = "Could not create directory $path: $!"; - return (); - } - - # Loop round for another try - - } - - } else { - - # Return true if the file can not be found - # Directory has been checked previously - - return (undef, $path) unless -e $path; - - # Try again until MAX_TRIES - - } - - # Did not successfully open the tempfile/dir - # so try again with a different set of random letters - # No point in trying to increment unless we have only - # 1 X say and the randomness could come up with the same - # file MAX_TRIES in a row. - - # Store current attempt - in principal this implies that the - # 3rd time around the open attempt that the first temp file - # name could be generated again. Probably should store each - # attempt and make sure that none are repeated - - my $original = $path; - my $counter = 0; # Stop infinite loop - my $MAX_GUESS = 50; - - do { - - # Generate new name from original template - $path = _replace_XX($template, $options{"suffixlen"}); - - $counter++; - - } until ($path ne $original || $counter > $MAX_GUESS); - - # Check for out of control looping - if ($counter > $MAX_GUESS) { - ${$options{ErrStr}} = "Tried to get a new temp name different to the previous value $MAX_GUESS times.\nSomething wrong with template?? ($template)"; - return (); - } - - } - - # If we get here, we have run out of tries - ${ $options{ErrStr} } = "Have exceeded the maximum number of attempts (" - . MAX_TRIES . ") to open temp file/dir"; - - return (); - -} - -# Internal routine to replace the XXXX... with random characters -# This has to be done by _gettemp() every time it fails to -# open a temp file/dir - -# Arguments: $template (the template with XXX), -# $ignore (number of characters at end to ignore) - -# Returns: modified template - -sub _replace_XX { - - croak 'Usage: _replace_XX($template, $ignore)' - unless scalar(@_) == 2; - - my ($path, $ignore) = @_; - - # Do it as an if, since the suffix adjusts which section to replace - # and suffixlen=0 returns nothing if used in the substr directly - # Alternatively, could simply set $ignore to length($path)-1 - # Don't want to always use substr when not required though. - my $end = ( $] >= 5.006 ? "\\z" : "\\Z" ); - - if ($ignore) { - substr($path, 0, - $ignore) =~ s/X(?=X*$end)/$CHARS[ int( rand( @CHARS ) ) ]/ge; - } else { - $path =~ s/X(?=X*$end)/$CHARS[ int( rand( @CHARS ) ) ]/ge; - } - return $path; -} - -# Internal routine to force a temp file to be writable after -# it is created so that we can unlink it. Windows seems to occasionally -# force a file to be readonly when written to certain temp locations -sub _force_writable { - my $file = shift; - chmod 0600, $file; -} - - -# internal routine to check to see if the directory is safe -# First checks to see if the directory is not owned by the -# current user or root. Then checks to see if anyone else -# can write to the directory and if so, checks to see if -# it has the sticky bit set - -# Will not work on systems that do not support sticky bit - -#Args: directory path to check -# Optionally: reference to scalar to contain error message -# Returns true if the path is safe and false otherwise. -# Returns undef if can not even run stat() on the path - -# This routine based on version written by Tom Christiansen - -# Presumably, by the time we actually attempt to create the -# file or directory in this directory, it may not be safe -# anymore... Have to run _is_safe directly after the open. - -sub _is_safe { - - my $path = shift; - my $err_ref = shift; - - # Stat path - my @info = stat($path); - unless (scalar(@info)) { - $$err_ref = "stat(path) returned no values"; - return 0; - } - ; - return 1 if $^O eq 'VMS'; # owner delete control at file level - - # Check to see whether owner is neither superuser (or a system uid) nor me - # Use the effective uid from the $> variable - # UID is in [4] - if ($info[4] > File::Temp->top_system_uid() && $info[4] != $>) { - - Carp::cluck(sprintf "uid=$info[4] topuid=%s euid=$> path='$path'", - File::Temp->top_system_uid()); - - $$err_ref = "Directory owned neither by root nor the current user" - if ref($err_ref); - return 0; - } - - # check whether group or other can write file - # use 066 to detect either reading or writing - # use 022 to check writability - # Do it with S_IWOTH and S_IWGRP for portability (maybe) - # mode is in info[2] - if (($info[2] & &Fcntl::S_IWGRP) || # Is group writable? - ($info[2] & &Fcntl::S_IWOTH) ) { # Is world writable? - # Must be a directory - unless (-d $path) { - $$err_ref = "Path ($path) is not a directory" - if ref($err_ref); - return 0; - } - # Must have sticky bit set - unless (-k $path) { - $$err_ref = "Sticky bit not set on $path when dir is group|world writable" - if ref($err_ref); - return 0; - } - } - - return 1; -} - -# Internal routine to check whether a directory is safe -# for temp files. Safer than _is_safe since it checks for -# the possibility of chown giveaway and if that is a possibility -# checks each directory in the path to see if it is safe (with _is_safe) - -# If _PC_CHOWN_RESTRICTED is not set, does the full test of each -# directory anyway. - -# Takes optional second arg as scalar ref to error reason - -sub _is_verysafe { - - # Need POSIX - but only want to bother if really necessary due to overhead - require POSIX; - - my $path = shift; - print "_is_verysafe testing $path\n" if $DEBUG; - return 1 if $^O eq 'VMS'; # owner delete control at file level - - my $err_ref = shift; - - # Should Get the value of _PC_CHOWN_RESTRICTED if it is defined - # and If it is not there do the extensive test - local($@); - my $chown_restricted; - $chown_restricted = &POSIX::_PC_CHOWN_RESTRICTED() - if eval { &POSIX::_PC_CHOWN_RESTRICTED(); 1}; - - # If chown_resticted is set to some value we should test it - if (defined $chown_restricted) { - - # Return if the current directory is safe - return _is_safe($path,$err_ref) if POSIX::sysconf( $chown_restricted ); - - } - - # To reach this point either, the _PC_CHOWN_RESTRICTED symbol - # was not available or the symbol was there but chown giveaway - # is allowed. Either way, we now have to test the entire tree for - # safety. - - # Convert path to an absolute directory if required - unless (File::Spec->file_name_is_absolute($path)) { - $path = File::Spec->rel2abs($path); - } - - # Split directory into components - assume no file - my ($volume, $directories, undef) = File::Spec->splitpath( $path, 1); - - # Slightly less efficient than having a function in File::Spec - # to chop off the end of a directory or even a function that - # can handle ../ in a directory tree - # Sometimes splitdir() returns a blank at the end - # so we will probably check the bottom directory twice in some cases - my @dirs = File::Spec->splitdir($directories); - - # Concatenate one less directory each time around - foreach my $pos (0.. $#dirs) { - # Get a directory name - my $dir = File::Spec->catpath($volume, - File::Spec->catdir(@dirs[0.. $#dirs - $pos]), - '' - ); - - print "TESTING DIR $dir\n" if $DEBUG; - - # Check the directory - return 0 unless _is_safe($dir,$err_ref); - - } - - return 1; -} - - - -# internal routine to determine whether unlink works on this -# platform for files that are currently open. -# Returns true if we can, false otherwise. - -# Currently WinNT, OS/2 and VMS can not unlink an opened file -# On VMS this is because the O_EXCL flag is used to open the -# temporary file. Currently I do not know enough about the issues -# on VMS to decide whether O_EXCL is a requirement. - -sub _can_unlink_opened_file { - - if (grep { $^O eq $_ } qw/MSWin32 os2 VMS dos MacOS haiku/) { - return 0; - } else { - return 1; - } - -} - -# internal routine to decide which security levels are allowed -# see safe_level() for more information on this - -# Controls whether the supplied security level is allowed - -# $cando = _can_do_level( $level ) - -sub _can_do_level { - - # Get security level - my $level = shift; - - # Always have to be able to do STANDARD - return 1 if $level == STANDARD; - - # Currently, the systems that can do HIGH or MEDIUM are identical - if ( $^O eq 'MSWin32' || $^O eq 'os2' || $^O eq 'cygwin' || $^O eq 'dos' || $^O eq 'MacOS' || $^O eq 'mpeix') { - return 0; - } else { - return 1; - } - -} - -# This routine sets up a deferred unlinking of a specified -# filename and filehandle. It is used in the following cases: -# - Called by unlink0 if an opened file can not be unlinked -# - Called by tempfile() if files are to be removed on shutdown -# - Called by tempdir() if directories are to be removed on shutdown - -# Arguments: -# _deferred_unlink( $fh, $fname, $isdir ); -# -# - filehandle (so that it can be explicitly closed if open -# - filename (the thing we want to remove) -# - isdir (flag to indicate that we are being given a directory) -# [and hence no filehandle] - -# Status is not referred to since all the magic is done with an END block - -{ - # Will set up two lexical variables to contain all the files to be - # removed. One array for files, another for directories They will - # only exist in this block. - - # This means we only have to set up a single END block to remove - # all files. - - # in order to prevent child processes inadvertently deleting the parent - # temp files we use a hash to store the temp files and directories - # created by a particular process id. - - # %files_to_unlink contains values that are references to an array of - # array references containing the filehandle and filename associated with - # the temp file. - my (%files_to_unlink, %dirs_to_unlink); - - # Set up an end block to use these arrays - END { - local($., $@, $!, $^E, $?); - cleanup(at_exit => 1); - } - - # Cleanup function. Always triggered on END (with at_exit => 1) but - # can be invoked manually. - sub cleanup { - my %h = @_; - my $at_exit = delete $h{at_exit}; - $at_exit = 0 if not defined $at_exit; - { my @k = sort keys %h; die "unrecognized parameters: @k" if @k } - - if (!$KEEP_ALL) { - # Files - my @files = (exists $files_to_unlink{$$} ? - @{ $files_to_unlink{$$} } : () ); - foreach my $file (@files) { - # close the filehandle without checking its state - # in order to make real sure that this is closed - # if its already closed then I don't care about the answer - # probably a better way to do this - close($file->[0]); # file handle is [0] - - if (-f $file->[1]) { # file name is [1] - _force_writable( $file->[1] ); # for windows - unlink $file->[1] or warn "Error removing ".$file->[1]; - } - } - # Dirs - my @dirs = (exists $dirs_to_unlink{$$} ? - @{ $dirs_to_unlink{$$} } : () ); - my ($cwd, $cwd_to_remove); - foreach my $dir (@dirs) { - if (-d $dir) { - # Some versions of rmtree will abort if you attempt to remove - # the directory you are sitting in. For automatic cleanup - # at program exit, we avoid this by chdir()ing out of the way - # first. If not at program exit, it's best not to mess with the - # current directory, so just let it fail with a warning. - if ($at_exit) { - $cwd = Cwd::abs_path(File::Spec->curdir) if not defined $cwd; - my $abs = Cwd::abs_path($dir); - if ($abs eq $cwd) { - $cwd_to_remove = $dir; - next; - } - } - eval { rmtree($dir, $DEBUG, 0); }; - warn $@ if ($@ && $^W); - } - } - - if (defined $cwd_to_remove) { - # We do need to clean up the current directory, and everything - # else is done, so get out of there and remove it. - chdir $cwd_to_remove or die "cannot chdir to $cwd_to_remove: $!"; - my $updir = File::Spec->updir; - chdir $updir or die "cannot chdir to $updir: $!"; - eval { rmtree($cwd_to_remove, $DEBUG, 0); }; - warn $@ if ($@ && $^W); - } - - # clear the arrays - @{ $files_to_unlink{$$} } = () - if exists $files_to_unlink{$$}; - @{ $dirs_to_unlink{$$} } = () - if exists $dirs_to_unlink{$$}; - } - } - - - # This is the sub called to register a file for deferred unlinking - # This could simply store the input parameters and defer everything - # until the END block. For now we do a bit of checking at this - # point in order to make sure that (1) we have a file/dir to delete - # and (2) we have been called with the correct arguments. - sub _deferred_unlink { - - croak 'Usage: _deferred_unlink($fh, $fname, $isdir)' - unless scalar(@_) == 3; - - my ($fh, $fname, $isdir) = @_; - - warn "Setting up deferred removal of $fname\n" - if $DEBUG; - - # make sure we save the absolute path for later cleanup - # OK to untaint because we only ever use this internally - # as a file path, never interpolating into the shell - $fname = Cwd::abs_path($fname); - ($fname) = $fname =~ /^(.*)$/; - - # If we have a directory, check that it is a directory - if ($isdir) { - - if (-d $fname) { - - # Directory exists so store it - # first on VMS turn []foo into [.foo] for rmtree - $fname = VMS::Filespec::vmspath($fname) if $^O eq 'VMS'; - $dirs_to_unlink{$$} = [] - unless exists $dirs_to_unlink{$$}; - push (@{ $dirs_to_unlink{$$} }, $fname); - - } else { - carp "Request to remove directory $fname could not be completed since it does not exist!\n" if $^W; - } - - } else { - - if (-f $fname) { - - # file exists so store handle and name for later removal - $files_to_unlink{$$} = [] - unless exists $files_to_unlink{$$}; - push(@{ $files_to_unlink{$$} }, [$fh, $fname]); - - } else { - carp "Request to remove file $fname could not be completed since it is not there!\n" if $^W; - } - - } - - } - - -} - -# normalize argument keys to upper case and do consistent handling -# of leading template vs TEMPLATE -sub _parse_args { - my $leading_template = (scalar(@_) % 2 == 1 ? shift(@_) : '' ); - my %args = @_; - %args = map { uc($_), $args{$_} } keys %args; - - # template (store it in an array so that it will - # disappear from the arg list of tempfile) - my @template = ( - exists $args{TEMPLATE} ? $args{TEMPLATE} : - $leading_template ? $leading_template : () - ); - delete $args{TEMPLATE}; - - return( \@template, \%args ); -} - - -sub new { - my $proto = shift; - my $class = ref($proto) || $proto; - - my ($maybe_template, $args) = _parse_args(@_); - - # see if they are unlinking (defaulting to yes) - my $unlink = (exists $args->{UNLINK} ? $args->{UNLINK} : 1 ); - delete $args->{UNLINK}; - - # Protect OPEN - delete $args->{OPEN}; - - # Open the file and retain file handle and file name - my ($fh, $path) = tempfile( @$maybe_template, %$args ); - - print "Tmp: $fh - $path\n" if $DEBUG; - - # Store the filename in the scalar slot - ${*$fh} = $path; - - # Cache the filename by pid so that the destructor can decide whether to remove it - $FILES_CREATED_BY_OBJECT{$$}{$path} = 1; - - # Store unlink information in hash slot (plus other constructor info) - %{*$fh} = %$args; - - # create the object - bless $fh, $class; - - # final method-based configuration - $fh->unlink_on_destroy( $unlink ); - - return $fh; -} - - -sub newdir { - my $self = shift; - - my ($maybe_template, $args) = _parse_args(@_); - - # handle CLEANUP without passing CLEANUP to tempdir - my $cleanup = (exists $args->{CLEANUP} ? $args->{CLEANUP} : 1 ); - delete $args->{CLEANUP}; - - my $tempdir = tempdir( @$maybe_template, %$args); - - # get a safe absolute path for cleanup, just like - # happens in _deferred_unlink - my $real_dir = Cwd::abs_path( $tempdir ); - ($real_dir) = $real_dir =~ /^(.*)$/; - - return bless { DIRNAME => $tempdir, - REALNAME => $real_dir, - CLEANUP => $cleanup, - LAUNCHPID => $$, - }, "File::Temp::Dir"; -} - - -sub filename { - my $self = shift; - return ${*$self}; -} - -sub STRINGIFY { - my $self = shift; - return $self->filename; -} - -# For reference, can't use '0+'=>\&Scalar::Util::refaddr directly because -# refaddr() demands one parameter only, whereas overload.pm calls with three -# even for unary operations like '0+'. -sub NUMIFY { - return refaddr($_[0]); -} - - -sub unlink_on_destroy { - my $self = shift; - if (@_) { - ${*$self}{UNLINK} = shift; - } - return ${*$self}{UNLINK}; -} - - -sub DESTROY { - local($., $@, $!, $^E, $?); - my $self = shift; - - # Make sure we always remove the file from the global hash - # on destruction. This prevents the hash from growing uncontrollably - # and post-destruction there is no reason to know about the file. - my $file = $self->filename; - my $was_created_by_proc; - if (exists $FILES_CREATED_BY_OBJECT{$$}{$file}) { - $was_created_by_proc = 1; - delete $FILES_CREATED_BY_OBJECT{$$}{$file}; - } - - if (${*$self}{UNLINK} && !$KEEP_ALL) { - print "# ---------> Unlinking $self\n" if $DEBUG; - - # only delete if this process created it - return unless $was_created_by_proc; - - # The unlink1 may fail if the file has been closed - # by the caller. This leaves us with the decision - # of whether to refuse to remove the file or simply - # do an unlink without test. Seems to be silly - # to do this when we are trying to be careful - # about security - _force_writable( $file ); # for windows - unlink1( $self, $file ) - or unlink($file); - } -} - - -sub tempfile { - if ( @_ && $_[0] eq 'File::Temp' ) { - croak "'tempfile' can't be called as a method"; - } - # Can not check for argument count since we can have any - # number of args - - # Default options - my %options = ( - "DIR" => undef, # Directory prefix - "SUFFIX" => '', # Template suffix - "UNLINK" => 0, # Do not unlink file on exit - "OPEN" => 1, # Open file - "TMPDIR" => 0, # Place tempfile in tempdir if template specified - "EXLOCK" => 1, # Open file with O_EXLOCK - ); - - # Check to see whether we have an odd or even number of arguments - my ($maybe_template, $args) = _parse_args(@_); - my $template = @$maybe_template ? $maybe_template->[0] : undef; - - # Read the options and merge with defaults - %options = (%options, %$args); - - # First decision is whether or not to open the file - if (! $options{"OPEN"}) { - - warn "tempfile(): temporary filename requested but not opened.\nPossibly unsafe, consider using tempfile() with OPEN set to true\n" - if $^W; - - } - - if ($options{"DIR"} and $^O eq 'VMS') { - - # on VMS turn []foo into [.foo] for concatenation - $options{"DIR"} = VMS::Filespec::vmspath($options{"DIR"}); - } - - # Construct the template - - # Have a choice of trying to work around the mkstemp/mktemp/tmpnam etc - # functions or simply constructing a template and using _gettemp() - # explicitly. Go for the latter - - # First generate a template if not defined and prefix the directory - # If no template must prefix the temp directory - if (defined $template) { - # End up with current directory if neither DIR not TMPDIR are set - if ($options{"DIR"}) { - - $template = File::Spec->catfile($options{"DIR"}, $template); - - } elsif ($options{TMPDIR}) { - - $template = File::Spec->catfile(File::Spec->tmpdir, $template ); - - } - - } else { - - if ($options{"DIR"}) { - - $template = File::Spec->catfile($options{"DIR"}, TEMPXXX); - - } else { - - $template = File::Spec->catfile(File::Spec->tmpdir, TEMPXXX); - - } - - } - - # Now add a suffix - $template .= $options{"SUFFIX"}; - - # Determine whether we should tell _gettemp to unlink the file - # On unix this is irrelevant and can be worked out after the file is - # opened (simply by unlinking the open filehandle). On Windows or VMS - # we have to indicate temporary-ness when we open the file. In general - # we only want a true temporary file if we are returning just the - # filehandle - if the user wants the filename they probably do not - # want the file to disappear as soon as they close it (which may be - # important if they want a child process to use the file) - # For this reason, tie unlink_on_close to the return context regardless - # of OS. - my $unlink_on_close = ( wantarray ? 0 : 1); - - # Create the file - my ($fh, $path, $errstr); - croak "Error in tempfile() using template $template: $errstr" - unless (($fh, $path) = _gettemp($template, - "open" => $options{'OPEN'}, - "mkdir"=> 0 , - "unlink_on_close" => $unlink_on_close, - "suffixlen" => length($options{'SUFFIX'}), - "ErrStr" => \$errstr, - "use_exlock" => $options{EXLOCK}, - ) ); - - # Set up an exit handler that can do whatever is right for the - # system. This removes files at exit when requested explicitly or when - # system is asked to unlink_on_close but is unable to do so because - # of OS limitations. - # The latter should be achieved by using a tied filehandle. - # Do not check return status since this is all done with END blocks. - _deferred_unlink($fh, $path, 0) if $options{"UNLINK"}; - - # Return - if (wantarray()) { - - if ($options{'OPEN'}) { - return ($fh, $path); - } else { - return (undef, $path); - } - - } else { - - # Unlink the file. It is up to unlink0 to decide what to do with - # this (whether to unlink now or to defer until later) - unlink0($fh, $path) or croak "Error unlinking file $path using unlink0"; - - # Return just the filehandle. - return $fh; - } - - -} - - -# ' - -sub tempdir { - if ( @_ && $_[0] eq 'File::Temp' ) { - croak "'tempdir' can't be called as a method"; - } - - # Can not check for argument count since we can have any - # number of args - - # Default options - my %options = ( - "CLEANUP" => 0, # Remove directory on exit - "DIR" => '', # Root directory - "TMPDIR" => 0, # Use tempdir with template - ); - - # Check to see whether we have an odd or even number of arguments - my ($maybe_template, $args) = _parse_args(@_); - my $template = @$maybe_template ? $maybe_template->[0] : undef; - - # Read the options and merge with defaults - %options = (%options, %$args); - - # Modify or generate the template - - # Deal with the DIR and TMPDIR options - if (defined $template) { - - # Need to strip directory path if using DIR or TMPDIR - if ($options{'TMPDIR'} || $options{'DIR'}) { - - # Strip parent directory from the filename - # - # There is no filename at the end - $template = VMS::Filespec::vmspath($template) if $^O eq 'VMS'; - my ($volume, $directories, undef) = File::Spec->splitpath( $template, 1); - - # Last directory is then our template - $template = (File::Spec->splitdir($directories))[-1]; - - # Prepend the supplied directory or temp dir - if ($options{"DIR"}) { - - $template = File::Spec->catdir($options{"DIR"}, $template); - - } elsif ($options{TMPDIR}) { - - # Prepend tmpdir - $template = File::Spec->catdir(File::Spec->tmpdir, $template); - - } - - } - - } else { - - if ($options{"DIR"}) { - - $template = File::Spec->catdir($options{"DIR"}, TEMPXXX); - - } else { - - $template = File::Spec->catdir(File::Spec->tmpdir, TEMPXXX); - - } - - } - - # Create the directory - my $tempdir; - my $suffixlen = 0; - if ($^O eq 'VMS') { # dir names can end in delimiters - $template =~ m/([\.\]:>]+)$/; - $suffixlen = length($1); - } - if ( ($^O eq 'MacOS') && (substr($template, -1) eq ':') ) { - # dir name has a trailing ':' - ++$suffixlen; - } - - my $errstr; - croak "Error in tempdir() using $template: $errstr" - unless ((undef, $tempdir) = _gettemp($template, - "open" => 0, - "mkdir"=> 1 , - "suffixlen" => $suffixlen, - "ErrStr" => \$errstr, - ) ); - - # Install exit handler; must be dynamic to get lexical - if ( $options{'CLEANUP'} && -d $tempdir) { - _deferred_unlink(undef, $tempdir, 1); - } - - # Return the dir name - return $tempdir; - -} - - - - -sub mkstemp { - - croak "Usage: mkstemp(template)" - if scalar(@_) != 1; - - my $template = shift; - - my ($fh, $path, $errstr); - croak "Error in mkstemp using $template: $errstr" - unless (($fh, $path) = _gettemp($template, - "open" => 1, - "mkdir"=> 0 , - "suffixlen" => 0, - "ErrStr" => \$errstr, - ) ); - - if (wantarray()) { - return ($fh, $path); - } else { - return $fh; - } - -} - - - -sub mkstemps { - - croak "Usage: mkstemps(template, suffix)" - if scalar(@_) != 2; - - - my $template = shift; - my $suffix = shift; - - $template .= $suffix; - - my ($fh, $path, $errstr); - croak "Error in mkstemps using $template: $errstr" - unless (($fh, $path) = _gettemp($template, - "open" => 1, - "mkdir"=> 0 , - "suffixlen" => length($suffix), - "ErrStr" => \$errstr, - ) ); - - if (wantarray()) { - return ($fh, $path); - } else { - return $fh; - } - -} - - -#' # for emacs - -sub mkdtemp { - - croak "Usage: mkdtemp(template)" - if scalar(@_) != 1; - - my $template = shift; - my $suffixlen = 0; - if ($^O eq 'VMS') { # dir names can end in delimiters - $template =~ m/([\.\]:>]+)$/; - $suffixlen = length($1); - } - if ( ($^O eq 'MacOS') && (substr($template, -1) eq ':') ) { - # dir name has a trailing ':' - ++$suffixlen; - } - my ($junk, $tmpdir, $errstr); - croak "Error creating temp directory from template $template\: $errstr" - unless (($junk, $tmpdir) = _gettemp($template, - "open" => 0, - "mkdir"=> 1 , - "suffixlen" => $suffixlen, - "ErrStr" => \$errstr, - ) ); - - return $tmpdir; - -} - - -sub mktemp { - - croak "Usage: mktemp(template)" - if scalar(@_) != 1; - - my $template = shift; - - my ($tmpname, $junk, $errstr); - croak "Error getting name to temp file from template $template: $errstr" - unless (($junk, $tmpname) = _gettemp($template, - "open" => 0, - "mkdir"=> 0 , - "suffixlen" => 0, - "ErrStr" => \$errstr, - ) ); - - return $tmpname; -} - - -sub tmpnam { - - # Retrieve the temporary directory name - my $tmpdir = File::Spec->tmpdir; - - croak "Error temporary directory is not writable" - if $tmpdir eq ''; - - # Use a ten character template and append to tmpdir - my $template = File::Spec->catfile($tmpdir, TEMPXXX); - - if (wantarray() ) { - return mkstemp($template); - } else { - return mktemp($template); - } - -} - - -sub tmpfile { - - # Simply call tmpnam() in a list context - my ($fh, $file) = tmpnam(); - - # Make sure file is removed when filehandle is closed - # This will fail on NFS - unlink0($fh, $file) - or return undef; - - return $fh; - -} - - -sub tempnam { - - croak 'Usage tempnam($dir, $prefix)' unless scalar(@_) == 2; - - my ($dir, $prefix) = @_; - - # Add a string to the prefix - $prefix .= 'XXXXXXXX'; - - # Concatenate the directory to the file - my $template = File::Spec->catfile($dir, $prefix); - - return mktemp($template); - -} - - -sub unlink0 { - - croak 'Usage: unlink0(filehandle, filename)' - unless scalar(@_) == 2; - - # Read args - my ($fh, $path) = @_; - - cmpstat($fh, $path) or return 0; - - # attempt remove the file (does not work on some platforms) - if (_can_unlink_opened_file()) { - - # return early (Without unlink) if we have been instructed to retain files. - return 1 if $KEEP_ALL; - - # XXX: do *not* call this on a directory; possible race - # resulting in recursive removal - croak "unlink0: $path has become a directory!" if -d $path; - unlink($path) or return 0; - - # Stat the filehandle - my @fh = stat $fh; - - print "Link count = $fh[3] \n" if $DEBUG; - - # Make sure that the link count is zero - # - Cygwin provides deferred unlinking, however, - # on Win9x the link count remains 1 - # On NFS the link count may still be 1 but we can't know that - # we are on NFS. Since we can't be sure, we'll defer it - - return 1 if $fh[3] == 0 || $^O eq 'cygwin'; - } - # fall-through if we can't unlink now - _deferred_unlink($fh, $path, 0); - return 1; -} - - -sub cmpstat { - - croak 'Usage: cmpstat(filehandle, filename)' - unless scalar(@_) == 2; - - # Read args - my ($fh, $path) = @_; - - warn "Comparing stat\n" - if $DEBUG; - - # Stat the filehandle - which may be closed if someone has manually - # closed the file. Can not turn off warnings without using $^W - # unless we upgrade to 5.006 minimum requirement - my @fh; - { - local ($^W) = 0; - @fh = stat $fh; - } - return unless @fh; - - if ($fh[3] > 1 && $^W) { - carp "unlink0: fstat found too many links; SB=@fh" if $^W; - } - - # Stat the path - my @path = stat $path; - - unless (@path) { - carp "unlink0: $path is gone already" if $^W; - return; - } - - # this is no longer a file, but may be a directory, or worse - unless (-f $path) { - confess "panic: $path is no longer a file: SB=@fh"; - } - - # Do comparison of each member of the array - # On WinNT dev and rdev seem to be different - # depending on whether it is a file or a handle. - # Cannot simply compare all members of the stat return - # Select the ones we can use - my @okstat = (0..$#fh); # Use all by default - if ($^O eq 'MSWin32') { - @okstat = (1,2,3,4,5,7,8,9,10); - } elsif ($^O eq 'os2') { - @okstat = (0, 2..$#fh); - } elsif ($^O eq 'VMS') { # device and file ID are sufficient - @okstat = (0, 1); - } elsif ($^O eq 'dos') { - @okstat = (0,2..7,11..$#fh); - } elsif ($^O eq 'mpeix') { - @okstat = (0..4,8..10); - } - - # Now compare each entry explicitly by number - for (@okstat) { - print "Comparing: $_ : $fh[$_] and $path[$_]\n" if $DEBUG; - # Use eq rather than == since rdev, blksize, and blocks (6, 11, - # and 12) will be '' on platforms that do not support them. This - # is fine since we are only comparing integers. - unless ($fh[$_] eq $path[$_]) { - warn "Did not match $_ element of stat\n" if $DEBUG; - return 0; - } - } - - return 1; -} - - -sub unlink1 { - croak 'Usage: unlink1(filehandle, filename)' - unless scalar(@_) == 2; - - # Read args - my ($fh, $path) = @_; - - cmpstat($fh, $path) or return 0; - - # Close the file - close( $fh ) or return 0; - - # Make sure the file is writable (for windows) - _force_writable( $path ); - - # return early (without unlink) if we have been instructed to retain files. - return 1 if $KEEP_ALL; - - # remove the file - return unlink($path); -} - - -{ - # protect from using the variable itself - my $LEVEL = STANDARD; - sub safe_level { - my $self = shift; - if (@_) { - my $level = shift; - if (($level != STANDARD) && ($level != MEDIUM) && ($level != HIGH)) { - carp "safe_level: Specified level ($level) not STANDARD, MEDIUM or HIGH - ignoring\n" if $^W; - } else { - # Don't allow this on perl 5.005 or earlier - if ($] < 5.006 && $level != STANDARD) { - # Cant do MEDIUM or HIGH checks - croak "Currently requires perl 5.006 or newer to do the safe checks"; - } - # Check that we are allowed to change level - # Silently ignore if we can not. - $LEVEL = $level if _can_do_level($level); - } - } - return $LEVEL; - } -} - - -{ - my $TopSystemUID = 10; - $TopSystemUID = 197108 if $^O eq 'interix'; # "Administrator" - sub top_system_uid { - my $self = shift; - if (@_) { - my $newuid = shift; - croak "top_system_uid: UIDs should be numeric" - unless $newuid =~ /^\d+$/s; - $TopSystemUID = $newuid; - } - return $TopSystemUID; - } -} - - -package File::Temp::Dir; - -use File::Path qw/ rmtree /; -use strict; -use overload '""' => "STRINGIFY", - '0+' => \&File::Temp::NUMIFY, - fallback => 1; - -# private class specifically to support tempdir objects -# created by File::Temp->newdir - -# ostensibly the same method interface as File::Temp but without -# inheriting all the IO::Seekable methods and other cruft - -# Read-only - returns the name of the temp directory - -sub dirname { - my $self = shift; - return $self->{DIRNAME}; -} - -sub STRINGIFY { - my $self = shift; - return $self->dirname; -} - -sub unlink_on_destroy { - my $self = shift; - if (@_) { - $self->{CLEANUP} = shift; - } - return $self->{CLEANUP}; -} - -sub DESTROY { - my $self = shift; - local($., $@, $!, $^E, $?); - if ($self->unlink_on_destroy && - $$ == $self->{LAUNCHPID} && !$File::Temp::KEEP_ALL) { - if (-d $self->{REALNAME}) { - # Some versions of rmtree will abort if you attempt to remove - # the directory you are sitting in. We protect that and turn it - # into a warning. We do this because this occurs during object - # destruction and so can not be caught by the user. - eval { rmtree($self->{REALNAME}, $File::Temp::DEBUG, 0); }; - warn $@ if ($@ && $^W); - } - } -} - -1; - -__END__ - -=pod - -=encoding utf-8 - -=head1 NAME - -File::Temp - return name and handle of a temporary file safely - -=head1 VERSION - -version 0.2304 - -=head1 SYNOPSIS - - use File::Temp qw/ tempfile tempdir /; - - $fh = tempfile(); - ($fh, $filename) = tempfile(); - - ($fh, $filename) = tempfile( $template, DIR => $dir); - ($fh, $filename) = tempfile( $template, SUFFIX => '.dat'); - ($fh, $filename) = tempfile( $template, TMPDIR => 1 ); - - binmode( $fh, ":utf8" ); - - $dir = tempdir( CLEANUP => 1 ); - ($fh, $filename) = tempfile( DIR => $dir ); - -Object interface: - - require File::Temp; - use File::Temp (); - use File::Temp qw/ :seekable /; - - $fh = File::Temp->new(); - $fname = $fh->filename; - - $fh = File::Temp->new(TEMPLATE => $template); - $fname = $fh->filename; - - $tmp = File::Temp->new( UNLINK => 0, SUFFIX => '.dat' ); - print $tmp "Some data\n"; - print "Filename is $tmp\n"; - $tmp->seek( 0, SEEK_END ); - -The following interfaces are provided for compatibility with -existing APIs. They should not be used in new code. - -MkTemp family: - - use File::Temp qw/ :mktemp /; - - ($fh, $file) = mkstemp( "tmpfileXXXXX" ); - ($fh, $file) = mkstemps( "tmpfileXXXXXX", $suffix); - - $tmpdir = mkdtemp( $template ); - - $unopened_file = mktemp( $template ); - -POSIX functions: - - use File::Temp qw/ :POSIX /; - - $file = tmpnam(); - $fh = tmpfile(); - - ($fh, $file) = tmpnam(); - -Compatibility functions: - - $unopened_file = File::Temp::tempnam( $dir, $pfx ); - -=head1 DESCRIPTION - -C can be used to create and open temporary files in a safe -way. There is both a function interface and an object-oriented -interface. The File::Temp constructor or the tempfile() function can -be used to return the name and the open filehandle of a temporary -file. The tempdir() function can be used to create a temporary -directory. - -The security aspect of temporary file creation is emphasized such that -a filehandle and filename are returned together. This helps guarantee -that a race condition can not occur where the temporary file is -created by another process between checking for the existence of the -file and its opening. Additional security levels are provided to -check, for example, that the sticky bit is set on world writable -directories. See L<"safe_level"> for more information. - -For compatibility with popular C library functions, Perl implementations of -the mkstemp() family of functions are provided. These are, mkstemp(), -mkstemps(), mkdtemp() and mktemp(). - -Additionally, implementations of the standard L -tmpnam() and tmpfile() functions are provided if required. - -Implementations of mktemp(), tmpnam(), and tempnam() are provided, -but should be used with caution since they return only a filename -that was valid when function was called, so cannot guarantee -that the file will not exist by the time the caller opens the filename. - -Filehandles returned by these functions support the seekable methods. - -=begin __INTERNALS - -=head1 PORTABILITY - -This section is at the top in order to provide easier access to -porters. It is not expected to be rendered by a standard pod -formatting tool. Please skip straight to the SYNOPSIS section if you -are not trying to port this module to a new platform. - -This module is designed to be portable across operating systems and it -currently supports Unix, VMS, DOS, OS/2, Windows and Mac OS -(Classic). When porting to a new OS there are generally three main -issues that have to be solved: -=over 4 - -=item * - -Can the OS unlink an open file? If it can not then the -C<_can_unlink_opened_file> method should be modified. - -=item * - -Are the return values from C reliable? By default all the -return values from C are compared when unlinking a temporary -file using the filename and the handle. Operating systems other than -unix do not always have valid entries in all fields. If utility function -C fails then the C comparison should be -modified accordingly. - -=item * - -Security. Systems that can not support a test for the sticky bit -on a directory can not use the MEDIUM and HIGH security tests. -The C<_can_do_level> method should be modified accordingly. - -=back - -=end __INTERNALS - -=head1 OBJECT-ORIENTED INTERFACE - -This is the primary interface for interacting with -C. Using the OO interface a temporary file can be created -when the object is constructed and the file can be removed when the -object is no longer required. - -Note that there is no method to obtain the filehandle from the -C object. The object itself acts as a filehandle. The object -isa C and isa C so all those methods are -available. - -Also, the object is configured such that it stringifies to the name of the -temporary file and so can be compared to a filename directly. It numifies -to the C the same as other handles and so can be compared to other -handles with C<==>. - - $fh eq $filename # as a string - $fh != \*STDOUT # as a number - -=over 4 - -=item B - -Create a temporary file object. - - my $tmp = File::Temp->new(); - -by default the object is constructed as if C -was called without options, but with the additional behaviour -that the temporary file is removed by the object destructor -if UNLINK is set to true (the default). - -Supported arguments are the same as for C: UNLINK -(defaulting to true), DIR, EXLOCK and SUFFIX. Additionally, the filename -template is specified using the TEMPLATE option. The OPEN option -is not supported (the file is always opened). - - $tmp = File::Temp->new( TEMPLATE => 'tempXXXXX', - DIR => 'mydir', - SUFFIX => '.dat'); - -Arguments are case insensitive. - -Can call croak() if an error occurs. - -=item B - -Create a temporary directory using an object oriented interface. - - $dir = File::Temp->newdir(); - -By default the directory is deleted when the object goes out of scope. - -Supports the same options as the C function. Note that directories -created with this method default to CLEANUP => 1. - - $dir = File::Temp->newdir( $template, %options ); - -A template may be specified either with a leading template or -with a TEMPLATE argument. - -=item B - -Return the name of the temporary file associated with this object -(if the object was created using the "new" constructor). - - $filename = $tmp->filename; - -This method is called automatically when the object is used as -a string. - -=item B - -Return the name of the temporary directory associated with this -object (if the object was created using the "newdir" constructor). - - $dirname = $tmpdir->dirname; - -This method is called automatically when the object is used in string context. - -=item B - -Control whether the file is unlinked when the object goes out of scope. -The file is removed if this value is true and $KEEP_ALL is not. - - $fh->unlink_on_destroy( 1 ); - -Default is for the file to be removed. - -=item B - -When the object goes out of scope, the destructor is called. This -destructor will attempt to unlink the file (using L) -if the constructor was called with UNLINK set to 1 (the default state -if UNLINK is not specified). - -No error is given if the unlink fails. - -If the object has been passed to a child process during a fork, the -file will be deleted when the object goes out of scope in the parent. - -For a temporary directory object the directory will be removed unless -the CLEANUP argument was used in the constructor (and set to false) or -C was modified after creation. Note that if a temp -directory is your current directory, it cannot be removed - a warning -will be given in this case. C out of the directory before -letting the object go out of scope. - -If the global variable $KEEP_ALL is true, the file or directory -will not be removed. - -=back - -=head1 FUNCTIONS - -This section describes the recommended interface for generating -temporary files and directories. - -=over 4 - -=item B - -This is the basic function to generate temporary files. -The behaviour of the file can be changed using various options: - - $fh = tempfile(); - ($fh, $filename) = tempfile(); - -Create a temporary file in the directory specified for temporary -files, as specified by the tmpdir() function in L. - - ($fh, $filename) = tempfile($template); - -Create a temporary file in the current directory using the supplied -template. Trailing `X' characters are replaced with random letters to -generate the filename. At least four `X' characters must be present -at the end of the template. - - ($fh, $filename) = tempfile($template, SUFFIX => $suffix) - -Same as previously, except that a suffix is added to the template -after the `X' translation. Useful for ensuring that a temporary -filename has a particular extension when needed by other applications. -But see the WARNING at the end. - - ($fh, $filename) = tempfile($template, DIR => $dir); - -Translates the template as before except that a directory name -is specified. - - ($fh, $filename) = tempfile($template, TMPDIR => 1); - -Equivalent to specifying a DIR of "File::Spec->tmpdir", writing the file -into the same temporary directory as would be used if no template was -specified at all. - - ($fh, $filename) = tempfile($template, UNLINK => 1); - -Return the filename and filehandle as before except that the file is -automatically removed when the program exits (dependent on -$KEEP_ALL). Default is for the file to be removed if a file handle is -requested and to be kept if the filename is requested. In a scalar -context (where no filename is returned) the file is always deleted -either (depending on the operating system) on exit or when it is -closed (unless $KEEP_ALL is true when the temp file is created). - -Use the object-oriented interface if fine-grained control of when -a file is removed is required. - -If the template is not specified, a template is always -automatically generated. This temporary file is placed in tmpdir() -(L) unless a directory is specified explicitly with the -DIR option. - - $fh = tempfile( DIR => $dir ); - -If called in scalar context, only the filehandle is returned and the -file will automatically be deleted when closed on operating systems -that support this (see the description of tmpfile() elsewhere in this -document). This is the preferred mode of operation, as if you only -have a filehandle, you can never create a race condition by fumbling -with the filename. On systems that can not unlink an open file or can -not mark a file as temporary when it is opened (for example, Windows -NT uses the C flag) the file is marked for deletion when -the program ends (equivalent to setting UNLINK to 1). The C -flag is ignored if present. - - (undef, $filename) = tempfile($template, OPEN => 0); - -This will return the filename based on the template but -will not open this file. Cannot be used in conjunction with -UNLINK set to true. Default is to always open the file -to protect from possible race conditions. A warning is issued -if warnings are turned on. Consider using the tmpnam() -and mktemp() functions described elsewhere in this document -if opening the file is not required. - -If the operating system supports it (for example BSD derived systems), the -filehandle will be opened with O_EXLOCK (open with exclusive file lock). -This can sometimes cause problems if the intention is to pass the filename -to another system that expects to take an exclusive lock itself (such as -DBD::SQLite) whilst ensuring that the tempfile is not reused. In this -situation the "EXLOCK" option can be passed to tempfile. By default EXLOCK -will be true (this retains compatibility with earlier releases). - - ($fh, $filename) = tempfile($template, EXLOCK => 0); - -Options can be combined as required. - -Will croak() if there is an error. - -=item B - -This is the recommended interface for creation of temporary -directories. By default the directory will not be removed on exit -(that is, it won't be temporary; this behaviour can not be changed -because of issues with backwards compatibility). To enable removal -either use the CLEANUP option which will trigger removal on program -exit, or consider using the "newdir" method in the object interface which -will allow the directory to be cleaned up when the object goes out of -scope. - -The behaviour of the function depends on the arguments: - - $tempdir = tempdir(); - -Create a directory in tmpdir() (see L). - - $tempdir = tempdir( $template ); - -Create a directory from the supplied template. This template is -similar to that described for tempfile(). `X' characters at the end -of the template are replaced with random letters to construct the -directory name. At least four `X' characters must be in the template. - - $tempdir = tempdir ( DIR => $dir ); - -Specifies the directory to use for the temporary directory. -The temporary directory name is derived from an internal template. - - $tempdir = tempdir ( $template, DIR => $dir ); - -Prepend the supplied directory name to the template. The template -should not include parent directory specifications itself. Any parent -directory specifications are removed from the template before -prepending the supplied directory. - - $tempdir = tempdir ( $template, TMPDIR => 1 ); - -Using the supplied template, create the temporary directory in -a standard location for temporary files. Equivalent to doing - - $tempdir = tempdir ( $template, DIR => File::Spec->tmpdir); - -but shorter. Parent directory specifications are stripped from the -template itself. The C option is ignored if C is set -explicitly. Additionally, C is implied if neither a template -nor a directory are supplied. - - $tempdir = tempdir( $template, CLEANUP => 1); - -Create a temporary directory using the supplied template, but -attempt to remove it (and all files inside it) when the program -exits. Note that an attempt will be made to remove all files from -the directory even if they were not created by this module (otherwise -why ask to clean it up?). The directory removal is made with -the rmtree() function from the L module. -Of course, if the template is not specified, the temporary directory -will be created in tmpdir() and will also be removed at program exit. - -Will croak() if there is an error. - -=back - -=head1 MKTEMP FUNCTIONS - -The following functions are Perl implementations of the -mktemp() family of temp file generation system calls. - -=over 4 - -=item B - -Given a template, returns a filehandle to the temporary file and the name -of the file. - - ($fh, $name) = mkstemp( $template ); - -In scalar context, just the filehandle is returned. - -The template may be any filename with some number of X's appended -to it, for example F. The trailing X's are replaced -with unique alphanumeric combinations. - -Will croak() if there is an error. - -=item B - -Similar to mkstemp(), except that an extra argument can be supplied -with a suffix to be appended to the template. - - ($fh, $name) = mkstemps( $template, $suffix ); - -For example a template of C and suffix of C<.dat> -would generate a file similar to F. - -Returns just the filehandle alone when called in scalar context. - -Will croak() if there is an error. - -=item B - -Create a directory from a template. The template must end in -X's that are replaced by the routine. - - $tmpdir_name = mkdtemp($template); - -Returns the name of the temporary directory created. - -Directory must be removed by the caller. - -Will croak() if there is an error. - -=item B - -Returns a valid temporary filename but does not guarantee -that the file will not be opened by someone else. - - $unopened_file = mktemp($template); - -Template is the same as that required by mkstemp(). - -Will croak() if there is an error. - -=back - -=head1 POSIX FUNCTIONS - -This section describes the re-implementation of the tmpnam() -and tmpfile() functions described in L -using the mkstemp() from this module. - -Unlike the L implementations, the directory used -for the temporary file is not specified in a system include -file (C) but simply depends on the choice of tmpdir() -returned by L. On some implementations this -location can be set using the C environment variable, which -may not be secure. -If this is a problem, simply use mkstemp() and specify a template. - -=over 4 - -=item B - -When called in scalar context, returns the full name (including path) -of a temporary file (uses mktemp()). The only check is that the file does -not already exist, but there is no guarantee that that condition will -continue to apply. - - $file = tmpnam(); - -When called in list context, a filehandle to the open file and -a filename are returned. This is achieved by calling mkstemp() -after constructing a suitable template. - - ($fh, $file) = tmpnam(); - -If possible, this form should be used to prevent possible -race conditions. - -See L for information on the choice of temporary -directory for a particular operating system. - -Will croak() if there is an error. - -=item B - -Returns the filehandle of a temporary file. - - $fh = tmpfile(); - -The file is removed when the filehandle is closed or when the program -exits. No access to the filename is provided. - -If the temporary file can not be created undef is returned. -Currently this command will probably not work when the temporary -directory is on an NFS file system. - -Will croak() if there is an error. - -=back - -=head1 ADDITIONAL FUNCTIONS - -These functions are provided for backwards compatibility -with common tempfile generation C library functions. - -They are not exported and must be addressed using the full package -name. - -=over 4 - -=item B - -Return the name of a temporary file in the specified directory -using a prefix. The file is guaranteed not to exist at the time -the function was called, but such guarantees are good for one -clock tick only. Always use the proper form of C -with C if you must open such a filename. - - $filename = File::Temp::tempnam( $dir, $prefix ); - -Equivalent to running mktemp() with $dir/$prefixXXXXXXXX -(using unix file convention as an example) - -Because this function uses mktemp(), it can suffer from race conditions. - -Will croak() if there is an error. - -=back - -=head1 UTILITY FUNCTIONS - -Useful functions for dealing with the filehandle and filename. - -=over 4 - -=item B - -Given an open filehandle and the associated filename, make a safe -unlink. This is achieved by first checking that the filename and -filehandle initially point to the same file and that the number of -links to the file is 1 (all fields returned by stat() are compared). -Then the filename is unlinked and the filehandle checked once again to -verify that the number of links on that file is now 0. This is the -closest you can come to making sure that the filename unlinked was the -same as the file whose descriptor you hold. - - unlink0($fh, $path) - or die "Error unlinking file $path safely"; - -Returns false on error but croaks() if there is a security -anomaly. The filehandle is not closed since on some occasions this is -not required. - -On some platforms, for example Windows NT, it is not possible to -unlink an open file (the file must be closed first). On those -platforms, the actual unlinking is deferred until the program ends and -good status is returned. A check is still performed to make sure that -the filehandle and filename are pointing to the same thing (but not at -the time the end block is executed since the deferred removal may not -have access to the filehandle). - -Additionally, on Windows NT not all the fields returned by stat() can -be compared. For example, the C and C fields seem to be -different. Also, it seems that the size of the file returned by stat() -does not always agree, with C being more accurate than -C, presumably because of caching issues even when -using autoflush (this is usually overcome by waiting a while after -writing to the tempfile before attempting to C it). - -Finally, on NFS file systems the link count of the file handle does -not always go to zero immediately after unlinking. Currently, this -command is expected to fail on NFS disks. - -This function is disabled if the global variable $KEEP_ALL is true -and an unlink on open file is supported. If the unlink is to be deferred -to the END block, the file is still registered for removal. - -This function should not be called if you are using the object oriented -interface since the it will interfere with the object destructor deleting -the file. - -=item B - -Compare C of filehandle with C of provided filename. This -can be used to check that the filename and filehandle initially point -to the same file and that the number of links to the file is 1 (all -fields returned by stat() are compared). - - cmpstat($fh, $path) - or die "Error comparing handle with file"; - -Returns false if the stat information differs or if the link count is -greater than 1. Calls croak if there is a security anomaly. - -On certain platforms, for example Windows, not all the fields returned by stat() -can be compared. For example, the C and C fields seem to be -different in Windows. Also, it seems that the size of the file -returned by stat() does not always agree, with C being more -accurate than C, presumably because of caching issues -even when using autoflush (this is usually overcome by waiting a while -after writing to the tempfile before attempting to C it). - -Not exported by default. - -=item B - -Similar to C except after file comparison using cmpstat, the -filehandle is closed prior to attempting to unlink the file. This -allows the file to be removed without using an END block, but does -mean that the post-unlink comparison of the filehandle state provided -by C is not available. - - unlink1($fh, $path) - or die "Error closing and unlinking file"; - -Usually called from the object destructor when using the OO interface. - -Not exported by default. - -This function is disabled if the global variable $KEEP_ALL is true. - -Can call croak() if there is a security anomaly during the stat() -comparison. - -=item B - -Calling this function will cause any temp files or temp directories -that are registered for removal to be removed. This happens automatically -when the process exits but can be triggered manually if the caller is sure -that none of the temp files are required. This method can be registered as -an Apache callback. - -Note that if a temp directory is your current directory, it cannot be -removed. C out of the directory first before calling -C. (For the cleanup at program exit when the CLEANUP flag -is set, this happens automatically.) - -On OSes where temp files are automatically removed when the temp file -is closed, calling this function will have no effect other than to remove -temporary directories (which may include temporary files). - - File::Temp::cleanup(); - -Not exported by default. - -=back - -=head1 PACKAGE VARIABLES - -These functions control the global state of the package. - -=over 4 - -=item B - -Controls the lengths to which the module will go to check the safety of the -temporary file or directory before proceeding. -Options are: - -=over 8 - -=item STANDARD - -Do the basic security measures to ensure the directory exists and is -writable, that temporary files are opened only if they do not already -exist, and that possible race conditions are avoided. Finally the -L function is used to remove files safely. - -=item MEDIUM - -In addition to the STANDARD security, the output directory is checked -to make sure that it is owned either by root or the user running the -program. If the directory is writable by group or by other, it is then -checked to make sure that the sticky bit is set. - -Will not work on platforms that do not support the C<-k> test -for sticky bit. - -=item HIGH - -In addition to the MEDIUM security checks, also check for the -possibility of ``chown() giveaway'' using the L -sysconf() function. If this is a possibility, each directory in the -path is checked in turn for safeness, recursively walking back to the -root directory. - -For platforms that do not support the L -C<_PC_CHOWN_RESTRICTED> symbol (for example, Windows NT) it is -assumed that ``chown() giveaway'' is possible and the recursive test -is performed. - -=back - -The level can be changed as follows: - - File::Temp->safe_level( File::Temp::HIGH ); - -The level constants are not exported by the module. - -Currently, you must be running at least perl v5.6.0 in order to -run with MEDIUM or HIGH security. This is simply because the -safety tests use functions from L that are not -available in older versions of perl. The problem is that the version -number for Fcntl is the same in perl 5.6.0 and in 5.005_03 even though -they are different versions. - -On systems that do not support the HIGH or MEDIUM safety levels -(for example Win NT or OS/2) any attempt to change the level will -be ignored. The decision to ignore rather than raise an exception -allows portable programs to be written with high security in mind -for the systems that can support this without those programs failing -on systems where the extra tests are irrelevant. - -If you really need to see whether the change has been accepted -simply examine the return value of C. - - $newlevel = File::Temp->safe_level( File::Temp::HIGH ); - die "Could not change to high security" - if $newlevel != File::Temp::HIGH; - -=item TopSystemUID - -This is the highest UID on the current system that refers to a root -UID. This is used to make sure that the temporary directory is -owned by a system UID (C, C, C etc) rather than -simply by root. - -This is required since on many unix systems C is not owned -by root. - -Default is to assume that any UID less than or equal to 10 is a root -UID. - - File::Temp->top_system_uid(10); - my $topid = File::Temp->top_system_uid; - -This value can be adjusted to reduce security checking if required. -The value is only relevant when C is set to MEDIUM or higher. - -=item B<$KEEP_ALL> - -Controls whether temporary files and directories should be retained -regardless of any instructions in the program to remove them -automatically. This is useful for debugging but should not be used in -production code. - - $File::Temp::KEEP_ALL = 1; - -Default is for files to be removed as requested by the caller. - -In some cases, files will only be retained if this variable is true -when the file is created. This means that you can not create a temporary -file, set this variable and expect the temp file to still be around -when the program exits. - -=item B<$DEBUG> - -Controls whether debugging messages should be enabled. - - $File::Temp::DEBUG = 1; - -Default is for debugging mode to be disabled. - -=back - -=head1 WARNING - -For maximum security, endeavour always to avoid ever looking at, -touching, or even imputing the existence of the filename. You do not -know that that filename is connected to the same file as the handle -you have, and attempts to check this can only trigger more race -conditions. It's far more secure to use the filehandle alone and -dispense with the filename altogether. - -If you need to pass the handle to something that expects a filename -then on a unix system you can use C<"/dev/fd/" . fileno($fh)> for -arbitrary programs. Perl code that uses the 2-argument version of -C<< open >> can be passed C<< "+<=&" . fileno($fh) >>. Otherwise you -will need to pass the filename. You will have to clear the -close-on-exec bit on that file descriptor before passing it to another -process. - - use Fcntl qw/F_SETFD F_GETFD/; - fcntl($tmpfh, F_SETFD, 0) - or die "Can't clear close-on-exec flag on temp fh: $!\n"; - -=head2 Temporary files and NFS - -Some problems are associated with using temporary files that reside -on NFS file systems and it is recommended that a local filesystem -is used whenever possible. Some of the security tests will most probably -fail when the temp file is not local. Additionally, be aware that -the performance of I/O operations over NFS will not be as good as for -a local disk. - -=head2 Forking - -In some cases files created by File::Temp are removed from within an -END block. Since END blocks are triggered when a child process exits -(unless C is used by the child) File::Temp takes care -to only remove those temp files created by a particular process ID. This -means that a child will not attempt to remove temp files created by the -parent process. - -If you are forking many processes in parallel that are all creating -temporary files, you may need to reset the random number seed using -srand(EXPR) in each child else all the children will attempt to walk -through the same set of random file names and may well cause -themselves to give up if they exceed the number of retry attempts. - -=head2 Directory removal - -Note that if you have chdir'ed into the temporary directory and it is -subsequently cleaned up (either in the END block or as part of object -destruction), then you will get a warning from File::Path::rmtree(). - -=head2 Taint mode - -If you need to run code under taint mode, updating to the latest -L is highly recommended. - -=head2 BINMODE - -The file returned by File::Temp will have been opened in binary mode -if such a mode is available. If that is not correct, use the C -function to change the mode of the filehandle. - -Note that you can modify the encoding of a file opened by File::Temp -also by using C. - -=head1 HISTORY - -Originally began life in May 1999 as an XS interface to the system -mkstemp() function. In March 2000, the OpenBSD mkstemp() code was -translated to Perl for total control of the code's -security checking, to ensure the presence of the function regardless of -operating system and to help with portability. The module was shipped -as a standard part of perl from v5.6.1. - -Thanks to Tom Christiansen for suggesting that this module -should be written and providing ideas for code improvements and -security enhancements. - -=head1 SEE ALSO - -L, L, L, L - -See L and L, L for -different implementations of temporary file handling. - -See L for an alternative object-oriented wrapper for -the C function. - -=for Pod::Coverage STRINGIFY NUMIFY top_system_uid - -# vim: ts=2 sts=2 sw=2 et: - -=for :stopwords cpan testmatrix url annocpan anno bugtracker rt cpants kwalitee diff irc mailto metadata placeholders metacpan - -=head1 SUPPORT - -=head2 Bugs / Feature Requests - -Please report any bugs or feature requests through the issue tracker -at L. -You will be notified automatically of any progress on your issue. - -=head2 Source Code - -This is open source software. The code repository is available for -public review and contribution under the terms of the license. - -L - - git clone https://github.com/Perl-Toolchain-Gang/File-Temp.git - -=head1 AUTHOR - -Tim Jenness - -=head1 CONTRIBUTORS - -=over 4 - -=item * - -Ben Tilly - -=item * - -David Golden - -=item * - -David Steinbrunner - -=item * - -Ed Avis - -=item * - -James E. Keenan - -=item * - -Karen Etheridge - -=item * - -Kevin Ryde - -=item * - -Olivier Mengue - -=item * - -Peter John Acklam - -=item * - -Peter Rabbitson - -=back - -=head1 COPYRIGHT AND LICENSE - -This software is copyright (c) 2013 by Tim Jenness and the UK Particle Physics and Astronomy Research Council. - -This is free software; you can redistribute it and/or modify it under -the same terms as the Perl 5 programming language system itself. - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/parent.pm b/dev-tools/src/main/resources/license-check/lib/parent.pm deleted file mode 100644 index f6e8cd497db..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/parent.pm +++ /dev/null @@ -1,119 +0,0 @@ -package parent; -use strict; -use vars qw($VERSION); -$VERSION = '0.234'; - -sub import { - my $class = shift; - - my $inheritor = caller(0); - - if ( @_ and $_[0] eq '-norequire' ) { - shift @_; - } else { - for ( my @filename = @_ ) { - s{::|'}{/}g; - require "$_.pm"; # dies if the file is not found - } - } - - { - no strict 'refs'; - push @{"$inheritor\::ISA"}, @_; - }; -}; - -"All your base are belong to us" - -__END__ - -=encoding utf8 - -=head1 NAME - -parent - Establish an ISA relationship with base classes at compile time - -=head1 SYNOPSIS - - package Baz; - use parent qw(Foo Bar); - -=head1 DESCRIPTION - -Allows you to both load one or more modules, while setting up inheritance from -those modules at the same time. Mostly similar in effect to - - package Baz; - BEGIN { - require Foo; - require Bar; - push @ISA, qw(Foo Bar); - } - -By default, every base class needs to live in a file of its own. -If you want to have a subclass and its parent class in the same file, you -can tell C not to load any modules by using the C<-norequire> switch: - - package Foo; - sub exclaim { "I CAN HAS PERL" } - - package DoesNotLoadFooBar; - use parent -norequire, 'Foo', 'Bar'; - # will not go looking for Foo.pm or Bar.pm - -This is equivalent to the following code: - - package Foo; - sub exclaim { "I CAN HAS PERL" } - - package DoesNotLoadFooBar; - push @DoesNotLoadFooBar::ISA, 'Foo', 'Bar'; - -This is also helpful for the case where a package lives within -a differently named file: - - package MyHash; - use Tie::Hash; - use parent -norequire, 'Tie::StdHash'; - -This is equivalent to the following code: - - package MyHash; - require Tie::Hash; - push @ISA, 'Tie::StdHash'; - -If you want to load a subclass from a file that C would -not consider an eligible filename (that is, it does not end in -either C<.pm> or C<.pmc>), use the following code: - - package MySecondPlugin; - require './plugins/custom.plugin'; # contains Plugin::Custom - use parent -norequire, 'Plugin::Custom'; - -=head1 HISTORY - -This module was forked from L to remove the cruft -that had accumulated in it. - -=head1 CAVEATS - -=head1 SEE ALSO - -L - -=head1 AUTHORS AND CONTRIBUTORS - -Rafaël Garcia-Suarez, Bart Lateur, Max Maischein, Anno Siegel, Michael Schwern - -=head1 MAINTAINER - -Max Maischein C< corion@cpan.org > - -Copyright (c) 2007-10 Max Maischein C<< >> -Based on the idea of C, which was introduced with Perl 5.004_04. - -=head1 LICENSE - -This module is released under the same terms as Perl itself. - -=cut diff --git a/dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml b/dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml deleted file mode 100644 index 88471f022cd..00000000000 --- a/dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - plugin - - zip - - false - - - ${project.basedir}/src/main/plugin-metadata - - plugin-security.policy - - - false - - - - - ${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties - - true - - - - - / - true - true - true - - - diff --git a/dev-tools/src/main/resources/site/site_en.xml b/dev-tools/src/main/resources/site/site_en.xml deleted file mode 100644 index f3011083e06..00000000000 --- a/dev-tools/src/main/resources/site/site_en.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - diff --git a/dev-tools/update_lucene.sh b/dev-tools/update_lucene.sh deleted file mode 100644 index a3f8b087de8..00000000000 --- a/dev-tools/update_lucene.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -mvn install -DskipTests -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update distribution/licenses/ distribution/zip/target/releases/elasticsearch-3.0.0-SNAPSHOT.zip elasticsearch-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-icu/licenses/ plugins/analysis-icu/target/releases/analysis-icu-3.0.0-SNAPSHOT.zip analysis-icu-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-kuromoji/licenses/ plugins/analysis-kuromoji/target/releases/analysis-kuromoji-3.0.0-SNAPSHOT.zip analysis-kuromoji-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-phonetic/licenses/ plugins/analysis-phonetic/target/releases/analysis-phonetic-3.0.0-SNAPSHOT.zip analysis-phonetic-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-smartcn/licenses/ plugins/analysis-smartcn/target/releases/analysis-smartcn-3.0.0-SNAPSHOT.zip analysis-smartcn-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-stempel/licenses/ plugins/analysis-stempel/target/releases/analysis-stempel-3.0.0-SNAPSHOT.zip analysis-stempel-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/lang-expression/licenses/ plugins/lang-expression/target/releases/lang-expression-3.0.0-SNAPSHOT.zip lang-expression-3.0.0-SNAPSHOT diff --git a/distribution/build.gradle b/distribution/build.gradle new file mode 100644 index 00000000000..4da164131f3 --- /dev/null +++ b/distribution/build.gradle @@ -0,0 +1,450 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.tools.ant.filters.FixCrLfFilter +import org.apache.tools.ant.taskdefs.condition.Os +import org.elasticsearch.gradle.precommit.DependencyLicensesTask +import org.elasticsearch.gradle.precommit.UpdateShasTask +import org.elasticsearch.gradle.test.RunTask +import org.elasticsearch.gradle.EmptyDirTask +import org.elasticsearch.gradle.MavenFilteringHack +import org.gradle.api.InvalidUserDataException +import org.gradle.internal.nativeintegration.filesystem.Chmod + +// for deb/rpm +buildscript { + repositories { + maven { + url "https://plugins.gradle.org/m2/" + } + } + dependencies { + classpath 'com.netflix.nebula:gradle-ospackage-plugin:3.1.0' + } +} + +// this is common configuration for distributions, but we also add it here for the license check to use +ext.dependencyFiles = project(':core').configurations.runtime.copyRecursive().exclude(module: 'slf4j-api') + + +/***************************************************************************** + * Modules * + *****************************************************************************/ + +task buildModules(type: Sync) { + into 'build/modules' +} + +ext.restTestExpansions = [ + 'expected.modules.count': 0, +] +// we create the buildModules task above so the distribution subprojects can +// depend on it, but we don't actually configure it until here so we can do a single +// loop over modules to also setup cross task dependencies and increment our modules counter +project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each { Project module -> + buildModules { + dependsOn({ project(module.path).bundlePlugin }) + into(module.name) { + from { zipTree(project(module.path).bundlePlugin.outputs.files.singleFile) } + } + } + // We would like to make sure integ tests for the distribution run after + // integ tests for the modules included in the distribution. However, gradle + // has a bug where depending on a task with a finalizer can sometimes not make + // the finalizer task follow the original task immediately. To work around this, + // we make the mustRunAfter the finalizer task itself. + // See https://discuss.gradle.org/t/cross-project-task-dependencies-ordering-screws-up-finalizers/13190 + project.configure(project.subprojects.findAll { it.name != 'integ-test-zip' }) { Project distribution -> + distribution.afterEvaluate({ + distribution.integTest.mustRunAfter("${module.path}:integTest#stop") + }) + } + // also want to make sure the module's integration tests run after the integ-test-zip (ie rest tests) + module.afterEvaluate({ + module.integTest.mustRunAfter(':distribution:integ-test-zip:integTest#stop') + }) + restTestExpansions['expected.modules.count'] += 1 +} + +// make sure we have a clean task since we aren't a java project, but we have tasks that +// put stuff in the build dir +task clean(type: Delete) { + delete 'build' +} + +subprojects { + /***************************************************************************** + * Rest test config * + *****************************************************************************/ + apply plugin: 'elasticsearch.rest-test' + project.integTest { + dependsOn project.assemble + includePackaged project.name == 'integ-test-zip' + cluster { + distribution = project.name + } + if (project.name != 'integ-test-zip') { + // see note above with module mustRunAfter about why integTest#stop is used here + mustRunAfter ':distribution:integ-test-zip:integTest#stop' + } + } + + processTestResources { + inputs.properties(project(':distribution').restTestExpansions) + MavenFilteringHack.filter(it, project(':distribution').restTestExpansions) + } + + /***************************************************************************** + * Maven config * + *****************************************************************************/ + // note: the group must be correct before applying the nexus plugin, or it will capture the wrong value... + project.group = "org.elasticsearch.distribution.${project.name}" + apply plugin: 'com.bmuschko.nexus' + // we must create our own install task, because it is only added when the java plugin is added + task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') { + configuration = configurations.archives + MavenRepositoryHandlerConvention repositoriesHandler = (MavenRepositoryHandlerConvention)getRepositories().getConvention().getPlugin(MavenRepositoryHandlerConvention) + repositoriesHandler.mavenInstaller() + } + + // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run... + /***************************************************************************** + * Properties to expand when copying packaging files * + *****************************************************************************/ + project.ext { + expansions = expansionsForDistribution(project.name) + + /***************************************************************************** + * Common files in all distributions * + *****************************************************************************/ + libFiles = copySpec { + into 'lib' + from project(':core').jar + from project(':distribution').dependencyFiles + } + + modulesFiles = copySpec { + into 'modules' + from project(':distribution').buildModules + } + + configFiles = copySpec { + from '../src/main/resources/config' + } + + commonFiles = copySpec { + // everything except windows files, and config is separate + from '../src/main/resources' + exclude 'bin/*.bat' + exclude 'bin/*.exe' + exclude 'config/**' + filesMatching('bin/*') { it.setMode(0755) } + MavenFilteringHack.filter(it, expansions) + } + } +} + +/***************************************************************************** + * Zip and tgz configuration * + *****************************************************************************/ +configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.name) }) { + project.ext.archivesFiles = copySpec { + into("elasticsearch-${version}") { + with libFiles + into('config') { + with configFiles + } + with copySpec { + with commonFiles + from('../src/main/resources') { + include 'bin/*.bat' + filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) + } + MavenFilteringHack.filter(it, expansions) + } + from('../src/main/resources') { + include 'bin/*.exe' + } + if (project.name != 'integ-test-zip') { + with modulesFiles + } + } + } +} + +/***************************************************************************** + * Deb and rpm configuration * + ***************************************************************************** + * + * The general strategy here is to build a directory on disk, packagingFiles + * that contains stuff that needs to be copied into the distributions. This is + * important for two reasons: + * 1. ospackage wants to copy the directory permissions that it sees off of the + * filesystem. If you ask it to create a directory that doesn't already + * exist on disk it petulantly creates it with 0755 permissions, no matter + * how hard you try to convince it otherwise. + * 2. Convincing ospackage to pick up an empty directory as part of a set of + * directories on disk is reasonably easy. Convincing it to just create an + * empty directory requires more wits than I have. + * 3. ospackage really wants to suck up some of the debian control scripts + * directly from the filesystem. It doesn't want to process them through + * MavenFilteringHack or any other copy-style action. + */ +configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) { + integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false + File packagingFiles = new File(buildDir, 'packaging') + project.ext.packagingFiles = packagingFiles + task processPackagingFiles(type: Copy) { + from '../src/main/packaging' + from 'src/main/packaging' + + MavenFilteringHack.filter(it, expansions) + into packagingFiles + /* Explicitly declare the outputs so that gradle won't skip this task if + one of the other tasks like createEtc run first and create the packaging + directory as a side effect. */ + outputs.dir("${packagingFiles}/scripts") + outputs.dir("${packagingFiles}/env") + outputs.dir("${packagingFiles}/systemd") + } + + task createEtc(type: EmptyDirTask) { + dir "${packagingFiles}/etc/elasticsearch" + dirMode 0750 + } + + task createEtcScripts(type: EmptyDirTask) { + dependsOn createEtc + dir "${packagingFiles}/etc/elasticsearch/scripts" + dirMode 0750 + } + + task fillEtc(type: Copy) { + dependsOn createEtc, createEtcScripts + with configFiles + into "${packagingFiles}/etc/elasticsearch" + /* Explicitly declare the output files so this task doesn't consider itself + up to date when the directory is created, which it would by default. And + that'll happen when createEtc runs. */ + outputs.file "${packagingFiles}/etc/elasticsearch/elasticsearch.yml" + outputs.file "${packagingFiles}/etc/elasticsearch/logging.yml" + } + + task createPidDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/run/elasticsearch" + } + task createLogDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/log/elasticsearch" + } + task createDataDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/lib/elasticsearch" + } + task createPluginsDir(type: EmptyDirTask) { + dir "${packagingFiles}/usr/share/elasticsearch/plugins" + } + + /** + * Setup the build/packaging directory to be like the target filesystem + * because ospackage will use arbitrary permissions if you try to create a + * directory that doesn't exist on the filesystem. + */ + task preparePackagingFiles { + dependsOn processPackagingFiles, fillEtc, createPidDir, createLogDir, + createDataDir, createPluginsDir + } + + apply plugin: 'nebula.ospackage-base' + ospackage { + packageName 'elasticsearch' + maintainer 'Elasticsearch Team ' + summary ''' + Elasticsearch is a distributed RESTful search engine built for the cloud. + Reference documentation can be found at + https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html + and the 'Elasticsearch: The Definitive Guide' book can be found at + https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html + '''.stripIndent().replace('\n', ' ').trim() + url 'https://www.elastic.co/' + + /* The version of the package can't contain -SNAPSHOT so we rip it off if + we see it. We'll add it back on to the file name though. */ + version project.version.replace('-SNAPSHOT', '') + + String scripts = "${packagingFiles}/scripts" + preInstall file("${scripts}/preinst") + postInstall file("${scripts}/postinst") + preUninstall file("${scripts}/prerm") + postUninstall file("${scripts}/postrm") + + into '/usr/share/elasticsearch' + user 'root' + permissionGroup 'root' + with libFiles + with modulesFiles + with copySpec { + with commonFiles + if (project.name == 'deb') { + // Deb gets a copyright file instead. + exclude 'LICENSE.txt' + } + } + + configurationFile '/etc/elasticsearch/elasticsearch.yml' + configurationFile '/etc/elasticsearch/logging.yml' + into('/etc') { + from "${packagingFiles}/etc" + fileMode 0750 + permissionGroup 'elasticsearch' + includeEmptyDirs true + createDirectoryEntry true + } + + into('/usr/lib/tmpfiles.d') { + from "${packagingFiles}/systemd/elasticsearch.conf" + } + configurationFile '/usr/lib/systemd/system/elasticsearch.service' + into('/usr/lib/systemd/system') { + from "${packagingFiles}/systemd/elasticsearch.service" + } + into('/usr/lib/sysctl.d') { + from "${packagingFiles}/systemd/sysctl/elasticsearch.conf" + } + configurationFile '/etc/init.d/elasticsearch' + into('/etc/init.d') { + from "${packagingFiles}/init.d/elasticsearch" + fileMode 0755 + } + configurationFile project.expansions['path.env'] + into(new File(project.expansions['path.env']).getParent()) { + from "${project.packagingFiles}/env/elasticsearch" + } + + /** + * Suck up all the empty directories that we need to install into the path. + */ + Closure suckUpEmptyDirectories = { path -> + into(path) { + from "${packagingFiles}/${path}" + includeEmptyDirs true + createDirectoryEntry true + /* All of these empty directories have this ownership. We're just + lucky! */ + user 'elasticsearch' + permissionGroup 'elasticsearch' + } + } + suckUpEmptyDirectories('/var/run') + suckUpEmptyDirectories('/var/log') + suckUpEmptyDirectories('/var/lib') + suckUpEmptyDirectories('/usr/share/elasticsearch') + } +} + +// TODO: dependency checks should really be when building the jar itself, which would remove the need +// for this hackery and instead we can do this inside the BuildPlugin +task dependencyLicenses(type: DependencyLicensesTask) { + dependsOn = [dependencyFiles] + dependencies = dependencyFiles + mapping from: /lucene-.*/, to: 'lucene' + mapping from: /jackson-.*/, to: 'jackson' +} +task check(group: 'Verification', description: 'Runs all checks.', dependsOn: dependencyLicenses) {} // dummy task! +task updateShas(type: UpdateShasTask) { + parentTask = dependencyLicenses +} + +task run(type: RunTask) {} + +/** + * Build some variables that are replaced in the packages. This includes both + * scripts like bin/elasticsearch and bin/plugin that a user might run and also + * scripts like postinst which are run as part of the installation. + * + *
    + *
    package.name
    + *
    The name of the project. Its sprinkled throughout the scripts.
    + *
    package.version
    + *
    The version of the project. Its mostly used to find the exact jar name. + * + *
    path.conf
    + *
    The default directory from which to load configuration. This is used in + * the packaging scripts, but in that context it is always + * /etc/elasticsearch. Its also used in bin/plugin, where it is + * /etc/elasticsearch for the os packages but $ESHOME/config otherwise.
    + *
    path.env
    + *
    The env file sourced before bin/elasticsearch to set environment + * variables. Think /etc/defaults/elasticsearch.
    + *
    heap.min and heap.max
    + *
    Default min and max heap
    + *
    scripts.footer
    + *
    Footer appended to control scripts embedded in the distribution that is + * (almost) entirely there for cosmetic reasons.
    + *
    stopping.timeout
    + *
    RPM's init script needs to wait for elasticsearch to stop before + * returning from stop and it needs a maximum time to wait. This is it. One + * day. DEB retries forever.
    + *
    + */ +Map expansionsForDistribution(distributionType) { + String footer = "# Built for ${project.name}-${project.version} " + + "(${distributionType})" + Map expansions = [ + 'project.name': project.name, + 'project.version': version, + + 'path.conf': [ + 'tar': '$ES_HOME/config', + 'zip': '$ES_HOME/config', + 'def': '/etc/elasticsearch', + ], + 'path.env': [ + 'deb': '/etc/default/elasticsearch', + 'rpm': '/etc/sysconfig/elasticsearch', + /* There isn't one of these files for tar or zip but its important to + make an empty string here so the script can properly skip it. */ + 'def': '', + ], + + 'heap.min': '256m', + 'heap.max': '1g', + + 'stopping.timeout': [ + 'rpm': 86400, + ], + + 'scripts.footer': [ + /* Debian needs exit 0 on these scripts so we add it here and preserve + the pretty footer. */ + 'deb': "exit 0\n${footer}", + 'def': footer + ], + ] + Map result = [:] + expansions = expansions.each { key, value -> + if (value instanceof Map) { + // 'def' is for default but its three characters like 'rpm' and 'deb' + value = value[distributionType] ?: value['def'] + if (value == null) { + return + } + } + result[key] = value + } + return result +} diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle new file mode 100644 index 00000000000..d9bd8447ab9 --- /dev/null +++ b/distribution/deb/build.gradle @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +task buildDeb(type: Deb) { + dependsOn preparePackagingFiles + baseName 'elasticsearch' // this is what pom generation uses for artifactId + // Follow elasticsearch's deb file naming convention + archiveName "${packageName}-${project.version}.deb" + packageGroup 'web' + requires 'libc6' + requires 'adduser' + + into('/usr/share/lintian/overrides') { + from("${project.packagingFiles}/lintian/elasticsearch") + } + into('/usr/share/doc/elasticsearch') { + from "${project.packagingFiles}/copyright" + } +} + +artifacts { + 'default' buildDeb + archives buildDeb +} + +integTest { + /* We use real deb tools to extract the deb file for testing so we have to + skip the test if they aren't around. */ + enabled = new File('/usr/bin/dpkg-deb').exists() || // Standard location + new File('/usr/local/bin/dpkg-deb').exists() // Homebrew location +} diff --git a/distribution/deb/pom.xml b/distribution/deb/pom.xml deleted file mode 100644 index c43e32be4f7..00000000000 --- a/distribution/deb/pom.xml +++ /dev/null @@ -1,332 +0,0 @@ - - - 4.0.0 - - org.elasticsearch.distribution - distributions - 3.0.0-SNAPSHOT - - - org.elasticsearch.distribution.deb - elasticsearch - Distribution: Deb - - - The Debian distribution of Elasticsearch - - - false - dpkg-sig - - - - - - ${project.basedir}/src/main/packaging/packaging.properties - - - - - org.apache.maven.plugins - maven-resources-plugin - - - - copy-resources-deb - prepare-package - - copy-resources - - - ${project.build.directory}/generated-packaging/deb/ - - ${project.basedir}/../src/main/packaging/packaging.properties - ${project.basedir}/src/main/packaging/packaging.properties - - - - ${project.basedir}/../src/main/packaging/ - true - - **/* - - - packaging.properties - - - - ${project.basedir}/src/main/packaging/ - true - - **/* - - - packaging.properties - - - - ${project.basedir}/../src/main/resources - true - - bin/elasticsearch - bin/elasticsearch.in.sh - bin/plugin - bin/elasticsearch-systemd-pre-exec - - - - - - - - - - - jdeb - org.vafer - - - ${project.build.directory}/releases/elasticsearch-${project.version}.deb - ${project.build.directory}/generated-packaging/deb/scripts - - - - package - - jdeb - - - ${deb.sign} - ${gpg.keyring} - ${gpg.key} - ${gpg.passphrase} - ${deb.sign.method} - - - - ${project.build.directory}/generated-packaging/deb/bin - directory - elasticsearch,elasticsearch.in.sh,plugin,elasticsearch-systemd-pre-exec - - perm - ${packaging.elasticsearch.bin.dir} - 755 - root - root - - - - - template - - ${packaging.elasticsearch.conf.dir} - - - perm - 750 - root - elasticsearch - - - - - ${project.basedir}/../src/main/resources/config - directory - .DS_Store - - perm - ${packaging.elasticsearch.conf.dir} - 750 - root - elasticsearch - - - - template - - ${packaging.elasticsearch.conf.dir}/scripts - - - perm - 750 - root - elasticsearch - - - - - ${project.build.directory}/generated-packaging/deb/env/elasticsearch - file - - perm - /etc/default - 644 - root - root - - - - - ${project.build.directory}/ - elasticsearch-${project.version}.jar - directory - - perm - ${packaging.elasticsearch.home.dir}/lib - root - root - - - - ${project.build.directory}/../target/lib - ${project.build.finalName}-sources.jar,${project.build.finalName}-tests.jar,${project.build.finalName}-test-sources.jar,slf4j-api-*.jar - directory - - perm - ${packaging.elasticsearch.home.dir}/lib - root - root - - - - - ${project.build.directory}/generated-packaging/deb/init.d/ - directory - .DS_Store - - perm - /etc/init.d - 755 - root - root - - - - - ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.service - ${packaging.elasticsearch.systemd.dir}/elasticsearch.service - file - - - - ${project.build.directory}/generated-packaging/deb/systemd/sysctl/elasticsearch.conf - ${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf - file - - - - ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.conf - ${packaging.elasticsearch.tmpfilesd.dir}/elasticsearch.conf - file - - - - ${project.build.directory}/generated-packaging/deb/lintian - directory - .DS_Store - - perm - /usr/share/lintian/overrides - root - root - - - - - ${project.basedir}/../src/main/resources/ - *.txt, *.textile - LICENSE.txt, .DS_Store - directory - - perm - ${packaging.elasticsearch.home.dir} - root - root - - - - - ${project.build.directory}/generated-packaging/deb/copyright - /usr/share/doc/elasticsearch/copyright - file - - - - template - - ${packaging.elasticsearch.data.dir} - ${packaging.elasticsearch.log.dir} - ${packaging.elasticsearch.plugins.dir} - ${packaging.elasticsearch.pid.dir} - - - perm - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - integ-setup - pre-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - integ-teardown - post-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - - - - - has_dpkg - - /usr/bin/dpkg-deb - - - true - - - - diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch index 3a82bbe7f76..9f1f1479a51 100755 --- a/distribution/deb/src/main/packaging/init.d/elasticsearch +++ b/distribution/deb/src/main/packaging/init.d/elasticsearch @@ -81,7 +81,7 @@ MAX_MAP_COUNT=262144 #ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log # Elasticsearch PID file directory -PID_DIR="${packaging.elasticsearch.pid.dir}" +PID_DIR="/var/run/elasticsearch" # End of variables that can be overwritten in $DEFAULT diff --git a/distribution/deb/src/main/packaging/packaging.properties b/distribution/deb/src/main/packaging/packaging.properties deleted file mode 100644 index 3635928c2ee..00000000000 --- a/distribution/deb/src/main/packaging/packaging.properties +++ /dev/null @@ -1,15 +0,0 @@ -# Properties used to build to the DEB package -# - -# Environment file -packaging.env.file=/etc/default/elasticsearch - -# Default configuration directory and file to use in bin/plugin script -packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir} - -# Simple marker to check that properties are correctly overridden -packaging.type=deb - -# Custom header for package scripts -packaging.scripts.header=#!/bin/sh${line.separator}set -e -packaging.scripts.footer=exit 0${line.separator}# Built for ${project.name}-${project.version} (${packaging.type}) diff --git a/distribution/deb/src/main/packaging/scripts/conffiles b/distribution/deb/src/main/packaging/scripts/conffiles deleted file mode 100644 index 9f658416784..00000000000 --- a/distribution/deb/src/main/packaging/scripts/conffiles +++ /dev/null @@ -1,5 +0,0 @@ -${packaging.env.file} -${packaging.elasticsearch.conf.dir}/elasticsearch.yml -${packaging.elasticsearch.conf.dir}/logging.yml -/etc/init.d/elasticsearch -/usr/lib/systemd/system/elasticsearch.service diff --git a/distribution/deb/src/main/packaging/scripts/control b/distribution/deb/src/main/packaging/scripts/control deleted file mode 100644 index 1913de78738..00000000000 --- a/distribution/deb/src/main/packaging/scripts/control +++ /dev/null @@ -1,9 +0,0 @@ -Package: elasticsearch -Version: [[version]] -Architecture: all -Maintainer: Elasticsearch Team -Depends: libc6, adduser -Section: web -Priority: optional -Homepage: https://www.elastic.co/ -Description: Elasticsearch is a distributed RESTful search engine built for the cloud. Reference documentation can be found at https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html and the 'Elasticsearch: The Definitive Guide' book can be found at https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html diff --git a/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..da68232f8d8 --- /dev/null +++ b/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.modules: ${expected.modules.count} } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalyzerProviderFactory.java b/distribution/integ-test-zip/build.gradle similarity index 79% rename from core/src/main/java/org/elasticsearch/index/analysis/AnalyzerProviderFactory.java rename to distribution/integ-test-zip/build.gradle index a4a3e7ffce5..23191ff03a4 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalyzerProviderFactory.java +++ b/distribution/integ-test-zip/build.gradle @@ -17,14 +17,15 @@ * under the License. */ -package org.elasticsearch.index.analysis; - -import org.elasticsearch.common.settings.Settings; - -/** - * - */ -public interface AnalyzerProviderFactory { - - AnalyzerProvider create(String name, Settings settings); +task buildZip(type: Zip) { + baseName = 'elasticsearch' + with archivesFiles } + +artifacts { + 'default' buildZip + archives buildZip +} + +integTest.dependsOn buildZip + diff --git a/core/src/test/java/org/elasticsearch/test/rest/Rest0IT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java similarity index 81% rename from core/src/test/java/org/elasticsearch/test/rest/Rest0IT.java rename to distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java index e73bf347093..fd12fd2e519 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/Rest0IT.java +++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java @@ -19,20 +19,20 @@ package org.elasticsearch.test.rest; -import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -/** Rest API tests subset 0 */ -public class Rest0IT extends ESRestTestCase { - public Rest0IT(@Name("yaml") RestTestCandidate testCandidate) { +/** Rest integration test. runs against external cluster in 'mvn verify' */ +public class RestIT extends ESRestTestCase { + public RestIT(RestTestCandidate testCandidate) { super(testCandidate); } + // we run them all sequentially: start simple! @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(0, 8); + return createParameters(0, 1); } } diff --git a/distribution/licenses/compress-lzf-1.0.2.jar.sha1 b/distribution/licenses/compress-lzf-1.0.2.jar.sha1 deleted file mode 100644 index 14e4a7f1476..00000000000 --- a/distribution/licenses/compress-lzf-1.0.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -62896e6fca184c79cc01a14d143f3ae2b4f4b4ae diff --git a/distribution/licenses/compress-lzf-LICENSE b/distribution/licenses/compress-lzf-LICENSE deleted file mode 100644 index 8d6813f770f..00000000000 --- a/distribution/licenses/compress-lzf-LICENSE +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Compress-LZF library is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/distribution/licenses/compress-lzf-NOTICE b/distribution/licenses/compress-lzf-NOTICE deleted file mode 100644 index 382a800c80d..00000000000 --- a/distribution/licenses/compress-lzf-NOTICE +++ /dev/null @@ -1,24 +0,0 @@ -# Compress LZF - -This library contains efficient implementation of LZF compression format, -as well as additional helper classes that build on JDK-provided gzip (deflat) -codec. - -## Licensing - -Library is licensed under Apache License 2.0, as per accompanying LICENSE file. - -## Credit - -Library has been written by Tatu Saloranta (tatu.saloranta@iki.fi). -It was started at Ning, inc., as an official Open Source process used by -platform backend, but after initial versions has been developed outside of -Ning by supporting community. - -Other contributors include: - -* Jon Hartlaub (first versions of streaming reader/writer; unit tests) -* Cedrik Lime: parallel LZF implementation - -Various community members have contributed bug reports, and suggested minor -fixes; these can be found from file "VERSION.txt" in SCM. diff --git a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 961bc6ff551..00000000000 --- a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5792c4b38aa2cf1f66c9dea8bf139907e33fa018 diff --git a/distribution/licenses/lucene-analyzers-common-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..5d95f64a15f --- /dev/null +++ b/distribution/licenses/lucene-analyzers-common-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +9f2b9811a4f4a57a1b3a98bdc1e1b63476b9f628 \ No newline at end of file diff --git a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 55354f421a5..00000000000 --- a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cd6fa25bc29718d8c964b0734fc9a009547453db diff --git a/distribution/licenses/lucene-backward-codecs-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..0ae258b597a --- /dev/null +++ b/distribution/licenses/lucene-backward-codecs-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +038071889a5dbeb279e37fa46225e194139a427c \ No newline at end of file diff --git a/distribution/licenses/lucene-core-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-core-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 938d27bd3cf..00000000000 --- a/distribution/licenses/lucene-core-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e4e0076ce4331309d1270a9c5b2edb51915fe32a diff --git a/distribution/licenses/lucene-core-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-core-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..aee7c10cffd --- /dev/null +++ b/distribution/licenses/lucene-core-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +b986d0ad8ee4dda8172a5a61875c47631e4b21d4 \ No newline at end of file diff --git a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 98041824d8d..00000000000 --- a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -49992ef742b6d3a24b551e06c96c0ab9cbad21e7 diff --git a/distribution/licenses/lucene-grouping-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-grouping-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..aa1011e007e --- /dev/null +++ b/distribution/licenses/lucene-grouping-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +f46574fbdfbcc81d936c77e15ba5b3af2c2b7253 \ No newline at end of file diff --git a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 7c9ca048217..00000000000 --- a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -85544da78c2d33b2fdfa6f76eb621c8c963eae37 diff --git a/distribution/licenses/lucene-highlighter-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-highlighter-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..561f17e773c --- /dev/null +++ b/distribution/licenses/lucene-highlighter-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +f620262d667a294d390e8df7575cc2cca2626559 \ No newline at end of file diff --git a/distribution/licenses/lucene-join-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-join-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index ed8c024b716..00000000000 --- a/distribution/licenses/lucene-join-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5c8b58d902a01cfce046e656eddee5c4e0578316 diff --git a/distribution/licenses/lucene-join-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-join-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..4735bdf1d2d --- /dev/null +++ b/distribution/licenses/lucene-join-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +4c44b07242fd706f6f7f14c9063a725e0e5b98cd \ No newline at end of file diff --git a/distribution/licenses/lucene-memory-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-memory-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 3edae47a468..00000000000 --- a/distribution/licenses/lucene-memory-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -55202617e88437dd6def4e42ceb42d18d08a9f6e diff --git a/distribution/licenses/lucene-memory-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-memory-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..9c19a6ad622 --- /dev/null +++ b/distribution/licenses/lucene-memory-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +1e33e0aa5fc227e90c8314f61b4cba1090035e33 \ No newline at end of file diff --git a/distribution/licenses/lucene-misc-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-misc-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index ce5a5e7d435..00000000000 --- a/distribution/licenses/lucene-misc-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -50b8439558061d0bbf09ddf8144d769143d33f00 diff --git a/distribution/licenses/lucene-misc-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-misc-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..c4a61bff68b --- /dev/null +++ b/distribution/licenses/lucene-misc-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +e416893f7b781239a15d3e2c7200ff26574d14de \ No newline at end of file diff --git a/distribution/licenses/lucene-queries-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-queries-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 938aa1d45a0..00000000000 --- a/distribution/licenses/lucene-queries-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ce470e38912676ebc63838635663f17d8db844f5 diff --git a/distribution/licenses/lucene-queries-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-queries-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..edc56751403 --- /dev/null +++ b/distribution/licenses/lucene-queries-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +b153b63b9333feedb18af2673eb6ccaf95bcc8bf \ No newline at end of file diff --git a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 25390b153da..00000000000 --- a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -318e192d61eb28de09cc62c42d4e9d044a497e8b diff --git a/distribution/licenses/lucene-queryparser-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-queryparser-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..eddd3d6cdcd --- /dev/null +++ b/distribution/licenses/lucene-queryparser-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +0aa2758d70a79f2e0f33a87624fd9d31e155c864 \ No newline at end of file diff --git a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 25b114cc5ef..00000000000 --- a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -043b3c3c03a5e0687884e5894754933d7e8f2f3e diff --git a/distribution/licenses/lucene-sandbox-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-sandbox-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..571903cc72c --- /dev/null +++ b/distribution/licenses/lucene-sandbox-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +873c716ba629dae389b12ddb1aedf2f5c5f57fea \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 823fe3ab4f5..00000000000 --- a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a9ab07d808456d9a2de248f3b4eba9765808b1cb diff --git a/distribution/licenses/lucene-spatial-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-spatial-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..5e6a27b7cd1 --- /dev/null +++ b/distribution/licenses/lucene-spatial-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +9d7e47c2fb73c614cc5ca41529b2c273c73b0ce7 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 84e38b7424d..00000000000 --- a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c9ae48e40c89364e0e69f27c514c924d52e0d57e diff --git a/distribution/licenses/lucene-spatial3d-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..cf841e18c5a --- /dev/null +++ b/distribution/licenses/lucene-spatial3d-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +4766305088797a66fe02d5aaa98e086867816e42 \ No newline at end of file diff --git a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index e2ba8be0c55..00000000000 --- a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8bd8fb7e75746fd7bce00b4d2813fca738d68f0b diff --git a/distribution/licenses/lucene-suggest-5.5.0-snapshot-1719088.jar.sha1 b/distribution/licenses/lucene-suggest-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..1fbb60a9d7a --- /dev/null +++ b/distribution/licenses/lucene-suggest-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +f0ee6fb780ea8aa9ec6d31e6a9cc7d48700bd2ca \ No newline at end of file diff --git a/distribution/licenses/securesm-1.0.jar.sha1 b/distribution/licenses/securesm-1.0.jar.sha1 new file mode 100644 index 00000000000..96d45d93e66 --- /dev/null +++ b/distribution/licenses/securesm-1.0.jar.sha1 @@ -0,0 +1 @@ +c0c6cf986ba0057390bfcc80c366a0e3157f944b diff --git a/distribution/licenses/securesm-LICENSE.txt b/distribution/licenses/securesm-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/distribution/licenses/securesm-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/distribution/licenses/securesm-NOTICE.txt b/distribution/licenses/securesm-NOTICE.txt new file mode 100644 index 00000000000..139597f9cb0 --- /dev/null +++ b/distribution/licenses/securesm-NOTICE.txt @@ -0,0 +1,2 @@ + + diff --git a/distribution/licenses/snakeyaml-1.15.jar.sha1 b/distribution/licenses/snakeyaml-1.15.jar.sha1 deleted file mode 100644 index 62d6943ca03..00000000000 --- a/distribution/licenses/snakeyaml-1.15.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3b132bea69e8ee099f416044970997bde80f4ea6 diff --git a/distribution/licenses/spatial4j-0.4.1.jar.sha1 b/distribution/licenses/spatial4j-0.4.1.jar.sha1 deleted file mode 100644 index 1c2883bd830..00000000000 --- a/distribution/licenses/spatial4j-0.4.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4234d12b1ba4d4b539fb3e29edd948a99539d9eb diff --git a/distribution/licenses/spatial4j-0.5.jar.sha1 b/distribution/licenses/spatial4j-0.5.jar.sha1 new file mode 100644 index 00000000000..4bcf7a33b15 --- /dev/null +++ b/distribution/licenses/spatial4j-0.5.jar.sha1 @@ -0,0 +1 @@ +6e16edaf6b1ba76db7f08c2f3723fce3b358ecc3 \ No newline at end of file diff --git a/distribution/licenses/spatial4j-ABOUT.txt b/distribution/licenses/spatial4j-ABOUT.txt new file mode 100644 index 00000000000..bee50a2b943 --- /dev/null +++ b/distribution/licenses/spatial4j-ABOUT.txt @@ -0,0 +1,15 @@ +About This Content + +May 22, 2015 + +License + +The Eclipse Foundation makes available all content in this plug-in ("Content"). Unless otherwise indicated below, the +Content is provided to you under the terms and conditions of the Apache License, Version 2.0. A copy of the Apache +License, Version 2.0 is available at http://www.apache.org/licenses/LICENSE-2.0.txt + +If you did not receive this Content directly from the Eclipse Foundation, the Content is being redistributed by another +party ("Redistributor") and different terms and conditions may apply to your use of any object code in the Content. +Check the Redistributor’s license that was provided with the Content. If no such license exists, contact the +Redistributor. Unless otherwise indicated below, the terms and conditions of the Apache License, Version 2.0 still apply +to any source code in the Content and such source code may be obtained at http://www.eclipse.org](http://www.eclipse.org. \ No newline at end of file diff --git a/distribution/licenses/spatial4j-NOTICE.txt b/distribution/licenses/spatial4j-NOTICE.txt index 8d1c8b69c3f..a8be036a412 100644 --- a/distribution/licenses/spatial4j-NOTICE.txt +++ b/distribution/licenses/spatial4j-NOTICE.txt @@ -1 +1,100 @@ - +Eclipse Foundation Software User Agreement + +April 9, 2014 + +Usage Of Content + +THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE +PROJECTS (COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR +THE TERMS AND CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE +THAT YOUR USE OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE +AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT +AND THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY +NOT USE THE CONTENT. + +Applicable Licenses + +Unless otherwise indicated, all Content made available by the Eclipse Foundation is provided to you under the terms and +conditions of the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is provided with this Content and is +also available at http://www.eclipse.org/legal/epl-v10.html. For purposes of the EPL, "Program" will mean the Content. + +Content includes, but is not limited to, source code, object code, documentation and other files maintained in the +Eclipse Foundation source code repository ("Repository") in software modules ("Modules") and made available as +downloadable archives ("Downloads"). + +* Content may be structured and packaged into modules to facilitate delivering, extending, and upgrading the Content. + Typical modules may include plug-ins ("Plug-ins"), plug-in fragments ("Fragments"), and features ("Features"). +* Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java™ ARchive) in a directory named "plugins". +* A Feature is a bundle of one or more Plug-ins and/or Fragments and associated material. Each Feature may be packaged + as a sub-directory in a directory named "features". Within a Feature, files named "feature.xml" may contain a list + of the names and version numbers of the Plug-ins and/or Fragments associated with that Feature. +* Features may also include other Features ("Included Features"). Within a Feature, files named "feature.xml" may + contain a list of the names and version numbers of Included Features. + +The terms and conditions governing Plug-ins and Fragments should be contained in files named "about.html" ("Abouts"). +The terms and conditions governing Features and Included Features should be contained in files named "license.html" +("Feature Licenses"). Abouts and Feature Licenses may be located in any directory of a Download or Module including, but +not limited to the following locations: + +* The top-level (root) directory +* Plug-in and Fragment directories +* Inside Plug-ins and Fragments packaged as JARs +* Sub-directories of the directory named "src" of certain Plug-ins +* Feature directories + +Note: if a Feature made available by the Eclipse Foundation is installed using the Provisioning Technology (as defined +below), you must agree to a license ("Feature Update License") during the installation process. If the Feature contains +Included Features, the Feature Update License should either provide you with the terms and conditions governing the +Included Features or inform you where you can locate them. Feature Update Licenses may be found in the "license" +property of files named "feature.properties" found within a Feature. Such Abouts, Feature Licenses, and Feature Update +Licenses contain the terms and conditions (or references to such terms and conditions) that govern your use of the +associated Content in that directory. + +THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR +TERMS AND CONDITIONS. SOME OF THESE OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO): + +* Eclipse Distribution License Version 1.0 (available at http://www.eclipse.org/licenses/edl-v10.html) +* Common Public License Version 1.0 (available at http://www.eclipse.org/legal/cpl-v10.html) +* Apache Software License 1.1 (available at http://www.apache.org/licenses/LICENSE) +* Apache Software License 2.0 (available at http://www.apache.org/licenses/LICENSE-2.0) +* Mozilla Public License Version 1.1 (available at http://www.mozilla.org/MPL/MPL-1.1.html) + +IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature +License, or Feature Update License is provided, please contact the Eclipse Foundation to determine what terms and +conditions govern that particular Content. + +### Use of Provisioning Technology + +The Eclipse Foundation makes available provisioning software, examples of which include, but are not limited to, p2 and +the Eclipse Update Manager ("Provisioning Technology") for the purpose of allowing users to install software, +documentation, information and/or other materials (collectively "Installable Software"). This capability is provided +with the intent of allowing such users to install, extend and update Eclipse-based products. Information about packaging +Installable Software is available at http://eclipse.org/equinox/p2/repository_packaging.html ("Specification"). + +You may use Provisioning Technology to allow other parties to install Installable Software. You shall be responsible for +enabling the applicable license agreements relating to the Installable Software to be presented to, and accepted by, the +users of the Provisioning Technology in accordance with the Specification. By using Provisioning Technology in such a +manner and making it available in accordance with the Specification, you further acknowledge your agreement to, and the +acquisition of all necessary rights to permit the following: + +1. A series of actions may occur ("Provisioning Process") in which a user may execute the Provisioning Technology on a + machine ("Target Machine") with the intent of installing, extending or updating the functionality of an + Eclipse-based product. +2. During the Provisioning Process, the Provisioning Technology may cause third party Installable Software or a portion + thereof to be accessed and copied to the Target Machine. +3. Pursuant to the Specification, you will provide to the user the terms and conditions that govern the use of the + Installable Software ("Installable Software Agreement") and such Installable Software Agreement shall be accessed + from the Target Machine in accordance with the Specification. Such Installable Software Agreement must inform the + user of the terms and conditions that govern the Installable Software and must solicit acceptance by the end user in + the manner prescribed in such Installable Software Agreement. Upon such indication of agreement by the user, the + provisioning Technology will complete installation of the Installable Software. + +Cryptography + +Content may contain encryption software. The country in which you are currently may have restrictions on the import, +possession, and use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, possession, or use, and re-export of +encryption software, to see if this is permitted. + +Java and all Java-based trademarks are trademarks of Oracle Corporation in the United States, other countries, +or both. \ No newline at end of file diff --git a/distribution/pom.xml b/distribution/pom.xml deleted file mode 100644 index 71575db9eef..00000000000 --- a/distribution/pom.xml +++ /dev/null @@ -1,255 +0,0 @@ - - - 4.0.0 - - org.elasticsearch - parent - 3.0.0-SNAPSHOT - - - org.elasticsearch.distribution - distributions - pom - Distribution: Parent POM - - - - /usr/share/elasticsearch - /usr/share/elasticsearch/bin - /etc/elasticsearch - /var/lib/elasticsearch - elasticsearch - elasticsearch - /var/log/elasticsearch - ${packaging.elasticsearch.home.dir}/plugins - /var/run/elasticsearch - /usr/lib/systemd/system - /usr/lib/sysctl.d - /usr/lib/tmpfiles.d - - - ${project.basedir}/../licenses - ${integ.scratch} - - - true - - - - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - test - - - - org.hamcrest - hamcrest-all - test - - - - org.apache.lucene - lucene-test-framework - test - - - - org.elasticsearch - elasticsearch - test - test-jar - - - - org.apache.httpcomponents - httpclient - test - - - - - org.elasticsearch - elasticsearch - - - - com.spatial4j - spatial4j - - - - com.vividsolutions - jts - - - - - com.github.spullara.mustache.java - compiler - - - - log4j - log4j - - - - log4j - apache-log4j-extras - - - - - - net.java.dev.jna - jna - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - prepare-package - - copy-dependencies - - - ${project.build.directory}/lib - runtime - - - - - - org.apache.maven.plugins - maven-eclipse-plugin - - - [groupId].[artifactId] - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - org.apache.maven.plugins - maven-resources-plugin - - - copy-resources - prepare-package - - copy-resources - - - ${project.build.directory}/bin - - - ${project.basedir}/../src/main/resources/bin - true - - *.exe - - - - ${project.basedir}/../src/main/resources/bin - false - - *.exe - - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - true - - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - - - integ-tests - - - 1 - - - localhost:${integ.transport.port} - - - - - - - - - - - tar - zip - deb - - - - - - macos_brew - - - - /usr/local/bin/rpmbuild - - - - rpm - - - - - rpm - - - - /usr/bin/rpmbuild - - - - rpm - - - - - diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle new file mode 100644 index 00000000000..2ab78fe7e41 --- /dev/null +++ b/distribution/rpm/build.gradle @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +task buildRpm(type: Rpm) { + dependsOn dependencyFiles, preparePackagingFiles + baseName 'elasticsearch' // this is what pom generation uses for artifactId + // Follow elasticsearch's rpm file naming convention + archiveName "${packageName}-${project.version}.rpm" + packageGroup 'Application/Internet' + prefix '/usr' + packager 'Elasticsearch' + release '1' + arch NOARCH + os LINUX + // TODO ospackage doesn't support icon but we used to have one +} + +artifacts { + 'default' buildRpm + archives buildRpm +} + +integTest { + /* We use real rpm tools to extract the rpm file for testing so we have to + skip the test if they aren't around. */ + enabled = new File('/bin/rpm').exists() || // Standard location + new File('/usr/bin/rpm').exists() || // Debian location + new File('/usr/local/bin/rpm').exists() // Homebrew location +} diff --git a/distribution/rpm/pom.xml b/distribution/rpm/pom.xml deleted file mode 100644 index 218e19e57b7..00000000000 --- a/distribution/rpm/pom.xml +++ /dev/null @@ -1,402 +0,0 @@ - - - 4.0.0 - - org.elasticsearch.distribution - distributions - 3.0.0-SNAPSHOT - - - org.elasticsearch.distribution.rpm - elasticsearch - Distribution: RPM - rpm - The RPM distribution of Elasticsearch - - - true - ${project.build.directory}/releases/ - - - - - - ${project.basedir}/src/main/packaging/packaging.properties - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - true - - - org.apache.maven.plugins - maven-resources-plugin - - - - copy-resources-rpm - prepare-package - - copy-resources - - - ${project.build.directory}/generated-packaging/rpm/ - - ${project.basedir}/../src/main/packaging/packaging.properties - ${project.basedir}/src/main/packaging/packaging.properties - - - - ${project.basedir}/../src/main/packaging/ - true - - **/* - - - packaging.properties - - - - ${project.basedir}/src/main/packaging/ - true - - **/* - - - packaging.properties - - - - ${project.basedir}/../src/main/resources - true - - bin/elasticsearch - bin/elasticsearch.in.sh - bin/plugin - bin/elasticsearch-systemd-pre-exec - - - - - - - - - - org.codehaus.mojo - rpm-maven-plugin - - false - elasticsearch - Elasticsearch - Application/Internet - Elasticsearch - /usr - - noarch - linux - src/changelog - - _unpackaged_files_terminate_build 0 - _binaries_in_noarch_packages_terminate_build 0 - - 644 - 755 - root - root - ${project.basedir}/src/main/resources/logo/elastic.gif - Elasticsearch is a distributed RESTful search engine built for the cloud. Reference documentation can be found at https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html and the 'Elasticsearch: The Definitive Guide' book can be found at https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html - - - - ${packaging.elasticsearch.bin.dir}/ - 755 - - - ${project.build.directory}/generated-packaging/rpm/bin - - elasticsearch - elasticsearch.in.sh - plugin - elasticsearch-systemd-pre-exec - - - - - - - - ${packaging.elasticsearch.conf.dir} - noreplace - elasticsearch - 750 - - - ${packaging.elasticsearch.conf.dir}/ - noreplace - elasticsearch - 750 - - - ${project.basedir}/../src/main/resources/config/ - - *.yml - - - - - - ${packaging.elasticsearch.conf.dir}/scripts - noreplace - elasticsearch - 750 - - - - /etc/sysconfig/ - false - noreplace - - - ${project.build.directory}/generated-packaging/rpm/env/ - - elasticsearch - - - - - - - ${packaging.elasticsearch.home.dir}/lib - - - target/lib/ - - ${project.build.finalName}-sources.jar - ${project.build.finalName}-tests.jar - ${project.build.finalName}-test-sources.jar - slf4j-api-*.jar - - - - - - - /etc/init.d - false - 755 - true - - - ${project.build.directory}/generated-packaging/rpm/init.d - - elasticsearch - - - - - - - ${packaging.elasticsearch.systemd.dir} - false - true - - - ${project.build.directory}/generated-packaging/rpm/systemd - - elasticsearch.service - - - - - - - ${packaging.elasticsearch.systemd.sysctl.dir} - true - - - ${project.build.directory}/generated-packaging/rpm/systemd/sysctl - - elasticsearch.conf - - - - - - - ${packaging.elasticsearch.tmpfilesd.dir} - true - - - ${project.build.directory}/generated-packaging/rpm/systemd/ - - elasticsearch.conf - - - - - - - ${packaging.elasticsearch.home.dir} - - - ${project.basedir}/../src/main/resources/ - - LICENSE.txt - NOTICE.txt - README.textile - - - - - - - ${packaging.elasticsearch.data.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.log.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.plugins.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.pid.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - - ${project.build.directory}/generated-packaging/rpm/scripts/preinst - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/postinst - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/prerm - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/postrm - utf-8 - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-rpm - package - - copy - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - true - ${rpm.outputDirectory} - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - integ-setup - pre-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - integ-teardown - post-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - - - - sign-rpm - - - rpm.sign - true - - - - - - org.codehaus.mojo - rpm-maven-plugin - - Application/Internet - ${gpg.key} - ${gpg.keyring} - - ${gpg.passphrase} - - - - - - - - diff --git a/distribution/rpm/src/main/packaging/init.d/elasticsearch b/distribution/rpm/src/main/packaging/init.d/elasticsearch index 924c67871af..12fed7dbc33 100644 --- a/distribution/rpm/src/main/packaging/init.d/elasticsearch +++ b/distribution/rpm/src/main/packaging/init.d/elasticsearch @@ -3,7 +3,7 @@ # elasticsearch # # chkconfig: 2345 80 20 -# description: Starts and stops a single elasticsearch instance on this system +# description: Starts and stops a single elasticsearch instance on this system # ### BEGIN INIT INFO @@ -32,19 +32,19 @@ if [ -f /etc/rc.d/init.d/functions ]; then fi # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" -ES_HOME="${packaging.elasticsearch.home.dir}" -MAX_OPEN_FILES=${packaging.os.max.open.files} -MAX_MAP_COUNT=${packaging.os.max.map.count} -LOG_DIR="${packaging.elasticsearch.log.dir}" -DATA_DIR="${packaging.elasticsearch.data.dir}" -CONF_DIR="${packaging.elasticsearch.conf.dir}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" +ES_HOME="/usr/share/elasticsearch" +MAX_OPEN_FILES=65535 +MAX_MAP_COUNT=262144 +LOG_DIR="/var/log/elasticsearch" +DATA_DIR="/var/lib/elasticsearch" +CONF_DIR="${path.conf}" -PID_DIR="${packaging.elasticsearch.pid.dir}" +PID_DIR="/var/run/elasticsearch" # Source the default env file -ES_ENV_FILE="${packaging.env.file}" +ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -64,12 +64,13 @@ export ES_HEAP_NEWSIZE export ES_DIRECT_SIZE export ES_JAVA_OPTS export ES_GC_LOG_FILE +export ES_STARTUP_SLEEP_TIME export JAVA_HOME lockfile=/var/lock/subsys/$prog # backwards compatibility for old config sysconfig files, pre 0.90.1 -if [ -n $USER ] && [ -z $ES_USER ] ; then +if [ -n $USER ] && [ -z $ES_USER ] ; then ES_USER=$USER fi @@ -125,7 +126,7 @@ start() { stop() { echo -n $"Stopping $prog: " # stop it here, often "killproc $prog" - killproc -p $pidfile -d ${packaging.elasticsearch.stopping.timeout} $prog + killproc -p $pidfile -d ${stopping.timeout} $prog retval=$? echo [ $retval -eq 0 ] && rm -f $lockfile diff --git a/distribution/rpm/src/main/packaging/packaging.properties b/distribution/rpm/src/main/packaging/packaging.properties deleted file mode 100644 index bc4af5f5ceb..00000000000 --- a/distribution/rpm/src/main/packaging/packaging.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Properties used to build to the RPM package -# - -# Environment file -packaging.env.file=/etc/sysconfig/elasticsearch - -# Default configuration directory and file to use in bin/plugin script -packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir} - -# Simple marker to check that properties are correctly overridden -packaging.type=rpm - -# Custom header for package scripts -packaging.scripts.header= -packaging.scripts.footer=# Built for ${project.name}-${project.version} (${packaging.type}) - -# Maximum time to wait for elasticsearch to stop (default to 1 day) -packaging.elasticsearch.stopping.timeout=86400 diff --git a/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..da68232f8d8 --- /dev/null +++ b/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.modules: ${expected.modules.count} } diff --git a/distribution/src/main/assemblies/common-bin.xml b/distribution/src/main/assemblies/common-bin.xml deleted file mode 100644 index f95368d3572..00000000000 --- a/distribution/src/main/assemblies/common-bin.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - /lib - true - false - false - - - - *:pom - - - - - - ../src/main/resources/config - config - - * - - - - true - ../src/main/resources/bin - bin - dos - - elasticsearch.in.bat - elasticsearch.bat - plugin.bat - service.bat - - - - false - ../src/main/resources/bin - bin - - *.exe - - - - true - ../src/main/resources/bin - bin - 0755 - 0755 - unix - - elasticsearch.in.sh - elasticsearch - plugin - - - - - - ../src/main/resources/README.textile - - - - ../src/main/resources/LICENSE.txt - - - - ../src/main/resources/NOTICE.txt - - - - diff --git a/distribution/src/main/packaging/env/elasticsearch b/distribution/src/main/packaging/env/elasticsearch index 0c01d4fb052..edacea3ec3b 100644 --- a/distribution/src/main/packaging/env/elasticsearch +++ b/distribution/src/main/packaging/env/elasticsearch @@ -3,21 +3,21 @@ ################################ # Elasticsearch home directory -#ES_HOME=${packaging.elasticsearch.home.dir} +#ES_HOME=/usr/share/elasticsearch # Elasticsearch configuration directory -#CONF_DIR=${packaging.elasticsearch.conf.dir} +#CONF_DIR=${path.conf} # Elasticsearch data directory -#DATA_DIR=${packaging.elasticsearch.data.dir} +#DATA_DIR=/var/lib/elasticsearch # Elasticsearch logs directory -#LOG_DIR=${packaging.elasticsearch.log.dir} +#LOG_DIR=/var/log/elasticsearch # Elasticsearch PID directory -#PID_DIR=${packaging.elasticsearch.pid.dir} +#PID_DIR=/var/run/elasticsearch -# Heap size defaults to ${packaging.elasticsearch.heap.min} min, ${packaging.elasticsearch.heap.max} max +# Heap size defaults to ${heap.min} min, ${heap.max} max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g #ES_HEAP_SIZE=2g @@ -34,7 +34,7 @@ #ES_RESTART_ON_UPGRADE=true # Path to the GC log file -#ES_GC_LOG_FILE=${packaging.elasticsearch.log.dir}/gc.log +#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log ################################ # Elasticsearch service @@ -43,12 +43,15 @@ # SysV init.d # # When executing the init script, this user will be used to run the elasticsearch service. -# The default value is '${packaging.elasticsearch.user}' and is declared in the init.d file. +# The default value is 'elasticsearch' and is declared in the init.d file. # Note that this setting is only used by the init script. If changed, make sure that # the configured user can read and write into the data, work, plugins and log directories. -# For systemd service, the user is usually configured in file ${packaging.elasticsearch.systemd.dir}/elasticsearch.service -#ES_USER=${packaging.elasticsearch.user} -#ES_GROUP=${packaging.elasticsearch.group} +# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service +#ES_USER=elasticsearch +#ES_GROUP=elasticsearch + +# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process +ES_STARTUP_SLEEP_TIME=5 ################################ # System properties @@ -56,17 +59,17 @@ # Specifies the maximum file descriptor number that can be opened by this process # When using Systemd, this setting is ignored and the LimitNOFILE defined in -# ${packaging.elasticsearch.systemd.dir}/elasticsearch.service takes precedence -#MAX_OPEN_FILES=${packaging.os.max.open.files} +# /usr/lib/systemd/system/elasticsearch.service takes precedence +#MAX_OPEN_FILES=65535 # The maximum number of bytes of memory that may be locked into RAM # Set to "unlimited" if you use the 'bootstrap.mlockall: true' option # in elasticsearch.yml (ES_HEAP_SIZE must also be set). # When using Systemd, the LimitMEMLOCK property must be set -# in ${packaging.elasticsearch.systemd.dir}/elasticsearch.service +# in /usr/lib/systemd/system/elasticsearch.service #MAX_LOCKED_MEMORY=unlimited # Maximum number of VMA (Virtual Memory Areas) a process can own # When using Systemd, this setting is ignored and the 'vm.max_map_count' -# property is set at boot time in ${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf -#MAX_MAP_COUNT=${packaging.os.max.map.count} +# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf +#MAX_MAP_COUNT=262144 diff --git a/distribution/src/main/packaging/packaging.properties b/distribution/src/main/packaging/packaging.properties deleted file mode 100644 index be5b60487ef..00000000000 --- a/distribution/src/main/packaging/packaging.properties +++ /dev/null @@ -1,27 +0,0 @@ -# Common properties for building ZIP,GZ,RPM and DEB packages -# -# Properties defined here can be overridden with specific settings, -# like in rpm/packaging.properties and deb/packaging.properties. - -# Environment file -packaging.env.file= - -# Default configuration directory and file to use in bin/plugin script -packaging.plugin.default.config.dir=$ES_HOME/config - -# Default values for min/max heap memory allocated to elasticsearch java process -packaging.elasticsearch.heap.min=256m -packaging.elasticsearch.heap.max=1g - -# Specifies the maximum file descriptor number -packaging.os.max.open.files=65535 - -# Maximum number of VMA (Virtual Memory Areas) a process can own -packaging.os.max.map.count=262144 - -# Simple marker to check that properties are correctly overridden -packaging.type=tar.gz - -# Custom header for package scripts -packaging.scripts.header= -packaging.scripts.footer= diff --git a/distribution/src/main/packaging/scripts/postinst b/distribution/src/main/packaging/scripts/postinst index 3d47a0338d4..61ac5f27cd7 100644 --- a/distribution/src/main/packaging/scripts/postinst +++ b/distribution/src/main/packaging/scripts/postinst @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the post-installation phase # @@ -13,11 +11,11 @@ ${packaging.scripts.header} # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" # Source the default env file -ES_ENV_FILE="${packaging.env.file}" +ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -98,4 +96,4 @@ elif [ "$RESTART_ON_UPGRADE" = "true" ]; then echo " OK" fi -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/scripts/postrm b/distribution/src/main/packaging/scripts/postrm index ee1c49b14ad..d4104845249 100644 --- a/distribution/src/main/packaging/scripts/postrm +++ b/distribution/src/main/packaging/scripts/postrm @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the post-removal phase # @@ -51,16 +49,16 @@ case "$1" in esac # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" -LOG_DIR="${packaging.elasticsearch.log.dir}" -PLUGINS_DIR="${packaging.elasticsearch.plugins.dir}" -PID_DIR="${packaging.elasticsearch.pid.dir}" -DATA_DIR="${packaging.elasticsearch.data.dir}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" +LOG_DIR="/var/log/elasticsearch" +PLUGINS_DIR="/usr/share/elasticsearch/plugins" +PID_DIR="/var/run/elasticsearch" +DATA_DIR="/var/lib/elasticsearch" # Source the default env file if [ "$SOURCE_ENV_FILE" = "true" ]; then - ES_ENV_FILE="${packaging.env.file}" + ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -116,4 +114,4 @@ if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then fi fi -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/scripts/preinst b/distribution/src/main/packaging/scripts/preinst index d3df84ecd02..21c4f137c85 100644 --- a/distribution/src/main/packaging/scripts/preinst +++ b/distribution/src/main/packaging/scripts/preinst @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the pre-installation phase # @@ -14,11 +12,11 @@ ${packaging.scripts.header} # Sets the default values for elasticsearch variables used in this script -ES_USER="${packaging.elasticsearch.user}" -ES_GROUP="${packaging.elasticsearch.group}" +ES_USER="elasticsearch" +ES_GROUP="elasticsearch" # Source the default env file -ES_ENV_FILE="${packaging.env.file}" +ES_ENV_FILE="${path.env}" if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi @@ -80,4 +78,4 @@ case "$1" in ;; esac -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/scripts/prerm b/distribution/src/main/packaging/scripts/prerm index 11d51c68637..07f39759aee 100644 --- a/distribution/src/main/packaging/scripts/prerm +++ b/distribution/src/main/packaging/scripts/prerm @@ -1,5 +1,3 @@ -${packaging.scripts.header} - # # This script is executed in the pre-remove phase # @@ -66,4 +64,4 @@ if [ "$STOP_REQUIRED" = "true" ]; then echo " OK" fi -${packaging.scripts.footer} +${scripts.footer} diff --git a/distribution/src/main/packaging/systemd/elasticsearch.conf b/distribution/src/main/packaging/systemd/elasticsearch.conf index 98dd5e61c25..d079f28b995 100644 --- a/distribution/src/main/packaging/systemd/elasticsearch.conf +++ b/distribution/src/main/packaging/systemd/elasticsearch.conf @@ -1 +1 @@ -d ${packaging.elasticsearch.pid.dir} 0755 ${packaging.elasticsearch.user} ${packaging.elasticsearch.group} - - +d /var/run/elasticsearch 0755 elasticsearch elasticsearch - - diff --git a/distribution/src/main/packaging/systemd/elasticsearch.service b/distribution/src/main/packaging/systemd/elasticsearch.service index d8f56f7d053..4a280a09d38 100644 --- a/distribution/src/main/packaging/systemd/elasticsearch.service +++ b/distribution/src/main/packaging/systemd/elasticsearch.service @@ -5,21 +5,21 @@ Wants=network-online.target After=network-online.target [Service] -Environment=ES_HOME=${packaging.elasticsearch.home.dir} -Environment=CONF_DIR=${packaging.elasticsearch.conf.dir} -Environment=DATA_DIR=${packaging.elasticsearch.data.dir} -Environment=LOG_DIR=${packaging.elasticsearch.log.dir} -Environment=PID_DIR=${packaging.elasticsearch.pid.dir} -EnvironmentFile=-${packaging.env.file} +Environment=ES_HOME=/usr/share/elasticsearch +Environment=CONF_DIR=${path.conf} +Environment=DATA_DIR=/var/lib/elasticsearch +Environment=LOG_DIR=/var/log/elasticsearch +Environment=PID_DIR=/var/run/elasticsearch +EnvironmentFile=-${path.env} -WorkingDirectory=${packaging.elasticsearch.home.dir} +WorkingDirectory=/usr/share/elasticsearch -User=${packaging.elasticsearch.user} -Group=${packaging.elasticsearch.group} +User=elasticsearch +Group=elasticsearch -ExecStartPre=${packaging.elasticsearch.bin.dir}/elasticsearch-systemd-pre-exec +ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec -ExecStart=${packaging.elasticsearch.bin.dir}/elasticsearch \ +ExecStart=/usr/share/elasticsearch/bin/elasticsearch \ -Des.pidfile=${PID_DIR}/elasticsearch.pid \ -Des.default.path.home=${ES_HOME} \ -Des.default.path.logs=${LOG_DIR} \ @@ -33,11 +33,11 @@ StandardOutput=null StandardError=journal # Specifies the maximum file descriptor number that can be opened by this process -LimitNOFILE=${packaging.os.max.open.files} +LimitNOFILE=65535 # Specifies the maximum number of bytes of memory that may be locked into RAM # Set to "infinity" if you use the 'bootstrap.mlockall: true' option -# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${packaging.env.file} +# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${path.env} #LimitMEMLOCK=infinity # Disable timeout logic and wait until process is stopped @@ -55,4 +55,4 @@ SuccessExitStatus=143 [Install] WantedBy=multi-user.target -# Built for ${project.name}-${project.version} (${packaging.type}) +# Built for ${project.name}-${project.version} (${project.name}) diff --git a/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf b/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf index 052cd89cf0b..62ea54d8697 100644 --- a/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf +++ b/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf @@ -1 +1 @@ -vm.max_map_count=${packaging.os.max.map.count} +vm.max_map_count=262144 diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch index 66f465765bf..459169bc3c4 100755 --- a/distribution/src/main/resources/bin/elasticsearch +++ b/distribution/src/main/resources/bin/elasticsearch @@ -42,12 +42,9 @@ # Be aware that you will be entirely responsible for populating the needed # environment variables. -# Maven will replace the project.name with elasticsearch below. If that -# hasn't been done, we assume that this is not a packaged version and the -# user has forgotten to run Maven to create a package. - -IS_PACKAGED_VERSION='${project.parent.artifactId}' -if [ "$IS_PACKAGED_VERSION" != "distributions" ]; then +# Check to see if you are trying to run this without building it first. Gradle +# will replace the project.name with _something_. +if echo '${project.name}' | grep project.name > /dev/null ; then cat >&2 << EOF Error: You must build the project with Maven or download a pre-built package before you can run Elasticsearch. See 'Building from Source' in README.textile @@ -142,6 +139,16 @@ if [ -z "$daemonized" ] ; then else exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \ org.elasticsearch.bootstrap.Elasticsearch start "$@" <&- & + retval=$? + pid=$! + [ $retval -eq 0 ] || exit $retval + if [ ! -z "$ES_STARTUP_SLEEP_TIME" ]; then + sleep $ES_STARTUP_SLEEP_TIME + fi + if ! ps -p $pid > /dev/null ; then + exit 1 + fi + exit 0 fi exit $? diff --git a/distribution/src/main/resources/bin/elasticsearch.in.bat b/distribution/src/main/resources/bin/elasticsearch.in.bat index 4e88d22f1dd..6f6550dcdf0 100644 --- a/distribution/src/main/resources/bin/elasticsearch.in.bat +++ b/distribution/src/main/resources/bin/elasticsearch.in.bat @@ -4,7 +4,7 @@ if DEFINED JAVA_HOME goto cont :err ECHO JAVA_HOME environment variable must be set! 1>&2 -EXIT /B 1 +EXIT /B 1 :cont set SCRIPT_DIR=%~dp0 @@ -14,11 +14,11 @@ for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI REM ***** JAVA options ***** if "%ES_MIN_MEM%" == "" ( -set ES_MIN_MEM=${packaging.elasticsearch.heap.min} +set ES_MIN_MEM=${heap.min} ) if "%ES_MAX_MEM%" == "" ( -set ES_MAX_MEM=${packaging.elasticsearch.heap.max} +set ES_MAX_MEM=${heap.max} ) if NOT "%ES_HEAP_SIZE%" == "" ( @@ -93,7 +93,7 @@ set JAVA_OPTS=%JAVA_OPTS% -Djna.nosys=true REM check in case a user was using this mechanism if "%ES_CLASSPATH%" == "" ( -set ES_CLASSPATH=%ES_HOME%/lib/${project.build.finalName}.jar;%ES_HOME%/lib/* +set ES_CLASSPATH=%ES_HOME%/lib/elasticsearch-${project.version}.jar;%ES_HOME%/lib/* ) else ( ECHO Error: Don't modify the classpath with ES_CLASSPATH, Best is to add 1>&2 ECHO additional elements via the plugin mechanism, or if code must really be 1>&2 diff --git a/distribution/src/main/resources/bin/elasticsearch.in.sh b/distribution/src/main/resources/bin/elasticsearch.in.sh index 5ac0025fe29..f859a06ffab 100644 --- a/distribution/src/main/resources/bin/elasticsearch.in.sh +++ b/distribution/src/main/resources/bin/elasticsearch.in.sh @@ -10,13 +10,13 @@ EOF exit 1 fi -ES_CLASSPATH="$ES_HOME/lib/${project.build.finalName}.jar:$ES_HOME/lib/*" +ES_CLASSPATH="$ES_HOME/lib/elasticsearch-${project.version}.jar:$ES_HOME/lib/*" if [ "x$ES_MIN_MEM" = "x" ]; then - ES_MIN_MEM=${packaging.elasticsearch.heap.min} + ES_MIN_MEM=${heap.min} fi if [ "x$ES_MAX_MEM" = "x" ]; then - ES_MAX_MEM=${packaging.elasticsearch.heap.max} + ES_MAX_MEM=${heap.max} fi if [ "x$ES_HEAP_SIZE" != "x" ]; then ES_MIN_MEM=$ES_HEAP_SIZE diff --git a/distribution/src/main/resources/bin/plugin b/distribution/src/main/resources/bin/plugin index 35dbe3a620a..95011870358 100755 --- a/distribution/src/main/resources/bin/plugin +++ b/distribution/src/main/resources/bin/plugin @@ -25,12 +25,12 @@ ES_HOME=`cd "$ES_HOME"; pwd` # Sets the default values for elasticsearch variables used in this script if [ -z "$CONF_DIR" ]; then - CONF_DIR="${packaging.plugin.default.config.dir}" + CONF_DIR="${path.conf}" fi # The default env file is defined at building/packaging time. -# For a ${packaging.type} package, the value is "${packaging.env.file}". -ES_ENV_FILE="${packaging.env.file}" +# For a ${project.name} package, the value is "${path.env}". +ES_ENV_FILE="${path.env}" # If an include is specified with the ES_INCLUDE environment variable, use it if [ -n "$ES_INCLUDE" ]; then diff --git a/distribution/src/main/resources/bin/service.bat b/distribution/src/main/resources/bin/service.bat index 9822e6bbdc0..5b5fbff7522 100644 --- a/distribution/src/main/resources/bin/service.bat +++ b/distribution/src/main/resources/bin/service.bat @@ -128,8 +128,8 @@ goto:eof ) :foundJVM -if "%ES_MIN_MEM%" == "" set ES_MIN_MEM=${packaging.elasticsearch.heap.min} -if "%ES_MAX_MEM%" == "" set ES_MAX_MEM=${packaging.elasticsearch.heap.max} +if "%ES_MIN_MEM%" == "" set ES_MIN_MEM=${heap.min} +if "%ES_MAX_MEM%" == "" set ES_MAX_MEM=${heap.max} if NOT "%ES_HEAP_SIZE%" == "" set ES_MIN_MEM=%ES_HEAP_SIZE% if NOT "%ES_HEAP_SIZE%" == "" set ES_MAX_MEM=%ES_HEAP_SIZE% diff --git a/distribution/src/main/resources/config/elasticsearch.yml b/distribution/src/main/resources/config/elasticsearch.yml index b1b11223f0f..4b335ce7a19 100644 --- a/distribution/src/main/resources/config/elasticsearch.yml +++ b/distribution/src/main/resources/config/elasticsearch.yml @@ -49,7 +49,7 @@ # # ---------------------------------- Network ----------------------------------- # -# Set the bind adress to a specific IP (IPv4 or IPv6): +# Set the bind address to a specific IP (IPv4 or IPv6): # # network.host: 192.168.0.1 # @@ -60,19 +60,8 @@ # For more information, see the documentation at: # # -# ---------------------------------- Gateway ----------------------------------- -# -# Block initial recovery after a full cluster restart until N nodes are started: -# -# gateway.recover_after_nodes: 3 -# -# For more information, see the documentation at: -# -# # --------------------------------- Discovery ---------------------------------- # -# Elasticsearch nodes will find each other via unicast, by default. -# # Pass an initial list of hosts to perform discovery when new node is started: # The default list of hosts is ["127.0.0.1", "[::1]"] # @@ -85,6 +74,15 @@ # For more information, see the documentation at: # # +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +# gateway.recover_after_nodes: 3 +# +# For more information, see the documentation at: +# +# # ---------------------------------- Various ----------------------------------- # # Disable starting multiple nodes on a single system: diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle new file mode 100644 index 00000000000..7230ab50799 --- /dev/null +++ b/distribution/tar/build.gradle @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +task buildTar(type: Tar) { + baseName = 'elasticsearch' + extension = 'tar.gz' + with archivesFiles + compression = Compression.GZIP +} + +artifacts { + 'default' buildTar + archives buildTar +} diff --git a/distribution/tar/pom.xml b/distribution/tar/pom.xml deleted file mode 100644 index f1ce8271c0c..00000000000 --- a/distribution/tar/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - org.elasticsearch.distribution - distributions - 3.0.0-SNAPSHOT - - - org.elasticsearch.distribution.tar - elasticsearch - Distribution: TAR - - - The TAR distribution of Elasticsearch - - - - ${project.basedir}/../src/main/packaging/packaging.properties - - - - - org.apache.maven.plugins - maven-assembly-plugin - - false - ${project.build.directory}/releases/ - - ${project.basedir}/src/main/assemblies/targz-bin.xml - - - - - package - - single - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - - integ-setup - pre-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - integ-teardown - post-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - diff --git a/distribution/tar/src/main/assemblies/targz-bin.xml b/distribution/tar/src/main/assemblies/targz-bin.xml deleted file mode 100644 index 7a42a2de986..00000000000 --- a/distribution/tar/src/main/assemblies/targz-bin.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - targz - - tar.gz - - - true - - - ../src/main/assemblies/common-bin.xml - - diff --git a/distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..da68232f8d8 --- /dev/null +++ b/distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.modules: ${expected.modules.count} } diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle new file mode 100644 index 00000000000..23191ff03a4 --- /dev/null +++ b/distribution/zip/build.gradle @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +task buildZip(type: Zip) { + baseName = 'elasticsearch' + with archivesFiles +} + +artifacts { + 'default' buildZip + archives buildZip +} + +integTest.dependsOn buildZip + diff --git a/distribution/zip/pom.xml b/distribution/zip/pom.xml deleted file mode 100644 index a24c4492d70..00000000000 --- a/distribution/zip/pom.xml +++ /dev/null @@ -1,103 +0,0 @@ - - - 4.0.0 - - org.elasticsearch.distribution - distributions - 3.0.0-SNAPSHOT - - - org.elasticsearch.distribution.zip - elasticsearch - Distribution: ZIP - - - The ZIP distribution of Elasticsearch - - - - ${project.basedir}/../src/main/packaging/packaging.properties - - - - - org.apache.maven.plugins - maven-assembly-plugin - - false - ${project.build.directory}/releases/ - - ${project.basedir}/src/main/assemblies/zip-bin.xml - - - - - package - - single - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - - execute - package - - run - - - - - - - - - - - - - - integ-setup - pre-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - integ-teardown - post-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - diff --git a/distribution/zip/src/main/assemblies/zip-bin.xml b/distribution/zip/src/main/assemblies/zip-bin.xml deleted file mode 100644 index 661d5540cf7..00000000000 --- a/distribution/zip/src/main/assemblies/zip-bin.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - zip - - zip - - - true - - - ../src/main/assemblies/common-bin.xml - - diff --git a/distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..da68232f8d8 --- /dev/null +++ b/distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.modules: ${expected.modules.count} } diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc index 56d658d8b7b..51789221be6 100644 --- a/docs/community-clients/index.asciidoc +++ b/docs/community-clients/index.asciidoc @@ -144,9 +144,6 @@ Also see the {client}/php-api/current/index.html[official Elasticsearch PHP clie Also see the {client}/python-api/current/index.html[official Elasticsearch Python client]. -* http://github.com/elasticsearch/elasticsearch-dsl-py[elasticsearch-dsl-py] - chainable query and filter construction built on top of official client. - * http://github.com/rhec/pyelasticsearch[pyelasticsearch]: Python client. @@ -173,7 +170,10 @@ The following projects appear to be abandoned: R client for Elasticsearch * https://github.com/ropensci/elastic[elastic]: - A general purpose R client for Elasticsearch + A low-level R client for Elasticsearch. + +* https://github.com/ropensci/elasticdsl[elasticdsl]: + A high-level R DSL for Elasticsearch, wrapping the elastic R client. [[ruby]] == Ruby diff --git a/docs/groovy-api/count.asciidoc b/docs/groovy-api/count.asciidoc deleted file mode 100644 index 48e8c188539..00000000000 --- a/docs/groovy-api/count.asciidoc +++ /dev/null @@ -1,22 +0,0 @@ -[[count]] -== Count API - -The count API is very similar to the -{java}/count.html[Java count API]. The Groovy -extension allows to provide the query to execute as a `Closure` (similar -to GORM criteria builder): - -[source,js] --------------------------------------------------- -def count = client.count { - indices "test" - types "type1" - query { - term { - test = "value" - } - } -} --------------------------------------------------- - -The query follows the same {ref}/query-dsl.html[Query DSL]. diff --git a/docs/groovy-api/delete.asciidoc b/docs/groovy-api/delete.asciidoc index 45020dfb131..e3320126966 100644 --- a/docs/groovy-api/delete.asciidoc +++ b/docs/groovy-api/delete.asciidoc @@ -2,7 +2,8 @@ == Delete API The delete API is very similar to the -{java}/delete.html[Java delete API], here is an +// {java}/java-docs-delete.html[] +Java delete API, here is an example: [source,js] diff --git a/docs/groovy-api/get.asciidoc b/docs/groovy-api/get.asciidoc index f5255beee4e..6bf476c16a8 100644 --- a/docs/groovy-api/get.asciidoc +++ b/docs/groovy-api/get.asciidoc @@ -2,7 +2,8 @@ == Get API The get API is very similar to the -{java}/get.html[Java get API]. The main benefit +// {java}/java-docs-get.html[] +Java get API. The main benefit of using groovy is handling the source content. It can be automatically converted to a `Map` which means using Groovy to navigate it is simple: diff --git a/docs/groovy-api/index.asciidoc b/docs/groovy-api/index.asciidoc index 7bab4d5d82a..a130556d0ad 100644 --- a/docs/groovy-api/index.asciidoc +++ b/docs/groovy-api/index.asciidoc @@ -46,5 +46,3 @@ include::get.asciidoc[] include::delete.asciidoc[] include::search.asciidoc[] - -include::count.asciidoc[] diff --git a/docs/groovy-api/index_.asciidoc b/docs/groovy-api/index_.asciidoc index 0e65a11863a..cd7f0ca4ac9 100644 --- a/docs/groovy-api/index_.asciidoc +++ b/docs/groovy-api/index_.asciidoc @@ -2,7 +2,8 @@ == Index API The index API is very similar to the -{java}/index_.html[Java index API]. The Groovy +// {java}/java-docs-index.html[] +Java index API. The Groovy extension to it is the ability to provide the indexed source using a closure. For example: diff --git a/docs/groovy-api/search.asciidoc b/docs/groovy-api/search.asciidoc index eb9b50185cb..946760d95cc 100644 --- a/docs/groovy-api/search.asciidoc +++ b/docs/groovy-api/search.asciidoc @@ -2,7 +2,8 @@ == Search API The search API is very similar to the -{java}/search.html[Java search API]. The Groovy +// {java}/java-search.html[] +Java search API. The Groovy extension allows to provide the search source to execute as a `Closure` including the query itself (similar to GORM criteria builder): diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc index cfc45b7bc4b..87aa7291b25 100644 --- a/docs/java-api/client.asciidoc +++ b/docs/java-api/client.asciidoc @@ -37,11 +37,10 @@ that can execute operations against elasticsearch. [source,java] -------------------------------------------------- -import static org.elasticsearch.node.NodeBuilder.*; // on startup -Node node = nodeBuilder().node(); +Node node = new Node(Settings.EMPTY).start(); Client client = node.client(); // on shutdown @@ -86,17 +85,15 @@ it): [source,java] -------------------------------------------------- -import static org.elasticsearch.node.NodeBuilder.*; // on startup // Embedded node clients behave just like standalone nodes, // which means that they will leave the HTTP port open! -Node node = - nodeBuilder() - .settings(Settings.settingsBuilder().put("http.enabled", false)) - .client(true) - .node(); +Node node = new Node(Settings.settingsBuilder() + .put("http.enabled", false) + .put("node.client", true).build()) + .start(); Client client = node.client(); @@ -115,11 +112,10 @@ and form a cluster. [source,java] -------------------------------------------------- -import static org.elasticsearch.node.NodeBuilder.*; // on startup -Node node = nodeBuilder().local(true).node(); +Node node = new Node(Settings.builder().put("node.local", true).build()).start(); Client client = node.client(); // on shutdown diff --git a/docs/java-api/count.asciidoc b/docs/java-api/count.asciidoc deleted file mode 100644 index 7b867b03f47..00000000000 --- a/docs/java-api/count.asciidoc +++ /dev/null @@ -1,37 +0,0 @@ -[[count]] -== Count API - -The count API allows one to easily execute a query and get the number of -matches for that query. It can be executed across one or more indices -and across one or more types. The query can be provided using the -{ref}/query-dsl.html[Query DSL]. - -[source,java] --------------------------------------------------- -import static org.elasticsearch.index.query.QueryBuilders.*; - -CountResponse response = client.prepareCount("test") - .setQuery(termQuery("_type", "type1")) - .execute() - .actionGet(); --------------------------------------------------- - -For more information on the count operation, check out the REST -{ref}/search-count.html[count] docs. - - -=== Operation Threading - -The count API allows one to set the threading model the operation will be -performed when the actual execution of the API is performed on the same -node (the API is executed on a shard that is allocated on the same -server). - -There are three threading modes.The `NO_THREADS` mode means that the -count operation will be executed on the calling thread. The -`SINGLE_THREAD` mode means that the count operation will be executed on -a single different thread for all local shards. The `THREAD_PER_SHARD` -mode means that the count operation will be executed on a different -thread for each local shard. - -The default mode is `SINGLE_THREAD`. diff --git a/docs/java-api/docs/update.asciidoc b/docs/java-api/docs/update.asciidoc index 874bdeb2c3a..1f63cd7001b 100644 --- a/docs/java-api/docs/update.asciidoc +++ b/docs/java-api/docs/update.asciidoc @@ -71,7 +71,7 @@ client.update(updateRequest).get(); [[java-docs-update-api-upsert]] ==== Upsert -There is also support for `upsert`. If the document already exists, the content of the `upsert` +There is also support for `upsert`. If the document does not exist, the content of the `upsert` element will be used to index the fresh doc: [source,java] diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc index 69ae88d3347..16403d5c147 100644 --- a/docs/java-api/index.asciidoc +++ b/docs/java-api/index.asciidoc @@ -34,6 +34,62 @@ For example, you can define the latest version in your `pom.xml` file: -------------------------------------------------- +== Dealing with JAR dependency conflicts + +If you want to use Elasticsearch in your Java application, you may have to deal with version conflicts with third party +dependencies like Guava and Joda. For instance, perhaps Elasticsearch uses Joda 2.8, while your code uses Joda 2.1. + +You have two choices: + +* The simplest solution is to upgrade. Newer module versions are likely to have fixed old bugs. +The further behind you fall, the harder it will be to upgrade later. Of course, it is possible that you are using a +third party dependency that in turn depends on an outdated version of a package, which prevents you from upgrading. + +* The second option is to relocate the troublesome dependencies and to shade them either with your own application +or with Elasticsearch and any plugins needed by the Elasticsearch client. + +The https://www.elastic.co/blog/to-shade-or-not-to-shade["To shade or not to shade" blog post] describes +all the steps for doing so. + +== Embedding jar with dependencies + +If you want to create a single jar containing your application and all dependencies, you should not +use `maven-assembly-plugin` for that because it can not deal with `META-INF/services` structure which is +required by Lucene jars. + +Instead, you can use `maven-shade-plugin` and configure it as follow: + +[source,xml] +-------------------------------------------------- + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + package + shade + + + + + + + + +-------------------------------------------------- + +Note that if you have a `main` class you want to automatically call when running `java -jar yourjar.jar`, just add +it to the `transformers`: + +[source,xml] +-------------------------------------------------- + + org.elasticsearch.demo.Generate + +-------------------------------------------------- + + == Deploying in JBoss EAP6 module Elasticsearch and Lucene classes need to be in the same JBoss module. @@ -84,8 +140,6 @@ include::docs.asciidoc[] include::search.asciidoc[] -include::count.asciidoc[] - include::aggs.asciidoc[] include::percolate.asciidoc[] diff --git a/docs/java-api/query-dsl/and-query.asciidoc b/docs/java-api/query-dsl/and-query.asciidoc deleted file mode 100644 index 02908ccc651..00000000000 --- a/docs/java-api/query-dsl/and-query.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[[java-query-dsl-and-query]] -==== And Query - -deprecated[2.0.0, Use the `bool` query instead] - -See {ref}/query-dsl-and-query.html[And Query] - -[source,java] --------------------------------------------------- -QueryBuilder query = andQuery( - rangeQuery("postDate").from("2010-03-01").to("2010-04-01"), <1> - prefixQuery("name.second", "ba")); <1> --------------------------------------------------- -<1> queries - diff --git a/docs/java-api/query-dsl/bool-query.asciidoc b/docs/java-api/query-dsl/bool-query.asciidoc index 249205dfbf3..cc55ada32a5 100644 --- a/docs/java-api/query-dsl/bool-query.asciidoc +++ b/docs/java-api/query-dsl/bool-query.asciidoc @@ -9,10 +9,11 @@ QueryBuilder qb = boolQuery() .must(termQuery("content", "test1")) <1> .must(termQuery("content", "test4")) <1> .mustNot(termQuery("content", "test2")) <2> - .should(termQuery("content", "test3")); <3> + .should(termQuery("content", "test3")) <3> + .filter(termQuery("content", "test5")); <4> -------------------------------------------------- <1> must query <2> must not query <3> should query - +<4> a query that must appear in the matching documents but doesn't contribute to scoring. diff --git a/docs/java-api/query-dsl/boosting-query.asciidoc b/docs/java-api/query-dsl/boosting-query.asciidoc index 58ecc596c8b..c07c1a51630 100644 --- a/docs/java-api/query-dsl/boosting-query.asciidoc +++ b/docs/java-api/query-dsl/boosting-query.asciidoc @@ -5,10 +5,10 @@ See {ref}/query-dsl-boosting-query.html[Boosting Query] [source,java] -------------------------------------------------- -QueryBuilder qb = boostingQuery() - .positive(termQuery("name","kimchy")) <1> - .negative(termQuery("name","dadoonet")) <2> - .negativeBoost(0.2f); <3> +QueryBuilder qb = boostingQuery( + termQuery("name","kimchy"), <1> + termQuery("name","dadoonet")) <2> + .negativeBoost(0.2f); <3> -------------------------------------------------- <1> query that will promote documents <2> query that will demote documents diff --git a/docs/java-api/query-dsl/compound-queries.asciidoc b/docs/java-api/query-dsl/compound-queries.asciidoc index 1335ef12de8..6d7e5e19b74 100644 --- a/docs/java-api/query-dsl/compound-queries.asciidoc +++ b/docs/java-api/query-dsl/compound-queries.asciidoc @@ -41,18 +41,6 @@ documents which also match a `negative` query. Execute one query for the specified indices, and another for other indices. -<>, <>, <>:: - -Synonyms for the `bool` query. - -<>:: - -Combine a query clause in query context with another in filter context. deprecated[2.0.0,Use the `bool` query instead] - -<>:: - -Limits the number of documents examined per shard. - include::constant-score-query.asciidoc[] include::bool-query.asciidoc[] @@ -60,10 +48,4 @@ include::dis-max-query.asciidoc[] include::function-score-query.asciidoc[] include::boosting-query.asciidoc[] include::indices-query.asciidoc[] -include::and-query.asciidoc[] -include::not-query.asciidoc[] -include::or-query.asciidoc[] -include::filtered-query.asciidoc[] -include::limit-query.asciidoc[] - diff --git a/docs/java-api/query-dsl/filtered-query.asciidoc b/docs/java-api/query-dsl/filtered-query.asciidoc deleted file mode 100644 index 47e0c06b6fd..00000000000 --- a/docs/java-api/query-dsl/filtered-query.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -[[java-query-dsl-filtered-query]] -==== Filtered Query - -deprecated[2.0.0, Use the `bool` query instead with a `must` clause for the query and a `filter` clause for the filter] - -See {ref}/query-dsl-filtered-query.html[Filtered Query]. - -[source,java] --------------------------------------------------- -QueryBuilder qb = filteredQuery( - matchQuery("name", "kimchy"), <1> - rangeQuery("dateOfBirth").from("1900").to("2100") <2> -); --------------------------------------------------- -<1> query which will be used for scoring -<2> query which will only be used for filtering the result set - diff --git a/docs/java-api/query-dsl/function-score-query.asciidoc b/docs/java-api/query-dsl/function-score-query.asciidoc index 1a1f92e1666..0915814ae1b 100644 --- a/docs/java-api/query-dsl/function-score-query.asciidoc +++ b/docs/java-api/query-dsl/function-score-query.asciidoc @@ -12,14 +12,13 @@ import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders. [source,java] -------------------------------------------------- -QueryBuilder qb = functionScoreQuery() - .add( - matchQuery("name", "kimchy"), <1> - randomFunction("ABCDEF") <2> - ) - .add( - exponentialDecayFunction("age", 0L, 1L) <3> - ); +FilterFunctionBuilder[] functions = { + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + matchQuery("name", "kimchy"), <1> + randomFunction("ABCDEF")), <2> + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + exponentialDecayFunction("age", 0L, 1L)) <3> +}; -------------------------------------------------- <1> Add a first function based on a query <2> And randomize the score based on a given seed diff --git a/docs/java-api/query-dsl/geo-bounding-box-query.asciidoc b/docs/java-api/query-dsl/geo-bounding-box-query.asciidoc index 75abad024f1..f877b11923a 100644 --- a/docs/java-api/query-dsl/geo-bounding-box-query.asciidoc +++ b/docs/java-api/query-dsl/geo-bounding-box-query.asciidoc @@ -6,8 +6,8 @@ See {ref}/query-dsl-geo-bounding-box-query.html[Geo Bounding Box Query] [source,java] -------------------------------------------------- QueryBuilder qb = geoBoundingBoxQuery("pin.location") <1> - .topLeft(40.73, -74.1) <2> - .bottomRight(40.717, -73.99); <3> + .setCorners(40.73, -74.1, <2> + 40.717, -73.99); <3> -------------------------------------------------- <1> field <2> bounding box top left point diff --git a/docs/java-api/query-dsl/geo-distance-range-query.asciidoc b/docs/java-api/query-dsl/geo-distance-range-query.asciidoc index 813989d20a5..1abe02a5e81 100644 --- a/docs/java-api/query-dsl/geo-distance-range-query.asciidoc +++ b/docs/java-api/query-dsl/geo-distance-range-query.asciidoc @@ -5,8 +5,8 @@ See {ref}/query-dsl-geo-distance-range-query.html[Geo Distance Range Query] [source,java] -------------------------------------------------- -QueryBuilder qb = geoDistanceRangeQuery("pin.location") <1> - .point(40, -70) <2> +QueryBuilder qb = geoDistanceRangeQuery("pin.location", <1> + new GeoPoint(40, -70)) <2> .from("200km") <3> .to("400km") <4> .includeLower(true) <5> diff --git a/docs/java-api/query-dsl/geo-polygon-query.asciidoc b/docs/java-api/query-dsl/geo-polygon-query.asciidoc index 0c10a1aa52b..1ee344c3098 100644 --- a/docs/java-api/query-dsl/geo-polygon-query.asciidoc +++ b/docs/java-api/query-dsl/geo-polygon-query.asciidoc @@ -5,11 +5,14 @@ See {ref}/query-dsl-geo-polygon-query.html[Geo Polygon Query] [source,java] -------------------------------------------------- -QueryBuilder qb = geoPolygonQuery("pin.location") <1> - .addPoint(40, -70) <2> - .addPoint(30, -80) <2> - .addPoint(20, -90); <2> --------------------------------------------------- -<1> field -<2> add your polygon of points a document should fall within +List points = new ArrayList(); <1> +points.add(new GeoPoint(40, -70)); +points.add(new GeoPoint(30, -80)); +points.add(new GeoPoint(20, -90)); + +QueryBuilder qb = + geoPolygonQuery("pin.location", points); <2> +-------------------------------------------------- +<1> add your polygon of points a document should fall within +<2> initialise the query with field and points diff --git a/docs/java-api/query-dsl/geo-shape-query.asciidoc b/docs/java-api/query-dsl/geo-shape-query.asciidoc index 386634d5942..a9b952ddf28 100644 --- a/docs/java-api/query-dsl/geo-shape-query.asciidoc +++ b/docs/java-api/query-dsl/geo-shape-query.asciidoc @@ -39,15 +39,16 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; [source,java] -------------------------------------------------- -QueryBuilder qb = geoShapeQuery( - "pin.location", <1> - ShapeBuilder.newMultiPoint() <2> - .point(0, 0) - .point(0, 10) - .point(10, 10) - .point(10, 0) - .point(0, 0), - ShapeRelation.WITHIN); <3> +GeoShapeQueryBuilder qb = geoShapeQuery( + "pin.location", <1> + ShapeBuilder.newMultiPoint() <2> + .point(0, 0) + .point(0, 10) + .point(10, 10) + .point(10, 0) + .point(0, 0)); +qb.relation(ShapeRelation.WITHIN); <3> + -------------------------------------------------- <1> field <2> shape @@ -56,11 +57,11 @@ QueryBuilder qb = geoShapeQuery( [source,java] -------------------------------------------------- // Using pre-indexed shapes -QueryBuilder qb = geoShapeQuery( +GeoShapeQueryBuilder qb = geoShapeQuery( "pin.location", <1> "DEU", <2> - "countries", <3> - ShapeRelation.WITHIN) <4> + "countries"); <3> +qb.relation(ShapeRelation.WITHIN)) <4> .indexedShapeIndex("shapes") <5> .indexedShapePath("location"); <6> -------------------------------------------------- diff --git a/docs/java-api/query-dsl/limit-query.asciidoc b/docs/java-api/query-dsl/limit-query.asciidoc deleted file mode 100644 index d2654f42e9c..00000000000 --- a/docs/java-api/query-dsl/limit-query.asciidoc +++ /dev/null @@ -1,11 +0,0 @@ -[[java-query-dsl-limit-query]] -==== Limit Query - -See {ref}/query-dsl-limit-query.html[Limit Query] - -[source,java] --------------------------------------------------- -QueryBuilder qb = limitQuery(100); <1> --------------------------------------------------- -<1> number of documents per shard - diff --git a/docs/java-api/query-dsl/missing-query.asciidoc b/docs/java-api/query-dsl/missing-query.asciidoc deleted file mode 100644 index 004a67ffdc8..00000000000 --- a/docs/java-api/query-dsl/missing-query.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[[java-query-dsl-missing-query]] -==== Missing Query - -See {ref}/query-dsl-missing-query.html[Missing Query] - -[source,java] --------------------------------------------------- -QueryBuilder qb = missingQuery("user"); <1> - .existence(true) <2> - .nullValue(true); <3> --------------------------------------------------- -<1> field -<2> find missing field that doesn’t exist -<3> find missing field with an explicit `null` value - diff --git a/docs/java-api/query-dsl/mlt-query.asciidoc b/docs/java-api/query-dsl/mlt-query.asciidoc index f6a258dd9d1..6be6cb7de2c 100644 --- a/docs/java-api/query-dsl/mlt-query.asciidoc +++ b/docs/java-api/query-dsl/mlt-query.asciidoc @@ -6,8 +6,11 @@ See: [source,java] -------------------------------------------------- -QueryBuilder qb = moreLikeThisQuery("name.first", "name.last") <1> - .like("text like this one") <2> +String[] fields = {"name.first", "name.last"}; <1> +String[] texts = {"text like this one"}; <2> +Item[] items = null; + +QueryBuilder qb = moreLikeThisQuery(fields, texts, items) .minTermFreq(1) <3> .maxQueryTerms(12); <4> -------------------------------------------------- diff --git a/docs/java-api/query-dsl/nested-query.asciidoc b/docs/java-api/query-dsl/nested-query.asciidoc index 00f876f1015..69fa1082c19 100644 --- a/docs/java-api/query-dsl/nested-query.asciidoc +++ b/docs/java-api/query-dsl/nested-query.asciidoc @@ -6,12 +6,12 @@ See {ref}/query-dsl-nested-query.html[Nested Query] [source,java] -------------------------------------------------- QueryBuilder qb = nestedQuery( - "obj1", <1> - boolQuery() <2> + "obj1", <1> + boolQuery() <2> .must(matchQuery("obj1.name", "blue")) .must(rangeQuery("obj1.count").gt(5)) ) - .scoreMode("avg"); <3> + .scoreMode(ScoreMode.Avg); <3> -------------------------------------------------- <1> path to nested document <2> your query. Any fields referenced inside the query must use the complete path (fully qualified). diff --git a/docs/java-api/query-dsl/not-query.asciidoc b/docs/java-api/query-dsl/not-query.asciidoc deleted file mode 100644 index b9c08d4136a..00000000000 --- a/docs/java-api/query-dsl/not-query.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[[java-query-dsl-not-query]] -==== Not Query - -See {ref}/query-dsl-not-query.html[Not Query] - - -[source,java] --------------------------------------------------- -QueryBuilder qb = notQuery( - rangeQuery("price").from("1").to("2") <1> -); --------------------------------------------------- -<1> query - - diff --git a/docs/java-api/query-dsl/or-query.asciidoc b/docs/java-api/query-dsl/or-query.asciidoc deleted file mode 100644 index c501a15b3da..00000000000 --- a/docs/java-api/query-dsl/or-query.asciidoc +++ /dev/null @@ -1,16 +0,0 @@ -[[java-query-dsl-or-query]] -==== Or Query - -deprecated[2.0.0, Use the `bool` query instead] - -See {ref}/query-dsl-or-query.html[Or Query] - -[source,java] --------------------------------------------------- -QueryBuilder qb = orQuery( - rangeQuery("price").from(1).to(2), <1> - matchQuery("name", "joe") <1> -); --------------------------------------------------- -<1> queries - diff --git a/docs/java-api/query-dsl/script-query.asciidoc b/docs/java-api/query-dsl/script-query.asciidoc index 2220296df09..534c803ab08 100644 --- a/docs/java-api/query-dsl/script-query.asciidoc +++ b/docs/java-api/query-dsl/script-query.asciidoc @@ -26,7 +26,7 @@ You can use it then with: QueryBuilder qb = scriptQuery( new Script( "mygroovyscript", <1> - ScriptService.ScriptType.FILE, <2> + ScriptType.FILE, <2> "groovy", <3> ImmutableMap.of("param1", 5)) <4> ); @@ -36,4 +36,4 @@ QueryBuilder qb = scriptQuery( <3> Scripting engine <4> Parameters as a `Map` of `` -æ + diff --git a/docs/java-api/query-dsl/span-containing-query.asciidoc b/docs/java-api/query-dsl/span-containing-query.asciidoc index 45d7174960e..81859eb93f3 100644 --- a/docs/java-api/query-dsl/span-containing-query.asciidoc +++ b/docs/java-api/query-dsl/span-containing-query.asciidoc @@ -5,15 +5,12 @@ See {ref}/query-dsl-span-containing-query.html[Span Containing Query] [source,java] -------------------------------------------------- -QueryBuilder qb = spanContainingQuery() - .little(spanTermQuery("field1","foo")) <1> - .big(spanNearQuery() <2> - .clause(spanTermQuery("field1","bar")) - .clause(spanTermQuery("field1","baz")) - .slop(5) - .inOrder(true) - ); +QueryBuilder qb = spanContainingQuery( + spanNearQuery(spanTermQuery("field1","bar"), 5) <1> + .clause(spanTermQuery("field1","baz")) + .inOrder(true), + spanTermQuery("field1","foo")); <2> -------------------------------------------------- -<1> `little` part -<2> `big` part +<1> `big` part +<2> `little` part diff --git a/docs/java-api/query-dsl/span-near-query.asciidoc b/docs/java-api/query-dsl/span-near-query.asciidoc index cfdc2fb3ee2..d18d2d74958 100644 --- a/docs/java-api/query-dsl/span-near-query.asciidoc +++ b/docs/java-api/query-dsl/span-near-query.asciidoc @@ -5,11 +5,11 @@ See {ref}/query-dsl-span-near-query.html[Span Near Query] [source,java] -------------------------------------------------- -QueryBuilder qb = spanNearQuery() - .clause(spanTermQuery("field","value1")) <1> +QueryBuilder qb = spanNearQuery( + spanTermQuery("field","value1"), <1> + 12) <2> .clause(spanTermQuery("field","value2")) <1> .clause(spanTermQuery("field","value3")) <1> - .slop(12) <2> .inOrder(false) <3> .collectPayloads(false); <4> -------------------------------------------------- diff --git a/docs/java-api/query-dsl/span-not-query.asciidoc b/docs/java-api/query-dsl/span-not-query.asciidoc index b279ccb8a7c..31026b7d8ea 100644 --- a/docs/java-api/query-dsl/span-not-query.asciidoc +++ b/docs/java-api/query-dsl/span-not-query.asciidoc @@ -5,9 +5,9 @@ See {ref}/query-dsl-span-not-query.html[Span Not Query] [source,java] -------------------------------------------------- -QueryBuilder qb = spanNotQuery() - .include(spanTermQuery("field","value1")) <1> - .exclude(spanTermQuery("field","value2")); <2> +QueryBuilder qb = spanNotQuery( + spanTermQuery("field","value1"), <1> + spanTermQuery("field","value2")); <2> -------------------------------------------------- <1> span query whose matches are filtered <2> span query whose matches must not overlap those returned diff --git a/docs/java-api/query-dsl/span-or-query.asciidoc b/docs/java-api/query-dsl/span-or-query.asciidoc index 7f7745d7c74..61f72a24fcf 100644 --- a/docs/java-api/query-dsl/span-or-query.asciidoc +++ b/docs/java-api/query-dsl/span-or-query.asciidoc @@ -5,8 +5,8 @@ See {ref}/query-dsl-span-or-query.html[Span Or Query] [source,java] -------------------------------------------------- -QueryBuilder qb = spanOrQuery() - .clause(spanTermQuery("field","value1")) <1> +QueryBuilder qb = spanOrQuery( + spanTermQuery("field","value1")) <1> .clause(spanTermQuery("field","value2")) <1> .clause(spanTermQuery("field","value3")); <1> -------------------------------------------------- diff --git a/docs/java-api/query-dsl/span-within-query.asciidoc b/docs/java-api/query-dsl/span-within-query.asciidoc index 4b7f1f90e25..345dabd8c12 100644 --- a/docs/java-api/query-dsl/span-within-query.asciidoc +++ b/docs/java-api/query-dsl/span-within-query.asciidoc @@ -5,14 +5,11 @@ See {ref}/query-dsl-span-within-query.html[Span Within Query] [source,java] -------------------------------------------------- -QueryBuilder qb = spanWithinQuery() - .little(spanTermQuery("field1", "foo")) <1> - .big(spanNearQuery() <2> - .clause(spanTermQuery("field1", "bar")) - .clause(spanTermQuery("field1", "baz")) - .slop(5) - .inOrder(true) - ); +QueryBuilder qb = spanWithinQuery( + spanNearQuery(spanTermQuery("field1", "bar"), 5) <1> + .clause(spanTermQuery("field1", "baz")) + .inOrder(true), + spanTermQuery("field1", "foo")); <2> -------------------------------------------------- -<1> `little` part -<2> `big` part +<1> `big` part +<2> `little` part diff --git a/docs/java-api/query-dsl/template-query.asciidoc b/docs/java-api/query-dsl/template-query.asciidoc index a0acae9fd0e..af950672e49 100644 --- a/docs/java-api/query-dsl/template-query.asciidoc +++ b/docs/java-api/query-dsl/template-query.asciidoc @@ -62,7 +62,7 @@ To execute an indexed templates, use `ScriptService.ScriptType.INDEXED`: -------------------------------------------------- QueryBuilder qb = templateQuery( "gender_template", <1> - ScriptService.ScriptType.INDEXED, <2> + ScriptType.INDEXED, <2> template_params); <3> -------------------------------------------------- <1> template name diff --git a/docs/java-api/query-dsl/term-level-queries.asciidoc b/docs/java-api/query-dsl/term-level-queries.asciidoc index 44fc3639072..e7d5ad4e52b 100644 --- a/docs/java-api/query-dsl/term-level-queries.asciidoc +++ b/docs/java-api/query-dsl/term-level-queries.asciidoc @@ -30,11 +30,6 @@ The queries in this group are: Find documents where the field specified contains any non-null value. -<>:: - - Find documents where the field specified does is missing or contains only - `null` values. - <>:: Find documents where the field specified contains terms which being with @@ -75,8 +70,6 @@ include::range-query.asciidoc[] include::exists-query.asciidoc[] -include::missing-query.asciidoc[] - include::prefix-query.asciidoc[] include::wildcard-query.asciidoc[] @@ -88,6 +81,3 @@ include::fuzzy-query.asciidoc[] include::type-query.asciidoc[] include::ids-query.asciidoc[] - - - diff --git a/docs/java-api/query-dsl/term-query.asciidoc b/docs/java-api/query-dsl/term-query.asciidoc index 658a715c8d3..47fcc522130 100644 --- a/docs/java-api/query-dsl/term-query.asciidoc +++ b/docs/java-api/query-dsl/term-query.asciidoc @@ -5,7 +5,7 @@ See {ref}/query-dsl-term-query.html[Term Query] [source,java] -------------------------------------------------- -QueryBuilder qb = term( +QueryBuilder qb = termQuery( "name", <1> "kimchy" <2> ); diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc index b15bfb8bd78..29fe9b4c7d1 100644 --- a/docs/plugins/analysis-phonetic.asciidoc +++ b/docs/plugins/analysis-phonetic.asciidoc @@ -42,7 +42,7 @@ The `phonetic` token filter takes the following settings: Which phonetic encoder to use. Accepts `metaphone` (default), `doublemetaphone`, `soundex`, `refinedsoundex`, `caverphone1`, `caverphone2`, `cologne`, `nysiis`, `koelnerphonetik`, `haasephonetik`, - `beidermorse`. + `beidermorse`, `daitch_mokotoff`. `replace`:: diff --git a/docs/plugins/api.asciidoc b/docs/plugins/api.asciidoc index 9e3b8f34da4..7ac0cdee834 100644 --- a/docs/plugins/api.asciidoc +++ b/docs/plugins/api.asciidoc @@ -44,6 +44,7 @@ A number of plugins have been contributed by our community: * https://github.com/hadashiA/elasticsearch-flavor[Elasticsearch Flavor Plugin] using http://mahout.apache.org/[Mahout] Collaboration filtering (by hadashiA) +* https://github.com/jurgc11/es-change-feed-plugin[WebSocket Change Feed Plugin] (by ForgeRock/Chris Clifton) These community plugins appear to have been abandoned: @@ -55,5 +56,6 @@ These community plugins appear to have been abandoned: * https://github.com/endgameinc/elasticsearch-term-plugin[Terms Component Plugin] (by Endgame Inc.) +* https://github.com/etsy/es-restlog[REST Request Logging Plugin] (by Etsy/Shikhar Bhushan) include::delete-by-query.asciidoc[] diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc index c0f310683f3..75b7776ec09 100644 --- a/docs/plugins/authors.asciidoc +++ b/docs/plugins/authors.asciidoc @@ -112,7 +112,7 @@ directory in the root of the plugin should be served. === Testing your plugin When testing a Java plugin, it will only be auto-loaded if it is in the -`plugins/` directory. Use `bin/plugin install file://path/to/your/plugin` +`plugins/` directory. Use `bin/plugin install file:///path/to/your/plugin` to install your plugin for testing. You may also load your plugin within the test framework for integration tests. diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 5ac208576df..bdd46fb72fd 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -64,16 +64,19 @@ cloud: protocol: https ---- -In addition, a proxy can be configured with the `proxy_host` and `proxy_port` settings (note that protocol can be -`http` or `https`): +In addition, a proxy can be configured with the `proxy.host`, `proxy.port`, `proxy.username` and `proxy.password` settings +(note that protocol can be `http` or `https`): [source,yaml] ---- cloud: aws: protocol: https - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself + password: theBestPasswordEver! ---- You can also set different proxies for `ec2` and `s3`: @@ -83,11 +86,17 @@ You can also set different proxies for `ec2` and `s3`: cloud: aws: s3: - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself1 + password: theBestPasswordEver1! ec2: - proxy_host: proxy2.company.com - proxy_port: 8083 + proxy: + host: proxy2.company.com + port: 8083 + username: myself2 + password: theBestPasswordEver2! ---- [[discovery-ec2-usage-region]] @@ -165,6 +174,11 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu Defaults to `3s`. If no unit like `ms`, `s` or `m` is specified, milliseconds are used. +`node_cache_time`:: + + How long the list of hosts is cached to prevent further requests to the AWS API. + Defaults to `10s`. + [IMPORTANT] .Binding the network host @@ -195,7 +209,6 @@ as valid network host settings: |`_ec2_` |equivalent to _ec2:privateIpv4_. |================================================================== - [[discovery-ec2-permissions]] ===== Recommended EC2 Permissions diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc index fef86462ae4..b9bac91f423 100644 --- a/docs/plugins/discovery-gce.asciidoc +++ b/docs/plugins/discovery-gce.asciidoc @@ -46,6 +46,19 @@ discovery: type: gce -------------------------------------------------- +The following gce settings (prefixed with `cloud.gce`) are supported: + + `retry`:: + + If set to `true`, client will use + https://developers.google.com/api-client-library/java/google-http-java-client/backoff[ExponentialBackOff] + policy to retry the failed http request. Defaults to `true`. + + `max_wait`:: + + The maximum elapsed time in milliseconds after the client instantiating retry. If the time elapsed goes past the + `max_wait`, client stops to retry. Defaults to 15 minutes (900000 milliseconds). + [IMPORTANT] .Binding the network host diff --git a/docs/plugins/discovery.asciidoc b/docs/plugins/discovery.asciidoc index 1fab9427d1a..cfc98e45dee 100644 --- a/docs/plugins/discovery.asciidoc +++ b/docs/plugins/discovery.asciidoc @@ -33,6 +33,7 @@ A number of discovery plugins have been contributed by our community: * https://github.com/grantr/elasticsearch-srv-discovery[DNS SRV Discovery Plugin] (by Grant Rodgers) * https://github.com/shikhar/eskka[eskka Discovery Plugin] (by Shikhar Bhushan) * https://github.com/grmblfrz/elasticsearch-zookeeper[ZooKeeper Discovery Plugin] (by Sonian Inc.) +* https://github.com/fabric8io/elasticsearch-cloud-kubernetes[Kubernetes Discovery Plugin] (by Jimmi Dyson, http://fabric8.io[fabric8]) include::discovery-ec2.asciidoc[] diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc index 52e31507b4f..fd44629dd29 100644 --- a/docs/plugins/integrations.asciidoc +++ b/docs/plugins/integrations.asciidoc @@ -32,11 +32,11 @@ Integrations are not plugins, but are external tools or modules that make it eas [[data-integrations]] === Data import/export and validation -NOTE: Rivers were used to import data from external systems into Elasticsearch prior to the 2.0 release. Elasticsearch +NOTE: Rivers were used to import data from external systems into Elasticsearch prior to the 2.0 release. Elasticsearch releases 2.0 and later do not support rivers. [float] -==== Supported by the community: +==== Supported by Elasticsearch: * https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html[Logstash output to Elasticsearch]: The Logstash `elasticsearch` output plugin. @@ -45,7 +45,7 @@ releases 2.0 and later do not support rivers. * https://www.elastic.co/guide/en/logstash/current/plugins-filters-elasticsearch.html[Elasticsearch event filtering in Logstash] The Logstash `elasticearch` filter plugin. * https://www.elastic.co/guide/en/logstash/current/plugins-codecs-es_bulk.html[Elasticsearch bulk codec] - The Logstash `es_bulk` plugin decodes the Elasticsearch bulk format into individual events. + The Logstash `es_bulk` plugin decodes the Elasticsearch bulk format into individual events. [float] ==== Supported by the community: @@ -53,15 +53,21 @@ releases 2.0 and later do not support rivers. * https://github.com/jprante/elasticsearch-jdbc[JDBC importer]: The Java Database Connection (JDBC) importer allows to fetch data from JDBC sources for indexing into Elasticsearch (by Jörg Prante) -* https://github.com/reachkrishnaraj/kafka-elasticsearch-standalone-consumer[Kafka Standalone Consumer]: - Easily Scalable & Extendable Kafka Standalone Consumer that reads messages from Kafka, then processes and indexes the messages in ElasticSearch +* https://github.com/reachkrishnaraj/kafka-elasticsearch-standalone-consumer/tree/branch2.0[Kafka Standalone Consumer(Indexer)]: + Kafka Standalone Consumer [Indexer] will read messages from Kafka in batches, processes(as implemented) and bulk-indexes them into ElasticSearch. Flexible and scalable. More documentation in above GitHub repo's Wiki.(Please use branch 2.0)! * https://github.com/ozlerhakan/mongolastic[Mongolastic]: A tool that clones data from ElasticSearch to MongoDB and vice versa * https://github.com/Aconex/scrutineer[Scrutineer]: A high performance consistency checker to compare what you've indexed - with your source of truth content + with your source of truth content (e.g. DB) + +* https://github.com/salyh/elasticsearch-imap[IMAP/POP3/Mail importer]: + The Mail importer allows to fetch data from IMAP and POP3 servers for indexing into Elasticsearch (by Hendrik Saly) + +* https://github.com/dadoonet/fscrawler[FS Crawler]: + The File System (FS) crawler allows to index documents (PDF, Open Office...) from your local file system and over SSH. (by David Pilato) [float] [[deployment]] diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc new file mode 100644 index 00000000000..c13d8ee0b4e --- /dev/null +++ b/docs/plugins/mapper-attachments.asciidoc @@ -0,0 +1,443 @@ +[[mapper-attachments]] +=== Mapper Attachments Plugin + +The mapper attachments plugin lets Elasticsearch index file attachments in common formats (such as PPT, XLS, PDF) +using the Apache text extraction library http://lucene.apache.org/tika/[Tika]. + +In practice, the plugin adds the `attachment` type when mapping properties so that documents can be populated with +file attachment contents (encoded as `base64`). + +[[mapper-attachments-install]] +[float] +==== Installation + +This plugin can be installed using the plugin manager: + +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin install mapper-attachments +---------------------------------------------------------------- + +The plugin must be installed on every node in the cluster, and each node must +be restarted after installation. + +[[mapper-attachments-remove]] +[float] +==== Removal + +The plugin can be removed with the following command: + +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin remove mapper-attachments +---------------------------------------------------------------- + +The node must be stopped before removing the plugin. + +[[mapper-attachments-helloworld]] +==== Hello, world + +Create a property mapping using the new type `attachment`: + +[source,js] +-------------------------- +POST /trying-out-mapper-attachments +{ + "mappings": { + "person": { + "properties": { + "cv": { "type": "attachment" } +}}}} +-------------------------- +// AUTOSENSE + +Index a new document populated with a `base64`-encoded attachment: + +[source,js] +-------------------------- +POST /trying-out-mapper-attachments/person/1 +{ + "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" +} +-------------------------- +// AUTOSENSE + +Search for the document using words in the attachment: + +[source,js] +-------------------------- +POST /trying-out-mapper-attachments/person/_search +{ + "query": { + "query_string": { + "query": "ipsum" +}}} +-------------------------- +// AUTOSENSE + +If you get a hit for your indexed document, the plugin should be installed and working. + +[[mapper-attachments-usage]] +==== Usage + +Using the attachment type is simple, in your mapping JSON, simply set a certain JSON element as attachment, for example: + +[source,js] +-------------------------- +PUT /test +PUT /test/person/_mapping +{ + "person" : { + "properties" : { + "my_attachment" : { "type" : "attachment" } + } + } +} +-------------------------- +// AUTOSENSE + +In this case, the JSON to index can be: + +[source,js] +-------------------------- +PUT /test/person/1 +{ + "my_attachment" : "... base64 encoded attachment ..." +} +-------------------------- +// AUTOSENSE + +Or it is possible to use more elaborated JSON if content type, resource name or language need to be set explicitly: + +[source,js] +-------------------------- +PUT /test/person/1 +{ + "my_attachment" : { + "_content_type" : "application/pdf", + "_name" : "resource/name/of/my.pdf", + "_language" : "en", + "_content" : "... base64 encoded attachment ..." + } +} +-------------------------- +// AUTOSENSE + +The `attachment` type not only indexes the content of the doc in `content` sub field, but also automatically adds meta +data on the attachment as well (when available). + +The metadata supported are: + +* `date` +* `title` +* `name` only available if you set `_name` see above +* `author` +* `keywords` +* `content_type` +* `content_length` is the original content_length before text extraction (aka file size) +* `language` + +They can be queried using the "dot notation", for example: `my_attachment.author`. + +Both the meta data and the actual content are simple core type mappers (string, date, …), thus, they can be controlled +in the mappings. For example: + +[source,js] +-------------------------- +PUT /test/person/_mapping +{ + "person" : { + "properties" : { + "file" : { + "type" : "attachment", + "fields" : { + "content" : {"index" : "no"}, + "title" : {"store" : "yes"}, + "date" : {"store" : "yes"}, + "author" : {"analyzer" : "myAnalyzer"}, + "keywords" : {"store" : "yes"}, + "content_type" : {"store" : "yes"}, + "content_length" : {"store" : "yes"}, + "language" : {"store" : "yes"} + } + } + } + } +} +-------------------------- +// AUTOSENSE + +In the above example, the actual content indexed is mapped under `fields` name `content`, and we decide not to index it, so +it will only be available in the `_all` field. The other fields map to their respective metadata names, but there is no +need to specify the `type` (like `string` or `date`) since it is already known. + +[[mapper-attachments-copy-to]] +==== Copy To feature + +If you want to use http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#copy-to[copy_to] +feature, you need to define it on each sub-field you want to copy to another field: + +[source,js] +-------------------------- +PUT /test/person/_mapping +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content": { + "type": "string", + "copy_to": "copy" + } + } + }, + "copy": { + "type": "string" + } + } + } +} +-------------------------- +// AUTOSENSE + +In this example, the extracted content will be copy as well to `copy` field. + +[[mapper-attachments-querying-metadata]] +==== Querying or accessing metadata + +If you need to query on metadata fields, use the attachment field name dot the metadata field. For example: + +[source,js] +-------------------------- +DELETE /test +PUT /test +PUT /test/person/_mapping +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content_type": { + "type": "string", + "store": true + } + } + } + } + } +} +PUT /test/person/1?refresh=true +{ + "file": "IkdvZCBTYXZlIHRoZSBRdWVlbiIgKGFsdGVybmF0aXZlbHkgIkdvZCBTYXZlIHRoZSBLaW5nIg==" +} +GET /test/person/_search +{ + "fields": [ "file.content_type" ], + "query": { + "match": { + "file.content_type": "text plain" + } + } +} +-------------------------- +// AUTOSENSE + +Will give you: + +[source,js] +-------------------------- +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": 0.16273327, + "hits": [ + { + "_index": "test", + "_type": "person", + "_id": "1", + "_score": 0.16273327, + "fields": { + "file.content_type": [ + "text/plain; charset=ISO-8859-1" + ] + } + } + ] + } +} +-------------------------- + +[[mapper-attachments-indexed-characters]] +==== Indexed Characters + +By default, `100000` characters are extracted when indexing the content. This default value can be changed by setting +the `index.mapping.attachment.indexed_chars` setting. It can also be provided on a per document indexed using the +`_indexed_chars` parameter. `-1` can be set to extract all text, but note that all the text needs to be allowed to be +represented in memory: + +[source,js] +-------------------------- +PUT /test/person/1 +{ + "my_attachment" : { + "_indexed_chars" : -1, + "_content" : "... base64 encoded attachment ..." + } +} +-------------------------- +// AUTOSENSE + +[[mapper-attachments-error-handling]] +==== Metadata parsing error handling + +While extracting metadata content, errors could happen for example when parsing dates. +Parsing errors are ignored so your document is indexed. + +You can disable this feature by setting the `index.mapping.attachment.ignore_errors` setting to `false`. + +[[mapper-attachments-language-detection]] +==== Language Detection + +By default, language detection is disabled (`false`) as it could come with a cost. +This default value can be changed by setting the `index.mapping.attachment.detect_language` setting. +It can also be provided on a per document indexed using the `_detect_language` parameter. + +Note that you can force language using `_language` field when sending your actual document: + +[source,js] +-------------------------- +{ + "my_attachment" : { + "_language" : "en", + "_content" : "... base64 encoded attachment ..." + } +} +-------------------------- + +[[mapper-attachments-highlighting]] +==== Highlighting attachments + +If you want to highlight your attachment content, you will need to set `"store": true` and +`"term_vector":"with_positions_offsets"` for your attachment field. Here is a full script which does it: + +[source,js] +-------------------------- +DELETE /test +PUT /test +PUT /test/person/_mapping +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content": { + "type": "string", + "term_vector":"with_positions_offsets", + "store": true + } + } + } + } + } +} +PUT /test/person/1?refresh=true +{ + "file": "IkdvZCBTYXZlIHRoZSBRdWVlbiIgKGFsdGVybmF0aXZlbHkgIkdvZCBTYXZlIHRoZSBLaW5nIg==" +} +GET /test/person/_search +{ + "fields": [], + "query": { + "match": { + "file.content": "king queen" + } + }, + "highlight": { + "fields": { + "file.content": { + } + } + } +} +-------------------------- +// AUTOSENSE + +It gives back: + +[source,js] +-------------------------- +{ + "took": 9, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": 0.13561106, + "hits": [ + { + "_index": "test", + "_type": "person", + "_id": "1", + "_score": 0.13561106, + "highlight": { + "file.content": [ + "\"God Save the Queen\" (alternatively \"God Save the King\"\n" + ] + } + } + ] + } +} +-------------------------- + +[[mapper-attachments-standalone]] +==== Stand alone runner + +If you want to run some tests within your IDE, you can use `StandaloneRunner` class. +It accepts arguments: + +* `-u file://URL/TO/YOUR/DOC` +* `--size` set extracted size (default to mapper attachment size) +* `BASE64` encoded binary + +Example: + +[source,sh] +-------------------------- +StandaloneRunner BASE64Text +StandaloneRunner -u /tmp/mydoc.pdf +StandaloneRunner -u /tmp/mydoc.pdf --size 1000000 +-------------------------- + +It produces something like: + +[source,text] +-------------------------- +## Extracted text +--------------------- BEGIN ----------------------- +This is the extracted text +---------------------- END ------------------------ +## Metadata +- author: null +- content_length: null +- content_type: application/pdf +- date: null +- keywords: null +- language: null +- name: null +- title: null +-------------------------- diff --git a/docs/plugins/mapper.asciidoc b/docs/plugins/mapper.asciidoc index c6a3a7b35aa..fcfc877f8f9 100644 --- a/docs/plugins/mapper.asciidoc +++ b/docs/plugins/mapper.asciidoc @@ -8,11 +8,10 @@ Mapper plugins allow new field datatypes to be added to Elasticsearch. The core mapper plugins are: -https://github.com/elasticsearch/elasticsearch-mapper-attachments[Mapper Attachments Type plugin]:: +<>:: -Integrates http://lucene.apache.org/tika/[Apache Tika] to provide a new field -type `attachment` to allow indexing of documents such as PDFs and Microsoft -Word. +The mapper-attachments integrates http://lucene.apache.org/tika/[Apache Tika] to provide a new field +type `attachment` to allow indexing of documents such as PDFs and Microsoft Word. <>:: @@ -25,5 +24,6 @@ indexes the size in bytes of the original The mapper-murmur3 plugin allows hashes to be computed at index-time and stored in the index for later use with the `cardinality` aggregation. +include::mapper-attachments.asciidoc[] include::mapper-size.asciidoc[] include::mapper-murmur3.asciidoc[] diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index 52ff574cfc7..58ba1e53d71 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -68,20 +68,19 @@ plugin from GitHub, run one of the following commands: [source,shell] ----------------------------------- sudo bin/plugin install lmenezes/elasticsearch-kopf <1> -sudo bin/plugin install lmenezes/elasticsearch-kopf/1.x <2> +sudo bin/plugin install lmenezes/elasticsearch-kopf/2.x <2> ----------------------------------- <1> Installs the latest version from GitHub. <2> Installs the 1.x version from GitHub. When installing from Maven Central/Sonatype, `[org]` should be replaced by the artifact `groupId`, and `[user|component]` by the `artifactId`. For -instance, to install the -https://github.com/elastic/elasticsearch-mapper-attachments[mapper attachment] +instance, to install the {plugins}/mapper-attachments.html[`mapper-attachments`] plugin from Sonatype, run: [source,shell] ----------------------------------- -sudo bin/plugin install org.elasticsearch/elasticsearch-mapper-attachments/2.6.0 <1> +sudo bin/plugin install org.elasticsearch.plugin/mapper-attachments/3.0.0 <1> ----------------------------------- <1> When installing from `download.elastic.co` or from Maven Central/Sonatype, the version is required. @@ -101,7 +100,16 @@ For instance, to install a plugin from your local file system, you could run: [source,shell] ----------------------------------- -sudo bin/plugin install file:/path/to/plugin.zip +sudo bin/plugin install file:///path/to/plugin.zip +----------------------------------- + +The plugin script will refuse to talk to an HTTPS URL with an untrusted +certificate. To use a self-signed HTTPS cert, you will need to add the CA cert +to a local Java truststore and pass the location to the script as follows: + +[source,shell] +----------------------------------- +sudo bin/plugin -Djavax.net.ssl.trustStore=/path/to/trustStore.jks install https://.... ----------------------------------- [[listing-removing]] @@ -229,7 +237,7 @@ example: [source,yaml] -------------------------------------------------- -plugin.mandatory: mapper-attachments,lang-groovy +plugin.mandatory: mapper-attachments,lang-python -------------------------------------------------- For safety reasons, a node will not start if it is missing a mandatory plugin. diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index fee308b3001..9846b5fbf58 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -41,22 +41,37 @@ To enable Azure repositories, you have first to set your azure storage settings cloud: azure: storage: - account: your_azure_storage_account - key: your_azure_storage_key + my_account: + account: your_azure_storage_account + key: your_azure_storage_key ---- -For information, in previous version of the azure plugin, settings were: +Note that you can also define more than one account: [source,yaml] ---- cloud: azure: - storage_account: your_azure_storage_account - storage_key: your_azure_storage_key + storage: + my_account1: + account: your_azure_storage_account1 + key: your_azure_storage_key1 + default: true + my_account2: + account: your_azure_storage_account2 + key: your_azure_storage_key2 ---- +`my_account1` is the default account which will be used by a repository unless you set an explicit one. + + The Azure repository supports following settings: +`account`:: + + Azure account settings to use. Defaults to the only one if you set a single + account or to the one marked as `default` if you have more than one. + `container`:: Container name. Defaults to `elasticsearch-snapshots` @@ -82,6 +97,11 @@ The Azure repository supports following settings: Makes repository read-only. coming[2.1.0] Defaults to `false`. +`location_mode`:: + + `primary_only` or `secondary_only`. Defaults to `primary_only`. Note that if you set it + to `secondary_only`, it will force `read_only` to true. + Some examples, using scripts: [source,json] @@ -97,12 +117,30 @@ PUT _snapshot/my_backup2 { "type": "azure", "settings": { - "container": "backup_container", + "container": "backup-container", "base_path": "backups", "chunk_size": "32m", "compress": true } } + + +# With two accounts defined in elasticsearch.yml (my_account1 and my_account2) +PUT _snapshot/my_backup3 +{ + "type": "azure", + "settings": { + "account": "my_account1" + } +} +PUT _snapshot/my_backup4 +{ + "type": "azure", + "settings": { + "account": "my_account2", + "location_mode": "primary_only" + } +} ---- // AUTOSENSE @@ -110,9 +148,9 @@ Example using Java: [source,java] ---- -client.admin().cluster().preparePutRepository("my_backup3") +client.admin().cluster().preparePutRepository("my_backup_java1") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Storage.CONTAINER, "backup_container") + .put(Storage.CONTAINER, "backup-container") .put(Storage.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB)) ).get(); ---- @@ -129,27 +167,3 @@ permitted in container names. * All letters in a container name must be lowercase. * Container names must be from 3 through 63 characters long. -[[repository-azure-testing]] -==== Testing Azure - -Integrations tests in this plugin require working Azure configuration and therefore disabled by default. -To enable tests prepare a config file `elasticsearch.yml` with the following content: - -[source,yaml] ----- -cloud: - azure: - storage: - account: "YOUR-AZURE-STORAGE-NAME" - key: "YOUR-AZURE-STORAGE-KEY" ----- - -Replaces `account`, `key` with your settings. Please, note that the test will delete all snapshot/restore related -files in the specified bucket. - -To run test: - -[source,sh] ----- -mvn -Dtests.azure=true -Dtests.config=/path/to/config/file/elasticsearch.yml clean test ----- diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc new file mode 100644 index 00000000000..ea13e5ad3a6 --- /dev/null +++ b/docs/plugins/repository-hdfs.asciidoc @@ -0,0 +1,115 @@ +[[repository-hdfs]] +=== Hadoop HDFS Repository Plugin + +The HDFS repository plugin adds support for using HDFS File System as a repository for +{ref}/modules-snapshots.html[Snapshot/Restore]. + +[[repository-hdfs-install]] +[float] +==== Installation + +This plugin can be installed using the plugin manager: + +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin install repository-hdfs +sudo bin/plugin install repository-hdfs-hadoop2 +sudo bin/plugin install repository-hdfs-lite +---------------------------------------------------------------- + +The plugin must be installed on every node in the cluster, and each node must +be restarted after installation. + +[[repository-hdfs-remove]] +[float] +==== Removal + +The plugin can be removed with the following command: + +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin remove repository-hdfs +sudo bin/plugin remove repository-hdfs-hadoop2 +sudo bin/plugin remove repository-hdfs-lite +---------------------------------------------------------------- + +The node must be stopped before removing the plugin. + +[[repository-hdfs-usage]] +==== Getting started with HDFS + +The HDFS snapshot/restore plugin comes in three _flavors_: + +* Default / Hadoop 1.x:: +The default version contains the plugin jar alongside Apache Hadoop 1.x (stable) dependencies. +* YARN / Hadoop 2.x:: +The `hadoop2` version contains the plugin jar plus the Apache Hadoop 2.x (also known as YARN) dependencies. +* Lite:: +The `lite` version contains just the plugin jar, without any Hadoop dependencies. The user should provide these (read below). + +[[repository-hdfs-flavor]] +===== What version to use? + +It depends on whether Hadoop is locally installed or not and if not, whether it is compatible with Apache Hadoop clients. + +* Are you using Apache Hadoop (or a _compatible_ distro) and do not have installed on the Elasticsearch nodes?:: ++ +If the answer is yes, for Apache Hadoop 1 use the default `repository-hdfs` or `repository-hdfs-hadoop2` for Apache Hadoop 2. ++ +* If you are have Hadoop installed locally on the Elasticsearch nodes or are using a certain distro:: ++ +Use the `lite` version and place your Hadoop _client_ jars and their dependencies in the plugin folder under `hadoop-libs`. +For large deployments, it is recommended to package the libraries in the plugin zip and deploy it manually across nodes +(and thus avoiding having to do the libraries setup on each node). + +[[repository-hdfs-security]] +==== Handling JVM Security and Permissions + +Out of the box, Elasticsearch runs in a JVM with the security manager turned _on_ to make sure that unsafe or sensitive actions +are allowed only from trusted code. Hadoop however is not really designed to run under one; it does not rely on privileged blocks +to execute sensitive code, of which it uses plenty. + +The `repository-hdfs` plugin provides the necessary permissions for both Apache Hadoop 1.x and 2.x (latest versions) to successfully +run in a secured JVM as one can tell from the number of permissions required when installing the plugin. +However using a certain Hadoop File-System (outside DFS), a certain distro or operating system (in particular Windows), might require +additional permissions which are not provided by the plugin. + +In this case there are several workarounds: +* add the permission into `plugin-security.policy` (available in the plugin folder) +* disable the security manager through `es.security.manager.enabled=false` configurations setting - NOT RECOMMENDED + +If you find yourself in such a situation, please let us know what Hadoop distro version and OS you are using and what permission is missing +by raising an issue. Thank you! + +[[repository-hdfs-config]] +==== Configuration Properties + +Once installed, define the configuration for the `hdfs` repository through `elasticsearch.yml` or the +{ref}/modules-snapshots.html[REST API]: + +[source] +---- +repositories + hdfs: + uri: "hdfs://:/" # optional - Hadoop file-system URI + path: "some/path" # required - path with the file-system where data is stored/loaded + load_defaults: "true" # optional - whether to load the default Hadoop configuration (default) or not + conf_location: "extra-cfg.xml" # optional - Hadoop configuration XML to be loaded (use commas for multi values) + conf. : "" # optional - 'inlined' key=value added to the Hadoop configuration + concurrent_streams: 5 # optional - the number of concurrent streams (defaults to 5) + compress: "false" # optional - whether to compress the metadata or not (default) + chunk_size: "10mb" # optional - chunk size (disabled by default) +---- + +NOTE: Be careful when including a paths within the `uri` setting; Some implementations ignore them completely while +others consider them. In general, we recommend keeping the `uri` to a minimum and using the `path` element instead. + +===== Plugging other file-systems + +Any HDFS-compatible file-systems (like Amazon `s3://` or Google `gs://`) can be used as long as the proper Hadoop +configuration is passed to the Elasticsearch plugin. In practice, this means making sure the correct Hadoop configuration +files (`core-site.xml` and `hdfs-site.xml`) and its jars are available in plugin classpath, just as you would with any +other Hadoop client or job. + +Otherwise, the plugin will only read the _default_, vanilla configuration of Hadoop and will not be able to recognized +the plugged-in file-system. diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 6819748365c..faaa87302ee 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -67,16 +67,19 @@ cloud: protocol: https ---- -In addition, a proxy can be configured with the `proxy_host` and `proxy_port` settings (note that protocol can be -`http` or `https`): +In addition, a proxy can be configured with the `proxy.host`, `proxy.port`, `proxy.username` and `proxy.password` settings +(note that protocol can be `http` or `https`): [source,yaml] ---- cloud: aws: protocol: https - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself + password: theBestPasswordEver! ---- You can also set different proxies for `ec2` and `s3`: @@ -86,11 +89,17 @@ You can also set different proxies for `ec2` and `s3`: cloud: aws: s3: - proxy_host: proxy1.company.com - proxy_port: 8083 + proxy: + host: proxy1.company.com + port: 8083 + username: myself1 + password: theBestPasswordEver1! ec2: - proxy_host: proxy2.company.com - proxy_port: 8083 + proxy: + host: proxy2.company.com + port: 8083 + username: myself2 + password: theBestPasswordEver2! ---- [[repository-s3-usage-region]] @@ -198,7 +207,7 @@ The following settings are supported: request. Beyond this threshold, the S3 repository will use the http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html[AWS Multipart Upload API] to split the chunk into several parts, each of `buffer_size` length, and - to upload each part in its own request. Note that positioning a buffer + to upload each part in its own request. Note that setting a buffer size lower than `5mb` is not allowed since it will prevents the use of the Multipart API and may result in upload errors. Defaults to `5mb`. @@ -210,6 +219,22 @@ The following settings are supported: Makes repository read-only. coming[2.1.0] Defaults to `false`. +`canned_acl`:: + + The S3 repository supports all http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl[S3 canned ACLs] + : `private`, `public-read`, `public-read-write`, `authenticated-read`, `log-delivery-write`, + `bucket-owner-read`, `bucket-owner-full-control`. Defaults to `private`. + You could specify a canned ACL using the `canned_acl` setting. When the S3 repository + creates buckets and objects, it adds the canned ACL into the buckets and objects. + +`storage_class`:: + + Sets the S3 storage class type for the backup files. Values may be + `standard`, `reduced_redundancy`, `standard_ia`. Defaults to `standard`. + Due to the extra complexity with the Glacier class lifecycle, it is not + currently supported by the plugin. For more information about the + different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] + The S3 repositories use the same credentials as the rest of the AWS services provided by this plugin (`discovery`). See <> for details. @@ -327,6 +352,14 @@ to your URL provider. Note that this setting will be used for all S3 repositorie Different `endpoint`, `region` and `protocol` settings can be set on a per-repository basis See <> for details. +[[repository-s3-aws-vpc]] +[float] +==== AWS VPC Bandwidth Settings + +AWS instances resolve S3 endpoints to a public IP. If the elasticsearch instances reside in a private subnet in an AWS VPC then all traffic to S3 will go through that VPC's NAT instance. If your VPC's NAT instance is a smaller instance size (e.g. a t1.micro) or is handling a high volume of network traffic your bandwidth to S3 may be limited by that NAT instance's networking bandwidth limitations. + +Instances residing in a public subnet in an AWS VPC will connect to S3 via the VPC's internet gateway and not be bandwidth limited by the VPC's NAT instance. + [[repository-s3-testing]] ==== Testing AWS @@ -371,4 +404,3 @@ To run test: ---- mvn -Dtests.aws=true -Dtests.config=/path/to/config/file/elasticsearch.yml clean test ---- - diff --git a/docs/plugins/security.asciidoc b/docs/plugins/security.asciidoc index 1d425a74651..06672812fc9 100644 --- a/docs/plugins/security.asciidoc +++ b/docs/plugins/security.asciidoc @@ -18,7 +18,10 @@ mind when it comes to protecting your data. [float] === Community contributed security plugins -The following plugin has been contributed by our community: +The following plugins have been contributed by our community: + +* https://github.com/codecentric/elasticsearch-shield-kerberos-realm[Kerberos/SPNEGO Realm]: + Custom Shield realm to Authenticate HTTP and Transport requests via Kerberos/SPNEGO (by codecentric AG) * https://github.com/sscarduzio/elasticsearch-readonlyrest-plugin[Readonly REST]: High performance access control for Elasticsearch native REST API (by Simone Scarduzio) @@ -26,4 +29,4 @@ The following plugin has been contributed by our community: This community plugin appears to have been abandoned: * https://github.com/sonian/elasticsearch-jetty[Jetty HTTP transport plugin]: - Uses Jetty to provide SSL connections, basic authentication, and request logging (by Sonian Inc.) \ No newline at end of file + Uses Jetty to provide SSL connections, basic authentication, and request logging (by Sonian Inc.) diff --git a/docs/python/index.asciidoc b/docs/python/index.asciidoc index 66b79d7fa2b..64756adc444 100644 --- a/docs/python/index.asciidoc +++ b/docs/python/index.asciidoc @@ -7,7 +7,29 @@ ground for all Elasticsearch-related code in Python; because of this it tries to be opinion-free and very extendable. The full documentation is available at http://elasticsearch-py.rtfd.org/ -It can be installed with: +.Elasticsearch DSL +************************************************************************************ +For a more high level client library with more limited scope, have a look at +http://elasticsearch-dsl.rtfd.org/[elasticsearch-dsl] - a more pythonic library +sitting on top of `elasticsearch-py`. + +It provides a more convenient and idiomatic way to write and manipulate +http://elasticsearch-dsl.readthedocs.org/en/latest/search_dsl.html[queries]. It +stays close to the Elasticsearch JSON DSL, mirroring its terminology and +structure while exposing the whole range of the DSL from Python either directly +using defined classes or a queryset-like expressions. + +It also provides an optional +http://elasticsearch-dsl.readthedocs.org/en/latest/persistence.html#doctype[persistence +layer] for working with documents as Python objects in an ORM-like fashion: +defining mappings, retrieving and saving documents, wrapping the document data +in user-defined classes. +************************************************************************************ + + +=== Installation + +It can be installed with pip: [source,sh] ------------------------------------ @@ -16,13 +38,24 @@ pip install elasticsearch === Versioning -There are two branches for development - `master` and `0.4`. Master branch is -used to track all the changes for Elasticsearch 1.0 and beyond whereas 0.4 -tracks Elasticsearch 0.90. +There are two branches for development - `master` and `1.x`. Master branch is +used to track all the changes for Elasticsearch 2.0 and beyond whereas 1.x +tracks Elasticsearch 1.*. Releases with major version 1 (1.X.Y) are to be used with Elasticsearch 1.* and later, 0.4 releases are meant to work with Elasticsearch 0.90.*. +The recommended way to set your requirements in your `setup.py` or +`requirements.txt` is: + +------------------------------------ + # Elasticsearch 2.x + elasticsearch>=2.0.0,<3.0.0 + + # Elasticsearch 1.x + elasticsearch>=1.0.0,<2.0.0 +------------------------------------ + === Example use Simple use-case: @@ -71,6 +104,10 @@ The client's features include: * pluggable architecture +The client also contains a convenient set of +http://elasticsearch-py.readthedocs.org/en/master/helpers.html[helpers] for +some of the more engaging tasks like bulk indexing and reindexing. + === License diff --git a/docs/reference/aggregations/metrics.asciidoc b/docs/reference/aggregations/metrics.asciidoc index f80c36f2ebe..ae6bee2eb7d 100644 --- a/docs/reference/aggregations/metrics.asciidoc +++ b/docs/reference/aggregations/metrics.asciidoc @@ -19,6 +19,8 @@ include::metrics/extendedstats-aggregation.asciidoc[] include::metrics/geobounds-aggregation.asciidoc[] +include::metrics/geocentroid-aggregation.asciidoc[] + include::metrics/max-aggregation.asciidoc[] include::metrics/min-aggregation.asciidoc[] diff --git a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc new file mode 100644 index 00000000000..8f871dc8dbc --- /dev/null +++ b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc @@ -0,0 +1,104 @@ +[[search-aggregations-metrics-geocentroid-aggregation]] +=== Geo Centroid Aggregation + +A metric aggregation that computes the weighted centroid from all coordinate values for a <> field. + + +Example: + +[source,js] +-------------------------------------------------- +{ + "query" : { + "match" : { "crime" : "burglary" } + }, + "aggs" : { + "centroid" : { + "geo_centroid" : { + "field" : "location" <1> + } + } + } +} +-------------------------------------------------- + +<1> The `geo_centroid` aggregation specifies the field to use for computing the centroid. (NOTE: field must be a <> type) + +The above aggregation demonstrates how one would compute the centroid of the location field for all documents with a crime type of burglary + +The response for the above aggregation: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "centroid": { + "location": { + "lat": 80.45, + "lon": -160.22 + } + } + } +} +-------------------------------------------------- + + +The `geo_centroid` aggregation is more interesting when combined as a sub-aggregation to other bucket aggregations. + +Example: + +[source,js] +-------------------------------------------------- +{ + "query" : { + "match" : { "crime" : "burglary" } + }, + "aggs" : { + "towns" : { + "terms" : { "field" : "town" }, + "aggs" : { + "centroid" : { + "geo_centroid" : { "field" : "location" } + } + } + } + } +} +-------------------------------------------------- + +The above example uses `geo_centroid` as a sub-aggregation to a <> bucket aggregation +for finding the central location for all crimes of type burglary in each town. + +The response for the above aggregation: + +[source,js] +-------------------------------------------------- +{ + ... + + "buckets": [ + { + "key": "Los Altos", + "doc_count": 113, + "centroid": { + "location": { + "lat": 37.3924582824111, + "lon": -122.12104808539152 + } + } + }, + { + "key": "Mountain View", + "doc_count": 92, + "centroid": { + "location": { + "lat": 37.382152481004596, + "lon": -122.08116559311748 + } + } + } + ] +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc b/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc index 968c596019c..0487eb25d0d 100644 --- a/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc @@ -363,7 +363,7 @@ as your buckets: "buckets_path": "the_sum", "window" : 30, "model" : "simple", - "predict" 10 + "predict" : 10 } } -------------------------------------------------- @@ -445,4 +445,4 @@ minimization is linear to the size of the window being processed: excessively la Finally, minimization fits the model to the last `n` values, where `n = window`. This generally produces better forecasts into the future, since the parameters are tuned around the end of the series. It can, however, generate poorer fitting moving averages at the beginning of the series. -====== \ No newline at end of file +====== diff --git a/docs/reference/analysis/analyzers/snowball-analyzer.asciidoc b/docs/reference/analysis/analyzers/snowball-analyzer.asciidoc index 64804fcb359..1f489bd0cf5 100644 --- a/docs/reference/analysis/analyzers/snowball-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/snowball-analyzer.asciidoc @@ -15,7 +15,7 @@ filter>>. The Snowball Analyzer is a stemming analyzer from Lucene that is originally based on the snowball project from -http://snowball.tartarus.org[snowball.tartarus.org]. +http://snowballstem.org[snowballstem.org]. Sample usage: diff --git a/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc index 8c6d767b6f9..b5d1b5cde10 100644 --- a/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc @@ -12,7 +12,7 @@ combined unigram+bigram approach. Bigrams are generated for characters in `han`, `hiragana`, `katakana` and `hangul`, but bigrams can be disabled for particular scripts with the -`ignore_scripts` parameter. All non-CJK input is passed through unmodified. +`ignored_scripts` parameter. All non-CJK input is passed through unmodified. [source,js] -------------------------------------------------- @@ -28,7 +28,7 @@ Bigrams are generated for characters in `han`, `hiragana`, `katakana` and "filter" : { "han_bigrams_filter" : { "type" : "cjk_bigram", - "ignore_scripts": [ + "ignored_scripts": [ "hiragana", "katakana", "hangul" diff --git a/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc index f0659e00868..eb1469af803 100644 --- a/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc @@ -1,7 +1,7 @@ [[analysis-common-grams-tokenfilter]] === Common Grams Token Filter -Token filter that generates bigrams for frequently occuring terms. +Token filter that generates bigrams for frequently occurring terms. Single terms are still indexed. It can be used as an alternative to the <> when we don't want to completely ignore common terms. diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 98fbcae2747..6bbd0419143 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -360,6 +360,22 @@ are: `s`:: Second `ms`:: Milli-second +[[size-units]] +[float] +=== Data size units + +Whenever the size of data needs to be specified, eg when setting a buffer size +parameter, the value must specify the unit, like `10kb` for 10 kilobytes. The +supported units are: + +[horizontal] +`b`:: Bytes +`kb`:: Kilobytes +`mb`:: Megabytes +`gb`:: Gigabytes +`tb`:: Terabytes +`pb`:: Petabytes + [[distance-units]] [float] === Distance Units diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc index 0e61c27618d..ed1f09b0f60 100644 --- a/docs/reference/cat.asciidoc +++ b/docs/reference/cat.asciidoc @@ -122,8 +122,12 @@ include::cat/plugins.asciidoc[] include::cat/recovery.asciidoc[] +include::cat/repositories.asciidoc[] + include::cat/thread_pool.asciidoc[] include::cat/shards.asciidoc[] include::cat/segments.asciidoc[] + +include::cat/snapshots.asciidoc[] diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc index 4b1cde5f472..a790b3d15cd 100644 --- a/docs/reference/cat/alias.asciidoc +++ b/docs/reference/cat/alias.asciidoc @@ -7,7 +7,7 @@ including filter and routing infos. [source,sh] -------------------------------------------------- % curl '192.168.56.10:9200/_cat/aliases?v' -alias index filter indexRouting searchRouting +alias index filter routing.index routing.search alias2 test1 * - - alias4 test1 - 2 1,2 alias1 test1 - - - diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc index 6b12a15dee2..bf9b3f17443 100644 --- a/docs/reference/cat/health.asciidoc +++ b/docs/reference/cat/health.asciidoc @@ -7,9 +7,9 @@ timestamping. [source,sh] -------------------------------------------------- -% curl 192.168.56.10:9200/_cat/health +% curl localhost:9200/_cat/health 1384308967 18:16:07 foo green 3 3 3 3 0 0 0 -% curl '192.168.56.10:9200/_cat/health?v&ts=0' +% curl 'localhost:9200/_cat/health?v&ts=0' cluster status nodeTotal nodeData shards pri relo init unassign tasks foo green 3 3 3 3 0 0 0 0 -------------------------------------------------- @@ -35,7 +35,7 @@ to track its progress is by using this command in a delayed loop: [source,sh] -------------------------------------------------- -% while true; do curl 192.168.56.10:9200/_cat/health; sleep 120; done +% while true; do curl localhost:9200/_cat/health; sleep 120; done 1384309446 18:24:06 foo red 3 3 20 20 0 0 1812 0 1384309566 18:26:06 foo yellow 3 3 950 916 0 12 870 0 1384309686 18:28:06 foo yellow 3 3 1328 916 0 12 492 0 diff --git a/docs/reference/cat/repositories.asciidoc b/docs/reference/cat/repositories.asciidoc new file mode 100644 index 00000000000..5fb68a8929c --- /dev/null +++ b/docs/reference/cat/repositories.asciidoc @@ -0,0 +1,14 @@ +[[cat-repositories]] +== cat repositories + +The `repositories` command shows the snapshot repositories registered in the cluster. + +[source,sh] +-------------------------------------------------- +% curl 'localhost:9200/_cat/repositories?v' +id type +repo1 fs +repo2 s3 +-------------------------------------------------- + +We can quickly see which repositories are registered and their type. diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc new file mode 100644 index 00000000000..3d34cd51e6d --- /dev/null +++ b/docs/reference/cat/snapshots.asciidoc @@ -0,0 +1,19 @@ +[[cat-snapshots]] +== cat snapshots + +The `snapshots` command shows all snapshots that belong to a specific repository. +To find a list of available repositories to query, the command `/_cat/repositories` can be used. +Querying the snapshots of a repository named `repo1` then looks as follows. + +[source,sh] +-------------------------------------------------- +% curl 'localhost:9200/_cat/snapshots/repo1?v' +id status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards +snap1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5 +snap2 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10 +-------------------------------------------------- + +Each snapshot contains information about when it was started and stopped. +Start and stop timestamps are available in two formats. +The `HH:MM:SS` output is simply for quick human consumption. +The epoch time retains more information, including date, and is machine sortable if the snapshot process spans days. \ No newline at end of file diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc index 508b3ee167b..f50dc369c3d 100644 --- a/docs/reference/cat/thread_pool.asciidoc +++ b/docs/reference/cat/thread_pool.asciidoc @@ -57,7 +57,7 @@ Currently available <>: |`get` |`g` |Thread pool used for <> operations |`index` |`i` |Thread pool used for <>/<> operations |`management` |`ma` |Thread pool used for management of Elasticsearch (e.g. cluster management) -|`optimize` |`o` |Thread pool used for <> operations +|`force_merge` |`fm` |Thread pool used for <> operations |`percolate` |`p` |Thread pool used for <> operations |`refresh` |`r` |Thread pool used for <> operations |`search` |`s` |Thread pool used for <>/<> operations @@ -107,4 +107,4 @@ other details like the `ip` of the responding node(s). |`host` |`h` |The hostname for the current node |`ip` |`i` |The IP address for the current node |`port` |`po` |The bound transport port for the current node -|======================================================================= \ No newline at end of file +|======================================================================= diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index 7d9bdc1b041..137b4ac48cd 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -87,6 +87,10 @@ The cluster health API accepts the following request parameters: A time based parameter controlling how long to wait if one of the wait_for_XXX are provided. Defaults to `30s`. +`local`:: + If `true` returns the local node information and does not provide + the state from master node. Default: `false`. + The following is an example of getting the cluster health at the `shards` level: diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc index 215e8449f37..a3072768ca6 100644 --- a/docs/reference/cluster/nodes-info.asciidoc +++ b/docs/reference/cluster/nodes-info.asciidoc @@ -54,6 +54,11 @@ the operating system: `os.available_processors`:: Number of processors available to the Java virtual machine +`os.allocated_processors`:: + The number of processors actually used to calculate thread pool size. This number can be set + with the `processors` setting of a node and defaults to the number of processors reported by the OS. + In both cases this number will never be larger than 32. + [float] [[process-info]] ==== Process information diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index b22312e2130..cd6c7db2d41 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -57,6 +57,9 @@ of `indices`, `os`, `process`, `jvm`, `transport`, `http`, `breaker`:: Statistics about the field data circuit breaker +`discovery`:: + Statistics about the discovery + [source,js] -------------------------------------------------- # return indices and os @@ -125,7 +128,10 @@ the operating system: `os.timestamp`:: Last time the operating system statistics have been refreshed -`os.load_average`:: +`os.cpu.percent`:: + Recent CPU usage for the whole system, or -1 if not supported + +`os.cpu.load_average`:: System load average for the last minute, or -1 if not supported `os.mem.total_in_bytes`:: diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 089af3e32ed..3aacdc7ed10 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -41,7 +41,7 @@ The index operation is successful in the case `successful` is at least 1. NOTE: Replica shards may not all be started when an indexing operation successfully returns (by default, a quorum is required). In that case, `total` will be equal to the total shards based on the index replica settings and - `successful` will be equal to the number of shard started (primary plus replicas). As there were no failures, + `successful` will be equal to the number of shards started (primary plus replicas). As there were no failures, the `failed` will be 0. [float] @@ -231,7 +231,7 @@ the `routing` parameter provided: "kimchy". When setting up explicit mapping, the `_routing` field can be optionally used to direct the index operation to extract the routing value from the document itself. This does come at the (very minimal) cost of an -additional document parsing pass. If the `_routing` mapping is defined, +additional document parsing pass. If the `_routing` mapping is defined and set to be `required`, the index operation will fail if no routing value is provided or extracted. diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index a30046e1333..8ff832c673f 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -115,7 +115,7 @@ Let's download the Elasticsearch {version} tar as follows (Windows users should ["source","sh",subs="attributes,callouts"] -------------------------------------------------- -curl -L -O https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-{version}.tar.gz +curl -L -O https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/{version}/elasticsearch-{version}.tar.gz -------------------------------------------------- Then extract it as follows (Windows users should unzip the zip package): @@ -1072,4 +1072,3 @@ There are a many other aggregations capabilities that we won't go into detail he == Conclusion Elasticsearch is both a simple and complex product. We've so far learned the basics of what it is, how to look inside of it, and how to work with it using some of the REST APIs. I hope that this tutorial has given you a better understanding of what Elasticsearch is and more importantly, inspired you to further experiment with the rest of its great features! - diff --git a/docs/reference/index-modules/allocation/delayed.asciidoc b/docs/reference/index-modules/allocation/delayed.asciidoc index 8d936383847..baaa3cb944d 100644 --- a/docs/reference/index-modules/allocation/delayed.asciidoc +++ b/docs/reference/index-modules/allocation/delayed.asciidoc @@ -58,7 +58,9 @@ With delayed allocation enabled, the above scenario changes to look like this: NOTE: This setting will not affect the promotion of replicas to primaries, nor will it affect the assignment of replicas that have not been assigned -previously. +previously. In particular, delayed allocation does not come into effect after a full cluster restart. +Also, in case of a master failover situation, elapsed delay time is forgotten +(i.e. reset to the full initial delay). ==== Cancellation of shard relocation diff --git a/docs/reference/index-modules/allocation/total_shards.asciidoc b/docs/reference/index-modules/allocation/total_shards.asciidoc index 3e1b3ab16e8..691ab8d937d 100644 --- a/docs/reference/index-modules/allocation/total_shards.asciidoc +++ b/docs/reference/index-modules/allocation/total_shards.asciidoc @@ -14,10 +14,17 @@ number of shards from a single index allowed per node: The maximum number of shards (replicas and primaries) that will be allocated to a single node. Defaults to unbounded. +You can also limit the amount of shards a node can have regardless of the index: + +`cluster.routing.allocation.total_shards_per_node`:: + + The maximum number of shards (replicas and primaries) that will be + allocated to a single node globally. Defaults to unbounded (-1). + [WARNING] ======================================= -This setting imposes a hard limit which can result in some shards not -being allocated. +Thess setting impose a hard limit which can result in some shards not being +allocated. Use with caution. ======================================= diff --git a/docs/reference/index-modules/analysis.asciidoc b/docs/reference/index-modules/analysis.asciidoc index 93f569b962a..709e5be5706 100644 --- a/docs/reference/index-modules/analysis.asciidoc +++ b/docs/reference/index-modules/analysis.asciidoc @@ -7,6 +7,6 @@ which are: * added to the inverted index in order to make the document searchable * used by high level queries such as the <> - to generate seach terms. + to generate search terms. See <> for configuration details. diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc index 82399a8cc79..ddec26b8030 100644 --- a/docs/reference/index-modules/similarity.asciidoc +++ b/docs/reference/index-modules/similarity.asciidoc @@ -112,7 +112,10 @@ Type name: `DFR` ==== IB similarity. http://lucene.apache.org/core/5_2_1/core/org/apache/lucene/search/similarities/IBSimilarity.html[Information -based model] . This similarity has the following options: +based model] . The algorithm is based on the concept that the information content in any symbolic 'distribution' +sequence is primarily determined by the repetitive usage of its basic elements. +For written texts this challenge would correspond to comparing the writing styles of diferent authors. +This similarity has the following options: [horizontal] `distribution`:: Possible values: `ll` and `spl`. @@ -138,11 +141,11 @@ Type name: `LMDirichlet` ==== LM Jelinek Mercer similarity. http://lucene.apache.org/core/5_2_1/core/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.html[LM -Jelinek Mercer similarity] . This similarity has the following options: +Jelinek Mercer similarity] . The algorithm attempts to capture important patterns in the text, while leaving out noise. This similarity has the following options: [horizontal] `lambda`:: The optimal value depends on both the collection and the query. The optimal value is around `0.1` -for title queries and `0.7` for long queries. Default to `0.1`. +for title queries and `0.7` for long queries. Default to `0.1`. When value approaches `0`, documents that match more query terms will be ranked higher than those that match fewer terms. Type name: `LMJelinekMercer` @@ -157,7 +160,7 @@ implementation used for these two methods, while not changing the `default`, it is possible to configure a similarity with the name `base`. This similarity will then be used for the two methods. -You can change the default similarity for all fields like this: +You can change the default similarity for all fields by putting the following setting into `elasticsearch.yml`: [source,js] -------------------------------------------------- diff --git a/docs/reference/index-modules/translog.asciidoc b/docs/reference/index-modules/translog.asciidoc index 5cb2e4b65a3..ad704299529 100644 --- a/docs/reference/index-modules/translog.asciidoc +++ b/docs/reference/index-modules/translog.asciidoc @@ -68,7 +68,7 @@ update, or bulk request. This setting accepts the following parameters: (default) `fsync` and commit after every request. In the event of hardware failure, all acknowledged writes will already have been - commited to disk. + committed to disk. `async`:: diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 8e34747ffdb..4acd1f16eab 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -7,6 +7,9 @@ :jdk: 1.8.0_25 :defguide: https://www.elastic.co/guide/en/elasticsearch/guide/current :plugins: https://www.elastic.co/guide/en/elasticsearch/plugins/master +:javaclient: https://www.elastic.co/guide/en/elasticsearch/client/java-api/master/ +:issue: https://github.com/elastic/elasticsearch/issues/ +:pull: https://github.com/elastic/elasticsearch/pull/ include::getting-started.asciidoc[] @@ -42,6 +45,10 @@ include::testing.asciidoc[] include::glossary.asciidoc[] +////////////////////////////////////////// + include::release-notes.asciidoc[] +////////////////////////////////////////// + include::redirects.asciidoc[] diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index 634e48801e5..da41ac52df5 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -59,7 +59,7 @@ and warmers. * <> * <> * <> -* <> +* <> * <> -- @@ -110,7 +110,7 @@ include::indices/flush.asciidoc[] include::indices/refresh.asciidoc[] -include::indices/optimize.asciidoc[] +include::indices/forcemerge.asciidoc[] include::indices/upgrade.asciidoc[] diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 9a65c89837d..57faa9718f9 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -63,7 +63,22 @@ curl -XPOST 'http://localhost:9200/_aliases' -d ' }' -------------------------------------------------- -Alternatively, you can use a glob pattern to associate an alias to +Multiple indices can be specified for an action with the `indices` array syntax: + +[source,js] +-------------------------------------------------- +curl -XPOST 'http://localhost:9200/_aliases' -d ' +{ + "actions" : [ + { "add" : { "indices" : ["test1", "test2"], "alias" : "alias1" } } + ] +}' +-------------------------------------------------- + +To specify multiple aliases in one action, the corresponding `aliases` array +syntax exists as well. + +For the example above, a glob pattern can also be used to associate an alias to more than one index that share a common name: [source,js] diff --git a/docs/reference/indices/analyze.asciidoc b/docs/reference/indices/analyze.asciidoc index 1a256a6330a..1e8cd77ef09 100644 --- a/docs/reference/indices/analyze.asciidoc +++ b/docs/reference/indices/analyze.asciidoc @@ -100,3 +100,74 @@ provided it doesn't start with `{` : -------------------------------------------------- curl -XGET 'localhost:9200/_analyze?tokenizer=keyword&token_filters=lowercase&char_filters=html_strip' -d 'this is a test' -------------------------------------------------- + +=== Explain Analyze + +If you want to get more advanced details, set `explain` to `true` (defaults to `false`). It will output all token attributes for each token. +You can filter token attributes you want to output by setting `attributes` option. + +experimental[The format of the additional detail information is experimental and can change at any time] + +[source,js] +-------------------------------------------------- +GET test/_analyze +{ + "tokenizer" : "standard", + "token_filters" : ["snowball"], + "text" : "detailed output", + "explain" : true, + "attributes" : ["keyword"] <1> +} +-------------------------------------------------- +// AUTOSENSE +<1> Set "keyword" to output "keyword" attribute only + +coming[2.0.0, body based parameters were added in 2.0.0] + +The request returns the following result: + +[source,js] +-------------------------------------------------- +{ + "detail" : { + "custom_analyzer" : true, + "charfilters" : [ ], + "tokenizer" : { + "name" : "standard", + "tokens" : [ { + "token" : "detailed", + "start_offset" : 0, + "end_offset" : 8, + "type" : "", + "position" : 0 + }, { + "token" : "output", + "start_offset" : 9, + "end_offset" : 15, + "type" : "", + "position" : 1 + } ] + }, + "tokenfilters" : [ { + "name" : "snowball", + "tokens" : [ { + "token" : "detail", + "start_offset" : 0, + "end_offset" : 8, + "type" : "", + "position" : 0, + "keyword" : false <1> + }, { + "token" : "output", + "start_offset" : 9, + "end_offset" : 15, + "type" : "", + "position" : 1, + "keyword" : false <1> + } ] + } ] + } +} +-------------------------------------------------- +<1> Output only "keyword" attribute, since specify "attributes" in the request. + diff --git a/docs/reference/indices/forcemerge.asciidoc b/docs/reference/indices/forcemerge.asciidoc new file mode 100644 index 00000000000..a33b7fdfe2c --- /dev/null +++ b/docs/reference/indices/forcemerge.asciidoc @@ -0,0 +1,51 @@ +[[indices-forcemerge]] +== Force Merge + +The force merge API allows to force merging of one or more indices through an +API. The merge relates to the number of segments a Lucene index holds within +each shard. The force merge operation allows to reduce the number of segments by +merging them. + +This call will block until the merge is complete. If the http connection is +lost, the request will continue in the background, and any new requests will +block until the previous force merge is complete. + +[source,js] +-------------------------------------------------- +$ curl -XPOST 'http://localhost:9200/twitter/_forcemerge' +-------------------------------------------------- + +[float] +[[forcemerge-parameters]] +=== Request Parameters + +The force merge API accepts the following request parameters: + +[horizontal] +`max_num_segments`:: The number of segments to merge to. To fully +merge the index, set it to `1`. Defaults to simply checking if a +merge needs to execute, and if so, executes it. + +`only_expunge_deletes`:: Should the merge process only expunge segments with +deletes in it. In Lucene, a document is not deleted from a segment, just marked +as deleted. During a merge process of segments, a new segment is created that +does not have those deletes. This flag allows to only merge segments that have +deletes. Defaults to `false`. Note that this won't override the +`index.merge.policy.expunge_deletes_allowed` threshold. + +`flush`:: Should a flush be performed after the forced merge. Defaults to +`true`. + +[float] +[[forcemerge-multi-index]] +=== Multi Index + +The force merge API can be applied to more than one index with a single call, or +even on `_all` the indices. + +[source,js] +-------------------------------------------------- +$ curl -XPOST 'http://localhost:9200/kimchy,elasticsearch/_forcemerge' + +$ curl -XPOST 'http://localhost:9200/_forcemerge' +-------------------------------------------------- diff --git a/docs/reference/indices/open-close.asciidoc b/docs/reference/indices/open-close.asciidoc index 29259d2b331..7f230b05a0c 100644 --- a/docs/reference/indices/open-close.asciidoc +++ b/docs/reference/indices/open-close.asciidoc @@ -26,4 +26,8 @@ or specifying patterns that identify them all (e.g. `*`). Identifying indices via wildcards or `_all` can be disabled by setting the `action.destructive_requires_name` flag in the config file to `true`. -This setting can also be changed via the cluster update settings api. \ No newline at end of file +This setting can also be changed via the cluster update settings api. + +Closed indices consume a significant amount of disk-space which can cause problems +issues in managed environments. Closing indices can be disabled via the cluster settings +API by setting `cluster.indices.close.enable` to `false`. The default is `true`. \ No newline at end of file diff --git a/docs/reference/indices/optimize.asciidoc b/docs/reference/indices/optimize.asciidoc deleted file mode 100644 index 799f0674fec..00000000000 --- a/docs/reference/indices/optimize.asciidoc +++ /dev/null @@ -1,52 +0,0 @@ -[[indices-optimize]] -== Optimize - -The optimize API allows to optimize one or more indices through an API. -The optimize process basically optimizes the index for faster search -operations (and relates to the number of segments a Lucene index holds -within each shard). The optimize operation allows to reduce the number -of segments by merging them. - -This call will block until the optimize is complete. If the http connection -is lost, the request will continue in the background, and -any new requests will block until the previous optimize is complete. - -[source,js] --------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/twitter/_optimize' --------------------------------------------------- - -[float] -[[optimize-parameters]] -=== Request Parameters - -The optimize API accepts the following request parameters as query arguments: - -[horizontal] -`max_num_segments`:: The number of segments to optimize to. To fully -optimize the index, set it to `1`. Defaults to simply checking if a -merge needs to execute, and if so, executes it. - -`only_expunge_deletes`:: Should the optimize process only expunge segments with -deletes in it. In Lucene, a document is not deleted from a segment, just marked -as deleted. During a merge process of segments, a new segment is created that -does not have those deletes. This flag allows to only merge segments that have -deletes. Defaults to `false`. Note that this won't override the -`index.merge.policy.expunge_deletes_allowed` threshold. - -`flush`:: Should a flush be performed after the optimize. Defaults to -`true`. - -[float] -[[optimize-multi-index]] -=== Multi Index - -The optimize API can be applied to more than one index with a single -call, or even on `_all` the indices. - -[source,js] --------------------------------------------------- -$ curl -XPOST 'http://localhost:9200/kimchy,elasticsearch/_optimize' - -$ curl -XPOST 'http://localhost:9200/_optimize?only_expunge_deletes=true' --------------------------------------------------- diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc index 7d6e6587786..7dd2389e824 100644 --- a/docs/reference/indices/put-mapping.asciidoc +++ b/docs/reference/indices/put-mapping.asciidoc @@ -99,7 +99,7 @@ PUT my_index <1> } } -PUT my_index/mapping/user +PUT my_index/_mapping/user { "properties": { "name": { diff --git a/docs/reference/indices/update-settings.asciidoc b/docs/reference/indices/update-settings.asciidoc index d5d00047e9c..574565192e4 100644 --- a/docs/reference/indices/update-settings.asciidoc +++ b/docs/reference/indices/update-settings.asciidoc @@ -63,11 +63,11 @@ curl -XPUT localhost:9200/test/_settings -d '{ } }' -------------------------------------------------- -And, an optimize should be called: +And, a force merge should be called: [source,js] -------------------------------------------------- -curl -XPOST 'http://localhost:9200/test/_optimize?max_num_segments=5' +curl -XPOST 'http://localhost:9200/test/_forcemerge?max_num_segments=5' -------------------------------------------------- [float] diff --git a/docs/reference/indices/upgrade.asciidoc b/docs/reference/indices/upgrade.asciidoc index f381526c3ef..c9d371bd8ca 100644 --- a/docs/reference/indices/upgrade.asciidoc +++ b/docs/reference/indices/upgrade.asciidoc @@ -70,7 +70,7 @@ $ curl -XPOST 'http://localhost:9200/twitter/_upgrade' NOTE: Upgrading is an I/O intensive operation, and is limited to processing a single shard per node at a time. It also is not allowed to run at the same -time as optimize. +time as an optimize/force-merge. This call will block until the upgrade is complete. If the http connection is lost, the request will continue in the background, and @@ -131,4 +131,4 @@ curl 'http://localhost:9200/twitter/_upgrade?pretty&human' The level of details in the upgrade status command can be controlled by setting `level` parameter to `cluster`, `index` (default) or `shard` levels. For example, you can run the upgrade status command with `level=shard` to -get detailed upgrade information of each individual shard. \ No newline at end of file +get detailed upgrade information of each individual shard. diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index e38fc31cb37..b60c5f0510e 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -148,13 +148,14 @@ PUT my_index/my_type/1 [[match-pattern]] ==== `match_pattern` -The `match_pattern` parameter behaves just like the `match` parameter, but -supports full Java regular expression matching on the field name instead of -simple wildcards, for instance: +The `match_pattern` parameter adjusts the behavior of the `match` parameter +such that it supports full Java regular expression matching on the field name +instead of simple wildcards, for instance: [source,js] -------------------------------------------------- - "match_pattern": "^profit_\d+$" + "match_pattern": "regex", + "match": "^profit_\d+$" -------------------------------------------------- [[path-match-unmatch]] diff --git a/docs/reference/mapping/fields/all-field.asciidoc b/docs/reference/mapping/fields/all-field.asciidoc index 00c8d3b245b..e206dcd125f 100644 --- a/docs/reference/mapping/fields/all-field.asciidoc +++ b/docs/reference/mapping/fields/all-field.asciidoc @@ -2,7 +2,7 @@ === `_all` field The `_all` field is a special _catch-all_ field which concatenates the values -of all of the other fields into one big string, which is then +of all of the other fields into one big string, using space as a delimiter, which is then <> and indexed, but not stored. This means that it can be searched, but not retrieved. diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc index 2c40f72bbea..bafc3e3f7d9 100644 --- a/docs/reference/mapping/fields/field-names-field.asciidoc +++ b/docs/reference/mapping/fields/field-names-field.asciidoc @@ -3,9 +3,8 @@ The `_field_names` field indexes the names of every field in a document that contains any value other than `null`. This field is used by the -<> and <> -queries to find documents that either have or don't have any non-+null+ value -for a particular field. +<> query to find documents that +either have or don't have any non-+null+ value for a particular field. The value of the `_field_name` field is accessible in queries, aggregations, and scripts: @@ -49,7 +48,6 @@ GET my_index/_search -------------------------- // AUTOSENSE -<1> Querying on the `_field_names` field (also see the <> and <> queries) +<1> Querying on the `_field_names` field (also see the <> query) <2> Aggregating on the `_field_names` field <3> Accessing the `_field_names` field in scripts (inline scripts must be <> for this example to work) - diff --git a/docs/reference/mapping/fields/index-field.asciidoc b/docs/reference/mapping/fields/index-field.asciidoc index 0a1928f15ff..bc506db1836 100644 --- a/docs/reference/mapping/fields/index-field.asciidoc +++ b/docs/reference/mapping/fields/index-field.asciidoc @@ -1,10 +1,17 @@ [[mapping-index-field]] === `_index` field -When performing queries across multiple indexes, it is sometimes desirable -to add query clauses that are associated with documents of only certain -indexes. The `_index` field allows matching on the index a document was -indexed into. Its value is accessible in queries, aggregations, scripts, and when sorting: +When performing queries across multiple indexes, it is sometimes desirable to +add query clauses that are associated with documents of only certain indexes. +The `_index` field allows matching on the index a document was indexed into. +Its value is accessible in `term`, or `terms` queries, aggregations, +scripts, and when sorting: + +NOTE: The `_index` is exposed as a virtual field -- it is not added to the +Lucene index as a real field. This means that you can use the `_index` field +in a `term` or `terms` query (or any query that is rewritten to a `term` +query, such as the `match`, `query_string` or `simple_query_string` query), +but it does not support `prefix`, `wildcard`, `regexp`, or `fuzzy` queries. [source,js] -------------------------- diff --git a/docs/reference/mapping/fields/ttl-field.asciidoc b/docs/reference/mapping/fields/ttl-field.asciidoc index 07ce8a86b9e..d81582c9078 100644 --- a/docs/reference/mapping/fields/ttl-field.asciidoc +++ b/docs/reference/mapping/fields/ttl-field.asciidoc @@ -62,7 +62,7 @@ PUT my_index "my_type": { "_ttl": { "enabled": true, - "defaut": "5m" + "default": "5m" } } } diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc index d47f7cbfe9f..a0108b899b9 100644 --- a/docs/reference/mapping/params/doc-values.asciidoc +++ b/docs/reference/mapping/params/doc-values.asciidoc @@ -9,11 +9,13 @@ of documents that contain the term. Sorting, aggregations, and access to field values in scripts requires a different data access pattern. Instead of lookup up the term and finding documents, we need to be able to look up the document and find the terms that -is has in a field. +it has in a field. Doc values are the on-disk data structure, built at document index time, which -makes this data access pattern possible. Doc values are supported on almost -all field types, with the __notable exception of `analyzed` string fields__. +makes this data access pattern possible. They store the same values as the +`_source` but in a column-oriented fashion that is way more efficient for +sorting and aggregations. Doc values are supported on almost all field types, +with the __notable exception of `analyzed` string fields__. All fields which support doc values have them enabled by default. If you are sure that you don't need to sort or aggregate on a field, or access the field diff --git a/docs/reference/mapping/params/multi-fields.asciidoc b/docs/reference/mapping/params/multi-fields.asciidoc index bcb85c6aef9..994d2fddbc1 100644 --- a/docs/reference/mapping/params/multi-fields.asciidoc +++ b/docs/reference/mapping/params/multi-fields.asciidoc @@ -83,12 +83,12 @@ PUT my_index "my_type": { "properties": { "text": { <1> - "type": "string" - }, - "fields": { - "english": { <2> - "type": "string", - "analyzer": "english" + "type": "string", + "fields": { + "english": { <2> + "type": "string", + "analyzer": "english" + } } } } diff --git a/docs/reference/mapping/params/null-value.asciidoc b/docs/reference/mapping/params/null-value.asciidoc index 552ce66ded8..4d70d4a6ac5 100644 --- a/docs/reference/mapping/params/null-value.asciidoc +++ b/docs/reference/mapping/params/null-value.asciidoc @@ -53,7 +53,3 @@ IMPORTANT: The `null_value` needs to be the same datatype as the field. For instance, a `long` field cannot have a string `null_value`. String fields which are `analyzed` will also pass the `null_value` through the configured analyzer. - -Also see the <> for its `null_value` support. - - diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 52cd41e37e5..60d96577a43 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -37,8 +37,8 @@ document: Attachment datatype:: - See the https://github.com/elastic/elasticsearch-mapper-attachments[mapper attachment plugin] - which supports indexing ``attachments'' like Microsoft Office formats, Open + See the {plugins}/mapper-attachments.html[`mapper-attachments`] plugin + which supports indexing `attachments` like Microsoft Office formats, Open Document formats, ePub, HTML, etc. into an `attachment` datatype. [float] diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc index ff76fbebf90..4e5f6b4bc27 100644 --- a/docs/reference/mapping/types/binary.asciidoc +++ b/docs/reference/mapping/types/binary.asciidoc @@ -40,8 +40,9 @@ The following parameters are accepted by `binary` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` or `false` (default). + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index 5ebcc651d09..9ff1aa13dde 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -98,8 +98,9 @@ The following parameters are accepted by `boolean` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index c8067a89fdf..118c1a85d4f 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -97,8 +97,9 @@ The following parameters are accepted by `date` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index 0049d8f93ac..ad7230e68d6 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -108,8 +108,9 @@ The following parameters are accepted by `geo_point` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index d974847a98a..1f0c76e1b93 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -17,13 +17,13 @@ The geo_shape mapping maps geo_json geometry objects to the geo_shape type. To enable it, users must explicitly map fields to the geo_shape type. -[cols="<,<",options="header",] +[cols="<,<,<",options="header",] |======================================================================= -|Option |Description +|Option |Description| Default |`tree` |Name of the PrefixTree implementation to be used: `geohash` for -GeohashPrefixTree and `quadtree` for QuadPrefixTree. Defaults to -`geohash`. +GeohashPrefixTree and `quadtree` for QuadPrefixTree. +| `geohash` |`precision` |This parameter may be used instead of `tree_levels` to set an appropriate value for the `tree_levels` parameter. The value @@ -31,7 +31,8 @@ specifies the desired precision and Elasticsearch will calculate the best tree_levels value to honor this precision. The value should be a number followed by an optional distance unit. Valid distance units include: `in`, `inch`, `yd`, `yard`, `mi`, `miles`, `km`, `kilometers`, -`m`,`meters` (default), `cm`,`centimeters`, `mm`, `millimeters`. +`m`,`meters`, `cm`,`centimeters`, `mm`, `millimeters`. +| `meters` |`tree_levels` |Maximum number of layers to be used by the PrefixTree. This can be used to control the precision of shape representations and @@ -41,27 +42,40 @@ certain level of understanding of the underlying implementation, users may use the `precision` parameter instead. However, Elasticsearch only uses the tree_levels parameter internally and this is what is returned via the mapping API even if you use the precision parameter. +| `50m` + +|`strategy` |The strategy parameter defines the approach for how to +represent shapes at indexing and search time. It also influences the +capabilities available so it is recommended to let Elasticsearch set +this parameter automatically. There are two strategies available: +`recursive` and `term`. Term strategy supports point types only (the +`points_only` parameter will be automatically set to true) while +Recursive strategy supports all shape types. (IMPORTANT: see +<> for more detailed information) +| `recursive` |`distance_error_pct` |Used as a hint to the PrefixTree about how precise it should be. Defaults to 0.025 (2.5%) with 0.5 as the maximum -supported value. PERFORMANCE NOTE: This value will be default to 0 if a `precision` or +supported value. PERFORMANCE NOTE: This value will default to 0 if a `precision` or `tree_level` definition is explicitly defined. This guarantees spatial precision at the level defined in the mapping. This can lead to significant memory usage for high resolution shapes with low error (e.g., large shapes at 1m with < 0.001 error). To improve indexing performance (at the cost of query accuracy) explicitly define `tree_level` or `precision` along with a reasonable `distance_error_pct`, noting that large shapes will have greater false positives. +| `0.025` |`orientation` |Optionally define how to interpret vertex order for polygons / multipolygons. This parameter defines one of two coordinate system rules (Right-hand or Left-hand) each of which can be specified in three -different ways. 1. Right-hand rule (default): `right`, `ccw`, `counterclockwise`, +different ways. 1. Right-hand rule: `right`, `ccw`, `counterclockwise`, 2. Left-hand rule: `left`, `cw`, `clockwise`. The default orientation (`counterclockwise`) complies with the OGC standard which defines outer ring vertices in counterclockwise order with inner ring(s) vertices (holes) in clockwise order. Setting this parameter in the geo_shape mapping explicitly sets vertex order for the coordinate list of a geo_shape field but can be overridden in each individual GeoJSON document. +| `ccw` |`points_only` |Setting this option to `true` (defaults to `false`) configures the `geo_shape` field type for point shapes only (NOTE: Multi-Points are not @@ -70,18 +84,21 @@ yet supported). This optimizes index and search performance for the `geohash` an queries can not be executed on `geo_point` field types. This option bridges the gap by improving point performance on a `geo_shape` field so that `geo_shape` queries are optimal on a point only field. +| `false` |======================================================================= +[[prefix-trees]] [float] ==== Prefix trees To efficiently represent shapes in the index, Shapes are converted into -a series of hashes representing grid squares using implementations of a -PrefixTree. The tree notion comes from the fact that the PrefixTree uses -multiple grid layers, each with an increasing level of precision to -represent the Earth. +a series of hashes representing grid squares (commonly referred to as "rasters") +using implementations of a PrefixTree. The tree notion comes from the fact that +the PrefixTree uses multiple grid layers, each with an increasing level of +precision to represent the Earth. This can be thought of as increasing the level +of detail of a map or image at higher zoom levels. Multiple PrefixTree implementations are provided: @@ -100,6 +117,29 @@ longitude the resulting hash is a bit set. A tree level in a quad tree represents 2 bits in this bit set, one for each coordinate. The maximum amount of levels for the quad trees in Elasticsearch is 50. +[[spatial-strategy]] +[float] +===== Spatial strategies +The PrefixTree implementations rely on a SpatialStrategy for decomposing +the provided Shape(s) into approximated grid squares. Each strategy answers +the following: + +* What type of Shapes can be indexed? +* What types of Query Operations and Shapes can be used? +* Does it support more than one Shape per field? + +The following Strategy implementations (with corresponding capabilities) +are provided: + +[cols="<,<,<,<",options="header",] +|======================================================================= +|Strategy |Supported Shapes |Supported Queries |Multiple Shapes + +|`recursive` |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes +|`term` |<> |`INTERSECTS` |Yes + +|======================================================================= + [float] ===== Accuracy @@ -149,6 +189,7 @@ between index size and a reasonable level of precision of 50m at the equator. This allows for indexing tens of millions of shapes without overly bloating the resulting index too much relative to the input size. +[[input-structure]] [float] ==== Input Structure @@ -189,6 +230,7 @@ differs from many Geospatial APIs (e.g., Google Maps) that generally use the colloquial latitude, longitude (Y, X). ============================================= +[[point]] [float] ===== http://geojson.org/geojson-spec.html#id2[Point] diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc index 9610466acc2..9b7443ef60a 100644 --- a/docs/reference/mapping/types/ip.asciidoc +++ b/docs/reference/mapping/types/ip.asciidoc @@ -54,8 +54,9 @@ The following parameters are accepted by `ip` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index f04efa16583..77f5808e6b0 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -52,8 +52,9 @@ The following parameters are accepted by numeric types: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/mapping/types/string.asciidoc b/docs/reference/mapping/types/string.asciidoc index d5d7b7a0fce..95c682c696f 100644 --- a/docs/reference/mapping/types/string.asciidoc +++ b/docs/reference/mapping/types/string.asciidoc @@ -82,9 +82,10 @@ The following parameters are accepted by `string` fields: <>:: - Can the field use on-disk index-time doc values for sorting, aggregations, - or scripting? Accepts `true` or `false`. Defaults to `true` for - `not_analyzed` fields. Analyzed fields do not support doc values. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + or `false`. Defaults to `true` for `not_analyzed` fields. Analyzed fields + do not support doc values. <>:: diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc index 6c1b93c34d9..ec02a647a6f 100644 --- a/docs/reference/mapping/types/token-count.asciidoc +++ b/docs/reference/mapping/types/token-count.asciidoc @@ -1,7 +1,7 @@ [[token-count]] === Token count datatype -A field of type `token_count` is really an <> field which +A field of type `token_count` is really an <> field which accepts string values, analyzes them, then indexes the number of tokens in the string. @@ -75,8 +75,9 @@ The following parameters are accepted by `token_count` fields: <>:: - Can the field value be used for sorting, aggregations, or scripting? - Accepts `true` (default) or `false`. + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. <>:: diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 1a26fe81a20..56c000c3d9a 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -18,6 +18,8 @@ See <> for more info. -- include::migrate_3_0.asciidoc[] +include::migrate_2_2.asciidoc[] + include::migrate_2_1.asciidoc[] include::migrate_2_0.asciidoc[] diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index fe0319cdd0a..adf12e7da5c 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -20,6 +20,26 @@ We have provided the https://github.com/elastic/elasticsearch-migration[Elastics to help you detect any issues that you may have when upgrading to Elasticsearch 2.0. Please install and run the plugin *before* upgrading. +[float] +=== Also see + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> + include::migrate_2_0/removals.asciidoc[] include::migrate_2_0/network.asciidoc[] diff --git a/docs/reference/migration/migrate_2_0/aggs.asciidoc b/docs/reference/migration/migrate_2_0/aggs.asciidoc index 8134812f912..1351b4cb4a3 100644 --- a/docs/reference/migration/migrate_2_0/aggs.asciidoc +++ b/docs/reference/migration/migrate_2_0/aggs.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_aggregation_changes]] === Aggregation changes ==== Min doc count defaults to zero diff --git a/docs/reference/migration/migrate_2_0/crud.asciidoc b/docs/reference/migration/migrate_2_0/crud.asciidoc index a7c947e769c..f79306a2bbd 100644 --- a/docs/reference/migration/migrate_2_0/crud.asciidoc +++ b/docs/reference/migration/migrate_2_0/crud.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_crud_and_routing_changes]] === CRUD and routing changes ==== Explicit custom routing diff --git a/docs/reference/migration/migrate_2_0/index_apis.asciidoc b/docs/reference/migration/migrate_2_0/index_apis.asciidoc index ffa2e9edea8..d305f7a9d8c 100644 --- a/docs/reference/migration/migrate_2_0/index_apis.asciidoc +++ b/docs/reference/migration/migrate_2_0/index_apis.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_index_api_changes]] === Index API changes ==== Index aliases diff --git a/docs/reference/migration/migrate_2_0/java.asciidoc b/docs/reference/migration/migrate_2_0/java.asciidoc index ef9c7efedf0..b2f5ee63e0d 100644 --- a/docs/reference/migration/migrate_2_0/java.asciidoc +++ b/docs/reference/migration/migrate_2_0/java.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_java_api_changes]] === Java API changes ==== Transport API construction @@ -39,6 +40,12 @@ the request. Now the exception is only thrown if all shards fail the request. The responses for these APIs will always have a `getShardFailures` method that you can and should check for failures. + +==== IndexMissingException removed. + +Use `IndexNotFoundException` instead. + + ==== Automatically thread client listeners Previously, the user had to set request listener threads to `true` when on the @@ -111,6 +118,16 @@ You can create an InetSocketAddress instance with `InetSocketAddress(String, int new InetSocketTransportAddress(new InetSocketAddress("127.0.0.1", 0)); ----------------------------- +==== Request Builders refactoring + +An `action` parameter has been added to various request builders: + +* Instead of `new SnapshotsStatusRequestBuilder(elasticSearchClient)` use `new SnapshotsStatusRequestBuilder(elasticSearchClient, SnapshotsStatusAction.INSTANCE)`. + +* Instead of `new CreateSnapshotRequestBuilder(elasticSearchClient)` use `new CreateSnapshotRequestBuilder(elasticSearchClient, CreateSnapshotAction.INSTANCE)`. + +* Instead of `new CreateIndexRequestBuilder(elasticSearchClient, index)` use `new CreateIndexRequestBuilder(elasticSearchClient, CreateIndexAction.INSTANCE, index)`. + ==== Shading and package relocation removed Elasticsearch used to shade its dependencies and to relocate packages. We no longer use shading or relocation. diff --git a/docs/reference/migration/migrate_2_0/mapping.asciidoc b/docs/reference/migration/migrate_2_0/mapping.asciidoc index ac9a9b546a3..09170a2e718 100644 --- a/docs/reference/migration/migrate_2_0/mapping.asciidoc +++ b/docs/reference/migration/migrate_2_0/mapping.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_mapping_changes]] === Mapping changes A number of changes have been made to mappings to remove ambiguity and to @@ -314,6 +315,10 @@ PUT my_index/my_type/1 ------------------------- <1> Has `format`: `"strict_date_optional_time||epoch_millis"`. +==== `mapping.date.round_ceil` setting + +The `mapping.date.round_ceil` setting for date math parsing has been removed. + [[migration-bool-fields]] ==== Boolean fields @@ -424,3 +429,11 @@ to use the old default of 0. This was done to prevent phrase queries from matching across different values of the same term unexpectedly. Specifically, 100 was chosen to cause phrase queries with slops up to 99 to match only within a single value of a field. + +==== copy_to and multi fields + +A <> within a <> is ignored from version 2.0 on. With any version after +2.1 or 2.0.1 creating a mapping that has a copy_to within a multi field will result +in an exception. + + diff --git a/docs/reference/migration/migrate_2_0/network.asciidoc b/docs/reference/migration/migrate_2_0/network.asciidoc index 78a482c3d17..2f23c3f924a 100644 --- a/docs/reference/migration/migrate_2_0/network.asciidoc +++ b/docs/reference/migration/migrate_2_0/network.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_network_changes]] === Network changes ==== Bind to localhost diff --git a/docs/reference/migration/migrate_2_0/packaging.asciidoc b/docs/reference/migration/migrate_2_0/packaging.asciidoc index 2d2e4365fbb..15b7b51d385 100644 --- a/docs/reference/migration/migrate_2_0/packaging.asciidoc +++ b/docs/reference/migration/migrate_2_0/packaging.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_plugin_and_packaging_changes]] === Plugin and packaging changes ==== Symbolic links and paths @@ -6,9 +7,9 @@ Elasticsearch 2.0 runs with the Java security manager enabled and is much more restrictive about which paths it is allowed to access. Various paths can be configured, e.g. `path.data`, `path.scripts`, `path.repo`. A configured path may itself be a symbolic link, but no symlinks under that path will be -followed (with the exception of `path.scripts`, which does follow symlinks). +followed. -==== Running `/bin/elasticsearch` +==== Running `bin/elasticsearch` The command line parameter parsing has been rewritten to deal properly with spaces in parameters. All config settings can still be specified on the @@ -19,10 +20,10 @@ For instance: [source,sh] ----------- -/bin/elasticsearch -d -p /tmp/foo.pid --http.cors.enabled=true --http.cors.allow-origin='*' +bin/elasticsearch -d -p /tmp/foo.pid --http.cors.enabled=true --http.cors.allow-origin='*' ----------- -For a list of static parameters, run `/bin/elasticsearch -h` +For a list of static parameters, run `bin/elasticsearch -h` ==== `-f` removed @@ -56,3 +57,28 @@ sudo bin/plugin install analysis-icu Community-provided plugins can be installed as before. +==== Plugins require descriptor file + +All plugins are now required to have a https://github.com/elastic/elasticsearch/blob/2.0/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties[plugin-descriptor.properties] file. If a node has a plugin installed which lacks this file, it will be unable to start. + +==== Repository naming structure changes + +Elasticsearch 2.0 changes the way the repository URLs are referenced. Instead +of specific repositories for both major and minor versions, the repositories will +use a major version reference only. + +The URL for apt packages now uses the following structure; + +[source,sh] +--------------- +deb http://packages.elastic.co/elasticsearch/2.x/debian stable main +--------------- + +And for yum packages it is; + +[source,sh] +--------------- +baseurl=http://packages.elastic.co/elasticsearch/2.x/centos +--------------- + +The <> page details this change. diff --git a/docs/reference/migration/migrate_2_0/parent_child.asciidoc b/docs/reference/migration/migrate_2_0/parent_child.asciidoc index fe198610e51..1addf883973 100644 --- a/docs/reference/migration/migrate_2_0/parent_child.asciidoc +++ b/docs/reference/migration/migrate_2_0/parent_child.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_parent_child_changes]] === Parent/Child changes Parent/child has been rewritten completely to reduce memory usage and to @@ -37,7 +38,6 @@ for the child type, but cannot be added before the child type. ==== `top_children` query removed The `top_children` query has been removed in favour of the `has_child` query. -It wasn't always faster than the `has_child` query and the was usually +It wasn't always faster than the `has_child` query and the results were usually inaccurate. The total hits and any aggregations in the same search request would be incorrect if `top_children` was used. - diff --git a/docs/reference/migration/migrate_2_0/query_dsl.asciidoc b/docs/reference/migration/migrate_2_0/query_dsl.asciidoc index 7555024b43c..352f000ac37 100644 --- a/docs/reference/migration/migrate_2_0/query_dsl.asciidoc +++ b/docs/reference/migration/migrate_2_0/query_dsl.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_query_dsl_changes]] === Query DSL changes ==== Queries and filters merged diff --git a/docs/reference/migration/migrate_2_0/removals.asciidoc b/docs/reference/migration/migrate_2_0/removals.asciidoc index 379565cb90e..55f76c6f30e 100644 --- a/docs/reference/migration/migrate_2_0/removals.asciidoc +++ b/docs/reference/migration/migrate_2_0/removals.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_removed_features]] === Removed features ==== Rivers have been removed diff --git a/docs/reference/migration/migrate_2_0/scripting.asciidoc b/docs/reference/migration/migrate_2_0/scripting.asciidoc index 4964ee05703..495d2daa2c5 100644 --- a/docs/reference/migration/migrate_2_0/scripting.asciidoc +++ b/docs/reference/migration/migrate_2_0/scripting.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_scripting_changes]] === Scripting changes ==== Scripting syntax diff --git a/docs/reference/migration/migrate_2_0/search.asciidoc b/docs/reference/migration/migrate_2_0/search.asciidoc index b9b5987f2e4..036313077ff 100644 --- a/docs/reference/migration/migrate_2_0/search.asciidoc +++ b/docs/reference/migration/migrate_2_0/search.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_search_changes]] === Search changes ==== Partial fields diff --git a/docs/reference/migration/migrate_2_0/settings.asciidoc b/docs/reference/migration/migrate_2_0/settings.asciidoc index 923d5069705..60f80b04e93 100644 --- a/docs/reference/migration/migrate_2_0/settings.asciidoc +++ b/docs/reference/migration/migrate_2_0/settings.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_setting_changes]] === Setting changes ==== Command line flags @@ -163,8 +164,6 @@ Log messages are now truncated at 10,000 characters. This can be changed in the `logging.yml` configuration file with the `file.layout.conversionPattern` setting. -Remove mapping.date.round_ceil setting for date math parsing #8889 (issues: #8556, #8598) - ==== Custom config file It is no longer possible to specify a custom config file with the `CONF_FILE` @@ -172,8 +171,21 @@ environment variable, or the `-Des.config`, `-Des.default.config`, or `-Delasticsearch.config` parameters. Instead, the config file must be named `elasticsearch.yml` and must be located -in the default `config/` directory, or in the directory specified in the -`CONF_DIR` environment variable. +in the default `config/` directory, unless a custom config directory is specified. + +The location of a custom config directory may be specified as follows: + +[source,sh] +-------------- +./bin/elasticsearch --path.conf=/path/to/conf/dir +./bin/plugin -Des.path.conf=/path/to/conf/dir install analysis-icu +-------------- + +When using the RPM or debian packages, the plugin script and the +init/service scripts will consult the `CONF_DIR` environment variable +to check for a custom config location. The value of the `CONF_DIR` +variable can be set in the environment config file which is located either in +`/etc/default/elasticsearch` or `/etc/sysconfig/elasticsearch`. ==== `ES_CLASSPATH removed` diff --git a/docs/reference/migration/migrate_2_0/snapshot_restore.asciidoc b/docs/reference/migration/migrate_2_0/snapshot_restore.asciidoc index 608cd8a0797..c9b222abdc8 100644 --- a/docs/reference/migration/migrate_2_0/snapshot_restore.asciidoc +++ b/docs/reference/migration/migrate_2_0/snapshot_restore.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_snapshot_and_restore_changes]] === Snapshot and Restore changes ==== File-system repositories must be whitelisted diff --git a/docs/reference/migration/migrate_2_0/stats.asciidoc b/docs/reference/migration/migrate_2_0/stats.asciidoc index b75246e4f11..dc80ecd83ec 100644 --- a/docs/reference/migration/migrate_2_0/stats.asciidoc +++ b/docs/reference/migration/migrate_2_0/stats.asciidoc @@ -1,3 +1,4 @@ +[[breaking_20_stats_info_and_literal_cat_literal_changes]] === Stats, info, and `cat` changes ==== Sigar removed diff --git a/docs/reference/migration/migrate_2_0/striping.asciidoc b/docs/reference/migration/migrate_2_0/striping.asciidoc index 7e0cc3686a5..2e80f29c774 100644 --- a/docs/reference/migration/migrate_2_0/striping.asciidoc +++ b/docs/reference/migration/migrate_2_0/striping.asciidoc @@ -1,6 +1,7 @@ -=== Multiple `data.path` striping +[[breaking_20_multiple_literal_data_path_literal_striping]] +=== Multiple `path.data` striping -Previously, if the `data.path` setting listed multiple data paths, then a +Previously, if the `path.data` setting listed multiple data paths, then a shard would be ``striped'' across all paths by writing a whole file to each path in turn (in accordance with the `index.store.distributor` setting). The result was that files from a single segment in a shard could be spread across diff --git a/docs/reference/migration/migrate_2_1.asciidoc b/docs/reference/migration/migrate_2_1.asciidoc index 1dbef8bfafc..454a57f96bc 100644 --- a/docs/reference/migration/migrate_2_1.asciidoc +++ b/docs/reference/migration/migrate_2_1.asciidoc @@ -4,6 +4,14 @@ This section discusses the changes that you need to be aware of when migrating your application to Elasticsearch 2.1. +* <> +* <> +* <> +* <> +* <> +* <> + +[[breaking_21_search_changes]] === Search changes ==== `search_type=scan` deprecated @@ -36,6 +44,7 @@ search. It's safest to leave this value as it is an use the scroll api for any deep scrolling but this setting is dynamic so it can raised or lowered as needed. +[[breaking_21_update_changes]] === Update changes ==== Updates now `detect_noop` by default @@ -46,6 +55,7 @@ source unless you explicitly set `"detect_noop": false`. `detect_noop` was always computationally cheap compared to the expense of the update which can be thought of as a delete operation followed by an index operation. +[[breaking_21_removed_features]] === Removed features ==== `indices.fielddata.cache.expire` @@ -53,16 +63,25 @@ thought of as a delete operation followed by an index operation. The experimental feature `indices.fielddata.cache.expire` has been removed. For indices that have this setting configured, this config will be ignored. +[[breaking_21_more_like_this]] === More Like This -The MoreLikeThisQueryBuilder#ignoreLike methods have been deprecating in favor -to using the unlike methods. +The MoreLikeThisQueryBuilder#ignoreLike methods have been deprecated in favor +of using the unlike methods. -MoreLikeThisBuilder#addItem has been deprecated in favor to using +MoreLikeThisBuilder#addItem has been deprecated in favor of using MoreLikeThisBuilder#addLikeItem. +[[breaking_21_nested_sorting]] === Nested sorting If sorting on field inside a nested object then the `nested_path` should be specified. Before there was an attempt to resolve the nested path automatically, but that was sometimes incorrect. -To avoid confusion the `nested_path` should always be specified. \ No newline at end of file +To avoid confusion the `nested_path` should always be specified. + +[[breaking_21_index_apis]] +=== Index APIs + +==== Optimize API + +The Optimize API has been deprecated, all new optimize actions should use the new Force Merge API. diff --git a/docs/reference/migration/migrate_2_2.asciidoc b/docs/reference/migration/migrate_2_2.asciidoc new file mode 100644 index 00000000000..c13358ecc15 --- /dev/null +++ b/docs/reference/migration/migrate_2_2.asciidoc @@ -0,0 +1,16 @@ +[[breaking-changes-2.2]] +== Breaking changes in 2.2 + +This section discusses the changes that you need to be aware of when migrating +your application to Elasticsearch 2.2. + +* <> + +[[breaking_22_index_apis]] +=== Index APIs + +==== Field stats API + +The field stats' response format has been changed for number based and date fields. The `min_value` and +`max_value` elements now return values as number and the new `min_value_as_string` and `max_value_as_string` +return the values as string. diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index db904199653..6588f22a85a 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -4,6 +4,18 @@ This section discusses the changes that you need to be aware of when migrating your application to Elasticsearch 3.0. +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> + +[[breaking_30_search_changes]] === Search changes ==== `search_type=count` removed @@ -13,6 +25,7 @@ In order to get the same benefits, you just need to set the value of the `size` parameter to `0`. For instance, the following request: + [source,sh] --------------- GET /my_index/_search?search_type=count @@ -28,6 +41,7 @@ GET /my_index/_search?search_type=count --------------- can be replaced with: + [source,sh] --------------- GET /my_index/_search @@ -63,6 +77,69 @@ Scroll requests sorted by `_doc` have been optimized to more efficiently resume from where the previous request stopped, so this will have the same performance characteristics as the former `scan` search type. +[[breaking_30_rest_api_changes]] +=== REST API changes + +==== search exists api removed + +The search exists api has been removed in favour of using the search api with +`size` set to `0` and `terminate_after` set to `1`. + +==== `/_optimize` endpoint removed + +The deprecated `/_optimize` endpoint has been removed. The `/_forcemerge` +endpoint should be used in lieu of optimize. + +The `GET` HTTP verb for `/_forcemerge` is no longer supported, please use the +`POST` HTTP verb. + +==== Deprecated queries removed + +The following deprecated queries have been removed: +* `filtered`: use `bool` query instead, which supports `filter` clauses too +* `and`: use `must` clauses in a `bool` query instead +* `or`: use should clauses in a `bool` query instead +* `limit`: use `terminate_after` parameter instead +* `fquery`: obsolete after filters and queries have been merged +* `query`: obsolete after filters and queries have been merged + +==== Unified fuzziness parameter + +* Removed support for the deprecated `min_similarity` parameter in `fuzzy query`, in favour of `similarity`. +* Removed support for the deprecated `fuzzy_min_sim` parameter in `query_string` query, in favour of `similarity`. +* Removed support for the deprecated `edit_distance` parameter in completion suggester, in favour of `similarity`. + +==== indices query + +Removed support for the deprecated `filter` and `no_match_filter` fields in `indices` query, +in favour of `query` and `no_match_query`. + +==== nested query + +Removed support for the deprecated `filter` fields in `nested` query, in favour of `query`. + +==== terms query + +Removed support for the deprecated `minimum_should_match` and `disable_coord` in `terms` query, use `bool` query instead. +Removed also support for the deprecated `execution` parameter. + +==== function_score query + +Removed support for the top level `filter` element in `function_score` query, replaced by `query`. + +==== highlighters + +Removed support for multiple highlighter names, the only supported ones are: `plain`, `fvh` and `postings`. + +==== top level filter + +Removed support for the deprecated top level `filter` in the search api, replaced by `post_filter`. + +==== `query_binary` and `filter_binary` removed + +Removed support for the undocumented `query_binary` and `filter_binary` sections of a search request. + +[[breaking_30_parent_child_changes]] === Parent/Child changes The `children` aggregation, parent child inner hits and `has_child` and `has_parent` queries will not work on indices @@ -80,7 +157,7 @@ which does the exact same thing. ==== `sum` score mode removed -The `sum` score mode has been removed in favour of the `total` mode which doesn the same and is already available in +The `sum` score mode has been removed in favour of the `total` mode which does the same and is already available in previous versions. ==== `max_children` option @@ -89,7 +166,9 @@ When `max_children` was set to `0` on the `has_child` query then there was no up are allowed to match. This has changed and `0` now really means to zero child documents are allowed. If no upper limit is needed then the `max_children` option shouldn't be defined at all on the `has_child` query. -=== Settings changes === + +[[breaking_30_settings_changes]] +=== Settings changes ==== Analysis settings @@ -104,7 +183,45 @@ Previously, there were three settings for the ping timeout: `discovery.zen.initi the only setting key for the ping timeout is now `discovery.zen.ping_timeout`. The default value for ping timeouts remains at three seconds. -=== Plugins + +==== Recovery settings + +Recovery settings deprecated in 1.x have been removed: + + * `index.shard.recovery.translog_size` is superseded by `indices.recovery.translog_size` + * `index.shard.recovery.translog_ops` is superseded by `indices.recovery.translog_ops` + * `index.shard.recovery.file_chunk_size` is superseded by `indices.recovery.file_chunk_size` + * `index.shard.recovery.concurrent_streams` is superseded by `indices.recovery.concurrent_streams` + * `index.shard.recovery.concurrent_small_file_streams` is superseded by `indices.recovery.concurrent_small_file_streams` + * `indices.recovery.max_size_per_sec` is superseded by `indices.recovery.max_bytes_per_sec` + +If you are using any of these settings please take the time and review their purpose. All of the settings above are considered +_expert settings_ and should only be used if absolutely necessary. If you have set any of the above setting as persistent +cluster settings please use the settings update API and set their superseded keys accordingly. + +[[breaking_30_mapping_changes]] +=== Mapping changes + +==== Transform removed + +The `transform` feature from mappings has been removed. It made issues very hard to debug. + +==== Default number mappings + +When a floating-point number is encountered, it is now dynamically mapped as a +float by default instead of a double. The reasoning is that floats should be +more than enough for most cases but would decrease storage requirements +significantly. + +==== `_source`'s `format` option + +The `_source` mapping does not support the `format` option anymore. This option +will still be accepted for indices created before the upgrade to 3.0 for backward +compatibility, but it will have no effect. Indices created on or after 3.0 will +reject this option. + +[[breaking_30_plugins]] +=== Plugin changes Plugins implementing custom queries need to implement the `fromXContent(QueryParseContext)` method in their `QueryParser` subclass rather than `parse`. This method will take care of parsing the query from `XContent` format @@ -127,14 +244,23 @@ function that it supports and it's able to parse. The function object can then t function through the new `toFunction(QueryShardContext)` method, which returns a lucene function to be executed on the data node. -==== Cloud AWS plugin +==== Cloud AWS plugin changes Cloud AWS plugin has been split in two plugins: * {plugins}/discovery-ec2.html[Discovery EC2 plugin] * {plugins}/repository-s3.html[Repository S3 plugin] -==== Cloud Azure plugin +Proxy settings for both plugins have been renamed: + +* from `cloud.aws.proxy_host` to `cloud.aws.proxy.host` +* from `cloud.aws.ec2.proxy_host` to `cloud.aws.ec2.proxy.host` +* from `cloud.aws.s3.proxy_host` to `cloud.aws.s3.proxy.host` +* from `cloud.aws.proxy_port` to `cloud.aws.proxy.port` +* from `cloud.aws.ec2.proxy_port` to `cloud.aws.ec2.proxy.port` +* from `cloud.aws.s3.proxy_port` to `cloud.aws.s3.proxy.port` + +==== Cloud Azure plugin changes Cloud Azure plugin has been split in three plugins: @@ -142,11 +268,54 @@ Cloud Azure plugin has been split in three plugins: * {plugins}/repository-azure.html[Repository Azure plugin] * {plugins}/store-smb.html[Store SMB plugin] -==== Cloud GCE plugin +If you were using the `cloud-azure` plugin for snapshot and restore, you had in `elasticsearch.yml`: + +[source,yaml] +----- +cloud: + azure: + storage: + account: your_azure_storage_account + key: your_azure_storage_key +----- + +You need to give a unique id to the storage details now as you can define multiple storage accounts: + +[source,yaml] +----- +cloud: + azure: + storage: + my_account: + account: your_azure_storage_account + key: your_azure_storage_key +----- + + +==== Cloud GCE plugin changes Cloud GCE plugin has been renamed to {plugins}/discovery-gce.html[Discovery GCE plugin]. -=== Java-API +[[breaking_30_java_api_changes]] +=== Java API changes + +==== Count api has been removed + +The deprecated count api has been removed from the Java api, use the search api instead and set size to 0. + +The following call + +[source,java] +----- +client.prepareCount(indices).setQuery(query).get(); +----- + +can be replaced with + +[source,java] +----- +client.prepareSearch(indices).setSource(new SearchSourceBuilder().size(0).query(query)).get(); +----- ==== BoostingQueryBuilder @@ -228,7 +397,7 @@ Also reusing new Operator enum. Removed `MoreLikeThisQueryBuilder.Item#id(String id)`, `Item#doc(BytesReference doc)`, `Item#doc(XContentBuilder doc)`. Use provided constructors instead. -Removed `MoreLikeThisQueryBuilder#addLike` in favor of texts and/or items beeing provided +Removed `MoreLikeThisQueryBuilder#addLike` in favor of texts and/or items being provided at construction time. Using arrays there instead of lists now. Removed `MoreLikeThisQueryBuilder#addUnlike` in favor to using the `unlike` methods @@ -265,9 +434,15 @@ Use the `field(String, float)` method instead. ==== MissingQueryBuilder -The two individual setters for existence() and nullValue() were removed in favour of -optional constructor settings in order to better capture and validate their interdependent -settings at construction time. +The MissingQueryBuilder which was deprecated in 2.2.0 is removed. As a replacement use ExistsQueryBuilder +inside a mustNot() clause. So instead of using `new ExistsQueryBuilder(name)` now use +`new BoolQueryBuilder().mustNot(new ExistsQueryBuilder(name))`. + +==== NotQueryBuilder + +The NotQueryBuilder which was deprecated in 2.1.0 is removed. As a replacement use BoolQueryBuilder +with added mustNot() clause. So instead of using `new NotQueryBuilder(filter)` now use +`new BoolQueryBuilder().mustNot(filter)`. ==== TermsQueryBuilder @@ -296,3 +471,47 @@ For simplicity, only one way of adding the ids to the existing list (empty by de `DocumentAlreadyExistsException` is removed and a `VersionConflictException` is thrown instead (with a better error description). This will influence code that use the `IndexRequest.opType()` or `IndexRequest.create()` to index a document only if it doesn't already exist. + +==== ShapeBuilders + +`InternalLineStringBuilder` is removed in favour of `LineStringBuilder`, `InternalPolygonBuilder` in favour of PolygonBuilder` and `Ring` has been replaced with `LineStringBuilder`. Also the abstract base classes `BaseLineStringBuilder` and `BasePolygonBuilder` haven been merged with their corresponding implementations. + +[[breaking_30_cache_concurrency]] +=== Cache concurrency level settings removed + +Two cache concurrency level settings `indices.requests.cache.concurrency_level` and +`indices.fielddata.cache.concurrency_level` because they no longer apply to the cache implementation used for the +request cache and the field data cache. + +[[breaking_30_non_loopback]] +=== Remove bind option of `non_loopback` + +This setting would arbitrarily pick the first interface not marked as loopback. Instead, specify by address +scope (e.g. `_local_,_site_` for all loopback and private network addresses) or by explicit interface names, +hostnames, or addresses. + +[[breaking_30_thread_pool]] +=== Forbid changing of thread pool types + +Previously, <> could be dynamically adjusted. The thread pool type effectively +controls the backing queue for the thread pool and modifying this is an expert setting with minimal practical benefits +and high risk of being misused. The ability to change the thread pool type for any thread pool has been removed; do note +that it is still possible to adjust relevant thread pool parameters for each of the thread pools (e.g., depending on +the thread pool type, `keep_alive`, `queue_size`, etc.). + +=== Adding system CPU percent to OS stats + +The recent CPU usage (as a percent) has been added to the OS stats reported under the node stats API and the cat nodes +API. The breaking change here is that there is a new object in the "os" object in the node stats response. This object +is called "cpu" and includes "percent" and "load_average" as fields. This moves the "load_average" field that was +previously a top-level field in the "os" object to the "cpu" object. Additionally, the "cpu" field in the cat nodes API +response is output by default. + +Finally, the API for org.elasticsearch.monitor.os.OsStats has changed. The `getLoadAverage` method has been removed. The +value for this can now be obtained from `OsStats.Cpu#getLoadAverage`. Additionally, the recent CPU usage can be obtained +from `OsStats.Cpu#getPercent`. + +=== Fields option +Only stored fields are retrievable with this option. +The fields option won't be able to load non stored fields from _source anymore. + diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc index 9f175cc2fa2..09ffb06fb68 100644 --- a/docs/reference/modules.asciidoc +++ b/docs/reference/modules.asciidoc @@ -67,6 +67,11 @@ The modules in this section are: Configure the transport networking layer, used internally by Elasticsearch to communicate between nodes. + +<>:: + + A tribe node joins one or more clusters and acts as a federated + client across them. -- diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 7105d2d60dd..5a710598206 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -1,86 +1,175 @@ [[modules-network]] == Network Settings -There are several modules within a Node that use network based -configuration, for example, the -<> and -<> modules. Node level -network settings allows to set common settings that will be shared among -all network based modules (unless explicitly overridden in each module). +Elasticsearch binds to localhost only by default. This is sufficient for you +to run a local development server (or even a development cluster, if you start +multiple nodes on the same machine), but you will need to configure some +<> in order to run a real +production cluster across multiple servers. -The `network.bind_host` setting allows to control the host different network -components will bind on. By default, the bind host will be `_local_` -(loopback addresses such as `127.0.0.1`, `::1`). +[WARNING] +.Be careful with the network configuration! +============================= +Never expose an unprotected node to the public internet. +============================= -The `network.publish_host` setting allows to control the host the node will -publish itself within the cluster so other nodes will be able to connect to it. -Currently an elasticsearch node may be bound to multiple addresses, but only -publishes one. If not specified, this defaults to the "best" address from -`network.bind_host`. By default, IPv4 addresses are preferred to IPv6, and -ordinary addresses are preferred to site-local or link-local addresses. +[float] +[[common-network-settings]] +=== Commonly Used Network Settings -The `network.host` setting is a simple setting to automatically set both -`network.bind_host` and `network.publish_host` to the same host value. +`network.host`:: -Both settings allows to be configured with either explicit host address -or host name. The settings also accept logical setting values explained -in the following table: +The node will bind to this hostname or IP address and _publish_ (advertise) +this host to other nodes in the cluster. Accepts an IP address, hostname, or a +<>. ++ +Defaults to `_local_`. -[cols="<,<",options="header",] -|======================================================================= -|Logical Host Setting Value |Description -|`_local_` |Will be resolved to loopback addresses +`discovery.zen.ping.unicast.hosts`:: -|`_local:ipv4_` |Will be resolved to loopback IPv4 addresses +In order to join a cluster, a node needs to know the hostname or IP address of +at least some of the other nodes in the cluster. This settting provides the +initial list of other nodes that this node will try to contact. Accepts IP +addresses or hostnames. ++ +Defaults to `["127.0.0.1", "[::1]"]`. -|`_local:ipv6_` |Will be resolved to loopback IPv6 addresses +`http.port`:: -|`_non_loopback_` |Addresses of the first non loopback interface +Port to bind to for incoming HTTP requests. Accepts a single value or a range. +If a range is specified, the node will bind to the first available port in the +range. ++ +Defaults to `9200-9300`. -|`_non_loopback:ipv4_` |IPv4 addresses of the first non loopback interface +`transport.tcp.port`:: -|`_non_loopback:ipv6_` |IPv6 addresses of the first non loopback interface +Port to bind for communication between nodes. Accepts a single value or a +range. If a range is specified, the node will bind to the first available port +in the range. ++ +Defaults to `9300-9400`. -|`_[networkInterface]_` |Resolves to the addresses of the provided -network interface. For example `_en0_`. +[float] +[[network-interface-values]] +=== Special values for `network.host` -|`_[networkInterface]:ipv4_` |Resolves to the ipv4 addresses of the -provided network interface. For example `_en0:ipv4_`. +The following special values may be passed to `network.host`: -|`_[networkInterface]:ipv6_` |Resolves to the ipv6 addresses of the -provided network interface. For example `_en0:ipv6_`. -|======================================================================= +[horizontal] +`_[networkInterface]_`:: -When the `discovery-ec2` plugin is installed, you can use -{plugins}/discovery-ec2-discovery.html#discovery-ec2-network-host[ec2 specific host settings]. + Addresses of a network interface, for example `_en0_`. -When the `discovery-gce` plugin is installed, you can use -{plugins}/discovery-gce-network-host.html[gce specific host settings]. +`_local_`:: + + Any loopback addresses on the system, for example `127.0.0.1`. + +`_site_`:: + + Any site-local addresses on the system, for example `192.168.0.1`. + +`_global_`:: + + Any globally-scoped addresses on the system, for example `8.8.8.8`. +[float] +==== IPv4 vs IPv6 + +These special values will work over both IPv4 and IPv6 by default, but you can +also limit this with the use of `:ipv4` of `:ipv6` specifiers. For example, +`_en0:ipv4_` would only bind to the IPv4 addresses of interface `en0`. + +[TIP] +.Discovery in the cloud +================================ + +More special settings are available when running in the cloud with either the +{plugins}/discovery-ec2-discovery.html#discovery-ec2-network-host[EC2 discovery plugin] or the +{plugins}/discovery-gce-network-host.html#discovery-gce-network-host[Google Compute Engine discovery plugin] +installed. + +================================ + +[float] +[[advanced-network-settings]] +=== Advanced network settings + +The `network.host` setting explained in <> +is a shortcut which sets the _bind host_ and the _publish host_ at the same +time. In advanced used cases, such as when running behind a proxy server, you +may need to set these settings to different values: + +`network.bind_host`:: + +This specifies which network interface(s) a node should bind to in order to +listen for incoming requests. A node can bind to multiple interfaces, e.g. +two network cards, or a site-local address and a local address. Defaults to +`network.host`. + +`network.publish_host`:: + +The publish host is the single interface that the node advertises to other +nodes in the cluster, so that those nodes can connect to it. Currently an +elasticsearch node may be bound to multiple addresses, but only publishes one. +If not specified, this defaults to the ``best'' address from +`network.bind_host`, sorted by IPv4/IPv6 stack preference, then by +reachability. + +Both of the above settings can be configured just like `network.host` -- they +accept IP addresses, host names, and +<>. + [float] [[tcp-settings]] -=== TCP Settings +=== Advanced TCP Settings -Any component that uses TCP (like the HTTP, Transport and Memcached) -share the following allowed settings: +Any component that uses TCP (like the <> and +<> modules) share the following settings: -[cols="<,<",options="header",] -|======================================================================= -|Setting |Description -|`network.tcp.no_delay` |Enable or disable tcp no delay setting. +[horizontal] +`network.tcp.no_delay`:: + +Enable or disable the https://en.wikipedia.org/wiki/Nagle%27s_algorithm[TCP no delay] +setting. Defaults to `true`. + +`network.tcp.keep_alive`:: + +Enable or disable https://en.wikipedia.org/wiki/Keepalive[TCP keep alive]. Defaults to `true`. -|`network.tcp.keep_alive` |Enable or disable tcp keep alive. Defaults -to `true`. +`network.tcp.reuse_address`:: -|`network.tcp.reuse_address` |Should an address be reused or not. -Defaults to `true` on non-windows machines. +Should an address be reused or not. Defaults to `true` on non-windows +machines. -|`network.tcp.send_buffer_size` |The size of the tcp send buffer size -(in size setting format). By default not explicitly set. +`network.tcp.send_buffer_size`:: -|`network.tcp.receive_buffer_size` |The size of the tcp receive buffer -size (in size setting format). By default not explicitly set. -|======================================================================= +The size of the TCP send buffer (specified with <>). +By default not explicitly set. + +`network.tcp.receive_buffer_size`:: + +The size of the TCP receive buffer (specified with <>). +By default not explicitly set. + +[float] +=== Transport and HTTP protocols + +An Elasticsearch node exposes two network protocols which inherit the above +settings, but may be further configured independently: + +TCP Transport:: + +Used for communication between nodes in the cluster and by the Java +{javaclient}/node-client.html[Node client], +{javaclient}/transport-client.html[Transport client], and by the +<>. See the <> +for more information. + +HTTP:: + +Exposes the JSON-over-HTTP interface used by all clients other than the Java +clients. See the <> for more information. diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index 50be5fdce48..aea7846202e 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -121,10 +121,12 @@ curl -XPOST localhost:9200/_search -d '{ "functions": [ { "script_score": { - "lang": "groovy", - "file": "calculate-score", - "params": { - "my_modifier": 8 + "script": { + "lang": "groovy", + "file": "calculate-score", + "params": { + "my_modifier": 8 + } } } } @@ -180,10 +182,12 @@ curl -XPOST localhost:9200/_search -d '{ "functions": [ { "script_score": { - "id": "indexedCalculateScore", - "lang" : "groovy", - "params": { - "my_modifier": 8 + "script": { + "id": "indexedCalculateScore", + "lang" : "groovy", + "params": { + "my_modifier": 8 + } } } } @@ -421,8 +425,10 @@ curl -XPOST localhost:9200/_search -d '{ "functions": [ { "script_score": { - "id": "my_script", - "lang" : "native" + "script": { + "id": "my_script", + "lang" : "native" + } } } ] diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 50ee4dfd1ea..dbb9f395402 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -45,6 +45,15 @@ which returns: } ----------------------------------- +Information about multiple repositories can be fetched in one go by using a comma-delimited list of repository names. +Star wildcards are supported as well. For example, information about repositories that start with `repo` or that contain `backup` +can be obtained using the following command: + +[source,js] +----------------------------------- +GET /_snapshot/repo*,*backup* +----------------------------------- + If a repository name is not specified, or `_all` is used as repository name Elasticsearch will return information about all repositories currently registered in the cluster: @@ -142,7 +151,7 @@ This setting supports wildcards in the place of host, path, query, and fragment. repositories.url.allowed_urls: ["http://www.example.org/root/*", "https://*.mydomain.com/*?*#*"] ----------------------------------- -URL repositories with `file:` URLs can only point to locations registered in the `path.repo` setting similiar to +URL repositories with `file:` URLs can only point to locations registered in the `path.repo` setting similar to shared file system repository. [float] @@ -251,6 +260,14 @@ GET /_snapshot/my_backup/snapshot_1 ----------------------------------- // AUTOSENSE +Similar as for repositories, information about multiple snapshots can be queried in one go, supporting wildcards as well: + +[source,sh] +----------------------------------- +GET /_snapshot/my_backup/snapshot_*,some_other_snapshot +----------------------------------- +// AUTOSENSE + All snapshots currently stored in the repository can be listed using the following command: [source,sh] @@ -259,6 +276,9 @@ GET /_snapshot/my_backup/_all ----------------------------------- // AUTOSENSE +The command fails if some of the snapshots are unavailable. The boolean parameter `ignore_unvailable` can be used to +return all snapshots that are currently available. + A currently running snapshot can be retrieved using the following command: [source,sh] diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index cde176f3425..bfd5474183c 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -9,87 +9,92 @@ of discarded. There are several thread pools, but the important ones include: +`generic`:: + For generic operations (e.g., background node discovery). + Thread pool type is `cached`. + `index`:: - For index/delete operations. Defaults to `fixed` + For index/delete operations. Thread pool type is `fixed` with a size of `# of available processors`, queue_size of `200`. `search`:: - For count/search operations. Defaults to `fixed` + For count/search operations. Thread pool type is `fixed` with a size of `int((# of available_processors * 3) / 2) + 1`, queue_size of `1000`. `suggest`:: - For suggest operations. Defaults to `fixed` + For suggest operations. Thread pool type is `fixed` with a size of `# of available processors`, queue_size of `1000`. `get`:: - For get operations. Defaults to `fixed` + For get operations. Thread pool type is `fixed` with a size of `# of available processors`, queue_size of `1000`. `bulk`:: - For bulk operations. Defaults to `fixed` + For bulk operations. Thread pool type is `fixed` with a size of `# of available processors`, queue_size of `50`. `percolate`:: - For percolate operations. Defaults to `fixed` + For percolate operations. Thread pool type is `fixed` with a size of `# of available processors`, queue_size of `1000`. `snapshot`:: - For snapshot/restore operations. Defaults to `scaling` with a + For snapshot/restore operations. Thread pool type is `scaling` with a keep-alive of `5m` and a size of `min(5, (# of available processors)/2)`. `warmer`:: - For segment warm-up operations. Defaults to `scaling` with a + For segment warm-up operations. Thread pool type is `scaling` with a keep-alive of `5m` and a size of `min(5, (# of available processors)/2)`. `refresh`:: - For refresh operations. Defaults to `scaling` with a + For refresh operations. Thread pool type is `scaling` with a keep-alive of `5m` and a size of `min(10, (# of available processors)/2)`. `listener`:: Mainly for java client executing of action when listener threaded is set to true. - Default size of `(# of available processors)/2`, max at 10. + Thread pool type is `scaling` with a default size of `min(10, (# of available processors)/2)`. -Changing a specific thread pool can be done by setting its type and -specific type parameters, for example, changing the `index` thread pool -to have more threads: +Changing a specific thread pool can be done by setting its type-specific parameters; for example, changing the `index` +thread pool to have more threads: [source,js] -------------------------------------------------- threadpool: index: - type: fixed size: 30 -------------------------------------------------- -NOTE: you can update threadpool settings live using - <>. - +NOTE: you can update thread pool settings dynamically using <>. [float] [[types]] === Thread pool types -The following are the types of thread pools that can be used and their -respective parameters: +The following are the types of thread pools and their respective parameters: [float] -==== `cache` +==== `cached` -The `cache` thread pool is an unbounded thread pool that will spawn a -thread if there are pending requests. Here is an example of how to set -it: +The `cached` thread pool is an unbounded thread pool that will spawn a +thread if there are pending requests. This thread pool is used to +prevent requests submitted to this pool from blocking or being +rejected. Unused threads in this thread pool will be terminated after +a keep alive expires (defaults to five minutes). The `cached` thread +pool is reserved for the <> thread pool. + +The `keep_alive` parameter determines how long a thread should be kept +around in the thread pool without doing any work. [source,js] -------------------------------------------------- threadpool: - index: - type: cached + generic: + keep_alive: 2m -------------------------------------------------- [float] @@ -111,7 +116,6 @@ full, it will abort the request. -------------------------------------------------- threadpool: index: - type: fixed size: 30 queue_size: 1000 -------------------------------------------------- @@ -130,7 +134,6 @@ around in the thread pool without it doing any work. -------------------------------------------------- threadpool: warmer: - type: scaling size: 8 keep_alive: 2m -------------------------------------------------- diff --git a/docs/reference/query-dsl.asciidoc b/docs/reference/query-dsl.asciidoc index b4a82afdc28..1eb6fbddd1e 100644 --- a/docs/reference/query-dsl.asciidoc +++ b/docs/reference/query-dsl.asciidoc @@ -20,7 +20,7 @@ Compound query clauses:: Compound query clauses wrap other leaf *or* compound queries and are used to combine multiple queries in a logical fashion (such as the <> or <> query), -or to alter their behaviour (such as the <> or +or to alter their behaviour (such as the <> query). Query clauses behave differently depending on whether they are used in diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index 8f2fdb0c99e..17bf74df1e1 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -51,7 +51,7 @@ final `_score` for each document. }, "filter": { "term" : { "tag" : "tech" } - } + }, "must_not" : { "range" : { "age" : { "from" : 10, "to" : 20 } diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc index aeecf39e6f7..a956c33c1ee 100644 --- a/docs/reference/query-dsl/common-terms-query.asciidoc +++ b/docs/reference/query-dsl/common-terms-query.asciidoc @@ -73,7 +73,7 @@ In this example, words that have a document frequency greater than 0.1% { "common": { "body": { - "query": "this is bonsai cool", + "query": "this is bonsai cool", "cutoff_frequency": 0.001 } } @@ -93,7 +93,7 @@ all terms required: { "common": { "body": { - "query": "nelly the elephant as a cartoon", + "query": "nelly the elephant as a cartoon", "cutoff_frequency": 0.001, "low_freq_operator": "and" } @@ -113,8 +113,8 @@ which is roughly equivalent to: { "term": { "body": "cartoon"}} ], "should": [ - { "term": { "body": "the"}}, - { "term": { "body": "as"}}, + { "term": { "body": "the"}} + { "term": { "body": "as"}} { "term": { "body": "a"}} ] } @@ -131,8 +131,8 @@ must be present, for instance: { "common": { "body": { - "query": "nelly the elephant as a cartoon", - "cutoff_frequency": 0.001, + "query": "nelly the elephant as a cartoon", + "cutoff_frequency": 0.001, "minimum_should_match": 2 } } @@ -156,8 +156,8 @@ which is roughly equivalent to: } }, "should": [ - { "term": { "body": "the"}}, - { "term": { "body": "as"}}, + { "term": { "body": "the"}} + { "term": { "body": "as"}} { "term": { "body": "a"}} ] } @@ -169,7 +169,7 @@ minimum_should_match A different <> can be applied for low and high frequency terms with the additional -`low_freq` and `high_freq` parameters Here is an example when providing +`low_freq` and `high_freq` parameters. Here is an example when providing additional parameters (note the change in structure): [source,js] @@ -177,8 +177,8 @@ additional parameters (note the change in structure): { "common": { "body": { - "query": "nelly the elephant not as a cartoon", - "cutoff_frequency": 0.001, + "query": "nelly the elephant not as a cartoon", + "cutoff_frequency": 0.001, "minimum_should_match": { "low_freq" : 2, "high_freq" : 3 @@ -230,8 +230,8 @@ for high frequency terms is when there are only high frequency terms: { "common": { "body": { - "query": "how not to be", - "cutoff_frequency": 0.001, + "query": "how not to be", + "cutoff_frequency": 0.001, "minimum_should_match": { "low_freq" : 2, "high_freq" : 3 diff --git a/docs/reference/query-dsl/compound-queries.asciidoc b/docs/reference/query-dsl/compound-queries.asciidoc index c6a1f0f2c3a..07624f02ab7 100644 --- a/docs/reference/query-dsl/compound-queries.asciidoc +++ b/docs/reference/query-dsl/compound-queries.asciidoc @@ -48,5 +48,3 @@ include::dis-max-query.asciidoc[] include::function-score-query.asciidoc[] include::boosting-query.asciidoc[] include::indices-query.asciidoc[] -include::not-query.asciidoc[] - diff --git a/docs/reference/query-dsl/exists-query.asciidoc b/docs/reference/query-dsl/exists-query.asciidoc index ca72887d048..404dce4a4ae 100644 --- a/docs/reference/query-dsl/exists-query.asciidoc +++ b/docs/reference/query-dsl/exists-query.asciidoc @@ -6,11 +6,7 @@ Returns documents that have at least one non-`null` value in the original field: [source,js] -------------------------------------------------- { - "constant_score" : { - "filter" : { - "exists" : { "field" : "user" } - } - } + "exists" : { "field" : "user" } } -------------------------------------------------- @@ -42,7 +38,7 @@ These documents would *not* match the above query: <3> The `user` field is missing completely. [float] -===== `null_value` mapping +==== `null_value` mapping If the field mapping includes the <> setting then explicit `null` values are replaced with the specified `null_value`. For @@ -74,3 +70,21 @@ no values in the `user` field and thus would not match the `exists` filter: { "foo": "bar" } -------------------------------------------------- +==== `missing` query + +'missing' query has been removed because it can be advantageously replaced by an `exists` query inside a must_not +clause as follows: + +[source,js] +-------------------------------------------------- +"bool": { + "must_not": { + "exists": { + "field": "user" + } + } +} +-------------------------------------------------- + +This query returns documents that have no value in the user field. + diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc index 6710461b5ba..c52bcb93e7d 100644 --- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc +++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc @@ -75,7 +75,7 @@ representation of the geo point, the filter can accept it as well: [source,js] -------------------------------------------------- { - "boost" : { + "bool" : { "must" : { "match_all" : {} }, diff --git a/docs/reference/query-dsl/geo-distance-range-query.asciidoc b/docs/reference/query-dsl/geo-distance-range-query.asciidoc index 901cca09829..48e9e10d5ca 100644 --- a/docs/reference/query-dsl/geo-distance-range-query.asciidoc +++ b/docs/reference/query-dsl/geo-distance-range-query.asciidoc @@ -6,7 +6,7 @@ Filters documents that exists within a range from a specific point: [source,js] -------------------------------------------------- { - "boost" : { + "bool" : { "must" : { "match_all" : {} }, diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc index 77deabcad91..d389380b781 100644 --- a/docs/reference/query-dsl/geo-shape-query.asciidoc +++ b/docs/reference/query-dsl/geo-shape-query.asciidoc @@ -50,7 +50,8 @@ The following query will find the point using the Elasticsearch's "shape": { "type": "envelope", "coordinates" : [[13.0, 53.0], [14.0, 52.0]] - } + }, + "relation": "within" } } } @@ -61,7 +62,7 @@ The following query will find the point using the Elasticsearch's ==== Pre-Indexed Shape -The Filter also supports using a shape which has already been indexed in +The Query also supports using a shape which has already been indexed in another index and/or index type. This is particularly useful for when you have a pre-defined list of shapes which are useful to your application and you want to reference this using a logical name (for @@ -101,3 +102,18 @@ shape: } -------------------------------------------------- +==== Spatial Relations + +The <> mapping parameter determines +which spatial relation operators may be used at search time. + +The following is a complete list of spatial relation operators available: + +* `INTERSECTS` - (default) Return all documents whose `geo_shape` field +intersects the query geometry. +* `DISJOINT` - Return all documents whose `geo_shape` field +has nothing in common with the query geometry. +* `WITHIN` - Return all documents whose `geo_shape` field +is within the query geometry. +* `CONTAINS` - Return all documents whose `geo_shape` field +contains the query geometry. \ No newline at end of file diff --git a/docs/reference/query-dsl/indices-query.asciidoc b/docs/reference/query-dsl/indices-query.asciidoc index f8e9e58d117..e3b604b7a39 100644 --- a/docs/reference/query-dsl/indices-query.asciidoc +++ b/docs/reference/query-dsl/indices-query.asciidoc @@ -1,10 +1,11 @@ [[query-dsl-indices-query]] === Indices Query -The `indices` query can be used when executed across multiple indices, -allowing to have a query that executes only when executed on an index -that matches a specific list of indices, and another query that executes -when it is executed on an index that does not match the listed indices. +The `indices` query is useful in cases where a search is executed across +multiple indices. It allows to specify a list of index names and an inner +query that is only executed for indices matching names on that list. +For other indices that are searched, but that don't match entries +on the list, the alternative `no_match_query` is executed. [source,js] -------------------------------------------------- @@ -27,11 +28,3 @@ You can use the `index` field to provide a single index. documents), and `all` (to match all). Defaults to `all`. `query` is mandatory, as well as `indices` (or `index`). - -[TIP] -==================================================================== -The fields order is important: if the `indices` are provided before `query` -or `no_match_query`, the related queries get parsed only against the indices -that they are going to be executed on. This is useful to avoid parsing queries -when it is not necessary and prevent potential mapping errors. -==================================================================== diff --git a/docs/reference/query-dsl/match-all-query.asciidoc b/docs/reference/query-dsl/match-all-query.asciidoc index b7f4251f0cd..d46b08f9e55 100644 --- a/docs/reference/query-dsl/match-all-query.asciidoc +++ b/docs/reference/query-dsl/match-all-query.asciidoc @@ -15,3 +15,14 @@ The `_score` can be changed with the `boost` parameter: -------------------------------------------------- { "match_all": { "boost" : 1.2 }} -------------------------------------------------- + +[[query-dsl-match-none-query]] +[float] +== Match None Query + +This is the inverse of the `match_all` query, which matches no documents. + +[source,js] +-------------------------------------------------- +{ "match_none": {} } +-------------------------------------------------- diff --git a/docs/reference/query-dsl/missing-query.asciidoc b/docs/reference/query-dsl/missing-query.asciidoc deleted file mode 100644 index 648da068189..00000000000 --- a/docs/reference/query-dsl/missing-query.asciidoc +++ /dev/null @@ -1,132 +0,0 @@ -[[query-dsl-missing-query]] -=== Missing Query - -Returns documents that have only `null` values or no value in the original field: - -[source,js] --------------------------------------------------- -{ - "constant_score" : { - "filter" : { - "missing" : { "field" : "user" } - } - } -} --------------------------------------------------- - -For instance, the following docs would match the above filter: - -[source,js] --------------------------------------------------- -{ "user": null } -{ "user": [] } <1> -{ "user": [null] } <2> -{ "foo": "bar" } <3> --------------------------------------------------- -<1> This field has no values. -<2> This field has no non-`null` values. -<3> The `user` field is missing completely. - -These documents would *not* match the above filter: - -[source,js] --------------------------------------------------- -{ "user": "jane" } -{ "user": "" } <1> -{ "user": "-" } <2> -{ "user": ["jane"] } -{ "user": ["jane", null ] } <3> --------------------------------------------------- -<1> An empty string is a non-`null` value. -<2> Even though the `standard` analyzer would emit zero tokens, the original field is non-`null`. -<3> This field has one non-`null` value. - -[float] -==== `null_value` mapping - -If the field mapping includes a <> then explicit `null` values -are replaced with the specified `null_value`. For instance, if the `user` field were mapped -as follows: - -[source,js] --------------------------------------------------- - "user": { - "type": "string", - "null_value": "_null_" - } --------------------------------------------------- - -then explicit `null` values would be indexed as the string `_null_`, and the -the following docs would *not* match the `missing` filter: - -[source,js] --------------------------------------------------- -{ "user": null } -{ "user": [null] } --------------------------------------------------- - -However, these docs--without explicit `null` values--would still have -no values in the `user` field and thus would match the `missing` filter: - -[source,js] --------------------------------------------------- -{ "user": [] } -{ "foo": "bar" } --------------------------------------------------- - -[float] -===== `existence` and `null_value` parameters - -When the field being queried has a `null_value` mapping, then the behaviour of -the `missing` filter can be altered with the `existence` and `null_value` -parameters: - -[source,js] --------------------------------------------------- -{ - "constant_score" : { - "filter" : { - "missing" : { - "field" : "user", - "existence" : true, - "null_value" : false - } - } - } -} --------------------------------------------------- - - -`existence`:: -+ --- -When the `existence` parameter is set to `true` (the default), the missing -filter will include documents where the field has *no* values, ie: - -[source,js] --------------------------------------------------- -{ "user": [] } -{ "foo": "bar" } --------------------------------------------------- - -When set to `false`, these documents will not be included. --- - -`null_value`:: -+ --- -When the `null_value` parameter is set to `true`, the missing -filter will include documents where the field contains a `null` value, ie: - -[source,js] --------------------------------------------------- -{ "user": null } -{ "user": [null] } -{ "user": ["jane",null] } <1> --------------------------------------------------- -<1> Matches because the field contains a `null` value, even though it also contains a non-`null` value. - -When set to `false` (the default), these documents will not be included. --- - -NOTE: Either `existence` or `null_value` or both must be set to `true`. diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc index d32705a0a7a..51f690c2cab 100644 --- a/docs/reference/query-dsl/nested-query.asciidoc +++ b/docs/reference/query-dsl/nested-query.asciidoc @@ -45,9 +45,9 @@ And here is a sample nested query usage: } -------------------------------------------------- -The query `path` points to the nested object path, and the `query` (or -`filter`) includes the query that will run on the nested docs matching -the direct path, and joining with the root parent docs. Note that any +The query `path` points to the nested object path, and the `query` +includes the query that will run on the nested docs matching the +direct path, and joining with the root parent docs. Note that any fields referenced inside the query must use the complete path (fully qualified). diff --git a/docs/reference/query-dsl/not-query.asciidoc b/docs/reference/query-dsl/not-query.asciidoc deleted file mode 100644 index 7854ee90afa..00000000000 --- a/docs/reference/query-dsl/not-query.asciidoc +++ /dev/null @@ -1,51 +0,0 @@ -[[query-dsl-not-query]] -=== Not Query - -A query that filters out matched documents using a query. For example: - -[source,js] --------------------------------------------------- -{ - "bool" : { - "must" : { - "term" : { "name.first" : "shay" } - }, - "filter" : { - "not" : { - "range" : { - "postDate" : { - "from" : "2010-03-01", - "to" : "2010-04-01" - } - } - } - } - } -} --------------------------------------------------- - -Or, in a longer form with a `filter` element: - -[source,js] --------------------------------------------------- -{ - "bool" : { - "must" : { - "term" : { "name.first" : "shay" } - }, - "filter" : { - "not" : { - "filter" : { - "range" : { - "postDate" : { - "from" : "2010-03-01", - "to" : "2010-04-01" - } - } - } - } - } - } -} --------------------------------------------------- - diff --git a/docs/reference/query-dsl/query-string-syntax.asciidoc b/docs/reference/query-dsl/query-string-syntax.asciidoc index 17198a1991f..6755b9e9efe 100644 --- a/docs/reference/query-dsl/query-string-syntax.asciidoc +++ b/docs/reference/query-dsl/query-string-syntax.asciidoc @@ -241,7 +241,7 @@ query. `((quick AND fox) OR (brown AND fox) OR fox) AND NOT news`:: This form now replicates the logic from the original query correctly, but -the relevance scoring bares little resemblance to the original. +the relevance scoring bears little resemblance to the original. In contrast, the same query rewritten using the <> would look like this: diff --git a/docs/reference/query-dsl/term-level-queries.asciidoc b/docs/reference/query-dsl/term-level-queries.asciidoc index 7e9f5e5ca3e..9c28a727b33 100644 --- a/docs/reference/query-dsl/term-level-queries.asciidoc +++ b/docs/reference/query-dsl/term-level-queries.asciidoc @@ -30,11 +30,6 @@ The queries in this group are: Find documents where the field specified contains any non-null value. -<>:: - - Find documents where the field specified does is missing or contains only - `null` values. - <>:: Find documents where the field specified contains terms which being with @@ -75,8 +70,6 @@ include::range-query.asciidoc[] include::exists-query.asciidoc[] -include::missing-query.asciidoc[] - include::prefix-query.asciidoc[] include::wildcard-query.asciidoc[] @@ -88,6 +81,3 @@ include::fuzzy-query.asciidoc[] include::type-query.asciidoc[] include::ids-query.asciidoc[] - - - diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 922216aeb93..823bdb70d07 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -80,9 +80,7 @@ in ``query context'' and as a filter in ``filter context'' (see <>). [role="exclude",id="query-dsl-not-filter"] === Not Filter -The `not` filter has been replaced by the <>. It behaves -as a query in ``query context'' and as a filter in ``filter context'' (see -<>). +The `not` query has been replaced by using a `mustNot` clause in a Boolean query. [role="exclude",id="query-dsl-bool-filter"] === Bool Filter @@ -98,14 +96,6 @@ The `exists` filter has been replaced by the <>. It beh as a query in ``query context'' and as a filter in ``filter context'' (see <>). -[role="exclude",id="query-dsl-missing-filter"] -=== Missing Filter - -The `missing` filter has been replaced by the <>. It behaves -as a query in ``query context'' and as a filter in ``filter context'' (see -<>). - - [role="exclude",id="query-dsl-geo-bounding-box-filter"] === Geo Bounding Box Filter @@ -364,7 +354,7 @@ The filter cache has been renamed <>. [role="exclude",id="query-dsl-filtered-query"] === Filtered query -The `filtered` query is replaced in favour of the <> query. Instead of +The `filtered` query is replaced by the <> query. Instead of the following: [source,js] @@ -443,3 +433,13 @@ parameter of search requests. The `limit` filter is replaced in favour of the <> parameter of search requests. + +[role="exclude",id="query-dsl-not-query"] +=== Not query + +The `not` query has been replaced by using a `mustNot` clause in a Boolean query. + +[role="exclude",id="mapping-nested-type"] +=== Nested type + +The docs for the `nested` field datatype have moved to <>. diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc new file mode 100644 index 00000000000..267525b1b3c --- /dev/null +++ b/docs/reference/release-notes.asciidoc @@ -0,0 +1,7 @@ +[[es-release-notes]] += Release Notes + +[partintro] +-- +This section will summarize the changes in released versions. +-- diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index f59444d739e..2d8a1f8bc9a 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -91,8 +91,6 @@ include::search/multi-search.asciidoc[] include::search/count.asciidoc[] -include::search/exists.asciidoc[] - include::search/validate.asciidoc[] include::search/explain.asciidoc[] diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc index 2f4b5dc48f8..9be219f5e74 100644 --- a/docs/reference/search/count.asciidoc +++ b/docs/reference/search/count.asciidoc @@ -73,8 +73,7 @@ not. Defaults to `true`. |`analyze_wildcard` |Should wildcard and prefix queries be analyzed or not. Defaults to `false`. -|`terminate_after` |experimental[The API for this feature may change in the future] -The maximum count for each shard, upon +|`terminate_after` |The maximum count for each shard, upon reaching which the query execution will terminate early. If set, the response will have a boolean field `terminated_early` to indicate whether the query execution has actually terminated_early. diff --git a/docs/reference/search/exists.asciidoc b/docs/reference/search/exists.asciidoc deleted file mode 100644 index 9bbd40134ec..00000000000 --- a/docs/reference/search/exists.asciidoc +++ /dev/null @@ -1,94 +0,0 @@ -[[search-exists]] -== Search Exists API - -The exists API allows to easily determine if any -matching documents exist for a provided query. It can be executed across one or more indices -and across one or more types. The query can either be provided using a -simple query string as a parameter, or using the -<> defined within the request -body. Here is an example: - -[source,js] --------------------------------------------------- -$ curl -XGET 'http://localhost:9200/twitter/tweet/_search/exists?q=user:kimchy' - -$ curl -XGET 'http://localhost:9200/twitter/tweet/_search/exists' -d ' -{ - "query" : { - "term" : { "user" : "kimchy" } - } -}' - --------------------------------------------------- - -NOTE: The query being sent in the body must be nested in a `query` key, same as -how the <> works. - -Both the examples above do the same thing, which is determine the existence of -tweets from the twitter index for a certain user. The response body will be of -the following format: - -[source,js] --------------------------------------------------- -{ - "exists" : true -} --------------------------------------------------- - -[float] -=== Multi index, Multi type - -The exists API can be applied to <>. - -[float] -=== Request Parameters - -When executing exists using the query parameter `q`, the query passed is -a query string using Lucene query parser. There are additional -parameters that can be passed: - -[cols="<,<",options="header",] -|======================================================================= -|Name |Description -|`df` |The default field to use when no field prefix is defined within the -query. - -|`analyzer` |The analyzer name to be used when analyzing the query string. - -|`default_operator` |The default operator to be used, can be `AND` or -`OR`. Defaults to `OR`. - -|`lenient` |If set to true will cause format based failures (like -providing text to a numeric field) to be ignored. Defaults to false. - -|`lowercase_expanded_terms` |Should terms be automatically lowercased or -not. Defaults to `true`. - -|`analyze_wildcard` |Should wildcard and prefix queries be analyzed or -not. Defaults to `false`. -|======================================================================= - -[float] -=== Request Body - -The exists API can use the <> within -its body in order to express the query that should be executed. The body -content can also be passed as a REST parameter named `source`. - -HTTP GET and HTTP POST can be used to execute exists with body. -Since not all clients support GET with body, POST is allowed as well. - -[float] -=== Distributed - -The exists operation is broadcast across all shards. For each shard id -group, a replica is chosen and executed against it. This means that -replicas increase the scalability of exists. The exists operation also -early terminates shard requests once the first shard reports matched -document existence. - -[float] -=== Routing - -The routing value (a comma separated list of the routing values) can be -specified to control which shards the exists request will be executed on. diff --git a/docs/reference/search/field-stats.asciidoc b/docs/reference/search/field-stats.asciidoc index fb29903ebeb..f3b76af038c 100644 --- a/docs/reference/search/field-stats.asciidoc +++ b/docs/reference/search/field-stats.asciidoc @@ -79,11 +79,21 @@ document and field. `min_value`:: -The lowest value in the field represented in a displayable form. +The lowest value in the field. + +`min_value_as_string`:: + +The lowest value in the field represented in a displayable form. All fields, +but string fields returns this. (since string fields, represent values already as strings) `max_value`:: -The highest value in the field represented in a displayable form. +The highest value in the field. + +`max_value_as_string`:: + +The highest value in the field represented in a displayable form. All fields, +but string fields returns this. (since string fields, represent values already as strings) NOTE: Documents marked as deleted (but not yet removed by the merge process) still affect all the mentioned statistics. @@ -240,7 +250,7 @@ curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{ "index_constraints" : { <2> "creation_date" : { <3> "min_value" : { <4> - "gte" : "2014-01-01T00:00:00.000Z", + "gte" : "2014-01-01T00:00:00.000Z" }, "max_value" : { "lt" : "2015-01-01T00:00:00.000Z" @@ -263,3 +273,27 @@ Each index constraint support the following comparisons: `gt`:: Greater-than `lte`:: Less-than or equal to `lt`:: Less-than + +Field stats index constraints on date fields optionally accept a `format` option, used to parse the constraint's value. +If missing, the format configured in the field's mapping is used. + +[source,js] +-------------------------------------------------- +curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{ + "fields" : ["answer_count"] + "index_constraints" : { + "creation_date" : { + "min_value" : { + "gte" : "2014-01-01", + "format" : "date_optional_time" <1> + }, + "max_value" : { + "lt" : "2015-01-01", + "format" : "date_optional_time" + } + } + } +}' +-------------------------------------------------- + +<1> Custom date format \ No newline at end of file diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 857047cd51c..325ae0d94a6 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -79,7 +79,6 @@ And here is a sample response: `terminate_after`:: - experimental[The API for this feature may change in the future] The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early. If set, the response will have a boolean field `terminated_early` to indicate whether @@ -95,6 +94,46 @@ parameter named `source`. Both HTTP GET and HTTP POST can be used to execute search with body. Since not all clients support GET with body, POST is allowed as well. +[float] +=== Fast check for any matching docs + +In case we only want to know if there are any documents matching a +specific query, we can set the `size` to `0` to indicate that we are not +interested in the search results. Also we can set `terminate_after` to `1` +to indicate that the query execution can be terminated whenever the first +matching document was found (per shard). + +[source,js] +-------------------------------------------------- +$ curl -XGET 'http://localhost:9200/_search?q=tag:wow&size=0&terminate_after=1' +-------------------------------------------------- + +The response will not contain any hits as the `size` was set to `0`. The +`hits.total` will be either equal to `0`, indicating that there were no +matching documents, or greater than `0` meaning that there were at least +as many documents matching the query when it was early terminated. +Also if the query was terminated early, the `terminated_early` flag will +be set to `true` in the response. + +[source,js] +-------------------------------------------------- +{ + "took": 3, + "timed_out": false, + "terminated_early": true, + "_shards": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": 0, + "hits": [] + } +} +-------------------------------------------------- + include::request/query.asciidoc[] diff --git a/docs/reference/search/request/fields.asciidoc b/docs/reference/search/request/fields.asciidoc index 56a3d49207b..e929928d427 100644 --- a/docs/reference/search/request/fields.asciidoc +++ b/docs/reference/search/request/fields.asciidoc @@ -1,6 +1,11 @@ [[search-request-fields]] === Fields +WARNING: The `fields` parameter is about fields that are explicitly marked as +stored in the mapping, which is off by default and generally not recommended. +Use <> instead to select +subsets of the original source document to be returned. + Allows to selectively load specific stored fields for each document represented by a search hit. diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc index 7a466405789..99742db77c1 100644 --- a/docs/reference/search/request/highlighting.asciidoc +++ b/docs/reference/search/request/highlighting.asciidoc @@ -2,9 +2,9 @@ === Highlighting Allows to highlight search results on one or more fields. The -implementation uses either the lucene `highlighter`, `fast-vector-highlighter` -or `postings-highlighter`. The following is an example of the search request -body: +implementation uses either the lucene `plain` highlighter, the +fast vector highlighter (`fvh`) or `postings` highlighter. +The following is an example of the search request body: [source,js] -------------------------------------------------- @@ -285,7 +285,7 @@ is required. Note that `fragment_size` is ignored in this case. } -------------------------------------------------- -When using `fast-vector-highlighter` one can use `fragment_offset` +When using `fvh` one can use `fragment_offset` parameter to control the margin to start highlighting from. In the case where there is no matching fragment to highlight, the default is @@ -554,7 +554,7 @@ to [[phrase-limit]] ==== Phrase Limit -The `fast-vector-highlighter` has a `phrase_limit` parameter that prevents +The fast vector highlighter has a `phrase_limit` parameter that prevents it from analyzing too many phrases and eating tons of memory. It defaults to 256 so only the first 256 matching phrases in the document scored considered. You can raise the limit with the `phrase_limit` parameter but diff --git a/docs/reference/search/request/post-filter.asciidoc b/docs/reference/search/request/post-filter.asciidoc index 7c352e9fd50..7bd95400312 100644 --- a/docs/reference/search/request/post-filter.asciidoc +++ b/docs/reference/search/request/post-filter.asciidoc @@ -78,7 +78,7 @@ curl -XGET localhost:9200/shirts/_search -d ' }, "aggs": { "colors": { - "terms": { "field": "color" }, <2> + "terms": { "field": "color" } <2> }, "color_red": { "filter": { diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc index f73181f19ed..8d0b6708979 100644 --- a/docs/reference/search/request/sort.asciidoc +++ b/docs/reference/search/request/sort.asciidoc @@ -84,13 +84,12 @@ field support has the following parameters on top of the already existing sort options: `nested_path`:: - Defines the on what nested object to sort. The actual - sort field must be a direct field inside this nested object. The default - is to use the most immediate inherited nested object from the sort - field. + Defines on which nested object to sort. The actual + sort field must be a direct field inside this nested object. + When sorting by nested field, this field is mandatory. `nested_filter`:: - A filter the inner objects inside the nested path + A filter that the inner objects inside the nested path should match with in order for its field values to be taken into account by sorting. Common case is to repeat the query / filter inside the nested filter or query. By default no `nested_filter` is active. @@ -98,7 +97,7 @@ existing sort options: ===== Nested sorting example In the below example `offer` is a field of type `nested`. -The `nested_path` needs to be specified other elasticsearch doesn't on what nested level sort values need to be captured. +The `nested_path` needs to be specified; otherwise, elasticsearch doesn't know on what nested level sort values need to be captured. [source,js] -------------------------------------------------- diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 4a5a867f08c..cad6f5a63b8 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -49,4 +49,3 @@ Or even search across all indices and all types: -------------------------------------------------- $ curl -XGET 'http://localhost:9200/_search?q=tag:wow' -------------------------------------------------- - diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index af93ea1598a..b0be107af1c 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -4,46 +4,37 @@ NOTE: In order to understand the format of suggestions, please read the <> page first. -The `completion` suggester is a so-called prefix suggester. It does not -do spell correction like the `term` or `phrase` suggesters but allows -basic `auto-complete` functionality. +The `completion` suggester provides auto-complete/search-as-you-type +functionality. This is a navigational feature to guide users to +relevant results as they are typing, improving search precision. +It is not meant for spell correction or did-you-mean functionality +like the `term` or `phrase` suggesters. -==== Why another suggester? Why not prefix queries? - -The first question which comes to mind when reading about a prefix -suggestion is, why you should use it at all, if you have prefix queries -already. The answer is simple: Prefix suggestions are fast. - -The data structures are internally backed by Lucenes -`AnalyzingSuggester`, which uses FSTs (finite state transducers) to -execute suggestions. Usually these data structures are costly to -create, stored in-memory and need to be rebuilt every now and then to -reflect changes in your indexed documents. The `completion` suggester -circumvents this by storing the FST (finite state transducer) as part -of your index during index time. This allows for really fast -loads and executions. +Ideally, auto-complete functionality should be as fast as a user +types to provide instant feedback relevant to what a user has already +typed in. Hence, `completion` suggester is optimized for speed. +The suggester uses data structures that enable fast lookups, +but are costly to build and are stored in-memory. [[completion-suggester-mapping]] ==== Mapping -In order to use this feature, you have to specify a special mapping for -this field, which enables the special storage of the field. +To use this feature, specify a special mapping for this field, +which indexes the field values for fast completions. [source,js] -------------------------------------------------- -curl -X PUT localhost:9200/music -curl -X PUT localhost:9200/music/song/_mapping -d '{ +PUT music/song/_mapping +{ "song" : { "properties" : { - "name" : { "type" : "string" }, - "suggest" : { "type" : "completion", - "analyzer" : "simple", - "search_analyzer" : "simple", - "payloads" : true + ... + "suggest" : { + "type" : "completion", } } } -}' +} -------------------------------------------------- Mapping supports the following parameters: @@ -58,17 +49,14 @@ Mapping supports the following parameters: `search_analyzer`:: The search analyzer to use, defaults to value of `analyzer`. -`payloads`:: - Enables the storing of payloads, defaults to `false` - `preserve_separators`:: Preserves the separators, defaults to `true`. If disabled, you could find a field starting with `Foo Fighters`, if you suggest for `foof`. `preserve_position_increments`:: - Enables position increments, defaults - to `true`. If disabled and using stopwords analyzer, you could get a + Enables position increments, defaults to `true`. + If disabled and using stopwords analyzer, you could get a field starting with `The Beatles`, if you suggest for `b`. *Note*: You could also achieve this by indexing two inputs, `Beatles` and `The Beatles`, no need to change a simple analyzer, if you are able to @@ -78,84 +66,88 @@ Mapping supports the following parameters: Limits the length of a single input, defaults to `50` UTF-16 code points. This limit is only used at index time to reduce the total number of characters per input string in order to prevent massive inputs from - bloating the underlying datastructure. The most usecases won't be influenced - by the default value since prefix completions hardly grow beyond prefixes longer - than a handful of characters. (Old name "max_input_len" is deprecated) + bloating the underlying datastructure. Most usecases won't be influenced + by the default value since prefix completions seldom grow beyond prefixes longer + than a handful of characters. [[indexing]] ==== Indexing +You index suggestions like any other field. A suggestion is made of an +`input` and an optional `weight` attribute. An `input` is the expected +text to be matched by a suggestion query and the `weight` determines how +the suggestions will be scored. Indexing a suggestion is as follows: + [source,js] -------------------------------------------------- -curl -X PUT 'localhost:9200/music/song/1?refresh=true' -d '{ - "name" : "Nevermind", +PUT music/song/1?refresh=true +{ "suggest" : { "input": [ "Nevermind", "Nirvana" ], - "output": "Nirvana - Nevermind", - "payload" : { "artistId" : 2321 }, "weight" : 34 } -}' +} -------------------------------------------------- The following parameters are supported: `input`:: - The input to store, this can be a an array of strings or just + The input to store, this can be an array of strings or just a string. This field is mandatory. -`output`:: - The string to return, if a suggestion matches. This is very - useful to normalize outputs (i.e. have them always in the format - `artist - songname`). This is optional. - *Note*: The result is de-duplicated if several documents - have the same output, i.e. only one is returned as part of the - suggest result. - -`payload`:: - An arbitrary JSON object, which is simply returned in the - suggest option. You could store data like the id of a document, in order - to load it from elasticsearch without executing another search (which - might not yield any results, if `input` and `output` differ strongly). - `weight`:: A positive integer or a string containing a positive integer, which defines a weight and allows you to rank your suggestions. This field is optional. -NOTE: Even though you will lose most of the features of the -completion suggest, you can choose to use the following shorthand form. -Keep in mind that you will not be able to use several inputs, an output, -payloads or weights. This form does still work inside of multi fields. +You can index multiple suggestions for a document as follows: + +[source,js] +-------------------------------------------------- +PUT music/song/1?refresh=true +{ + "suggest" : [ + { + "input": "Nevermind", + "weight" : 10 + }, + { + "input": "Nirvana", + "weight" : 3 + } + ] +} +-------------------------------------------------- + +You can use the following shorthand form. Note that you can not specify +a weight with suggestion(s). [source,js] -------------------------------------------------- { - "suggest" : "Nirvana" + "suggest" : [ "Nevermind", "Nirvana" ] } -------------------------------------------------- -NOTE: The suggest data structure might not reflect deletes on -documents immediately. You may need to do an <> for that. -You can call optimize with the `only_expunge_deletes=true` to only target -deletions for merging. - [[querying]] ==== Querying Suggesting works as usual, except that you have to specify the suggest -type as `completion`. +type as `completion`. Suggestions are near real-time, which means +new suggestions can be made visible by <> and +documents once deleted are never shown. [source,js] -------------------------------------------------- -curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ +POST music/_suggest?pretty +{ "song-suggest" : { - "text" : "n", + "prefix" : "n", "completion" : { "field" : "suggest" } } -}' +} { "_shards" : { @@ -168,39 +160,97 @@ curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ "offset" : 0, "length" : 1, "options" : [ { - "text" : "Nirvana - Nevermind", - "score" : 34.0, "payload" : {"artistId":2321} + "text" : "Nirvana", + "score" : 34.0 } ] } ] } -------------------------------------------------- -As you can see, the payload is included in the response, if configured -appropriately. If you configured a weight for a suggestion, this weight -is used as `score`. Also the `text` field uses the `output` of your -indexed suggestion, if configured, otherwise the matched part of the -`input` field. +The configured weight for a suggestion is returned as `score`. +The `text` field uses the `input` of your indexed suggestion. -The basic completion suggester query supports the following two parameters: +Suggestions are document oriented, you can specify fields to be +returned as part of suggestion payload. All field types (`string`, +`numeric`, `date`, etc) are supported. + +For example, if you index a "title" field along with the suggestion +as follows: + +[source,js] +-------------------------------------------------- +POST music/song +{ + "suggest" : "Nirvana", + "title" : "Nevermind" +} +-------------------------------------------------- + +You can get the "title" as part of the suggestion +payload by specifying it as a `payload`: + +[source,js] +-------------------------------------------------- +POST music/_suggest?pretty +{ + "song-suggest" : { + "prefix" : "n", + "completion" : { + "field" : "suggest" + "payload" : [ "title" ] <1> + } + } +} + +{ + "_shards" : { + "total" : 5, + "successful" : 5, + "failed" : 0 + }, + "song-suggest" : [ { + "text" : "n", + "offset" : 0, + "length" : 1, + "options" : [ { + "text" : "Nirvana", + "score" : 34.0, + "payload" : { + "title" : [ "Nevermind" ] + } + } ] + } ] +} +-------------------------------------------------- +<1> The fields to be returned as part of each suggestion payload. + +The basic completion suggester query supports the following parameters: `field`:: The name of the field on which to run the query (required). `size`:: The number of suggestions to return (defaults to `5`). +`payload`:: The name of the field or field name array to be returned + as payload (defaults to no fields). NOTE: The completion suggester considers all documents in the index. See <> for an explanation of how to query a subset of documents instead. +NOTE: Specifying `payload` fields will incur additional search performance +hit. The `payload` fields are retrieved eagerly (single pass) for top +suggestions at the shard level using field data or from doc values. + [[fuzzy]] ==== Fuzzy queries The completion suggester also supports fuzzy queries - this means, -you can actually have a typo in your search and still get results back. +you can have a typo in your search and still get results back. [source,js] -------------------------------------------------- -curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ +POST music/_suggest?pretty +{ "song-suggest" : { - "text" : "n", + "prefix" : "n", "completion" : { "field" : "suggest", "fuzzy" : { @@ -208,9 +258,12 @@ curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ } } } -}' +} -------------------------------------------------- +Suggestions that share the longest prefix to the query `prefix` will +be scored higher. + The fuzzy query can take specific fuzzy parameters. The following parameters are supported: @@ -232,10 +285,48 @@ The following parameters are supported: checked for fuzzy alternatives, defaults to `1` `unicode_aware`:: - Sets all are measurements (like edit distance, - transpositions and lengths) in unicode code points - (actual letters) instead of bytes. + If `true`, all measurements (like fuzzy edit + distance, transpositions, and lengths) are + measured in Unicode code points instead of + in bytes. This is slightly slower than raw + bytes, so it is set to `false` by default. NOTE: If you want to stick with the default values, but still use fuzzy, you can either use `fuzzy: {}` or `fuzzy: true`. + +[[regex]] +==== Regex queries + +The completion suggester also supports regex queries meaning +you can express a prefix as a regular expression + +[source,js] +-------------------------------------------------- +POST music/_suggest?pretty +{ + "song-suggest" : { + "regex" : "n[ever|i]r", + "completion" : { + "field" : "suggest", + } + } +} +-------------------------------------------------- + +The regex query can take specific regex parameters. +The following parameters are supported: + +[horizontal] +`flags`:: + Possible flags are `ALL` (default), `ANYSTRING`, `COMPLEMENT`, + `EMPTY`, `INTERSECTION`, `INTERVAL`, or `NONE`. See <> + for their meaning + +`max_determinized_states`:: + Regular expressions are dangerous because it's easy to accidentally + create an innocuous looking one that requires an exponential number of + internal determinized automaton states (and corresponding RAM and CPU) + for Lucene to execute. Lucene prevents these using the + `max_determinized_states` setting (defaults to 10000). You can raise + this limit to allow more complex regular expressions to execute. diff --git a/docs/reference/search/suggesters/context-suggest.asciidoc b/docs/reference/search/suggesters/context-suggest.asciidoc index c09659a43a9..a492c37b5b4 100644 --- a/docs/reference/search/suggesters/context-suggest.asciidoc +++ b/docs/reference/search/suggesters/context-suggest.asciidoc @@ -1,237 +1,290 @@ [[suggester-context]] === Context Suggester -The context suggester is an extension to the suggest API of Elasticsearch. Namely the -suggester system provides a very fast way of searching documents by handling these -entirely in memory. But this special treatment does not allow the handling of -traditional queries and filters, because those would have notable impact on the -performance. So the context extension is designed to take so-called context information -into account to specify a more accurate way of searching within the suggester system. -Instead of using the traditional query and filter system a predefined ``context`` is -configured to limit suggestions to a particular subset of suggestions. -Such a context is defined by a set of context mappings which can either be a simple -*category* or a *geo location*. The information used by the context suggester is -configured in the type mapping with the `context` parameter, which lists all of the -contexts that need to be specified in each document and in each suggestion request. -For instance: +The completion suggester considers all documents in the index, but it is often +desirable to serve suggestions filtered and/or boosted by some criteria. +For example, you want to suggest song titles filtered by certain artists or +you want to boost song titles based on their genre. + +To achieve suggestion filtering and/or boosting, you can add context mappings while +configuring a completion field. You can define multiple context mappings for a +completion field. +Every context mapping has a unique name and a type. There are two types: `category` +and `geo`. Context mappings are configured under the `contexts` parameter in +the field mapping. + +The following defines two context mappings for a completion field: [source,js] -------------------------------------------------- -PUT services/_mapping/service +PUT place/shops/_mapping { - "service": { - "properties": { - "name": { - "type" : "string" - }, - "tag": { - "type" : "string" - }, - "suggest_field": { - "type": "completion", - "context": { - "color": { <1> + "shops" : { + "properties" : { + ... + "suggest" : { + "type" : "completion", + "contexts": [ + { <1> + "name": "place_type", "type": "category", - "path": "color_field", - "default": ["red", "green", "blue"] + "path": "cat" }, - "location": { <2> + { <2> + "name": "location" "type": "geo", - "precision": "5m", - "neighbors": true, - "default": "u33" + "path": "loc" } - } + ] } } } } -------------------------------------------------- -<1> See <> -<2> See <> +<1> Defines a `category` context named 'place_type', which will index values from field 'cat'. + See <> +<2> Defines a `geo` context named 'location', which will index values from field 'loc'. + See <> -However contexts are specified (as type `category` or `geo`, which are discussed below), each -context value generates a new sub-set of documents which can be queried by the completion -suggester. All three types accept a `default` parameter which provides a default value to use -if the corresponding context value is absent. - -The basic structure of this element is that each field forms a new context and the fieldname -is used to reference this context information later on during indexing or querying. All context -mappings have the `default` and the `type` option in common. The value of the `default` field -is used, when ever no specific is provided for the certain context. Note that a context is -defined by at least one value. The `type` option defines the kind of information hold by this -context. These type will be explained further in the following sections. +NOTE: Adding context mappings increases the index size for completion field. The completion index +is entirely heap resident, you can monitor the completion field index size using <>. [[suggester-context-category]] [float] ==== Category Context -The `category` context allows you to specify one or more categories in the document at index time. -The document will be assigned to each named category, which can then be queried later. The category -type also allows to specify a field to extract the categories from. The `path` parameter is used to -specify this field of the documents that should be used. If the referenced field contains multiple -values, all these values will be used as alternative categories. + +The `category` context allows you to associate one or more categories with suggestions at index +time. At query time, suggestions can be filtered and boosted by their associated categories. [float] ===== Category Mapping -The mapping for a category is simply defined by its `default` values. These can either be -defined as list of *default* categories: +A `category` context mapping, where categories are provided explicitly with suggestions +can be defined as follows: [source,js] -------------------------------------------------- -"context": { - "color": { +"contexts": [ + { + "name": "cat_context", "type": "category", - "default": ["red", "orange"] } -} +] -------------------------------------------------- -or as a single value +Alternatively, A `category` context mapping that references another field within a document +can be defined as follows: [source,js] -------------------------------------------------- -"context": { - "color": { +"contexts": [ + { + "name": "cat_context", "type": "category", - "default": "red" + "path": "cat_field" } -} +] -------------------------------------------------- -or as reference to another field within the documents indexed: - -[source,js] --------------------------------------------------- -"context": { - "color": { - "type": "category", - "default": "red", - "path": "color_field" - } -} --------------------------------------------------- - -in this case the *default* categories will only be used, if the given field does not -exist within the document. In the example above the categories are received from a -field named `color_field`. If this field does not exist a category *red* is assumed for -the context *color*. - [float] ===== Indexing category contexts -Within a document the category is specified either as an `array` of values, a -single value or `null`. A list of values is interpreted as alternative categories. So -a document belongs to all the categories defined. If the category is `null` or remains -unset the categories will be retrieved from the documents field addressed by the `path` -parameter. If this value is not set or the field is missing, the default values of the -mapping will be assigned to the context. + +Category contexts can be specified explicitly when indexing suggestions. If a suggestion has +multiple categories, the suggestion will be indexed for each category: [source,js] -------------------------------------------------- -PUT services/service/1 +PUT place/shops/1 { - "name": "knapsack", - "suggest_field": { - "input": ["knacksack", "backpack", "daypack"], + ... + "suggest": { + "input": ["timmy's", "starbucks", "dunkin donuts"], "context": { - "color": ["red", "yellow"] + "place_type": ["cafe", "food"] <1> } } } -------------------------------------------------- -[float] -===== Category Query -A query within a category works similar to the configuration. If the value is `null` -the mappings default categories will be used. Otherwise the suggestion takes place -for all documents that have at least one category in common with the query. +<1> These suggestions will be associated with 'cafe' and 'food' category. + +Category contexts can also be referenced from another indexed field in the document via +the `path` parameter in the field mapping: [source,js] -------------------------------------------------- -POST services/_suggest?pretty' +"contexts": [ + { + "name": "cat_context", + "type": "category", + "path": "cat" + } +] +-------------------------------------------------- + +With the above mapping, the following will index the suggestions, treating the values of the +'cat' field as category contexts: + +[source,js] +-------------------------------------------------- +PUT place/shops/1 +{ + ... + "suggest": ["timmy's", "starbucks", "dunkin donuts"], + "cat": ["cafe", "food"] <1> +} +-------------------------------------------------- + +<1> These suggestions will be associated with 'cafe' and 'food' category. + +NOTE: If context mapping references another field and the categories +are explicitly indexed, the suggestions are indexed with both set +of categories. + + +[float] +===== Category Query + +Suggestions can be filtered by one or more categories. The following +filters suggestions by multiple categories: + +[source,js] +-------------------------------------------------- +POST place/_suggest?pretty { "suggest" : { - "text" : "m", + "prefix" : "tim", "completion" : { - "field" : "suggest_field", + "field" : "suggest", "size": 10, - "context": { - "color": "red" + "contexts": { + "place_type": [ "cafe", "restaurants" ] } } } } -------------------------------------------------- +NOTE: When no categories are provided at query-time, all indexed documents are considered. +Querying with no categories on a category enabled completion field should be avoided, as it +will degrade search performance. + +Suggestions with certain categories can be boosted higher than others. +The following filters suggestions by categories and additionally boosts +suggestions associated with some categories: + +[source,js] +-------------------------------------------------- +POST place/_suggest?pretty +{ + "suggest" : { + "prefix" : "tim", + "completion" : { + "field" : "suggest", + "size": 10, + "contexts": { + "place_type": [ <1> + { "context" : "cafe" }, + { "context" : "restaurants", "boost": 2 } + ] + } + } + } +} +-------------------------------------------------- +<1> The context query filter suggestions associated with + categories 'cafe' and 'restaurants' and boosts the + suggestions associated with 'restaurants' by a + factor of `2` + +In addition to accepting category values, a context query can be composed of +multiple category context clauses. The following parameters are supported for a +`category` context clause: + +[horizontal] +`context`:: + The value of the category to filter/boost on. + This is mandatory. + +`boost`:: + The factor by which the score of the suggestion + should be boosted, the score is computed by + multiplying the boost with the suggestion weight, + defaults to `1` + +`prefix`:: + Whether the category value should be treated as a + prefix or not. For example, if set to `true`, + you can filter category of 'type1', 'type2' and + so on, by specifying a category prefix of 'type'. + Defaults to `false` + [[suggester-context-geo]] [float] ==== Geo location Context -A `geo` context allows you to limit results to those that lie within a certain distance -of a specified geolocation. At index time, a lat/long geo point is converted into a -geohash of a certain precision, which provides the context. + +A `geo` context allows you to associate one or more geo points or geohashes with suggestions +at index time. At query time, suggestions can be filtered and boosted if they are within +a certain distance of a specified geo location. + +Internally, geo points are encoded as geohashes with the specified precision. +See <> for more background details. [float] -===== Geo location Mapping -The mapping for a geo context accepts four settings, only of which `precision` is required: +===== Geo Mapping + +In addition to the `path` setting, `geo` context mapping accepts the following settings: [horizontal] -`precision`:: This defines the precision of the geohash and can be specified as `5m`, `10km`, - or as a raw geohash precision: `1`..`12`. It's also possible to setup multiple - precisions by defining a list of precisions: `["5m", "10km"]` -`neighbors`:: Geohashes are rectangles, so a geolocation, which in reality is only 1 metre - away from the specified point, may fall into the neighbouring rectangle. Set - `neighbours` to `true` to include the neighbouring geohashes in the context. - (default is *on*) -`path`:: Optionally specify a field to use to look up the geopoint. -`default`:: The geopoint to use if no geopoint has been specified. +`precision`:: + This defines the precision of the geohash to be indexed and can be specified + as a distance value (`5m`, `10km` etc.), or as a raw geohash precision (`1`..`12`). + Defaults to a raw geohash precision value of `6`. -Since all locations of this mapping are translated into geohashes, each location matches -a geohash cell. So some results that lie within the specified range but not in the same -cell as the query location will not match. To avoid this the `neighbors` option allows a -matching of cells that join the bordering regions of the documents location. This option -is turned on by default. -If a document or a query doesn't define a location a value to use instead can defined by -the `default` option. The value of this option supports all the ways a `geo_point` can be -defined. The `path` refers to another field within the document to retrieve the -location. If this field contains multiple values, the document will be linked to all these -locations. +NOTE: The index time `precision` setting sets the maximum geohash precision that +can be used at query time. + +The following defines a `geo` context mapping with an index time precision of `4` +indexing values from a geo point field 'pin': [source,js] -------------------------------------------------- -"context": { - "location": { +"contexts": [ + { + "name": "location" "type": "geo", - "precision": ["1km", "5m"], - "neighbors": true, + "precision": 4, "path": "pin", - "default": { - "lat": 0.0, - "lon": 0.0 - } } -} +] -------------------------------------------------- [float] -===== Geo location Config +===== Indexing geo contexts -Within a document a geo location retrieved from the mapping definition can be overridden -by another location. In this case the context mapped to a geo location supports all -variants of defining a `geo_point`. +`geo` contexts can be explicitly set with suggestions or be indexed from a geo point field in the +document via the `path` parameter, similar to `category` contexts. Associating multiple geo location context +with a suggestion, will index the suggestion for every geo location. The following indexes a suggestion +with two geo location contexts: [source,js] -------------------------------------------------- -PUT services/service/1 +PUT place/shops/1 { - "name": "some hotel 1", - "suggest_field": { - "input": ["my hotel", "this hotel"], - "context": { - "location": { - "lat": 0, - "lon": 0 - } - } + "suggest": { + "input": "timmy's", + "context": [ + "location": [ + { + "lat": 43.6624803, + "lon": -79.3863353 + }, + { + "lat": 43.6624718, + "lon": -79.3873227 + } + ] + ] } } -------------------------------------------------- @@ -239,24 +292,23 @@ PUT services/service/1 [float] ===== Geo location Query -Like in the configuration, querying with a geo location in context, the geo location -query supports all representations of a `geo_point` to define the location. In this -simple case all precision values defined in the mapping will be applied to the given -location. +Suggestions can be filtered and boosted with respect to how close they are to one or +more geo points. The following filters suggestions that fall within the area represented by +the encoded geohash of a geo point: [source,js] -------------------------------------------------- -POST services/_suggest +POST place/_suggest { "suggest" : { - "text" : "m", + "prefix" : "tim", "completion" : { - "field" : "suggest_field", + "field" : "suggest", "size": 10, "context": { "location": { - "lat": 0, - "lon": 0 + "lat": 43.662, + "lon": -79.380 } } } @@ -264,54 +316,71 @@ POST services/_suggest } -------------------------------------------------- -But it also possible to set a subset of the precisions set in the mapping, by using the -`precision` parameter. Like in the mapping, this parameter is allowed to be set to a -single precision value or a list of these. +NOTE: When a location with a lower precision at query time is specified, all suggestions +that fall within the area will be considered. + +Suggestions that are within an area represented by a geohash can also be boosted higher +than others, as shown by the following: [source,js] -------------------------------------------------- -POST services/_suggest +POST place/_suggest?pretty { "suggest" : { - "text" : "m", + "prefix" : "tim", "completion" : { - "field" : "suggest_field", + "field" : "suggest", "size": 10, - "context": { - "location": { - "value": { - "lat": 0, - "lon": 0 + "contexts": { + "location": [ <1> + { + "lat": 43.6624803, + "lon": -79.3863353, + "precision": 2 }, - "precision": "1km" - } + { + "context": { + "lat": 43.6624803, + "lon": -79.3863353 + }, + "boost": 2 + } + ] } } } } -------------------------------------------------- +<1> The context query filters for suggestions that fall under + the geo location represented by a geohash of '(43.662, -79.380)' + with a precision of '2' and boosts suggestions + that fall under the geohash representation of '(43.6624803, -79.3863353)' + with a default precision of '6' by a factor of `2` -A special form of the query is defined by an extension of the object representation of -the `geo_point`. Using this representation allows to set the `precision` parameter within -the location itself: +In addition to accepting context values, a context query can be composed of +multiple context clauses. The following parameters are supported for a +`category` context clause: -[source,js] --------------------------------------------------- -POST services/_suggest -{ - "suggest" : { - "text" : "m", - "completion" : { - "field" : "suggest_field", - "size": 10, - "context": { - "location": { - "lat": 0, - "lon": 0, - "precision": "1km" - } - } - } - } -} --------------------------------------------------- +[horizontal] +`context`:: + A geo point object or a geo hash string to filter or + boost the suggestion by. This is mandatory. + +`boost`:: + The factor by which the score of the suggestion + should be boosted, the score is computed by + multiplying the boost with the suggestion weight, + defaults to `1` + +`precision`:: + The precision of the geohash to encode the query geo point. + This can be specified as a distance value (`5m`, `10km` etc.), + or as a raw geohash precision (`1`..`12`). + Defaults to index time precision level. + +`neighbours`:: + Accepts an array of precision values at which + neighbouring geohashes should be taken into account. + precision value can be a distance value (`5m`, `10km` etc.) + or a raw geohash precision (`1`..`12`). Defaults to + generating neighbours for index time precision level. diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc index c2322e53cf9..e43c3edf94a 100644 --- a/docs/reference/search/uri-request.asciidoc +++ b/docs/reference/search/uri-request.asciidoc @@ -91,8 +91,7 @@ scores and return them as part of each hit. within the specified time value and bail with the hits accumulated up to that point when expired. Defaults to no timeout. -|`terminate_after` |experimental[The API for this feature may change in the future] -The maximum number of documents to collect for +|`terminate_after` |The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early. If set, the response will have a boolean field `terminated_early` to indicate whether the query execution has actually terminated_early. diff --git a/docs/reference/setup/cluster_restart.asciidoc b/docs/reference/setup/cluster_restart.asciidoc index 32e82008814..219a6450848 100644 --- a/docs/reference/setup/cluster_restart.asciidoc +++ b/docs/reference/setup/cluster_restart.asciidoc @@ -64,7 +64,7 @@ upgraded following the same procedure described in <>. If you have dedicated master nodes -- nodes with `node.master` set to `true`(the default) and `node.data` set to `false` -- then it is a good idea to start them first. Wait for them to form a cluster and to elect a master -before proceding with the data nodes. You can check progress by looking at the +before proceeding with the data nodes. You can check progress by looking at the logs. As soon as the <> diff --git a/docs/reference/setup/dir-layout.asciidoc b/docs/reference/setup/dir-layout.asciidoc index 85b7ae1d935..5e290e48794 100644 --- a/docs/reference/setup/dir-layout.asciidoc +++ b/docs/reference/setup/dir-layout.asciidoc @@ -57,7 +57,7 @@ Below are the default paths that elasticsearch will use, if not explicitly chang | conf | Configuration files `elasticsearch.yml` and `logging.yml`. | /etc/elasticsearch | /etc/elasticsearch -| conf | Environment variables including heap size, file descriptors. | /etc/default/elasticseach | /etc/sysconfig/elasticsearch +| conf | Environment variables including heap size, file descriptors. | /etc/default/elasticsearch | /etc/sysconfig/elasticsearch | data | The location of the data files of each index / shard allocated on the node. | /var/lib/elasticsearch/data | /var/lib/elasticsearch @@ -83,8 +83,6 @@ on the node. | /var/lib/elasticsearch/data | /var/lib/elasticsearch | conf | Configuration files `elasticsearch.yml` and `logging.yml` | {extract.path}/config -| conf | Environment variables including heap size, file descriptors | {extract.path}/config - | data | The location of the data files of each index / shard allocated on the node | {extract.path}/data diff --git a/docs/reference/setup/rolling_upgrade.asciidoc b/docs/reference/setup/rolling_upgrade.asciidoc index 2ac2963a239..b3c00d337f8 100644 --- a/docs/reference/setup/rolling_upgrade.asciidoc +++ b/docs/reference/setup/rolling_upgrade.asciidoc @@ -60,7 +60,7 @@ default. It is a good idea to place these directories in a different location so that there is no chance of deleting them when upgrading Elasticsearch. These -custom paths can be <> with the `path.config` and +custom paths can be <> with the `path.conf` and `path.data` settings. The Debian and RPM packages place these directories in the @@ -80,7 +80,7 @@ To upgrade using a zip or compressed tarball: overwrite the `config` or `data` directories. * Either copy the files in the `config` directory from your old installation - to your new installation, or use the `--path.config` option on the command + to your new installation, or use the `--path.conf` option on the command line to point to an external config directory. * Either copy the files in the `data` directory from your old installation diff --git a/docs/reference/setup/upgrade.asciidoc b/docs/reference/setup/upgrade.asciidoc index 15cd90f98dd..894f82a6db5 100644 --- a/docs/reference/setup/upgrade.asciidoc +++ b/docs/reference/setup/upgrade.asciidoc @@ -21,12 +21,10 @@ consult this table: [cols="1> -|< 0.90.7 |0.90.x |<> -|>= 0.90.7 |0.90.x |<> -|1.0.0 - 1.3.1 |1.x |<> (if <> set to `false`) -|>= 1.3.2 |1.x |<> +|0.90.x |2.x |<> |1.x |2.x |<> +|2.x |2.y |<> (where `y > x `) +|2.x |3.x |<> |======================================================================= TIP: Take plugins into consideration as well when upgrading. Most plugins will have to be upgraded alongside Elasticsearch, although some plugins accessed primarily through the browser (`_site` plugins) may continue to work given that API changes are compatible. diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc index a2d3f39d5c1..9c0e5f4f10d 100644 --- a/docs/reference/testing/testing-framework.asciidoc +++ b/docs/reference/testing/testing-framework.asciidoc @@ -81,7 +81,7 @@ There are a couple of helper methods in `ESIntegTestCase`, which will make your `createIndex(name)`:: Creates an index with the specified name `flush()`:: Flushes all indices in a cluster `flushAndRefresh()`:: Combines `flush()` and `refresh()` calls -`optimize()`:: Waits for all relocations and optimized all indices in the cluster to one segment. +`forceMerge()`:: Waits for all relocations and force merges all indices in the cluster to one segment. `indexExists(name)`:: Checks if given index exists `admin()`:: Returns an `AdminClient` for administrative tasks `clusterService()`:: Returns the cluster service java class diff --git a/docs/resiliency/index.asciidoc b/docs/resiliency/index.asciidoc index 14fadeb35b7..3929bc7c8a8 100644 --- a/docs/resiliency/index.asciidoc +++ b/docs/resiliency/index.asciidoc @@ -56,7 +56,7 @@ If you encounter an issue, https://github.com/elasticsearch/elasticsearch/issues We are committed to tracking down and fixing all the issues that are posted. [float] -=== Use two phase commit for Cluster State publishing (STATUS: ONGOING) +=== Use two phase commit for Cluster State publishing (STATUS: ONGOING, v3.0.0) A master node in Elasticsearch continuously https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#fault-detection[monitors the cluster nodes] and removes any node from the cluster that doesn't respond to its pings in a timely @@ -103,38 +103,6 @@ Further issues remain with the retry mechanism: See {GIT}9967[#9967]. (STATUS: ONGOING) -[float] -=== Wait on incoming joins before electing local node as master (STATUS: ONGOING) - -During master election each node pings in order to discover other nodes and validate the liveness of existing -nodes. Based on this information the node either discovers an existing master or, if enough nodes are found -(see https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#master-election[`discovery.zen.minimum_master_nodes`]) a new master will be elected. Currently, the node that is -elected as master will update the cluster state to indicate the result of the election. Other nodes will submit -a join request to the newly elected master node. Instead of immediately processing the election result, the elected master -node should wait for the incoming joins from other nodes, thus validating that the result of the election is properly applied. As soon as enough -nodes have sent their joins request (based on the `minimum_master_nodes` settings) the cluster state is updated. -{GIT}12161[#12161] - - -[float] -=== Write index metadata on data nodes where shards allocated (STATUS: ONGOING) - -Today, index metadata is written only on nodes that are master-eligible, not on -data-only nodes. This is not a problem when running with multiple master nodes, -as recommended, as the loss of all but one master node is still recoverable. -However, users running with a single master node are at risk of losing -their index metadata if the master fails. Instead, this metadata should -also be written on any node where a shard is allocated. {GIT}8823[#8823] - -[float] -=== Better file distribution with multiple data paths (STATUS: ONGOING) - -Today, a node configured with multiple data paths distributes writes across -all paths by writing one file to each path in turn. This can mean that the -failure of a single disk corrupts many shards at once. Instead, by allocating -an entire shard to a single data path, the extent of the damage can be limited -to just the shards on that disk. {GIT}9498[#9498] - [float] === OOM resiliency (STATUS: ONGOING) @@ -142,21 +110,10 @@ The family of circuit breakers has greatly reduced the occurrence of OOM exceptions, but it is still possible to cause a node to run out of heap space. The following issues have been identified: -* Set a hard limit on `from`/`size` parameters {GIT}9311[#9311]. (STATUS: ONGOING) +* Set a hard limit on `from`/`size` parameters {GIT}9311[#9311]. (STATUS: DONE, v2.1.0) * Prevent combinatorial explosion in aggregations from causing OOM {GIT}8081[#8081]. (STATUS: ONGOING) * Add the byte size of each hit to the request circuit breaker {GIT}9310[#9310]. (STATUS: ONGOING) -[float] -=== Mapping changes should be applied synchronously (STATUS: ONGOING) - -When introducing new fields using dynamic mapping, it is possible that the same -field can be added to different shards with different data types. Each shard -will operate with its local data type but, if the shard is relocated, the -data type from the cluster state will be applied to the new shard, which -can result in a corrupt shard. To prevent this, new fields should not -be added to a shard's mapping until confirmed by the master. -{GIT}8688[#8688] (STATUS: DONE) - [float] === Loss of documents during network partition (STATUS: ONGOING) @@ -166,26 +123,6 @@ If the node hosts a primary shard at the moment of partition, and ends up being A test to replicate this condition was added in {GIT}7493[#7493]. -[float] -=== Lucene checksums phase 3 (STATUS:ONGOING) - -Almost all files in Elasticsearch now have checksums which are validated before use. A few changes remain: - -* {GIT}7586[#7586] adds checksums for cluster and index state files. (STATUS: DONE, Fixed in v1.5.0) -* {GIT}9183[#9183] supports validating the checksums on all files when starting a node. (STATUS: DONE, Fixed in v2.0.0) -* {JIRA}5894[LUCENE-5894] lays the groundwork for extending more efficient checksum validation to all files during optimized bulk merges. (STATUS: DONE, Fixed in v2.0.0) -* {GIT}8403[#8403] to add validation of checksums on Lucene `segments_N` files. (STATUS: NOT STARTED) - -[float] -=== Add per-segment and per-commit ID to help replication (STATUS: ONGOING) - -{JIRA}5895[LUCENE-5895] adds a unique ID for each segment and each commit point. File-based replication (as performed by snapshot/restore) can use this ID to know whether the segment/commit on the source and destination machines are the same. Fixed in Lucene 5.0. - -[float] -=== Report shard-level statuses on write operations (STATUS: ONGOING) - -Make write calls return the number of total/successful/missing shards in the same way that we do in search, which ensures transparency in the consistency of write operations. {GIT}7994[#7994]. (STATUS: DONE, v2.0.0) - [float] === Jepsen Test Failures (STATUS: ONGOING) @@ -196,16 +133,96 @@ We have increased our test coverage to include scenarios tested by Jepsen. We ma This status page is a start, but we can do a better job of explicitly documenting the processes at work in Elasticsearch, and what happens in the case of each type of failure. The plan is to have a test case that validates each behavior under simulated conditions. Every test will document the expected results, the associated test code and an explicit PASS or FAIL status for each simulated case. - [float] -=== Take filter cache key size into account (STATUS: ONGOING) +=== Do not allow stale shards to automatically be promoted to primary (STATUS: ONGOING) -Commonly used filters are cached in Elasticsearch. That cache is limited in size (10% of node's memory by default) and is being evicted based on a least recently used policy. The amount of memory used by the cache depends on two primary components - the values it stores and the keys associated with them. Calculating the memory footprint of the values is easy enough but the keys accounting is trickier to achieve as they are, by default, raw Lucene objects. This is largely not a problem as the keys are dominated by the values. However, recent optimizations in Lucene have changed the balance causing the filter cache to grow beyond it's size. - -While we are working on a longer term solution ({GIT}9176[#9176]), we introduced a minimum weight of 1k for each cache entry. This puts an effective limit on the number of entries in the cache. See {GIT}8304[#8304] (STATUS: DONE, fixed in v1.4.0) +In some scenarios, after the loss of all valid copies, a stale replica shard can be assigned as a primary. This can lead to +a loss of acknowledged writes if the valid copies are not lost but are rather temporarily isolated. Work is underway +({GIT}14671[#14671]) to prevent the automatic promotion of a stale primary and only allow such promotion to occur when +a system operator manually intervenes. == Completed +[float] +=== Wait on incoming joins before electing local node as master (STATUS: DONE, v2.0.0) + +During master election each node pings in order to discover other nodes and validate the liveness of existing +nodes. Based on this information the node either discovers an existing master or, if enough nodes are found +(see https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#master-election[`discovery.zen.minimum_master_nodes`]) a new master will be elected. Currently, the node that is +elected as master will update the cluster state to indicate the result of the election. Other nodes will submit +a join request to the newly elected master node. Instead of immediately processing the election result, the elected master +node should wait for the incoming joins from other nodes, thus validating that the result of the election is properly applied. As soon as enough +nodes have sent their joins request (based on the `minimum_master_nodes` settings) the cluster state is updated. +{GIT}12161[#12161] + +[float] +=== Mapping changes should be applied synchronously (STATUS: DONE, v2.0.0) + +When introducing new fields using dynamic mapping, it is possible that the same +field can be added to different shards with different data types. Each shard +will operate with its local data type but, if the shard is relocated, the +data type from the cluster state will be applied to the new shard, which +can result in a corrupt shard. To prevent this, new fields should not +be added to a shard's mapping until confirmed by the master. +{GIT}8688[#8688] (STATUS: DONE) + +[float] +=== Add per-segment and per-commit ID to help replication (STATUS: DONE, v2.0.0) + +{JIRA}5895[LUCENE-5895] adds a unique ID for each segment and each commit point. File-based replication (as performed by snapshot/restore) can use this ID to know whether the segment/commit on the source and destination machines are the same. Fixed in Lucene 5.0. + +[float] +=== Write index metadata on data nodes where shards allocated (STATUS: DONE, v2.0.0) + +Today, index metadata is written only on nodes that are master-eligible, not on +data-only nodes. This is not a problem when running with multiple master nodes, +as recommended, as the loss of all but one master node is still recoverable. +However, users running with a single master node are at risk of losing +their index metadata if the master fails. Instead, this metadata should +also be written on any node where a shard is allocated. {GIT}8823[#8823], {GIT}9952[#9952] + +[float] +=== Better file distribution with multiple data paths (STATUS: DONE, v2.0.0) + +Today, a node configured with multiple data paths distributes writes across +all paths by writing one file to each path in turn. This can mean that the +failure of a single disk corrupts many shards at once. Instead, by allocating +an entire shard to a single data path, the extent of the damage can be limited +to just the shards on that disk. {GIT}9498[#9498] + +[float] +=== Lucene checksums phase 3 (STATUS: DONE, v2.0.0) + +Almost all files in Elasticsearch now have checksums which are validated before use. A few changes remain: + +* {GIT}7586[#7586] adds checksums for cluster and index state files. (STATUS: DONE, Fixed in v1.5.0) +* {GIT}9183[#9183] supports validating the checksums on all files when starting a node. (STATUS: DONE, Fixed in v2.0.0) +* {JIRA}5894[LUCENE-5894] lays the groundwork for extending more efficient checksum validation to all files during optimized bulk merges. (STATUS: DONE, Fixed in v2.0.0) +* {GIT}8403[#8403] to add validation of checksums on Lucene `segments_N` files. (STATUS: DONE, v2.0.0) + +[float] +=== Report shard-level statuses on write operations (STATUS: DONE, v2.0.0) + +Make write calls return the number of total/successful/missing shards in the same way that we do in search, which ensures transparency in the consistency of write operations. {GIT}7994[#7994]. (STATUS: DONE, v2.0.0) + +[float] +=== Take filter cache key size into account (STATUS: DONE, v2.0.0) + +Commonly used filters are cached in Elasticsearch. That cache is limited in size +(10% of node's memory by default) and is being evicted based on a least recently +used policy. The amount of memory used by the cache depends on two primary +components - the values it stores and the keys associated with them. Calculating +the memory footprint of the values is easy enough but the keys accounting is +trickier to achieve as they are, by default, raw Lucene objects. This is largely +not a problem as the keys are dominated by the values. However, recent +optimizations in Lucene have changed the balance causing the filter cache to +grow beyond it's size. + +As a temporary solution, we introduced a minimum weight of 1k for each cache entry. +This puts an effective limit on the number of entries in the cache. See {GIT}8304[#8304] (STATUS: DONE, fixed in v1.4.0) + +The issue has been completely solved by the move to Lucene's query cache. See {GIT}10897[#10897] + [float] === Ensure shard state ID is incremental (STATUS: DONE, v1.5.1) @@ -444,7 +461,7 @@ The Snapshot/Restore API supports a number of different repository types for sto [float] === Circuit Breaker: Fielddata (STATUS: DONE, v1.0.0) -Currently, the https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-fielddata.html[circuit breaker] protects against loading too much field data by estimating how much memory the field data will take to load, then aborting the request if the memory requirements are too high. This feature was added in Elasticsearch version 1.0.0. +Currently, the circuit breaker protects against loading too much field data by estimating how much memory the field data will take to load, then aborting the request if the memory requirements are too high. This feature was added in Elasticsearch version 1.0.0. [float] === Use of Paginated Data Structures to Ease Garbage Collection (STATUS: DONE, v1.0.0 & v1.2.0) @@ -483,4 +500,3 @@ At Elasticsearch, we live the philosophy that we can miss a bug once, but never === Lucene Loses Data On File Descriptors Failure (STATUS: DONE, v0.90.0) When a process runs out of file descriptors, Lucene can causes an index to be completely deleted. This issue was fixed in Lucene ({JIRA}4870[version 4.2.1]) and fixed in an early version of Elasticsearch. See issue {GIT}2812[#2812]. - diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000000..6b1823d86a6 --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +org.gradle.daemon=false diff --git a/modules/build.gradle b/modules/build.gradle new file mode 100644 index 00000000000..41f7a8873b4 --- /dev/null +++ b/modules/build.gradle @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +subprojects { + apply plugin: 'elasticsearch.esplugin' + + esplugin { + // for local ES plugins, the name of the plugin is the same as the directory + name project.name + } + + if (project.file('src/main/packaging').exists()) { + throw new InvalidModelException("Modules cannot contain packaging files") + } + if (project.file('src/main/bin').exists()) { + throw new InvalidModelException("Modules cannot contain bin files") + } + if (project.file('src/main/config').exists()) { + throw new InvalidModelException("Modules cannot contain config files") + } + + project.afterEvaluate { + if (esplugin.isolated == false) { + throw new InvalidModelException("Modules cannot disable isolation") + } + if (esplugin.jvm == false) { + throw new InvalidModelException("Modules must be jvm plugins") + } + } +} diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle new file mode 100644 index 00000000000..9f62e34687d --- /dev/null +++ b/modules/lang-expression/build.gradle @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Lucene expressions integration for Elasticsearch' + classname 'org.elasticsearch.script.expression.ExpressionPlugin' +} + +dependencies { + compile "org.apache.lucene:lucene-expressions:${versions.lucene}" + compile 'org.antlr:antlr4-runtime:4.5.1-1' + compile 'org.ow2.asm:asm:5.0.4' + compile 'org.ow2.asm:asm-commons:5.0.4' +} + +dependencyLicenses { + mapping from: /lucene-.*/, to: 'lucene' +} + +compileJava.options.compilerArgs << '-Xlint:-rawtypes' +compileTestJava.options.compilerArgs << '-Xlint:-rawtypes' + diff --git a/modules/lang-expression/licenses/antlr4-runtime-4.5.1-1.jar.sha1 b/modules/lang-expression/licenses/antlr4-runtime-4.5.1-1.jar.sha1 new file mode 100644 index 00000000000..f15e50069ba --- /dev/null +++ b/modules/lang-expression/licenses/antlr4-runtime-4.5.1-1.jar.sha1 @@ -0,0 +1 @@ +66144204f9d6d7d3f3f775622c2dd7e9bd511d97 diff --git a/plugins/lang-expression/licenses/antlr4-runtime-LICENSE.txt b/modules/lang-expression/licenses/antlr4-runtime-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/antlr4-runtime-LICENSE.txt rename to modules/lang-expression/licenses/antlr4-runtime-LICENSE.txt diff --git a/plugins/lang-expression/licenses/antlr4-runtime-NOTICE.txt b/modules/lang-expression/licenses/antlr4-runtime-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/antlr4-runtime-NOTICE.txt rename to modules/lang-expression/licenses/antlr4-runtime-NOTICE.txt diff --git a/plugins/lang-expression/licenses/asm-5.0.4.jar.sha1 b/modules/lang-expression/licenses/asm-5.0.4.jar.sha1 similarity index 100% rename from plugins/lang-expression/licenses/asm-5.0.4.jar.sha1 rename to modules/lang-expression/licenses/asm-5.0.4.jar.sha1 diff --git a/plugins/lang-expression/licenses/asm-LICENSE.txt b/modules/lang-expression/licenses/asm-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-LICENSE.txt rename to modules/lang-expression/licenses/asm-LICENSE.txt diff --git a/plugins/lang-expression/licenses/asm-NOTICE.txt b/modules/lang-expression/licenses/asm-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-NOTICE.txt rename to modules/lang-expression/licenses/asm-NOTICE.txt diff --git a/plugins/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 b/modules/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 similarity index 100% rename from plugins/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 rename to modules/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 diff --git a/plugins/lang-expression/licenses/asm-commons-LICENSE.txt b/modules/lang-expression/licenses/asm-commons-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-commons-LICENSE.txt rename to modules/lang-expression/licenses/asm-commons-LICENSE.txt diff --git a/plugins/lang-expression/licenses/asm-commons-NOTICE.txt b/modules/lang-expression/licenses/asm-commons-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-commons-NOTICE.txt rename to modules/lang-expression/licenses/asm-commons-NOTICE.txt diff --git a/plugins/lang-expression/licenses/lucene-LICENSE.txt b/modules/lang-expression/licenses/lucene-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/lucene-LICENSE.txt rename to modules/lang-expression/licenses/lucene-LICENSE.txt diff --git a/plugins/lang-expression/licenses/lucene-NOTICE.txt b/modules/lang-expression/licenses/lucene-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/lucene-NOTICE.txt rename to modules/lang-expression/licenses/lucene-NOTICE.txt diff --git a/modules/lang-expression/licenses/lucene-expressions-5.5.0-snapshot-1719088.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..50bb58f443d --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +787356d4ae6142bb8ca7e9713d0a281a797b57fb \ No newline at end of file diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java similarity index 99% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java index c1e2ed47dc4..c72428c4c4c 100644 --- a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java @@ -23,7 +23,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; public class ExpressionPlugin extends Plugin { - + @Override public String name() { return "lang-expression"; diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java similarity index 90% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 72a1dd7d5c6..a7f93925119 100644 --- a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.index.mapper.core.NumberFieldMapper; +import org.elasticsearch.script.ClassPermission; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptEngineService; @@ -44,6 +45,7 @@ import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.lookup.SearchLookup; +import java.security.AccessControlContext; import java.security.AccessController; import java.security.PrivilegedAction; import java.text.ParseException; @@ -95,7 +97,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements @Override public Object compile(String script) { // classloader created here - SecurityManager sm = System.getSecurityManager(); + final SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new SpecialPermission()); } @@ -103,8 +105,24 @@ public class ExpressionScriptEngineService extends AbstractComponent implements @Override public Expression run() { try { + // snapshot our context here, we check on behalf of the expression + AccessControlContext engineContext = AccessController.getContext(); + ClassLoader loader = getClass().getClassLoader(); + if (sm != null) { + loader = new ClassLoader(loader) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + return super.loadClass(name, resolve); + } + }; + } // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here - return JavascriptCompiler.compile(script); + return JavascriptCompiler.compile(script, JavascriptCompiler.DEFAULT_FUNCTIONS, loader); } catch (ParseException e) { throw new ScriptException("Failed to parse expression: " + script, e); } diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java diff --git a/plugins/lang-groovy/src/main/plugin-metadata/plugin-security.policy b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy similarity index 60% rename from plugins/lang-groovy/src/main/plugin-metadata/plugin-security.policy rename to modules/lang-expression/src/main/plugin-metadata/plugin-security.policy index 55c2fab13f7..c11af51e464 100644 --- a/plugins/lang-groovy/src/main/plugin-metadata/plugin-security.policy +++ b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy @@ -20,10 +20,13 @@ grant { // needed to generate runtime classes permission java.lang.RuntimePermission "createClassLoader"; - // needed by groovy engine - permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect"; - // needed by GroovyScriptEngineService to close its classloader (why?) - permission java.lang.RuntimePermission "closeClassLoader"; - // Allow executing groovy scripts with codesource of /untrusted - permission groovy.security.GroovyCodeSourcePermission "/untrusted"; + + // expression runtime + permission org.elasticsearch.script.ClassPermission "java.lang.String"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.queries.function.FunctionValues"; + // available functions + permission org.elasticsearch.script.ClassPermission "java.lang.Math"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; }; diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java similarity index 100% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java similarity index 100% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java similarity index 81% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java index b91450ffd23..65b47b92331 100644 --- a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java @@ -19,13 +19,14 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.Collection; @@ -35,7 +36,6 @@ import static org.hamcrest.Matchers.containsString; //TODO: please convert to unit tests! public class IndexedExpressionTests extends ESIntegTestCase { - @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); @@ -45,13 +45,12 @@ public class IndexedExpressionTests extends ESIntegTestCase { builder.put("script.engine.expression.indexed.mapping", "off"); return builder.build(); } - + @Override protected Collection> nodePlugins() { return Collections.singleton(ExpressionPlugin.class); } - @Test public void testAllOpsDisabledIndexedScripts() throws IOException { if (randomBoolean()) { client().preparePutIndexedScript(ExpressionScriptEngineService.NAME, "script1", "{\"script\":\"2\"}").get(); @@ -68,16 +67,20 @@ public class IndexedExpressionTests extends ESIntegTestCase { assertThat(e.getCause().getMessage(), containsString("scripts of type [indexed], operation [update] and lang [expression] are disabled")); } try { - String query = "{ \"script_fields\" : { \"test1\" : { \"script_id\" : \"script1\", \"lang\":\"expression\" }}}"; - client().prepareSearch().setSource(new BytesArray(query)).setIndices("test").setTypes("scriptTest").get(); + client().prepareSearch() + .setSource( + new SearchSourceBuilder().scriptField("test1", new Script("script1", ScriptType.INDEXED, "expression", null))) + .setIndices("test").setTypes("scriptTest").get(); fail("search script should have been rejected"); } catch(Exception e) { assertThat(e.toString(), containsString("scripts of type [indexed], operation [search] and lang [expression] are disabled")); } try { - String source = "{\"aggs\": {\"test\": { \"terms\" : { \"script_id\":\"script1\", \"script_lang\":\"expression\" } } } }"; - client().prepareSearch("test").setSource(new BytesArray(source)).get(); - } catch(Exception e) { + client().prepareSearch("test") + .setSource( + new SearchSourceBuilder().aggregation(AggregationBuilders.terms("test").script( + new Script("script1", ScriptType.INDEXED, "expression", null)))).get(); + } catch (Exception e) { assertThat(e.toString(), containsString("scripts of type [indexed], operation [aggs] and lang [expression] are disabled")); } } diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java similarity index 95% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java index baf4ab04a90..89a5be7ff1c 100644 --- a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java @@ -97,6 +97,16 @@ public class MoreExpressionTests extends ESIntegTestCase { assertEquals(1, rsp.getHits().getTotalHits()); assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); } + + public void testFunction() throws Exception { + createIndex("test"); + ensureGreen("test"); + client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get(); + SearchResponse rsp = buildRequest("doc['foo'] + abs(1)").get(); + assertSearchResponse(rsp); + assertEquals(1, rsp.getHits().getTotalHits()); + assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); + } public void testBasicUsingDotValue() throws Exception { createIndex("test"); @@ -504,32 +514,6 @@ public class MoreExpressionTests extends ESIntegTestCase { } } - // test to make sure expressions are not allowed to be used as mapping scripts - public void testInvalidMappingScript() throws Exception{ - try { - createIndex("test_index"); - ensureGreen("test_index"); - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - builder.startObject("transform"); - builder.field("script", "1.0"); - builder.field("lang", ExpressionScriptEngineService.NAME); - builder.endObject(); - builder.startObject("properties"); - builder.startObject("double_field"); - builder.field("type", "double"); - builder.endObject(); - builder.endObject(); - builder.endObject(); - client().admin().indices().preparePutMapping("test_index").setType("trans_test").setSource(builder).get(); - client().prepareIndex("test_index", "trans_test", "1").setSource("double_field", 0.0).get(); - fail("Expression scripts should not be allowed to run as mapping scripts."); - } catch (Exception e) { - String message = ExceptionsHelper.detailedMessage(e); - assertThat(message + " should have contained failed to parse", message.contains("failed to parse"), equalTo(true)); - assertThat(message + " should have contained not supported", message.contains("not supported"), equalTo(true)); - } - } - // test to make sure expressions are allowed to be used for reduce in pipeline aggregations public void testPipelineAggregationScript() throws Exception { createIndex("agg_index"); diff --git a/plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml similarity index 62% rename from plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml rename to modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml index 9c7819d925d..1550f2a7f81 100644 --- a/plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml +++ b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml @@ -10,5 +10,5 @@ - do: nodes.info: {} - - match: { nodes.$master.plugins.0.name: lang-expression } - - match: { nodes.$master.plugins.0.jvm: true } + - match: { nodes.$master.modules.0.name: lang-expression } + - match: { nodes.$master.modules.0.jvm: true } diff --git a/plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml similarity index 76% rename from plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml rename to modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml index a0953a25972..36ff7f58ea6 100644 --- a/plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml +++ b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml @@ -22,6 +22,6 @@ setup: --- "Expressions scripting test": - - do: { search: { body: { script_fields : { my_field : { lang: expression, script: 'doc["age"].value + 19' } } } } } + - do: { search: { body: { script_fields : { my_field : { script: { lang: expression, inline: 'doc["age"].value + 19' } } } } } } - match: { hits.hits.0.fields.my_field.0: 42.0 } diff --git a/modules/lang-groovy/build.gradle b/modules/lang-groovy/build.gradle new file mode 100644 index 00000000000..341dcbf0d6c --- /dev/null +++ b/modules/lang-groovy/build.gradle @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Groovy scripting integration for Elasticsearch' + classname 'org.elasticsearch.script.groovy.GroovyPlugin' +} + +dependencies { + compile 'org.codehaus.groovy:groovy-all:2.4.4:indy' +} + +compileJava.options.compilerArgs << '-Xlint:-rawtypes,-unchecked,-cast,-deprecation' +compileTestJava.options.compilerArgs << '-Xlint:-rawtypes,-unchecked,-cast,-deprecation' + +integTest { + cluster { + systemProperty 'es.script.inline', 'on' + systemProperty 'es.script.indexed', 'on' + } +} diff --git a/plugins/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 b/modules/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 rename to modules/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-NOTICE.txt b/modules/lang-groovy/licenses/groovy-all-NOTICE.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-NOTICE.txt rename to modules/lang-groovy/licenses/groovy-all-NOTICE.txt diff --git a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java similarity index 100% rename from plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java rename to modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java diff --git a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java similarity index 85% rename from plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java rename to modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java index d1e7160282b..85f57694ce6 100644 --- a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java +++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java @@ -51,6 +51,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.security.AccessControlContext; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.HashMap; @@ -65,16 +66,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri * The name of the scripting engine/language. */ public static final String NAME = "groovy"; - /** - * The setting to enable or disable invokedynamic instruction support in Java 7+. - *

    - * Note: If this is disabled because invokedynamic is causing issues, then the Groovy - * indy jar needs to be replaced by the non-indy variant of it on the classpath (e.g., - * groovy-all-2.4.4-indy.jar should be replaced by groovy-all-2.4.4.jar). - *

    - * Defaults to {@code true}. - */ - public static final String GROOVY_INDY_ENABLED = "script.groovy.indy"; /** * The name of the Groovy compiler setting to use associated with activating invokedynamic support. */ @@ -96,22 +87,33 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri // Add BigDecimal -> Double transformer config.addCompilationCustomizers(new GroovyBigDecimalTransformer(CompilePhase.CONVERSION)); - // Implicitly requires Java 7u60 or later to get valid support - if (settings.getAsBoolean(GROOVY_INDY_ENABLED, true)) { - // maintain any default optimizations - config.getOptimizationOptions().put(GROOVY_INDY_SETTING_NAME, true); - } + // always enable invokeDynamic, not the crazy softreference-based stuff + config.getOptimizationOptions().put(GROOVY_INDY_SETTING_NAME, true); // Groovy class loader to isolate Groovy-land code // classloader created here - SecurityManager sm = System.getSecurityManager(); + final SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new SpecialPermission()); } this.loader = AccessController.doPrivileged(new PrivilegedAction() { @Override public GroovyClassLoader run() { - return new GroovyClassLoader(getClass().getClassLoader(), config); + // snapshot our context (which has permissions for classes), since the script has none + final AccessControlContext engineContext = AccessController.getContext(); + return new GroovyClassLoader(new ClassLoader(getClass().getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (sm != null) { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + } + return super.loadClass(name, resolve); + } + }, config); } }); } @@ -172,13 +174,15 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri } String fake = MessageDigests.toHexString(MessageDigests.sha1().digest(script.getBytes(StandardCharsets.UTF_8))); // same logic as GroovyClassLoader.parseClass() but with a different codesource string: - GroovyCodeSource gcs = AccessController.doPrivileged(new PrivilegedAction() { - public GroovyCodeSource run() { - return new GroovyCodeSource(script, fake, BootstrapInfo.UNTRUSTED_CODEBASE); + return AccessController.doPrivileged(new PrivilegedAction() { + public Class run() { + GroovyCodeSource gcs = new GroovyCodeSource(script, fake, BootstrapInfo.UNTRUSTED_CODEBASE); + gcs.setCachable(false); + // TODO: we could be more complicated and paranoid, and move this to separate block, to + // sandbox the compilation process itself better. + return loader.parseClass(gcs); } }); - gcs.setCachable(false); - return loader.parseClass(gcs); } catch (Throwable e) { if (logger.isTraceEnabled()) { logger.trace("exception compiling Groovy script:", e); @@ -293,7 +297,14 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri @Override public Object run() { try { - return script.run(); + // NOTE: we truncate the stack because IndyInterface has security issue (needs getClassLoader) + // we don't do a security check just as a tradeoff, it cannot really escalate to anything. + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Object run() { + return script.run(); + } + }); } catch (Throwable e) { if (logger.isTraceEnabled()) { logger.trace("failed to run " + compiledScript, e); diff --git a/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..e1fd920d119 --- /dev/null +++ b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grant { + // needed to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; + // needed by IndyInterface + permission java.lang.RuntimePermission "getClassLoader"; + // needed by groovy engine + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect"; + // needed by GroovyScriptEngineService to close its classloader (why?) + permission java.lang.RuntimePermission "closeClassLoader"; + // Allow executing groovy scripts with codesource of /untrusted + permission groovy.security.GroovyCodeSourcePermission "/untrusted"; + + // Standard set of classes + permission org.elasticsearch.script.ClassPermission "<>"; + // groovy runtime (TODO: clean these up if possible) + permission org.elasticsearch.script.ClassPermission "groovy.grape.GrabAnnotationTransformation"; + permission org.elasticsearch.script.ClassPermission "groovy.json.JsonOutput"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.Binding"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.GroovyObject"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.GString"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.Script"; + permission org.elasticsearch.script.ClassPermission "groovy.util.GroovyCollections"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.ast.builder.AstBuilderTransformation"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.reflection.ClassInfo"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.GStringImpl"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.powerassert.ValueRecorder"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.powerassert.AssertionRenderer"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.ScriptBytecodeAdapter"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.vmplugin.v7.IndyInterface"; + permission org.elasticsearch.script.ClassPermission "sun.reflect.ConstructorAccessorImpl"; +}; diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java similarity index 98% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java index af17e469b28..06119fd6a76 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -70,7 +69,7 @@ public class BucketScriptTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -103,9 +102,7 @@ public class BucketScriptTests extends ESIntegTestCase { return jsonBuilder; } - @Test - public void inlineScript() { - + public void testInlineScript() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -149,9 +146,7 @@ public class BucketScriptTests extends ESIntegTestCase { } } - @Test - public void inlineScript2() { - + public void testInlineScript2() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -195,9 +190,7 @@ public class BucketScriptTests extends ESIntegTestCase { } } - @Test - public void inlineScriptSingleVariable() { - + public void testInlineScriptSingleVariable() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -233,9 +226,7 @@ public class BucketScriptTests extends ESIntegTestCase { } } - @Test - public void inlineScriptNamedVars() { - + public void testInlineScriptNamedVars() { Map bucketsPathsMap = new HashMap<>(); bucketsPathsMap.put("foo", "field2Sum"); bucketsPathsMap.put("bar", "field3Sum"); @@ -283,9 +274,7 @@ public class BucketScriptTests extends ESIntegTestCase { } } - @Test - public void inlineScriptWithParams() { - + public void testInlineScriptWithParams() { Map params = new HashMap<>(); params.put("factor", 3); SearchResponse response = client() @@ -331,9 +320,7 @@ public class BucketScriptTests extends ESIntegTestCase { } } - @Test - public void inlineScriptInsertZeros() { - + public void testInlineScriptInsertZeros() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -379,9 +366,7 @@ public class BucketScriptTests extends ESIntegTestCase { } } - @Test - public void indexedScript() { - + public void testIndexedScript() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -425,8 +410,7 @@ public class BucketScriptTests extends ESIntegTestCase { } } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx_unmapped") .addAggregation( @@ -449,8 +433,7 @@ public class BucketScriptTests extends ESIntegTestCase { assertThat(deriv.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx", "idx_unmapped") .addAggregation( diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java similarity index 97% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java index 7b14bca98b3..2883b74cc1d 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram. import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -70,7 +69,7 @@ public class BucketSelectorTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -104,9 +103,7 @@ public class BucketSelectorTests extends ESIntegTestCase { return jsonBuilder; } - @Test - public void inlineScript() { - + public void testInlineScript() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -140,9 +137,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void inlineScriptNoBucketsPruned() { - + public void testInlineScriptNoBucketsPruned() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -176,9 +171,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void inlineScriptNoBucketsLeft() { - + public void testInlineScriptNoBucketsLeft() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -201,9 +194,7 @@ public class BucketSelectorTests extends ESIntegTestCase { assertThat(buckets.size(), equalTo(0)); } - @Test - public void inlineScript2() { - + public void testInlineScript2() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -236,9 +227,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void inlineScriptSingleVariable() { - + public void testInlineScriptSingleVariable() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -267,9 +256,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void inlineScriptNamedVars() { - + public void testInlineScriptNamedVars() { Map bucketPathsMap = new HashMap<>(); bucketPathsMap.put("my_value1", "field2Sum"); bucketPathsMap.put("my_value2", "field3Sum"); @@ -307,9 +294,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void inlineScriptWithParams() { - + public void testInlineScriptWithParams() { Map params = new HashMap<>(); params.put("threshold", 100); SearchResponse response = client() @@ -345,9 +330,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void inlineScriptInsertZeros() { - + public void testInlineScriptInsertZeros() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -380,9 +363,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void indexedScript() { - + public void testIndexedScript() { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -414,9 +395,7 @@ public class BucketSelectorTests extends ESIntegTestCase { } } - @Test - public void unmapped() throws Exception { - + public void testUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx_unmapped") .addAggregation( @@ -439,9 +418,7 @@ public class BucketSelectorTests extends ESIntegTestCase { assertThat(deriv.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { - + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx", "idx_unmapped") .addAggregation( diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java similarity index 96% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java index fa6e91bed10..d893b2767ca 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; @@ -45,7 +45,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -54,18 +53,23 @@ import java.util.Collections; import java.util.concurrent.CyclicBarrier; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; public class BulkTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - @Test - public void testBulkUpdate_simple() throws Exception { + + public void testBulkUpdateSimple() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); @@ -156,7 +160,6 @@ public class BulkTests extends ESIntegTestCase { assertThat(((Long) getResponse.getField("field").getValue()), equalTo(4l)); } - @Test public void testBulkVersioning() throws Exception { createIndex("test"); ensureGreen(); @@ -203,9 +206,7 @@ public class BulkTests extends ESIntegTestCase { assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(21l)); } - @Test - public void testBulkUpdate_malformedScripts() throws Exception { - + public void testBulkUpdateMalformedScripts() throws Exception { createIndex("test"); ensureGreen(); @@ -244,8 +245,7 @@ public class BulkTests extends ESIntegTestCase { assertThat(bulkResponse.getItems()[2].getResponse(), nullValue()); } - @Test - public void testBulkUpdate_largerVolume() throws Exception { + public void testBulkUpdateLargerVolume() throws Exception { createIndex("test"); ensureGreen(); @@ -378,9 +378,7 @@ public class BulkTests extends ESIntegTestCase { } } - @Test public void testBulkIndexingWhileInitializing() throws Exception { - int replica = randomInt(2); internalCluster().ensureAtLeastNumDataNodes(1 + replica); @@ -407,14 +405,13 @@ public class BulkTests extends ESIntegTestCase { refresh(); - CountResponse countResponse = client().prepareCount().get(); + SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); } /* Test for https://github.com/elasticsearch/elasticsearch/issues/3444 */ - @Test public void testBulkUpdateDocAsUpsertWithParent() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("parent", "{\"parent\":{}}") @@ -452,7 +449,6 @@ public class BulkTests extends ESIntegTestCase { /* Test for https://github.com/elasticsearch/elasticsearch/issues/3444 */ - @Test public void testBulkUpdateUpsertWithParent() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent", "{\"parent\":{}}") @@ -487,7 +483,6 @@ public class BulkTests extends ESIntegTestCase { /* * Test for https://github.com/elasticsearch/elasticsearch/issues/8365 */ - @Test public void testBulkUpdateChildMissingParentRouting() throws Exception { assertAcked(prepareCreate("test").addMapping("parent", "{\"parent\":{}}").addMapping("child", "{\"child\": {\"_parent\": {\"type\": \"parent\"}}}")); @@ -518,7 +513,6 @@ public class BulkTests extends ESIntegTestCase { assertThat(bulkResponse.getItems()[3].isFailed(), equalTo(false)); } - @Test public void testFailingVersionedUpdatedOnBulk() throws Exception { createIndex("test"); index("test", "type", "1", "field", "1"); @@ -561,8 +555,8 @@ public class BulkTests extends ESIntegTestCase { assertThat(successes, equalTo(1)); } - @Test // issue 4745 - public void preParsingSourceDueToMappingShouldNotBreakCompleteBulkRequest() throws Exception { + // issue 4745 + public void testPreParsingSourceDueToMappingShouldNotBreakCompleteBulkRequest() throws Exception { XContentBuilder builder = jsonBuilder().startObject() .startObject("type") .startObject("_timestamp") @@ -587,8 +581,8 @@ public class BulkTests extends ESIntegTestCase { assertExists(get("test", "type", "2")); } - @Test // issue 4745 - public void preParsingSourceDueToRoutingShouldNotBreakCompleteBulkRequest() throws Exception { + // issue 4745 + public void testPreParsingSourceDueToRoutingShouldNotBreakCompleteBulkRequest() throws Exception { XContentBuilder builder = jsonBuilder().startObject() .startObject("type") .startObject("_routing") @@ -615,8 +609,8 @@ public class BulkTests extends ESIntegTestCase { } - @Test // issue 4745 - public void preParsingSourceDueToIdShouldNotBreakCompleteBulkRequest() throws Exception { + // issue 4745 + public void testPreParsingSourceDueToIdShouldNotBreakCompleteBulkRequest() throws Exception { XContentBuilder builder = jsonBuilder().startObject() .startObject("type") .startObject("_id") @@ -641,7 +635,7 @@ public class BulkTests extends ESIntegTestCase { assertExists(get("test", "type", "48")); } - @Test // issue 4987 + // issue 4987 public void testThatInvalidIndexNamesShouldNotBreakCompleteBulkRequest() { int bulkEntryCount = randomIntBetween(10, 50); BulkRequestBuilder builder = client().prepareBulk(); @@ -669,7 +663,7 @@ public class BulkTests extends ESIntegTestCase { } } - @Test // issue 6630 + // issue 6630 public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception { BulkResponse indexBulkItemResponse = client().prepareBulk() .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }")) @@ -704,7 +698,7 @@ public class BulkTests extends ESIntegTestCase { return randomBoolean() ? "test" : "alias"; } - @Test // issue 6410 + // issue 6410 public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception{ createIndex("bulkindex1", "bulkindex2"); ensureYellow(); @@ -727,7 +721,7 @@ public class BulkTests extends ESIntegTestCase { assertThat(bulkResponse.getItems().length, is(5)); } - @Test // issue 9821 + // issue 9821 public void testFailedRequestsOnClosedIndex() throws Exception { createIndex("bulkindex1"); ensureYellow(); @@ -749,7 +743,7 @@ public class BulkTests extends ESIntegTestCase { assertThat(responseItems[2].getOpType(), is("delete")); } - @Test // issue 9821 + // issue 9821 public void testInvalidIndexNamesCorrectOpType() { BulkResponse bulkResponse = client().prepareBulk() .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1").setSource("field", 1)) diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java similarity index 92% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java index 37ed5a68907..4c034b9dcbb 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -52,7 +51,7 @@ public class CardinalityTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public Settings indexSettings() { return Settings.builder() @@ -132,8 +131,7 @@ public class CardinalityTests extends ESIntegTestCase { return randomBoolean() ? "l_values" : "d_values"; } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value")) .execute().actionGet(); @@ -146,8 +144,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, 0); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx", "idx_unmapped").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value")) .execute().actionGet(); @@ -160,8 +157,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void singleValuedString() throws Exception { + public void testSingleValuedString() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value")) .execute().actionGet(); @@ -174,8 +170,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void singleValuedNumeric() throws Exception { + public void testSingleValuedNumeric() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField())) .execute().actionGet(); @@ -188,9 +183,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void singleValuedNumeric_getProperty() throws Exception { - + public void testSingleValuedNumericGetProperty() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation( global("global").subAggregation( @@ -216,8 +209,7 @@ public class CardinalityTests extends ESIntegTestCase { assertThat((double) cardinality.getProperty("value"), equalTo((double) cardinality.getValue())); } - @Test - public void singleValuedNumericHashed() throws Exception { + public void testSingleValuedNumericHashed() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField())) .execute().actionGet(); @@ -230,8 +222,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void multiValuedString() throws Exception { + public void testMultiValuedString() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values")) .execute().actionGet(); @@ -244,8 +235,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void multiValuedNumeric() throws Exception { + public void testMultiValuedNumeric() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(false))) .execute().actionGet(); @@ -258,8 +248,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void multiValuedNumericHashed() throws Exception { + public void testMultiValuedNumericHashed() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(true))) .execute().actionGet(); @@ -272,8 +261,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void singleValuedStringScript() throws Exception { + public void testSingleValuedStringScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).script(new Script("doc['str_value'].value"))) @@ -287,8 +275,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void multiValuedStringScript() throws Exception { + public void testMultiValuedStringScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).script(new Script("doc['str_values'].values"))) @@ -302,8 +289,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void singleValuedNumericScript() throws Exception { + public void testSingleValuedNumericScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).script( @@ -318,8 +304,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void multiValuedNumericScript() throws Exception { + public void testMultiValuedNumericScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).script( @@ -334,8 +319,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void singleValuedStringValueScript() throws Exception { + public void testSingleValuedStringValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value").script(new Script("_value"))) @@ -349,8 +333,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void multiValuedStringValueScript() throws Exception { + public void testMultiValuedStringValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values").script(new Script("_value"))) @@ -364,8 +347,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void singleValuedNumericValueScript() throws Exception { + public void testSingleValuedNumericValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()) @@ -380,8 +362,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void multiValuedNumericValueScript() throws Exception { + public void testMultiValuedNumericValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(false)) @@ -396,8 +377,7 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void asSubAgg() throws Exception { + public void testAsSubAgg() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms").field("str_value") .collectMode(randomFrom(SubAggCollectionMode.values())) @@ -414,5 +394,4 @@ public class CardinalityTests extends ESIntegTestCase { assertCount(count, 2); } } - } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java similarity index 65% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java index a1ed4b74955..728a932d2b5 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java @@ -27,16 +27,13 @@ import org.elasticsearch.action.ActionModule; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptRequest; import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptResponse; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; @@ -45,9 +42,8 @@ import org.elasticsearch.client.FilterClient; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.GeoShapeQueryBuilder; @@ -55,35 +51,26 @@ import org.elasticsearch.index.query.MoreLikeThisQueryBuilder; import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.index.query.functionscore.script.ScriptScoreFunctionBuilder; import org.elasticsearch.indices.cache.query.terms.TermsLookup; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.Template; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.script.groovy.GroovyScriptEngineService; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders; -import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; import org.elasticsearch.test.rest.client.http.HttpResponse; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -105,7 +92,6 @@ import static org.hamcrest.Matchers.is; @ClusterScope(scope = SUITE) public class ContextAndHeaderTransportTests extends ESIntegTestCase { - private static final List requests = new CopyOnWriteArrayList<>(); private String randomHeaderKey = randomAsciiOfLength(10); private String randomHeaderValue = randomAsciiOfLength(20); @@ -154,7 +140,6 @@ public class ContextAndHeaderTransportTests extends ESIntegTestCase { assertRequestsContainHeader(RefreshRequest.class); } - @Test public void testThatTermsLookupGetRequestContainsContextAndHeaders() throws Exception { transportClient().prepareIndex(lookupIndex, "type", "1") .setSource(jsonBuilder().startObject().array("followers", "foo", "bar", "baz").endObject()).get(); @@ -175,7 +160,6 @@ public class ContextAndHeaderTransportTests extends ESIntegTestCase { assertGetRequestsContainHeaders(); } - @Test public void testThatGeoShapeQueryGetRequestContainsContextAndHeaders() throws Exception { transportClient().prepareIndex(lookupIndex, "type", "1").setSource(jsonBuilder().startObject() .field("name", "Munich Suburban Area") @@ -217,7 +201,6 @@ public class ContextAndHeaderTransportTests extends ESIntegTestCase { assertGetRequestsContainHeaders(); } - @Test public void testThatMoreLikeThisQueryMultiTermVectorRequestContainsContextAndHeaders() throws Exception { transportClient().prepareIndex(lookupIndex, "type", "1") .setSource(jsonBuilder().startObject().field("name", "Star Wars - The new republic").endObject()) @@ -245,7 +228,6 @@ public class ContextAndHeaderTransportTests extends ESIntegTestCase { assertRequestsContainHeader(MultiTermVectorsRequest.class); } - @Test public void testThatPercolatingExistingDocumentGetRequestContainsContextAndHeaders() throws Exception { transportClient().prepareIndex(lookupIndex, ".percolator", "1") .setSource(jsonBuilder().startObject().startObject("query").startObject("match").field("name", "star wars").endObject().endObject().endObject()) @@ -262,7 +244,6 @@ public class ContextAndHeaderTransportTests extends ESIntegTestCase { assertGetRequestsContainHeaders(); } - @Test public void testThatIndexedScriptGetRequestContainsContextAndHeaders() throws Exception { PutIndexedScriptResponse scriptResponse = transportClient().preparePutIndexedScript(GroovyScriptEngineService.NAME, "my_script", jsonBuilder().startObject().field("script", "_score * 10").endObject().string() @@ -274,14 +255,12 @@ public class ContextAndHeaderTransportTests extends ESIntegTestCase { .get(); transportClient().admin().indices().prepareRefresh(queryIndex).get(); - // custom content, not sure how to specify "script_id" otherwise in the API - XContentBuilder builder = jsonBuilder().startObject().startObject("function_score").field("boost_mode", "replace").startArray("functions") - .startObject().startObject("script_score").field("script_id", "my_script").field("lang", "groovy").endObject().endObject().endArray().endObject().endObject(); - SearchResponse searchResponse = transportClient() .prepareSearch(queryIndex) - .setQuery(builder) - .get(); + .setQuery( + QueryBuilders.functionScoreQuery( + new ScriptScoreFunctionBuilder(new Script("my_script", ScriptType.INDEXED, "groovy", null))).boostMode( + CombineFunction.REPLACE)).get(); assertNoFailures(searchResponse); assertHitCount(searchResponse, 1); assertThat(searchResponse.getHits().getMaxScore(), is(10.0f)); @@ -290,176 +269,6 @@ public class ContextAndHeaderTransportTests extends ESIntegTestCase { assertRequestsContainHeader(PutIndexedScriptRequest.class); } - @Test - public void testThatIndexedScriptGetRequestInTemplateQueryContainsContextAndHeaders() throws Exception { - PutIndexedScriptResponse scriptResponse = transportClient() - .preparePutIndexedScript( - MustacheScriptEngineService.NAME, - "my_script", - jsonBuilder().startObject().field("script", "{ \"match\": { \"name\": \"Star Wars\" }}").endObject() - .string()).get(); - assertThat(scriptResponse.isCreated(), is(true)); - - transportClient().prepareIndex(queryIndex, "type", "1") - .setSource(jsonBuilder().startObject().field("name", "Star Wars - The new republic").endObject()).get(); - transportClient().admin().indices().prepareRefresh(queryIndex).get(); - - SearchResponse searchResponse = transportClient() - .prepareSearch(queryIndex) - .setQuery( - QueryBuilders.templateQuery(new Template("my_script", ScriptType.INDEXED, - MustacheScriptEngineService.NAME, null, null))).get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 1); - - assertGetRequestsContainHeaders(".scripts"); - assertRequestsContainHeader(PutIndexedScriptRequest.class); - } - - @Test - public void testThatIndexedScriptGetRequestInReducePhaseContainsContextAndHeaders() throws Exception { - PutIndexedScriptResponse scriptResponse = transportClient().preparePutIndexedScript(GroovyScriptEngineService.NAME, "my_script", - jsonBuilder().startObject().field("script", "_value0 * 10").endObject().string()).get(); - assertThat(scriptResponse.isCreated(), is(true)); - - transportClient().prepareIndex(queryIndex, "type", "1") - .setSource(jsonBuilder().startObject().field("s_field", "foo").field("l_field", 10).endObject()).get(); - transportClient().admin().indices().prepareRefresh(queryIndex).get(); - - SearchResponse searchResponse = transportClient() - .prepareSearch(queryIndex) - .addAggregation( - AggregationBuilders - .terms("terms") - .field("s_field") - .subAggregation(AggregationBuilders.max("max").field("l_field")) - .subAggregation( - PipelineAggregatorBuilders.bucketScript("scripted").setBucketsPaths("max").script( - new Script("my_script", ScriptType.INDEXED, GroovyScriptEngineService.NAME, null)))).get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 1); - - assertGetRequestsContainHeaders(".scripts"); - assertRequestsContainHeader(PutIndexedScriptRequest.class); - } - - @Test - public void testThatSearchTemplatesWithIndexedTemplatesGetRequestContainsContextAndHeaders() throws Exception { - PutIndexedScriptResponse scriptResponse = transportClient().preparePutIndexedScript(MustacheScriptEngineService.NAME, "the_template", - jsonBuilder().startObject().startObject("template").startObject("query").startObject("match") - .field("name", "{{query_string}}").endObject().endObject().endObject().endObject().string() - ).get(); - assertThat(scriptResponse.isCreated(), is(true)); - - transportClient().prepareIndex(queryIndex, "type", "1") - .setSource(jsonBuilder().startObject().field("name", "Star Wars - The new republic").endObject()) - .get(); - transportClient().admin().indices().prepareRefresh(queryIndex).get(); - - Map params = new HashMap<>(); - params.put("query_string", "star wars"); - - SearchResponse searchResponse = transportClient().prepareSearch(queryIndex).setTemplate(new Template("the_template", ScriptType.INDEXED, MustacheScriptEngineService.NAME, null, params)) - .get(); - - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 1); - - assertGetRequestsContainHeaders(".scripts"); - assertRequestsContainHeader(PutIndexedScriptRequest.class); - } - - @Test - public void testThatIndexedScriptGetRequestInPhraseSuggestContainsContextAndHeaders() throws Exception { - CreateIndexRequestBuilder builder = transportClient().admin().indices().prepareCreate("test").setSettings(settingsBuilder() - .put(indexSettings()) - .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable. - .put("index.analysis.analyzer.text.tokenizer", "standard") - .putArray("index.analysis.analyzer.text.filter", "lowercase", "my_shingle") - .put("index.analysis.filter.my_shingle.type", "shingle") - .put("index.analysis.filter.my_shingle.output_unigrams", true) - .put("index.analysis.filter.my_shingle.min_shingle_size", 2) - .put("index.analysis.filter.my_shingle.max_shingle_size", 3)); - - XContentBuilder mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("title") - .field("type", "string") - .field("analyzer", "text") - .endObject() - .endObject() - .endObject() - .endObject(); - assertAcked(builder.addMapping("type1", mapping)); - ensureGreen(); - - List titles = new ArrayList<>(); - - titles.add("United States House of Representatives Elections in Washington 2006"); - titles.add("United States House of Representatives Elections in Washington 2005"); - titles.add("State"); - titles.add("Houses of Parliament"); - titles.add("Representative Government"); - titles.add("Election"); - - List builders = new ArrayList<>(); - for (String title: titles) { - transportClient().prepareIndex("test", "type1").setSource("title", title).get(); - } - transportClient().admin().indices().prepareRefresh("test").get(); - - String filterStringAsFilter = XContentFactory.jsonBuilder() - .startObject() - .startObject("match_phrase") - .field("title", "{{suggestion}}") - .endObject() - .endObject() - .string(); - - PutIndexedScriptResponse scriptResponse = transportClient() - .preparePutIndexedScript( - MustacheScriptEngineService.NAME, - "my_script", - jsonBuilder().startObject().field("script", filterStringAsFilter).endObject() - .string()).get(); - assertThat(scriptResponse.isCreated(), is(true)); - - PhraseSuggestionBuilder suggest = phraseSuggestion("title") - .field("title") - .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title") - .suggestMode("always") - .maxTermFreq(.99f) - .size(10) - .maxInspections(200) - ) - .confidence(0f) - .maxErrors(2f) - .shardSize(30000) - .size(10); - - PhraseSuggestionBuilder filteredFilterSuggest = suggest.collateQuery(new Template("my_script", ScriptType.INDEXED, - MustacheScriptEngineService.NAME, null, null)); - - SearchRequestBuilder searchRequestBuilder = transportClient().prepareSearch("test").setSize(0); - String suggestText = "united states house of representatives elections in washington 2006"; - if (suggestText != null) { - searchRequestBuilder.setSuggestText(suggestText); - } - searchRequestBuilder.addSuggestion(filteredFilterSuggest); - SearchResponse actionGet = searchRequestBuilder.execute().actionGet(); - assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(0)); - Suggest searchSuggest = actionGet.getSuggest(); - - assertSuggestionSize(searchSuggest, 0, 2, "title"); - - assertGetRequestsContainHeaders(".scripts"); - assertRequestsContainHeader(PutIndexedScriptRequest.class); - } - - - @Test public void testThatRelevantHttpHeadersBecomeRequestHeaders() throws Exception { String releventHeaderName = "relevant_" + randomHeaderKey; for (RestController restController : internalCluster().getDataNodeInstances(RestController.class)) { diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java similarity index 97% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java index f6425fc6b28..ba4ca38d16e 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -66,7 +65,7 @@ public class DateRangeTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + private static IndexRequestBuilder indexDoc(int month, int day, int value) throws Exception { return client().prepareIndex("idx", "type").setSource(jsonBuilder() .startObject() @@ -112,8 +111,7 @@ public class DateRangeTests extends ESIntegTestCase { ensureSearchable(); } - @Test - public void dateMath() throws Exception { + public void testDateMath() throws Exception { DateRangeBuilder rangeBuilder = dateRange("range"); if (randomBoolean()) { rangeBuilder.field("date"); @@ -152,8 +150,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0L)); } - @Test - public void singleValueField() throws Exception { + public void testSingleValueField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("date") @@ -199,8 +196,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 4l)); } - @Test - public void singleValueField_WithStringDates() throws Exception { + public void testSingleValueFieldWithStringDates() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("date") @@ -246,8 +242,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 4l)); } - @Test - public void singleValueField_WithStringDates_WithCustomFormat() throws Exception { + public void testSingleValueFieldWithStringDatesWithCustomFormat() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("date") @@ -294,8 +289,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 4l)); } - @Test - public void singleValueField_WithDateMath() throws Exception { + public void testSingleValueFieldWithDateMath() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("date") @@ -341,8 +335,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 4l)); } - @Test - public void singleValueField_WithCustomKey() throws Exception { + public void testSingleValueFieldWithCustomKey() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("date") @@ -397,8 +390,7 @@ public class DateRangeTests extends ESIntegTestCase { Mar 23, 6 */ - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("date") @@ -464,8 +456,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat((long) propertiesDocCounts[2], equalTo(numDocs - 4l)); } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregationInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("date") @@ -530,8 +521,7 @@ public class DateRangeTests extends ESIntegTestCase { Mar 23, Apr 24 6 */ - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("dates") @@ -587,8 +577,7 @@ public class DateRangeTests extends ESIntegTestCase { */ - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("dates") @@ -632,7 +621,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 1l)); } - + /* Feb 2, Mar 3, 1 @@ -643,8 +632,7 @@ public class DateRangeTests extends ESIntegTestCase { Apr 23, May 24 6 */ - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .field("dates") @@ -696,8 +684,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(max, notNullValue()); } - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .script(new Script("doc['date'].value")) @@ -743,8 +730,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 4l)); } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -796,7 +782,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(max, notNullValue()); } - + /* Jan 2, Feb 3, 1 @@ -807,8 +793,7 @@ public class DateRangeTests extends ESIntegTestCase { Mar 23, Apr 24 6 */ - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -851,8 +836,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 2l)); } - @Test - public void script_MultiValued_WithAggregatorInherited() throws Exception { + public void testScriptMultiValuedWithAggregatorInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(dateRange("range") .script(new Script("doc['dates'].values")).addUnboundedTo(date(2, 15)) @@ -904,8 +888,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(min.getValue(), equalTo((double) date(2, 15).getMillis())); } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { client().admin().cluster().prepareHealth("idx_unmapped").setWaitForYellowStatus().execute().actionGet(); SearchResponse response = client().prepareSearch("idx_unmapped") @@ -953,8 +936,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0l)); } - @Test - public void unmapped_WithStringDates() throws Exception { + public void testUnmappedWithStringDates() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped") .addAggregation(dateRange("range") .field("date") @@ -1000,8 +982,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0l)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") .addAggregation(dateRange("range") .field("date") @@ -1047,8 +1028,7 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 4l)); } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(dateRange("date_range").addRange("0-1", 0, 1))) @@ -1071,6 +1051,5 @@ public class DateRangeTests extends ESIntegTestCase { assertThat(((DateTime) buckets.get(0).getTo()).getMillis(), equalTo(1l)); assertThat(buckets.get(0).getDocCount(), equalTo(0l)); assertThat(buckets.get(0).getAggregations().asList().isEmpty(), is(true)); - } } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java similarity index 93% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java index 78398a1f6ab..d0de4c7fd85 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStat import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -77,7 +76,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + private static final int NUM_DOCS = 5; // TODO: randomize the size? private static final String SINGLE_VALUED_FIELD_NAME = "d_value"; private static final String MULTI_VALUED_FIELD_NAME = "d_values"; @@ -233,9 +232,8 @@ public class DoubleTermsTests extends AbstractTermsTestCase { return bucket.getKeyAsString(); } - @Test // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard - public void sizeIsZero() { + public void testSizeIsZero() { SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -252,8 +250,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().size(), equalTo(100)); } - @Test - public void singleValueField() throws Exception { + public void testSingleValueField() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -277,8 +274,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_WithMaxSize() throws Exception { + public void testSingleValueFieldWithMaxSize() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -303,9 +299,8 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(bucket.getDocCount(), equalTo(1l)); } } - - @Test - public void singleValueFieldWithFiltering() throws Exception { + + public void testSingleValueFieldWithFiltering() throws Exception { double includes[] = { 1, 2, 3, 98.2 }; double excludes[] = { 2, 4, 99 }; double empty[] = {}; @@ -334,10 +329,8 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(bucket.getDocCount(), equalTo(1l)); } } - - @Test - public void singleValueField_OrderedByTermAsc() throws Exception { + public void testSingleValueFieldOrderedByTermAsc() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -363,8 +356,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_OrderedByTermDesc() throws Exception { + public void testSingleValueFieldOrderedByTermDesc() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -390,8 +382,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -425,8 +416,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregationInherited() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -454,8 +444,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -480,8 +469,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -509,8 +497,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -539,8 +526,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField_WithValueScript_NotUnique() throws Exception { + public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -580,8 +566,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { */ - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -618,8 +603,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -643,8 +627,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -672,8 +655,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -701,9 +683,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_MultiValued_WithAggregatorInherited_NoExplicitType() throws Exception { - + public void testScriptMultiValuedWithAggregatorInheritedNoExplicitType() throws Exception { // since no type is explicitly defined, es will assume all values returned by the script to be strings (bytes), // so the aggregation should fail, since the "sum" aggregation can only operation on numeric values. @@ -725,8 +705,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } - @Test - public void script_MultiValued_WithAggregatorInherited_WithExplicitType() throws Exception { + public void testScriptMultiValuedWithAggregatorInheritedWithExplicitType() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .collectMode(randomFrom(SubAggCollectionMode.values())) @@ -763,8 +742,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -781,8 +759,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped", "idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -806,8 +783,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1l).minDocCount(0) @@ -826,8 +802,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().isEmpty(), is(true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -856,8 +831,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscWithSubTermsAgg() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscWithSubTermsAgg() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -902,8 +876,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleBucketSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleBucketSubAggregationAsc() throws Exception { boolean asc = randomBoolean(); SearchResponse response = client() .prepareSearch("idx") @@ -940,8 +913,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(filter.getDocCount(), equalTo(asc ? 3l : 2l)); } - @Test - public void singleValuedField_OrderedBySubAggregationAsc_MultiHierarchyLevels() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels() throws Exception { boolean asc = randomBoolean(); SearchResponse response = client() .prepareSearch("idx") @@ -998,8 +970,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(max.getValue(), equalTo(asc ? 4.0 : 2.0)); } - @Test - public void singleValuedField_OrderedByMissingSubAggregation() throws Exception { + public void testSingleValuedFieldOrderedByMissingSubAggregation() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index) @@ -1016,8 +987,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByNonMetricsOrMultiBucketSubAggregation() throws Exception { + public void testSingleValuedFieldOrderedByNonMetricsOrMultiBucketSubAggregation() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index) @@ -1039,8 +1009,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregation_WithUknownMetric() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithUknownMetric() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index) @@ -1059,8 +1028,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregation_WithoutMetric() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithoutMetric() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index) @@ -1079,8 +1047,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception { boolean asc = false; SearchResponse response = client() .prepareSearch("idx") @@ -1109,11 +1076,9 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(avg, notNullValue()); assertThat(avg.getValue(), equalTo((double) i)); } - } - @Test - public void singleValuedField_OrderedByMultiValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -1140,11 +1105,9 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(stats, notNullValue()); assertThat(stats.getMax(), equalTo((double) i)); } - } - @Test - public void singleValuedField_OrderedByMultiValueSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueSubAggregationDesc() throws Exception { boolean asc = false; SearchResponse response = client() .prepareSearch("idx") @@ -1171,11 +1134,9 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(stats, notNullValue()); assertThat(stats.getMax(), equalTo((double) i)); } - } - @Test - public void singleValuedField_OrderedByMultiValueExtendedStatsAsc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueExtendedStatsAsc() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -1205,8 +1166,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } - @Test - public void script_Score() { + public void testScriptScore() { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -1232,44 +1192,37 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAndTermsDesc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndTermsDesc() throws Exception { double[] expectedKeys = new double[] { 1, 2, 4, 3, 7, 6, 5 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true), Terms.Order.term(false)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAndTermsAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndTermsAsc() throws Exception { double[] expectedKeys = new double[] { 1, 2, 3, 4, 5, 6, 7 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true), Terms.Order.term(true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationDescAndTermsAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationDescAndTermsAsc() throws Exception { double[] expectedKeys = new double[] { 5, 6, 7, 3, 4, 2, 1 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true)); } - @Test - public void singleValuedField_OrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception { double[] expectedKeys = new double[] { 6, 7, 3, 4, 5, 1, 2 }; assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception { double[] expectedKeys = new double[] { 6, 7, 3, 5, 4, 1, 2 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true)); } - @Test - public void singleValuedField_OrderedByThreeCriteria() throws Exception { + public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception { double[] expectedKeys = new double[] { 2, 1, 4, 5, 3, 6, 7 }; assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAsCompound() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception { double[] expectedKeys = new double[] { 1, 2, 3, 4, 5, 6, 7 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true)); } @@ -1305,8 +1258,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } } - @Test - public void otherDocCount() { + public void testOtherDocCount() { testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME); } } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java similarity index 87% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java index 47cdc42987e..b610f9648b5 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -49,7 +48,6 @@ import static org.hamcrest.Matchers.sameInstance; * */ public class ExtendedStatsTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); @@ -70,9 +68,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test public void testEmptyAggregation() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(extendedStats("stats"))) @@ -99,7 +95,6 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test public void testUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx_unmapped") .setQuery(matchAllQuery()) @@ -124,7 +119,6 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") @@ -148,9 +142,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { checkUpperLowerBounds(stats, sigma); } - @Test public void testSingleValuedFieldDefaultSigma() throws Exception { - // Same as previous test, but uses a default value for sigma SearchResponse searchResponse = client().prepareSearch("idx") @@ -204,9 +196,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { - + public void testSingleValuedFieldGetProperty() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(global("global").subAggregation(extendedStats("stats").field("value"))).execute().actionGet(); @@ -252,8 +242,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") .setQuery(matchAllQuery()) @@ -277,8 +266,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -302,8 +290,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); double sigma = randomDouble() * randomIntBetween(1, 10); @@ -331,7 +318,6 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") @@ -356,8 +342,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -381,8 +366,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); double sigma = randomDouble() * randomIntBetween(1, 10); @@ -410,8 +394,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -435,8 +418,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); double sigma = randomDouble() * randomIntBetween(1, 10); @@ -463,36 +445,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - double sigma = randomDouble() * randomIntBetween(1, 10); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - extendedStats("stats").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params)).sigma(sigma)) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - ExtendedStats stats = searchResponse.getAggregations().get("stats"); - assertThat(stats, notNullValue()); - assertThat(stats.getName(), equalTo("stats")); - assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); - assertThat(stats.getMin(), equalTo(2.0)); - assertThat(stats.getMax(), equalTo(11.0)); - assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11)); - assertThat(stats.getCount(), equalTo(10l)); - assertThat(stats.getSumOfSquares(), equalTo((double) 4+9+16+25+36+49+64+81+100+121)); - assertThat(stats.getVariance(), equalTo(variance(2, 3, 4, 5, 6, 7, 8 ,9, 10, 11))); - assertThat(stats.getStdDeviation(), equalTo(stdDev(2, 3, 4, 5, 6, 7, 8 ,9, 10, 11))); - checkUpperLowerBounds(stats, sigma); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -516,35 +469,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - double sigma = randomDouble() * randomIntBetween(1, 10); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(extendedStats("stats").script(new Script("doc['values'].values")).sigma(sigma)) - .execute().actionGet(); - - assertShardExecutionState(searchResponse, 0); - assertHitCount(searchResponse, 10); - - ExtendedStats stats = searchResponse.getAggregations().get("stats"); - assertThat(stats, notNullValue()); - assertThat(stats.getName(), equalTo("stats")); - assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12) / 20)); - assertThat(stats.getMin(), equalTo(2.0)); - assertThat(stats.getMax(), equalTo(12.0)); - assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12)); - assertThat(stats.getCount(), equalTo(20l)); - assertThat(stats.getSumOfSquares(), equalTo((double) 4+9+16+25+36+49+64+81+100+121+9+16+25+36+49+64+81+100+121+144)); - assertThat(stats.getVariance(), equalTo(variance(2, 3, 4, 5, 6, 7, 8 ,9, 10, 11, 3, 4, 5, 6, 7, 8 ,9, 10, 11, 12))); - assertThat(stats.getStdDeviation(), equalTo(stdDev(2, 3, 4, 5, 6, 7, 8 ,9, 10, 11, 3, 4, 5, 6, 7, 8 ,9, 10, 11, 12))); - checkUpperLowerBounds(stats, sigma); - - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); double sigma = randomDouble() * randomIntBetween(1, 10); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java similarity index 99% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java index 51fc5a4de8b..22bb778c7be 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -52,7 +51,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class FunctionScoreTests extends ESIntegTestCase { - static final String TYPE = "type"; static final String INDEX = "index"; @@ -61,8 +59,6 @@ public class FunctionScoreTests extends ESIntegTestCase { return Collections.singleton(GroovyPlugin.class); } - - @Test public void testScriptScoresNested() throws IOException { createIndex(INDEX); ensureYellow(); @@ -84,7 +80,6 @@ public class FunctionScoreTests extends ESIntegTestCase { assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); } - @Test public void testScriptScoresWithAgg() throws IOException { createIndex(INDEX); ensureYellow(); @@ -132,7 +127,6 @@ public class FunctionScoreTests extends ESIntegTestCase { } } - @Test public void testMinScoreFunctionScoreManyDocsAndRandomMinScore() throws IOException, ExecutionException, InterruptedException { List docs = new ArrayList<>(); int numDocs = randomIntBetween(1, 100); @@ -175,7 +169,6 @@ public class FunctionScoreTests extends ESIntegTestCase { } } - @Test public void testWithEmptyFunctions() throws IOException, ExecutionException, InterruptedException { assertAcked(prepareCreate("test")); ensureYellow(); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java similarity index 86% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java index 4e601c23892..d79beb186cf 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java @@ -19,14 +19,18 @@ package org.elasticsearch.messy.tests; -import org.apache.lucene.util.XGeoHashUtils; +import org.apache.lucene.util.GeoHashUtils; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoDistance; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.query.GeoDistanceQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; @@ -36,7 +40,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.ArrayList; @@ -66,19 +70,21 @@ import static org.hamcrest.Matchers.equalTo; /** */ public class GeoDistanceTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - @Test - public void simpleDistanceTests() throws Exception { + + public void testSimpleDistance() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true) - .startObject("fielddata").field("format", randomNumericFieldDataFormat()).endObject().endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); indexRandom(true, client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -225,14 +231,16 @@ public class GeoDistanceTests extends ESIntegTestCase { assertOrderedSearchHits(searchResponse, "7", "2", "6", "5", "4", "3", "1"); } - @Test public void testDistanceSortingMVFields() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("locations").field("type", "geo_point").field("lat_lon", true) - .field("ignore_malformed", true).field("coerce", true).startObject("fielddata") - .field("format", randomNumericFieldDataFormat()).endObject().endObject().endObject().endObject().endObject(); - assertAcked(prepareCreate("test") - .addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("locations").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true).field("coerce", true); + } + xContentBuilder.field("ignore_malformed", true).endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -360,14 +368,17 @@ public class GeoDistanceTests extends ESIntegTestCase { containsString("sort_mode [sum] isn't supported for sorting by geo distance")); } - @Test // Regression bug: https://github.com/elasticsearch/elasticsearch/issues/2851 public void testDistanceSortingWithMissingGeoPoint() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("locations").field("type", "geo_point").field("lat_lon", true) - .startObject("fielddata").field("format", randomNumericFieldDataFormat()).endObject().endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("locations").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -408,17 +419,21 @@ public class GeoDistanceTests extends ESIntegTestCase { assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), closeTo(5286d, 10d)); } - @Test - public void distanceScriptTests() throws Exception { + public void testDistanceScript() throws Exception { double source_lat = 32.798; double source_long = -117.151; double target_lat = 32.81; double target_long = -117.21; + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true).endObject().endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder)); + .startObject("properties").startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() @@ -433,28 +448,28 @@ public class GeoDistanceTests extends ESIntegTestCase { .actionGet(); Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance1, - closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.0001d)); + closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d)); SearchResponse searchResponse2 = client().prepareSearch().addField("_source") .addScriptField("distance", new Script("doc['location'].distance(" + target_lat + "," + target_long + ")")).execute() .actionGet(); Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance2, - closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.0001d)); + closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d)); SearchResponse searchResponse3 = client().prepareSearch().addField("_source") .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + target_lat + "," + target_long + ")")) .execute().actionGet(); Double resultArcDistance3 = searchResponse3.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance3, - closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.0001d)); + closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); SearchResponse searchResponse4 = client().prepareSearch().addField("_source") .addScriptField("distance", new Script("doc['location'].distanceInKm(" + target_lat + "," + target_long + ")")).execute() .actionGet(); Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance4, - closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.0001d)); + closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); SearchResponse searchResponse5 = client() .prepareSearch() @@ -463,7 +478,7 @@ public class GeoDistanceTests extends ESIntegTestCase { .execute().actionGet(); Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance5, - closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.0001d)); + closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); SearchResponse searchResponse6 = client() .prepareSearch() @@ -472,26 +487,26 @@ public class GeoDistanceTests extends ESIntegTestCase { .execute().actionGet(); Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance6, - closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.0001d)); + closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); SearchResponse searchResponse7 = client().prepareSearch().addField("_source") .addScriptField("distance", new Script("doc['location'].arcDistanceInMiles(" + target_lat + "," + target_long + ")")) .execute().actionGet(); Double resultDistance7 = searchResponse7.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance7, - closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.0001d)); + closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d)); SearchResponse searchResponse8 = client().prepareSearch().addField("_source") .addScriptField("distance", new Script("doc['location'].distanceInMiles(" + target_lat + "," + target_long + ")")) .execute().actionGet(); Double resultDistance8 = searchResponse8.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance8, - closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.0001d)); + closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d)); } - - @Test public void testDistanceSortingNestedFields() throws Exception { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("company") .startObject("properties") .startObject("name").field("type", "string").endObject() @@ -499,14 +514,17 @@ public class GeoDistanceTests extends ESIntegTestCase { .field("type", "nested") .startObject("properties") .startObject("name").field("type", "string").endObject() - .startObject("location").field("type", "geo_point").field("lat_lon", true) - .startObject("fielddata").field("format", randomNumericFieldDataFormat()).endObject().endObject() + .startObject("location").field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + xContentBuilder.field("lat_lon", true); + } + xContentBuilder.endObject() .endObject() .endObject() .endObject() .endObject().endObject(); - assertAcked(prepareCreate("companies").addMapping("company", xContentBuilder)); + assertAcked(prepareCreate("companies").setSettings(settings).addMapping("company", xContentBuilder)); ensureGreen(); indexRandom(true, client().prepareIndex("companies", "company", "1").setSource(jsonBuilder().startObject() @@ -654,34 +672,29 @@ public class GeoDistanceTests extends ESIntegTestCase { /** * Issue 3073 */ - @Test public void testGeoDistanceFilter() throws IOException { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); double lat = 40.720611; double lon = -73.998776; XContentBuilder mapping = JsonXContent.contentBuilder() .startObject() - .startObject("location") - .startObject("properties") - .startObject("pin") - .field("type", "geo_point") - .field("geohash", true) - .field("geohash_precision", 24) - .field("lat_lon", true) - .startObject("fielddata") - .field("format", randomNumericFieldDataFormat()) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); + .startObject("location") + .startObject("properties") + .startObject("pin") + .field("type", "geo_point"); + if (version.before(Version.V_2_2_0)) { + mapping.field("lat_lon", true); + } + mapping.endObject().endObject().endObject().endObject(); XContentBuilder source = JsonXContent.contentBuilder() .startObject() - .field("pin", XGeoHashUtils.stringEncode(lon, lat)) + .field("pin", GeoHashUtils.stringEncode(lon, lat)) .endObject(); - assertAcked(prepareCreate("locations").addMapping("location", mapping)); + assertAcked(prepareCreate("locations").setSettings(settings).addMapping("location", mapping)); client().prepareIndex("locations", "location", "1").setCreate(true).setSource(source).execute().actionGet(); refresh(); client().prepareGet("locations", "location", "1").execute().actionGet(); @@ -706,11 +719,19 @@ public class GeoDistanceTests extends ESIntegTestCase { } public void testDuelOptimizations() throws Exception { - assertAcked(prepareCreate("index").addMapping("type", "location", "type=geo_point,lat_lon=true")); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + if (version.before(Version.V_2_2_0)) { + assertAcked(prepareCreate("index").setSettings(settings) + .addMapping("type", "location", "type=geo_point,lat_lon=true")); + } else { + assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point")); + } final int numDocs = scaledRandomIntBetween(3000, 10000); List docs = new ArrayList<>(); for (int i = 0; i < numDocs; ++i) { - docs.add(client().prepareIndex("index", "type").setSource(jsonBuilder().startObject().startObject("location").field("lat", randomLat()).field("lon", randomLon()).endObject().endObject())); + docs.add(client().prepareIndex("index", "type").setSource(jsonBuilder().startObject().startObject("location") + .field("lat", randomLat()).field("lon", randomLon()).endObject().endObject())); } indexRandom(true, docs); ensureSearchable(); @@ -718,23 +739,37 @@ public class GeoDistanceTests extends ESIntegTestCase { for (int i = 0; i < 10; ++i) { final double originLat = randomLat(); final double originLon = randomLon(); - final String distance = DistanceUnit.KILOMETERS.toString(randomInt(10000)); + final String distance = DistanceUnit.KILOMETERS.toString(randomIntBetween(1, 10000)); for (GeoDistance geoDistance : Arrays.asList(GeoDistance.ARC, GeoDistance.SLOPPY_ARC)) { logger.info("Now testing GeoDistance={}, distance={}, origin=({}, {})", geoDistance, distance, originLat, originLon); - long matches = -1; - for (String optimizeBbox : Arrays.asList("none", "memory", "indexed")) { - SearchResponse resp = client().prepareSearch("index").setSize(0).setQuery(QueryBuilders.constantScoreQuery( - QueryBuilders.geoDistanceQuery("location").point(originLat, originLon).distance(distance).geoDistance(geoDistance).optimizeBbox(optimizeBbox))).execute().actionGet(); - assertSearchResponse(resp); - logger.info("{} -> {} hits", optimizeBbox, resp.getHits().totalHits()); - if (matches < 0) { - matches = resp.getHits().totalHits(); - } else { - assertEquals(matches, resp.getHits().totalHits()); + GeoDistanceQueryBuilder qb = QueryBuilders.geoDistanceQuery("location").point(originLat, originLon).distance(distance).geoDistance(geoDistance); + long matches; + if (version.before(Version.V_2_2_0)) { + for (String optimizeBbox : Arrays.asList("none", "memory", "indexed")) { + qb.optimizeBbox(optimizeBbox); + SearchResponse resp = client().prepareSearch("index").setSize(0) + .setQuery(QueryBuilders.constantScoreQuery(qb)).execute().actionGet(); + matches = assertDuelOptimization(resp); + logger.info("{} -> {} hits", optimizeBbox, matches); } + } else { + SearchResponse resp = client().prepareSearch("index").setSize(0) + .setQuery(QueryBuilders.constantScoreQuery(qb)).execute().actionGet(); + matches = assertDuelOptimization(resp); + logger.info("{} hits", matches); } } } } -} \ No newline at end of file + private long assertDuelOptimization(SearchResponse resp) { + long matches = -1; + assertSearchResponse(resp); + if (matches < 0) { + matches = resp.getHits().totalHits(); + } else { + assertEquals(matches, matches = resp.getHits().totalHits()); + } + return matches; + } +} diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java new file mode 100644 index 00000000000..37132c5a923 --- /dev/null +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.messy.tests; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Collection; +import java.util.Collections; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + */ +public class GeoShapeIntegrationTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(GroovyPlugin.class); + } + + /** + * Test that orientation parameter correctly persists across cluster restart + */ + public void testOrientationPersistence() throws Exception { + String idxName = "orientation"; + String mapping = XContentFactory.jsonBuilder().startObject().startObject("shape") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject().string(); + + // create index + assertAcked(prepareCreate(idxName).addMapping("shape", mapping)); + + mapping = XContentFactory.jsonBuilder().startObject().startObject("shape") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject().string(); + + assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping)); + ensureGreen(idxName, idxName+"2"); + + internalCluster().fullRestart(); + ensureGreen(idxName, idxName+"2"); + + // left orientation test + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName)); + IndexService indexService = indicesService.indexService(idxName); + MappedFieldType fieldType = indexService.mapperService().smartNameFieldType("location"); + assertThat(fieldType, instanceOf(GeoShapeFieldMapper.GeoShapeFieldType.class)); + + GeoShapeFieldMapper.GeoShapeFieldType gsfm = (GeoShapeFieldMapper.GeoShapeFieldType)fieldType; + ShapeBuilder.Orientation orientation = gsfm.orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); + + // right orientation test + indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName+"2")); + indexService = indicesService.indexService(idxName+"2"); + fieldType = indexService.mapperService().smartNameFieldType("location"); + assertThat(fieldType, instanceOf(GeoShapeFieldMapper.GeoShapeFieldType.class)); + + gsfm = (GeoShapeFieldMapper.GeoShapeFieldType)fieldType; + orientation = gsfm.orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); + } + + private String findNodeName(String index) { + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0); + String nodeId = shard.assignedShards().get(0).currentNodeId(); + return state.getNodes().get(nodeId).name(); + } +} diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java similarity index 87% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java index b22455ad0bb..7e7e8cb76fa 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod; -import org.junit.Test; import java.util.Arrays; import java.util.Collection; @@ -42,22 +41,26 @@ import java.util.Map; import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.percentileRanks; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; /** * */ public class HDRPercentileRanksTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - private static double[] randomPercents(long minValue, long maxValue) { + private static double[] randomPercents(long minValue, long maxValue) { final int length = randomIntBetween(1, 20); final double[] percents = new double[length]; for (int i = 0; i < percents.length; ++i) { @@ -107,9 +110,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testEmptyAggregation() throws Exception { - int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx") @@ -137,7 +138,6 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testUnmapped() throws Exception { int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -159,7 +159,6 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValue, maxValue); @@ -177,8 +176,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { + public void testSingleValuedFieldGetProperty() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValue, maxValue); SearchResponse searchResponse = client() @@ -205,7 +203,6 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } - @Test public void testSingleValuedFieldOutsideRange() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = new double[] { minValue - 1, maxValue + 1 }; @@ -223,8 +220,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValue, maxValue); SearchResponse searchResponse = client() @@ -241,8 +237,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValue - 1, maxValue - 1); SearchResponse searchResponse = client() @@ -259,8 +254,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { int sigDigits = randomSignificantDigits(); Map params = new HashMap<>(); params.put("dec", 1); @@ -280,7 +274,6 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValues, maxValues); @@ -298,8 +291,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValues - 1, maxValues - 1); SearchResponse searchResponse = client() @@ -315,8 +307,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits); } - @Test - public void testMultiValuedField_WithValueScript_Reverse() throws Exception { + public void testMultiValuedFieldWithValueScriptReverse() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(20 - maxValues, 20 - minValues); SearchResponse searchResponse = client() @@ -333,8 +324,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { int sigDigits = randomSignificantDigits(); Map params = new HashMap<>(); params.put("dec", 1); @@ -354,8 +344,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValue, maxValue); SearchResponse searchResponse = client() @@ -372,8 +361,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { int sigDigits = randomSignificantDigits(); Map params = new HashMap<>(); params.put("dec", 1); @@ -393,29 +381,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - int sigDigits = randomSignificantDigits(); - Map params = new HashMap<>(); - params.put("dec", 1); - final double[] pcts = randomPercents(minValue - 1, maxValue - 1); - SearchResponse searchResponse = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits) - .script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).percentiles(pcts)) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { int sigDigits = randomSignificantDigits(); final double[] pcts = randomPercents(minValues, maxValues); SearchResponse searchResponse = client() @@ -432,26 +398,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - int sigDigits = randomSignificantDigits(); - final double[] pcts = randomPercents(minValues, maxValues); - SearchResponse searchResponse = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits) - .script(new Script("doc['values'].values")).percentiles(pcts)).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { int sigDigits = randomSignificantDigits(); Map params = new HashMap<>(); params.put("dec", 1); @@ -473,7 +420,6 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits); } - @Test public void testOrderBySubAggregation() { int sigDigits = randomSignificantDigits(); boolean asc = randomBoolean(); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java similarity index 87% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java index 07e9d1097f5..bfd094ac78c 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod; -import org.junit.Test; import java.util.Arrays; import java.util.Collection; @@ -42,20 +41,26 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; /** * */ public class HDRPercentilesTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + private static double[] randomPercentiles() { final int length = randomIntBetween(1, 20); final double[] percentiles = new double[length]; @@ -106,7 +111,6 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testEmptyAggregation() throws Exception { int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -136,7 +140,6 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testUnmapped() throws Exception { int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -158,7 +161,6 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomIntBetween(1, 5); @@ -177,8 +179,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { + public void testSingleValuedFieldGetProperty() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -207,8 +208,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -226,8 +226,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -244,8 +243,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); @@ -265,7 +263,6 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); @@ -284,8 +281,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -301,8 +297,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits); } - @Test - public void testMultiValuedField_WithValueScript_Reverse() throws Exception { + public void testMultiValuedFieldWithValueScriptReverse() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -319,8 +314,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); @@ -340,8 +334,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -358,8 +351,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); @@ -379,29 +371,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("dec", 1); - final double[] pcts = randomPercentiles(); - int sigDigits = randomSignificantDigits(); - SearchResponse searchResponse = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR) - .script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).percentiles(pcts)) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final Percentiles percentiles = searchResponse.getAggregations().get("percentiles"); - assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1, sigDigits); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { final double[] pcts = randomPercentiles(); int sigDigits = randomSignificantDigits(); SearchResponse searchResponse = client() @@ -418,26 +388,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - final double[] pcts = randomPercentiles(); - int sigDigits = randomSignificantDigits(); - SearchResponse searchResponse = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR) - .script(new Script("doc['values'].values")).percentiles(pcts)).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final Percentiles percentiles = searchResponse.getAggregations().get("percentiles"); - assertConsistent(pcts, percentiles, minValues, maxValues, sigDigits); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); @@ -459,7 +410,6 @@ public class HDRPercentilesTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1, sigDigits); } - @Test public void testOrderBySubAggregation() { int sigDigits = randomSignificantDigits(); boolean asc = randomBoolean(); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java similarity index 92% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java index e36e38e956c..dd3d2e99fcd 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -79,7 +78,7 @@ public class HistogramTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -128,8 +127,7 @@ public class HistogramTests extends ESIntegTestCase { ensureSearchable(); } - @Test - public void singleValuedField() throws Exception { + public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)) .execute().actionGet(); @@ -182,8 +180,7 @@ public class HistogramTests extends ESIntegTestCase { * Shift buckets by random offset between [2..interval]. From setup we have 1 doc per values from 1..numdocs. * Special care needs to be taken for expecations on counts in first and last bucket. */ - @Test - public void singleValuedField_withRandomOffset() throws Exception { + public void testSingleValuedFieldWithRandomOffset() throws Exception { int offset = randomIntBetween(2, interval); SearchResponse response = client() .prepareSearch("idx") @@ -218,8 +215,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByKeyAsc() throws Exception { + public void testSingleValuedFieldOrderedByKeyAsc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.KEY_ASC)) .execute().actionGet(); @@ -242,8 +238,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByKeyDesc() throws Exception { + public void testsingleValuedFieldOrderedByKeyDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.KEY_DESC)) .execute().actionGet(); @@ -266,8 +261,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByCountAsc() throws Exception { + public void testSingleValuedFieldOrderedByCountAsc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.COUNT_ASC)) .execute().actionGet(); @@ -296,8 +290,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByCountDesc() throws Exception { + public void testSingleValuedFieldOrderedByCountDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.COUNT_DESC)) .execute().actionGet(); @@ -326,8 +319,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -367,8 +359,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregationInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) .subAggregation(sum("sum"))) @@ -402,8 +393,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedBySubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationAsc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("sum", true)) .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -443,8 +433,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedBySubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("sum", false)) .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))) @@ -484,8 +473,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregationAsc_Inherited() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationAscInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("stats.sum", true)) .subAggregation(stats("stats"))) @@ -525,8 +513,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("stats.sum", false)) .subAggregation(stats("stats").field(SINGLE_VALUED_FIELD_NAME))) @@ -566,8 +553,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_OrderedBySubAggregationDesc_DeepOrderPath() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationDescDeepOrderPath() throws Exception { boolean asc = randomBoolean(); SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("filter>max", asc)) @@ -605,8 +591,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void singleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).script(new Script("_value + 1")).interval(interval)) .execute().actionGet(); @@ -634,8 +619,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(MULTI_VALUED_FIELD_NAME).interval(interval)) .execute().actionGet(); @@ -657,8 +641,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void multiValuedField_OrderedByKeyDesc() throws Exception { + public void testMultiValuedFieldOrderedByKeyDesc() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(MULTI_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.KEY_DESC)) .execute().actionGet(); @@ -681,8 +664,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").field(MULTI_VALUED_FIELD_NAME).script(new Script("_value + 1")).interval(interval)) .execute().actionGet(); @@ -715,8 +697,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation( histogram("histo") @@ -765,8 +746,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").script(new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")).interval(interval)) .execute().actionGet(); @@ -787,8 +767,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -822,8 +801,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(histogram("histo").script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']")).interval(interval)) .execute().actionGet(); @@ -844,8 +822,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void script_MultiValued_WithAggregatorInherited() throws Exception { + public void testScriptMultiValuedWithAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -879,8 +856,7 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)) .execute().actionGet(); @@ -894,8 +870,7 @@ public class HistogramTests extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)) .execute().actionGet(); @@ -917,8 +892,40 @@ public class HistogramTests extends ESIntegTestCase { } } - @Test - public void emptyAggregation() throws Exception { + public void testPartiallyUnmappedWithExtendedBounds() throws Exception { + SearchResponse response = client() + .prepareSearch("idx", "idx_unmapped") + .addAggregation( + histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval) + .extendedBounds((long) -1 * 2 * interval, (long) valueCounts.length * interval)).execute().actionGet(); + + assertSearchResponse(response); + + Histogram histo = response.getAggregations().get("histo"); + assertThat(histo, notNullValue()); + assertThat(histo.getName(), equalTo("histo")); + List buckets = histo.getBuckets(); + assertThat(buckets.size(), equalTo(numValueBuckets + 3)); + + Histogram.Bucket bucket = buckets.get(0); + assertThat(bucket, notNullValue()); + assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) -1 * 2 * interval)); + assertThat(bucket.getDocCount(), equalTo(0l)); + + bucket = buckets.get(1); + assertThat(bucket, notNullValue()); + assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) -1 * interval)); + assertThat(bucket.getDocCount(), equalTo(0l)); + + for (int i = 2; i < numValueBuckets + 2; ++i) { + bucket = buckets.get(i); + assertThat(bucket, notNullValue()); + assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) (i - 2) * interval)); + assertThat(bucket.getDocCount(), equalTo(valueCounts[i - 2])); + } + } + + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1l).minDocCount(0) @@ -938,8 +945,7 @@ public class HistogramTests extends ESIntegTestCase { assertThat(histo.getBuckets().isEmpty(), is(true)); } - @Test - public void singleValuedField_WithExtendedBounds() throws Exception { + public void testSingleValuedFieldWithExtendedBounds() throws Exception { int lastDataBucketKey = (numValueBuckets - 1) * interval; // randomizing the number of buckets on the min bound diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java similarity index 95% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java index 7ade58ce6c5..b93d090b56a 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -61,7 +60,7 @@ public class IPv4RangeTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public void setupSuiteScopeCluster() throws Exception { { @@ -122,8 +121,7 @@ public class IPv4RangeTests extends ESIntegTestCase { ensureSearchable(); } - @Test - public void singleValueField() throws Exception { + public void testSingleValueField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(ipRange("range") .field("ip") @@ -143,25 +141,25 @@ public class IPv4RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) (String) bucket.getKey(), equalTo("*-10.0.0.100")); - assertThat(((Number) ((Number) bucket.getFrom())).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); + assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100")); + assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(bucket.getFromAsString(), nullValue()); assertThat(bucket.getToAsString(), equalTo("10.0.0.100")); - assertThat(((Number) ((Number) bucket.getTo())).doubleValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.100"))); + assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.100"))); assertThat(bucket.getDocCount(), equalTo(100l)); bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) (String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200")); + assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200")); assertThat(bucket.getFromAsString(), equalTo("10.0.0.100")); - assertThat(((Number) ((Number) bucket.getFrom())).doubleValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.100"))); + assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.100"))); assertThat(bucket.getToAsString(), equalTo("10.0.0.200")); - assertThat(((Number) ((Number) bucket.getTo())).doubleValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.200"))); + assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.200"))); assertThat(bucket.getDocCount(), equalTo(100l)); bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) (String) bucket.getKey(), equalTo("10.0.0.200-*")); + assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*")); assertThat(bucket.getFromAsString(), equalTo("10.0.0.200")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.200"))); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); @@ -169,8 +167,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(55l)); } - @Test - public void singleValueField_WithMaskRange() throws Exception { + public void testSingleValueFieldWithMaskRange() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(ipRange("range") .field("ip") @@ -206,8 +203,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(127l)); // include 10.0.0.128 } - @Test - public void singleValueField_WithCustomKey() throws Exception { + public void testSingleValueFieldWithCustomKey() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(ipRange("range") .field("ip") @@ -253,8 +249,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(55l)); } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(ipRange("range") .field("ip") @@ -322,8 +317,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat((double) propertiesCounts[2], equalTo((double) 55 * 3)); } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregationInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(ipRange("range") .field("ip") @@ -379,8 +373,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(max.getValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.254"))); } - @Test - public void singleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -440,8 +433,7 @@ public class IPv4RangeTests extends ESIntegTestCase { [255, 256] */ - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(ipRange("range") .field("ips") @@ -487,8 +479,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(56l)); } - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -532,8 +523,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(56l)); } - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -586,8 +576,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat((long) max.getValue(), equalTo(IpFieldMapper.ipToLong("10.0.0.255"))); } - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -631,8 +620,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(55l)); } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -686,8 +674,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(max.getValue(), equalTo((double) IpFieldMapper.ipToLong("10.0.0.254"))); } - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -731,8 +718,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(56l)); } - @Test - public void script_MultiValued_WithAggregatorInherited() throws Exception { + public void testScriptMultiValuedWithAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -786,8 +772,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat((long) max.getValue(), equalTo(IpFieldMapper.ipToLong("10.0.0.255"))); } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped") .addAggregation(ipRange("range") .field("ip") @@ -833,8 +818,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0l)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") .addAggregation(ipRange("range") .field("ip") @@ -880,8 +864,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(55l)); } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) @@ -906,8 +889,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertThat(buckets.get(0).getDocCount(), equalTo(0l)); } - @Test - public void mask0() { + public void testMask0() { SearchResponse response = client().prepareSearch("idx") .addAggregation(ipRange("range") .field("ip") @@ -930,10 +912,7 @@ public class IPv4RangeTests extends ESIntegTestCase { assertEquals(255l, bucket.getDocCount()); } - - @Test - public void mask0SpecialIps() { - + public void testMask0SpecialIps() { SearchResponse response = client().prepareSearch("range_idx") .addAggregation(ipRange("range") .field("ip") diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java similarity index 99% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java index b1c2a332f13..f972f3b8944 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -49,7 +48,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC import static org.hamcrest.Matchers.equalTo; public class IndexLookupTests extends ESIntegTestCase { - String includeAllFlag = "_FREQUENCIES | _OFFSETS | _PAYLOADS | _POSITIONS | _CACHE"; String includeAllWithoutRecordFlag = "_FREQUENCIES | _OFFSETS | _PAYLOADS | _POSITIONS "; private HashMap> expectedEndOffsetsArray; @@ -62,7 +60,7 @@ public class IndexLookupTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + void initTestData() throws InterruptedException, ExecutionException, IOException { emptyArray = new HashMap<>(); List empty1 = new ArrayList<>(); @@ -154,9 +152,7 @@ public class IndexLookupTests extends ESIntegTestCase { ensureGreen(); } - @Test public void testTwoScripts() throws Exception { - initTestData(); // check term frequencies for 'a' @@ -176,9 +172,7 @@ public class IndexLookupTests extends ESIntegTestCase { } - @Test public void testCallWithDifferentFlagsFails() throws Exception { - initTestData(); // should throw an exception, we cannot call with different flags twice @@ -212,9 +206,7 @@ public class IndexLookupTests extends ESIntegTestCase { } } - @Test public void testDocumentationExample() throws Exception { - initTestData(); Script script = new Script("term = _index['float_payload_field'].get('b'," + includeAllFlag @@ -237,9 +229,7 @@ public class IndexLookupTests extends ESIntegTestCase { checkValueInEachDoc(script, zeroArray, 3); } - @Test public void testIteratorAndRecording() throws Exception { - initTestData(); // call twice with record: should work as expected @@ -300,9 +290,7 @@ public class IndexLookupTests extends ESIntegTestCase { return new Script(script); } - @Test public void testFlags() throws Exception { - initTestData(); // check default flag @@ -409,7 +397,6 @@ public class IndexLookupTests extends ESIntegTestCase { assertThat(nullCounter, equalTo(expectedArray.size())); } - @Test public void testAllExceptPosAndOffset() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("float_payload_field").field("type", "string").field("index_options", "offsets").field("term_vector", "no") diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java similarity index 78% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java index 1bba7bfc03c..a3a786a140e 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java @@ -24,23 +24,26 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptResponse; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.script.groovy.GroovyScriptEngineService; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -49,12 +52,11 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; public class IndexedScriptTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); @@ -69,7 +71,6 @@ public class IndexedScriptTests extends ESIntegTestCase { return builder.build(); } - @Test public void testFieldIndexedScript() throws ExecutionException, InterruptedException { List builders = new ArrayList<>(); builders.add(client().prepareIndex(ScriptService.SCRIPT_INDEX, "groovy", "script1").setSource("{" + @@ -91,17 +92,23 @@ public class IndexedScriptTests extends ESIntegTestCase { builders.add(client().prepareIndex("test", "scriptTest", "5").setSource("{\"theField\":\"bar\"}")); indexRandom(true, builders); - String query = "{ \"query\" : { \"match_all\": {}} , \"script_fields\" : { \"test1\" : { \"script_id\" : \"script1\", \"lang\":\"groovy\" }, \"test2\" : { \"script_id\" : \"script2\", \"lang\":\"groovy\", \"params\":{\"factor\":3} }}, size:1}"; - SearchResponse searchResponse = client().prepareSearch().setSource(new BytesArray(query)).setIndices("test").setTypes("scriptTest").get(); + Map script2Params = new HashMap<>(); + script2Params.put("factor", 3); + SearchResponse searchResponse = client() + .prepareSearch() + .setSource( + new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(1) + .scriptField("test1", new Script("script1", ScriptType.INDEXED, "groovy", null)) + .scriptField("test2", new Script("script2", ScriptType.INDEXED, "groovy", script2Params))) + .setIndices("test").setTypes("scriptTest").get(); assertHitCount(searchResponse, 5); assertTrue(searchResponse.getHits().hits().length == 1); SearchHit sh = searchResponse.getHits().getAt(0); - assertThat((Integer)sh.field("test1").getValue(), equalTo(2)); - assertThat((Integer)sh.field("test2").getValue(), equalTo(6)); + assertThat((Integer) sh.field("test1").getValue(), equalTo(2)); + assertThat((Integer) sh.field("test2").getValue(), equalTo(6)); } // Relates to #10397 - @Test public void testUpdateScripts() { createIndex("test_index"); ensureGreen("test_index"); @@ -110,21 +117,21 @@ public class IndexedScriptTests extends ESIntegTestCase { int iterations = randomIntBetween(2, 11); for (int i = 1; i < iterations; i++) { - PutIndexedScriptResponse response = + PutIndexedScriptResponse response = client().preparePutIndexedScript(GroovyScriptEngineService.NAME, "script1", "{\"script\":\"" + i + "\"}").get(); assertEquals(i, response.getVersion()); - - String query = "{" - + " \"query\" : { \"match_all\": {}}, " - + " \"script_fields\" : { \"test_field\" : { \"script_id\" : \"script1\", \"lang\":\"groovy\" } } }"; - SearchResponse searchResponse = client().prepareSearch().setSource(new BytesArray(query)).setIndices("test_index").setTypes("test_type").get(); + SearchResponse searchResponse = client() + .prepareSearch() + .setSource( + new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).scriptField("test_field", + new Script("script1", ScriptType.INDEXED, "groovy", null))).setIndices("test_index") + .setTypes("test_type").get(); assertHitCount(searchResponse, 1); SearchHit sh = searchResponse.getHits().getAt(0); assertThat((Integer)sh.field("test_field").getValue(), equalTo(i)); } } - @Test public void testDisabledUpdateIndexedScriptsOnly() { if (randomBoolean()) { client().preparePutIndexedScript(GroovyScriptEngineService.NAME, "script1", "{\"script\":\"2\"}").get(); @@ -143,7 +150,6 @@ public class IndexedScriptTests extends ESIntegTestCase { } } - @Test public void testDisabledAggsDynamicScripts() { //dynamic scripts don't need to be enabled for an indexed script to be indexed and later on executed if (randomBoolean()) { @@ -153,8 +159,11 @@ public class IndexedScriptTests extends ESIntegTestCase { } client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}").get(); refresh(); - String source = "{\"aggs\": {\"test\": { \"terms\" : { \"script_id\":\"script1\" } } } }"; - SearchResponse searchResponse = client().prepareSearch("test").setSource(new BytesArray(source)).get(); + SearchResponse searchResponse = client() + .prepareSearch("test") + .setSource( + new SearchSourceBuilder().aggregation(AggregationBuilders.terms("test").script( + new Script("script1", ScriptType.INDEXED, null, null)))).get(); assertHitCount(searchResponse, 1); assertThat(searchResponse.getAggregations().get("test"), notNullValue()); } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java similarity index 92% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java index 041838552e7..66a764dd75a 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java @@ -31,6 +31,8 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.TransportShardFlushAction; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; @@ -39,8 +41,6 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexAction; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.optimize.OptimizeAction; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -59,8 +59,6 @@ import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteAction; import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.exists.ExistsAction; -import org.elasticsearch.action.exists.ExistsRequest; import org.elasticsearch.action.explain.ExplainAction; import org.elasticsearch.action.explain.ExplainRequest; import org.elasticsearch.action.get.GetAction; @@ -78,7 +76,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.suggest.SuggestAction; import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsAction; @@ -101,10 +98,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.*; -import java.util.concurrent.Callable; import java.util.function.Supplier; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -113,7 +108,6 @@ import static org.hamcrest.Matchers.*; @ClusterScope(scope = Scope.SUITE, numClientNodes = 1, minNumDataNodes = 2) public class IndicesRequestTests extends ESIntegTestCase { - private final List indices = new ArrayList<>(); @Override @@ -159,7 +153,6 @@ public class IndicesRequestTests extends ESIntegTestCase { indices.clear(); } - @Test public void testGetFieldMappings() { String getFieldMappingsShardAction = GetFieldMappingsAction.NAME + "[index][s]"; interceptTransportActions(getFieldMappingsShardAction); @@ -172,7 +165,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(getFieldMappingsRequest, getFieldMappingsShardAction); } - @Test public void testAnalyze() { String analyzeShardAction = AnalyzeAction.NAME + "[s]"; interceptTransportActions(analyzeShardAction); @@ -185,9 +177,8 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(analyzeRequest, analyzeShardAction); } - @Test public void testIndex() { - String[] indexShardActions = new String[]{IndexAction.NAME, IndexAction.NAME + "[r]"}; + String[] indexShardActions = new String[]{IndexAction.NAME, IndexAction.NAME + "[p]", IndexAction.NAME + "[r]"}; interceptTransportActions(indexShardActions); IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id").source("field", "value"); @@ -197,9 +188,8 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(indexRequest, indexShardActions); } - @Test public void testDelete() { - String[] deleteShardActions = new String[]{DeleteAction.NAME, DeleteAction.NAME + "[r]"}; + String[] deleteShardActions = new String[]{DeleteAction.NAME, DeleteAction.NAME + "[p]", DeleteAction.NAME + "[r]"}; interceptTransportActions(deleteShardActions); DeleteRequest deleteRequest = new DeleteRequest(randomIndexOrAlias(), "type", "id"); @@ -209,7 +199,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(deleteRequest, deleteShardActions); } - @Test public void testUpdate() { //update action goes to the primary, index op gets executed locally, then replicated String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", IndexAction.NAME + "[r]"}; @@ -225,7 +214,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(updateRequest, updateShardActions); } - @Test public void testUpdateUpsert() { //update action goes to the primary, index op gets executed locally, then replicated String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", IndexAction.NAME + "[r]"}; @@ -240,7 +228,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(updateRequest, updateShardActions); } - @Test public void testUpdateDelete() { //update action goes to the primary, delete op gets executed locally, then replicated String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", DeleteAction.NAME + "[r]"}; @@ -256,9 +243,8 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(updateRequest, updateShardActions); } - @Test public void testBulk() { - String[] bulkShardActions = new String[]{BulkAction.NAME + "[s]", BulkAction.NAME + "[s][r]"}; + String[] bulkShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"}; interceptTransportActions(bulkShardActions); List indices = new ArrayList<>(); @@ -288,7 +274,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertIndicesSubset(indices, bulkShardActions); } - @Test public void testGet() { String getShardAction = GetAction.NAME + "[s]"; interceptTransportActions(getShardAction); @@ -300,19 +285,17 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(getRequest, getShardAction); } - @Test public void testExplain() { String explainShardAction = ExplainAction.NAME + "[s]"; interceptTransportActions(explainShardAction); - ExplainRequest explainRequest = new ExplainRequest(randomIndexOrAlias(), "type", "id").source(new QuerySourceBuilder().setQuery(QueryBuilders.matchAllQuery())); + ExplainRequest explainRequest = new ExplainRequest(randomIndexOrAlias(), "type", "id").query(QueryBuilders.matchAllQuery()); internalCluster().clientNodeClient().explain(explainRequest).actionGet(); clearInterceptedActions(); assertSameIndices(explainRequest, explainShardAction); } - @Test public void testTermVector() { String termVectorShardAction = TermVectorsAction.NAME + "[s]"; interceptTransportActions(termVectorShardAction); @@ -324,7 +307,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(termVectorsRequest, termVectorShardAction); } - @Test public void testMultiTermVector() { String multiTermVectorsShardAction = MultiTermVectorsAction.NAME + "[shard][s]"; interceptTransportActions(multiTermVectorsShardAction); @@ -343,7 +325,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertIndicesSubset(indices, multiTermVectorsShardAction); } - @Test public void testMultiGet() { String multiGetShardAction = MultiGetAction.NAME + "[shard][s]"; interceptTransportActions(multiGetShardAction); @@ -362,21 +343,8 @@ public class IndicesRequestTests extends ESIntegTestCase { assertIndicesSubset(indices, multiGetShardAction); } - @Test - public void testExists() { - String existsShardAction = ExistsAction.NAME + "[s]"; - interceptTransportActions(existsShardAction); - - ExistsRequest existsRequest = new ExistsRequest(randomIndicesOrAliases()); - internalCluster().clientNodeClient().exists(existsRequest).actionGet(); - - clearInterceptedActions(); - assertSameIndices(existsRequest, existsShardAction); - } - - @Test public void testFlush() { - String[] indexShardActions = new String[]{TransportShardFlushAction.NAME + "[r]", TransportShardFlushAction.NAME}; + String[] indexShardActions = new String[]{TransportShardFlushAction.NAME, TransportShardFlushAction.NAME + "[r]", TransportShardFlushAction.NAME + "[p]"}; interceptTransportActions(indexShardActions); FlushRequest flushRequest = new FlushRequest(randomIndicesOrAliases()); @@ -387,21 +355,19 @@ public class IndicesRequestTests extends ESIntegTestCase { assertIndicesSubset(Arrays.asList(indices), indexShardActions); } - @Test - public void testOptimize() { - String optimizeShardAction = OptimizeAction.NAME + "[n]"; - interceptTransportActions(optimizeShardAction); + public void testForceMerge() { + String mergeShardAction = ForceMergeAction.NAME + "[n]"; + interceptTransportActions(mergeShardAction); - OptimizeRequest optimizeRequest = new OptimizeRequest(randomIndicesOrAliases()); - internalCluster().clientNodeClient().admin().indices().optimize(optimizeRequest).actionGet(); + ForceMergeRequest mergeRequest = new ForceMergeRequest(randomIndicesOrAliases()); + internalCluster().clientNodeClient().admin().indices().forceMerge(mergeRequest).actionGet(); clearInterceptedActions(); - assertSameIndices(optimizeRequest, optimizeShardAction); + assertSameIndices(mergeRequest, mergeShardAction); } - @Test public void testRefresh() { - String[] indexShardActions = new String[]{TransportShardRefreshAction.NAME + "[r]", TransportShardRefreshAction.NAME}; + String[] indexShardActions = new String[]{TransportShardRefreshAction.NAME, TransportShardRefreshAction.NAME + "[r]", TransportShardRefreshAction.NAME + "[p]"}; interceptTransportActions(indexShardActions); RefreshRequest refreshRequest = new RefreshRequest(randomIndicesOrAliases()); @@ -412,7 +378,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertIndicesSubset(Arrays.asList(indices), indexShardActions); } - @Test public void testClearCache() { String clearCacheAction = ClearIndicesCacheAction.NAME + "[n]"; interceptTransportActions(clearCacheAction); @@ -424,7 +389,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(clearIndicesCacheRequest, clearCacheAction); } - @Test public void testRecovery() { String recoveryAction = RecoveryAction.NAME + "[n]"; interceptTransportActions(recoveryAction); @@ -436,7 +400,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(recoveryRequest, recoveryAction); } - @Test public void testSegments() { String segmentsAction = IndicesSegmentsAction.NAME + "[n]"; interceptTransportActions(segmentsAction); @@ -448,7 +411,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(segmentsRequest, segmentsAction); } - @Test public void testIndicesStats() { String indicesStats = IndicesStatsAction.NAME + "[n]"; interceptTransportActions(indicesStats); @@ -460,7 +422,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(indicesStatsRequest, indicesStats); } - @Test public void testSuggest() { String suggestAction = SuggestAction.NAME + "[s]"; interceptTransportActions(suggestAction); @@ -472,7 +433,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(suggestRequest, suggestAction); } - @Test public void testValidateQuery() { String validateQueryShardAction = ValidateQueryAction.NAME + "[s]"; interceptTransportActions(validateQueryShardAction); @@ -484,7 +444,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(validateQueryRequest, validateQueryShardAction); } - @Test public void testPercolate() { String percolateShardAction = PercolateAction.NAME + "[s]"; interceptTransportActions(percolateShardAction); @@ -503,7 +462,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(percolateRequest, percolateShardAction); } - @Test public void testMultiPercolate() { String multiPercolateShardAction = MultiPercolateAction.NAME + "[shard][s]"; interceptTransportActions(multiPercolateShardAction); @@ -531,7 +489,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertIndicesSubset(indices, multiPercolateShardAction); } - @Test public void testOpenIndex() { interceptTransportActions(OpenIndexAction.NAME); @@ -542,7 +499,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(openIndexRequest, OpenIndexAction.NAME); } - @Test public void testCloseIndex() { interceptTransportActions(CloseIndexAction.NAME); @@ -553,7 +509,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(closeIndexRequest, CloseIndexAction.NAME); } - @Test public void testDeleteIndex() { interceptTransportActions(DeleteIndexAction.NAME); @@ -565,7 +520,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(deleteIndexRequest, DeleteIndexAction.NAME); } - @Test public void testGetMappings() { interceptTransportActions(GetMappingsAction.NAME); @@ -576,7 +530,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(getMappingsRequest, GetMappingsAction.NAME); } - @Test public void testPutMapping() { interceptTransportActions(PutMappingAction.NAME); @@ -587,7 +540,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(putMappingRequest, PutMappingAction.NAME); } - @Test public void testGetSettings() { interceptTransportActions(GetSettingsAction.NAME); @@ -598,7 +550,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(getSettingsRequest, GetSettingsAction.NAME); } - @Test public void testUpdateSettings() { interceptTransportActions(UpdateSettingsAction.NAME); @@ -609,7 +560,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(updateSettingsRequest, UpdateSettingsAction.NAME); } - @Test public void testSearchQueryThenFetch() throws Exception { interceptTransportActions(SearchServiceTransportAction.QUERY_ACTION_NAME, SearchServiceTransportAction.FETCH_ID_ACTION_NAME, SearchServiceTransportAction.FREE_CONTEXT_ACTION_NAME); @@ -631,7 +581,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndicesOptionalRequests(searchRequest, SearchServiceTransportAction.FREE_CONTEXT_ACTION_NAME); } - @Test public void testSearchDfsQueryThenFetch() throws Exception { interceptTransportActions(SearchServiceTransportAction.DFS_ACTION_NAME, SearchServiceTransportAction.QUERY_ID_ACTION_NAME, SearchServiceTransportAction.FETCH_ID_ACTION_NAME, SearchServiceTransportAction.FREE_CONTEXT_ACTION_NAME); @@ -654,7 +603,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndicesOptionalRequests(searchRequest, SearchServiceTransportAction.FREE_CONTEXT_ACTION_NAME); } - @Test public void testSearchQueryAndFetch() throws Exception { interceptTransportActions(SearchServiceTransportAction.QUERY_FETCH_ACTION_NAME, SearchServiceTransportAction.FREE_CONTEXT_ACTION_NAME); @@ -676,7 +624,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndicesOptionalRequests(searchRequest, SearchServiceTransportAction.FREE_CONTEXT_ACTION_NAME); } - @Test public void testSearchDfsQueryAndFetch() throws Exception { interceptTransportActions(SearchServiceTransportAction.QUERY_QUERY_FETCH_ACTION_NAME, SearchServiceTransportAction.FREE_CONTEXT_ACTION_NAME); @@ -719,24 +666,6 @@ public class IndicesRequestTests extends ESIntegTestCase { } } } - - private static void assertSameIndicesOptionalRequests(String[] indices, String... actions) { - assertSameIndices(indices, true, actions); - } - - private static void assertSameIndices(String[] indices, boolean optional, String... actions) { - for (String action : actions) { - List requests = consumeTransportRequests(action); - if (!optional) { - assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0)); - } - for (TransportRequest internalRequest : requests) { - assertThat(internalRequest, instanceOf(IndicesRequest.class)); - assertThat(internalRequest.getClass().getName(), ((IndicesRequest)internalRequest).indices(), equalTo(indices)); - } - } - } - private static void assertIndicesSubset(List indices, String... actions) { //indices returned by each bulk shard request need to be a subset of the original indices for (String action : actions) { @@ -855,26 +784,26 @@ public class IndicesRequestTests extends ESIntegTestCase { @Override public void registerRequestHandler(String action, Supplier request, String executor, boolean forceExecution, TransportRequestHandler handler) { - super.registerRequestHandler(action, request, executor, forceExecution, new InterceptingRequestHandler(action, handler)); + super.registerRequestHandler(action, request, executor, forceExecution, new InterceptingRequestHandler<>(action, handler)); } @Override public void registerRequestHandler(String action, Supplier requestFactory, String executor, TransportRequestHandler handler) { - super.registerRequestHandler(action, requestFactory, executor, new InterceptingRequestHandler(action, handler)); + super.registerRequestHandler(action, requestFactory, executor, new InterceptingRequestHandler<>(action, handler)); } - private class InterceptingRequestHandler implements TransportRequestHandler { + private class InterceptingRequestHandler implements TransportRequestHandler { - private final TransportRequestHandler requestHandler; + private final TransportRequestHandler requestHandler; private final String action; - InterceptingRequestHandler(String action, TransportRequestHandler requestHandler) { + InterceptingRequestHandler(String action, TransportRequestHandler requestHandler) { this.requestHandler = requestHandler; this.action = action; } @Override - public void messageReceived(TransportRequest request, TransportChannel channel) throws Exception { + public void messageReceived(T request, TransportChannel channel) throws Exception { synchronized (InterceptingTransportService.this) { if (actions.contains(action)) { List requestList = requests.get(action); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java similarity index 93% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java index 8657534e3d0..638d9238b45 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java @@ -37,7 +37,6 @@ import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStat import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; @@ -75,7 +74,7 @@ public class LongTermsTests extends AbstractTermsTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + private static final int NUM_DOCS = 5; // TODO randomize the size? private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; private static final String MULTI_VALUED_FIELD_NAME = "l_values"; @@ -235,9 +234,8 @@ public class LongTermsTests extends AbstractTermsTestCase { return bucket.getKeyAsString(); } - @Test // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard - public void sizeIsZero() { + public void testSizeIsZero() { SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -254,8 +252,7 @@ public class LongTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().size(), equalTo(100)); } - @Test - public void singleValueField() throws Exception { + public void testSingleValueField() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -279,8 +276,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueFieldWithFiltering() throws Exception { + public void testSingleValueFieldWithFiltering() throws Exception { long includes[] = { 1, 2, 3, 98 }; long excludes[] = { -1, 2, 4 }; long empty[] = {}; @@ -310,8 +306,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_WithMaxSize() throws Exception { + public void testSingleValueFieldWithMaxSize() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -337,8 +332,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_OrderedByTermAsc() throws Exception { + public void testSingleValueFieldOrderedByTermAsc() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -362,8 +356,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_OrderedByTermDesc() throws Exception { + public void testSingleValueFieldOrderedByTermDesc() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -389,8 +382,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -424,8 +416,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregationInherited() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -453,8 +444,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -479,8 +469,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -508,8 +497,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -538,8 +526,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField_WithValueScript_NotUnique() throws Exception { + public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -579,8 +566,7 @@ public class LongTermsTests extends AbstractTermsTestCase { */ - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) @@ -617,12 +603,11 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .collectMode(randomFrom(SubAggCollectionMode.values())) -.script( + .script( new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value"))) .execute().actionGet(); @@ -643,8 +628,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -672,12 +656,11 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .collectMode(randomFrom(SubAggCollectionMode.values())) -.script( + .script( new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']"))) .execute().actionGet(); @@ -702,30 +685,23 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_MultiValued_WithAggregatorInherited_NoExplicitType() throws Exception { - + public void testScriptMultiValuedWithAggregatorInheritedNoExplicitType() throws Exception { // since no type ie explicitly defined, es will assume all values returned by the script to be strings (bytes), // so the aggregation should fail, since the "sum" aggregation can only operation on numeric values. - try { - client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .collectMode(randomFrom(SubAggCollectionMode.values())) .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']")) .subAggregation(sum("sum"))) .execute().actionGet(); - fail("expected to fail as sub-aggregation sum requires a numeric value source context, but there is none"); - } catch (Exception e) { // expected } } - @Test - public void script_MultiValued_WithAggregatorInherited_WithExplicitType() throws Exception { + public void testScriptMultiValuedWithAggregatorInheritedWithExplicitType() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .collectMode(randomFrom(SubAggCollectionMode.values())) @@ -762,8 +738,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -780,8 +755,7 @@ public class LongTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped", "idx").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) @@ -805,8 +779,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1l).minDocCount(0) @@ -825,8 +798,7 @@ public class LongTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().isEmpty(), is(true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") @@ -855,8 +827,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscWithTermsSubAgg() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscWithTermsSubAgg() throws Exception { boolean asc = true; SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") @@ -898,8 +869,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleBucketSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleBucketSubAggregationAsc() throws Exception { boolean asc = randomBoolean(); SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("num_tags") @@ -936,8 +906,7 @@ public class LongTermsTests extends AbstractTermsTestCase { assertThat(filter.getDocCount(), equalTo(asc ? 3l : 2l)); } - @Test - public void singleValuedField_OrderedBySubAggregationAsc_MultiHierarchyLevels() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels() throws Exception { boolean asc = randomBoolean(); SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("tags") @@ -991,8 +960,7 @@ public class LongTermsTests extends AbstractTermsTestCase { assertThat(max.getValue(), equalTo(asc ? 4.0 : 2.0)); } - @Test - public void singleValuedField_OrderedByMissingSubAggregation() throws Exception { + public void testSingleValuedFieldOrderedByMissingSubAggregation() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index).setTypes("type") @@ -1010,8 +978,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByNonMetricsOrMultiBucketSubAggregation() throws Exception { + public void testSingleValuedFieldOrderedByNonMetricsOrMultiBucketSubAggregation() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index).setTypes("type") @@ -1031,8 +998,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregation_WithUknownMetric() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithUknownMetric() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index).setTypes("type") @@ -1052,8 +1018,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregation_WithoutMetric() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithoutMetric() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index).setTypes("type") @@ -1073,8 +1038,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception { boolean asc = false; SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") @@ -1106,8 +1070,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedByMultiValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") @@ -1137,8 +1100,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedByMultiValueSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueSubAggregationDesc() throws Exception { boolean asc = false; SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") @@ -1168,8 +1130,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedByMultiValueExtendedStatsAsc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueExtendedStatsAsc() throws Exception { boolean asc = true; SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") @@ -1199,44 +1160,37 @@ public class LongTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAndTermsDesc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndTermsDesc() throws Exception { long[] expectedKeys = new long[] { 1, 2, 4, 3, 7, 6, 5 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true), Terms.Order.term(false)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAndTermsAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndTermsAsc() throws Exception { long[] expectedKeys = new long[] { 1, 2, 3, 4, 5, 6, 7 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true), Terms.Order.term(true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationDescAndTermsAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationDescAndTermsAsc() throws Exception { long[] expectedKeys = new long[] { 5, 6, 7, 3, 4, 2, 1 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true)); } - @Test - public void singleValuedField_OrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception { long[] expectedKeys = new long[] { 6, 7, 3, 4, 5, 1, 2 }; assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception { long[] expectedKeys = new long[] { 6, 7, 3, 5, 4, 1, 2 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true)); } - @Test - public void singleValuedField_OrderedByThreeCriteria() throws Exception { + public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception { long[] expectedKeys = new long[] { 2, 1, 4, 5, 3, 6, 7 }; assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAsCompound() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception { long[] expectedKeys = new long[] { 1, 2, 3, 4, 5, 6, 7 }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true)); } @@ -1273,8 +1227,7 @@ public class LongTermsTests extends AbstractTermsTestCase { } } - @Test - public void otherDocCount() { + public void testOtherDocCount() { testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME); } } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java similarity index 83% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java index ec21ddc05cf..7ecd99062aa 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.max.Max; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -46,16 +45,13 @@ import static org.hamcrest.Matchers.notNullValue; * */ public class MaxTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - @Override - @Test - public void testEmptyAggregation() throws Exception { + @Override + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(max("max"))) @@ -72,10 +68,9 @@ public class MaxTests extends AbstractNumericTestCase { assertThat(max.getName(), equalTo("max")); assertThat(max.getValue(), equalTo(Double.NEGATIVE_INFINITY)); } - @Override - @Test - public void testUnmapped() throws Exception { + @Override + public void testUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx_unmapped") .setQuery(matchAllQuery()) .addAggregation(max("max").field("value")) @@ -90,7 +85,6 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -105,8 +99,7 @@ public class MaxTests extends AbstractNumericTestCase { assertThat(max.getValue(), equalTo(10.0)); } - @Test - public void testSingleValuedField_WithFormatter() throws Exception { + public void testSingleValuedFieldWithFormatter() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(max("max").format("0000.0").field("value")).execute().actionGet(); @@ -120,9 +113,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { - + public void testSingleValuedFieldGetProperty() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(global("global").subAggregation(max("max").field("value"))).execute().actionGet(); @@ -146,8 +137,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") .setQuery(matchAllQuery()) .addAggregation(max("max").field("value")) @@ -162,8 +152,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(max("max").field("value").script(new Script("_value + 1"))) @@ -178,8 +167,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") @@ -196,7 +184,6 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -212,8 +199,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(max("max").field("values").script(new Script("_value + 1"))) @@ -228,8 +214,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") @@ -246,8 +231,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(max("max").script(new Script("doc['value'].value"))) @@ -262,8 +246,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") @@ -280,26 +263,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(max("max").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Max max = searchResponse.getAggregations().get("max"); - assertThat(max, notNullValue()); - assertThat(max.getName(), equalTo("max")); - assertThat(max.getValue(), equalTo(11.0)); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(max("max").script(new Script("doc['values'].values"))) @@ -314,24 +278,7 @@ public class MaxTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(max("max").script(new Script("doc['values'].values"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Max max = searchResponse.getAggregations().get("max"); - assertThat(max, notNullValue()); - assertThat(max.getName(), equalTo("max")); - assertThat(max.getValue(), equalTo(12.0)); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java similarity index 84% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java index 27086bbc8ab..b5f0105537e 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.min.Min; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -46,16 +45,13 @@ import static org.hamcrest.Matchers.notNullValue; * */ public class MinTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - @Override - @Test - public void testEmptyAggregation() throws Exception { + @Override + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(min("min"))) @@ -74,7 +70,6 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test public void testUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx_unmapped") .setQuery(matchAllQuery()) @@ -90,7 +85,6 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -105,8 +99,7 @@ public class MinTests extends AbstractNumericTestCase { assertThat(min.getValue(), equalTo(1.0)); } - @Test - public void testSingleValuedField_WithFormatter() throws Exception { + public void testSingleValuedFieldWithFormatter() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(min("min").format("0000.0").field("value")).execute().actionGet(); @@ -120,8 +113,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { + public void testSingleValuedFieldGetProperty() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(global("global").subAggregation(min("min").field("value"))).execute().actionGet(); @@ -146,8 +138,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") .setQuery(matchAllQuery()) .addAggregation(min("min").field("value")) @@ -162,8 +153,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(min("min").field("value").script(new Script("_value - 1"))) @@ -178,8 +168,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); SearchResponse searchResponse = client().prepareSearch("idx") @@ -196,7 +185,6 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -212,8 +200,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(min("min").field("values").script(new Script("_value - 1"))).execute().actionGet(); @@ -226,8 +213,7 @@ public class MinTests extends AbstractNumericTestCase { assertThat(min.getValue(), equalTo(1.0)); } - @Test - public void testMultiValuedField_WithValueScript_Reverse() throws Exception { + public void testMultiValuedFieldWithValueScriptReverse() throws Exception { // test what happens when values arrive in reverse order since the min // aggregator is optimized to work on sorted values SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) @@ -242,8 +228,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) @@ -259,8 +244,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(min("min").script(new Script("doc['value'].value"))).execute().actionGet(); @@ -273,8 +257,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) @@ -290,25 +273,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("dec", 1); - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(min("min").script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params))).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - - Min min = searchResponse.getAggregations().get("min"); - assertThat(min, notNullValue()); - assertThat(min.getName(), equalTo("min")); - assertThat(min.getValue(), equalTo(0.0)); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(min("min").script(new Script("doc['values'].values"))).execute().actionGet(); @@ -321,22 +286,7 @@ public class MinTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(min("min").script(new Script("doc['values'].values"))).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Min min = searchResponse.getAggregations().get("min"); - assertThat(min, notNullValue()); - assertThat(min.getName(), equalTo("min")); - assertThat(min.getValue(), equalTo(2.0)); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); SearchResponse searchResponse = client() @@ -355,5 +305,4 @@ public class MinTests extends AbstractNumericTestCase { assertThat(min.getName(), equalTo("min")); assertThat(min.getValue(), equalTo(1.0)); } - } \ No newline at end of file diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java similarity index 96% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java index 0ab0b1b9ec5..5d78b99ded9 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.search.aggregations.metrics.avg.Avg; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -67,7 +66,7 @@ public class RangeTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -93,8 +92,7 @@ public class RangeTests extends ESIntegTestCase { ensureSearchable(); } - @Test - public void rangeAsSubAggregation() throws Exception { + public void testRangeAsSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field(MULTI_VALUED_FIELD_NAME).size(100) .collectMode(randomFrom(SubAggCollectionMode.values())).subAggregation( @@ -157,8 +155,7 @@ public class RangeTests extends ESIntegTestCase { } } - @Test - public void singleValueField() throws Exception { + public void testSingleValueField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(range("range") .field(SINGLE_VALUED_FIELD_NAME) @@ -179,8 +176,8 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); assertThat((String) bucket.getKey(), equalTo("*-3.0")); - assertThat(((Number) ((Number) bucket.getFrom())).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); - assertThat(((Number) ((Number) bucket.getTo())).doubleValue(), equalTo(3.0)); + assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); + assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); assertThat(bucket.getToAsString(), equalTo("3.0")); assertThat(bucket.getDocCount(), equalTo(2l)); @@ -204,8 +201,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 5L)); } - @Test - public void singleValueField_WithFormat() throws Exception { + public void testSingleValueFieldWithFormat() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -249,8 +245,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 5L)); } - @Test - public void singleValueField_WithCustomKey() throws Exception { + public void testSingleValueFieldWithCustomKey() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(range("range") .field(SINGLE_VALUED_FIELD_NAME) @@ -296,8 +291,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 5L)); } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(range("range") .field(SINGLE_VALUED_FIELD_NAME) @@ -369,8 +363,7 @@ public class RangeTests extends ESIntegTestCase { assertThat((double) propertiesCounts[2], equalTo((double) total)); } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregationInherited() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(range("range") .field(SINGLE_VALUED_FIELD_NAME) @@ -430,8 +423,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(avg.getValue(), equalTo((double) total / (numDocs - 5))); // (6 + 7 + 8 + 9 + 10) / 5 } - @Test - public void singleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -487,8 +479,7 @@ public class RangeTests extends ESIntegTestCase { [10, 11] */ - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(range("range") .field(MULTI_VALUED_FIELD_NAME) @@ -547,8 +538,7 @@ public class RangeTests extends ESIntegTestCase { [11, 12] */ - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -609,8 +599,7 @@ public class RangeTests extends ESIntegTestCase { r3: 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 */ - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -670,8 +659,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(sum.getValue(), equalTo((double) total)); } - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -715,8 +703,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 5l)); } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -773,8 +760,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(avg.getValue(), equalTo((double) total / (numDocs - 5))); // (6 + 7 + 8 + 9 + 10) / 5 } - @Test - public void emptyRange() throws Exception { + public void testEmptyRange() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(range("range") .field(MULTI_VALUED_FIELD_NAME) @@ -810,8 +796,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0l)); } - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -854,7 +839,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getToAsString(), nullValue()); assertThat(bucket.getDocCount(), equalTo(numDocs - 4l)); } - + /* [1, 2] [2, 3] @@ -866,14 +851,13 @@ public class RangeTests extends ESIntegTestCase { [8, 9] [9, 10] [10, 11] - + r1: 1, 2, 2 r2: 3, 3, 4, 4, 5, 5 r3: 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11 */ - @Test - public void script_MultiValued_WithAggregatorInherited() throws Exception { + public void testScriptMultiValuedWithAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( @@ -933,8 +917,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(sum.getValue(), equalTo((double) total)); } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped") .addAggregation(range("range") .field(SINGLE_VALUED_FIELD_NAME) @@ -980,8 +963,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(0l)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { client().admin().cluster().prepareHealth("idx_unmapped").setWaitForYellowStatus().execute().actionGet(); SearchResponse response = client().prepareSearch("idx", "idx_unmapped") @@ -1029,8 +1011,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 5l)); } - @Test - public void overlappingRanges() throws Exception { + public void testOverlappingRanges() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(range("range") .field(MULTI_VALUED_FIELD_NAME) @@ -1086,8 +1067,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(bucket.getDocCount(), equalTo(numDocs - 2l)); } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1l).minDocCount(0) diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java similarity index 91% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java index 06db7e89eea..34ca4f49662 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.indexedscripts.get.GetIndexedScriptResponse; import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.index.IndexNotFoundException; @@ -31,20 +30,19 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collection; import java.util.Collections; +import static org.hamcrest.Matchers.is; + @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class ScriptIndexSettingsTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - @Test public void testScriptIndexSettings() { PutIndexedScriptResponse putIndexedScriptResponse = client().preparePutIndexedScript().setId("foobar").setScriptLang("groovy").setSource("{ \"script\": 1 }") @@ -77,7 +75,6 @@ public class ScriptIndexSettingsTests extends ESIntegTestCase { assertEquals("Auto expand replicas should be 0-all", "0-all", numberOfReplicas); } - @Test public void testDeleteScriptIndex() { PutIndexedScriptResponse putIndexedScriptResponse = client().preparePutIndexedScript().setId("foobar").setScriptLang("groovy").setSource("{ \"script\": 1 }") @@ -87,13 +84,10 @@ public class ScriptIndexSettingsTests extends ESIntegTestCase { assertTrue(deleteResponse.isAcknowledged()); ensureGreen(); try { - GetIndexedScriptResponse response = client().prepareGetIndexedScript("groovy","foobar").get(); - assertTrue(false); //This should not happen - } catch (IndexNotFoundException ime) { - assertTrue(true); + client().prepareGetIndexedScript("groovy","foobar").get(); + fail("Expected IndexNotFoundException"); + } catch (IndexNotFoundException e) { + assertThat(e.getMessage(), is("no such index")); } } - - - } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java similarity index 93% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java index 27262177d43..f3574982377 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java @@ -21,15 +21,13 @@ package org.elasticsearch.messy.tests; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.cache.IndexCacheModule; -import org.elasticsearch.index.cache.query.index.IndexQueryCache; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -38,7 +36,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; import static org.hamcrest.Matchers.equalTo; @@ -47,22 +44,20 @@ import static org.hamcrest.Matchers.equalTo; */ @ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE) public class ScriptQuerySearchTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)) // aggressive filter caching so that we can assert on the number of iterations of the script filters - .put(IndexCacheModule.QUERY_CACHE_TYPE, IndexCacheModule.INDEX_QUERY_CACHE) - .put(IndexCacheModule.QUERY_CACHE_EVERYTHING, true) + .put(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE) + .put(IndexModule.QUERY_CACHE_EVERYTHING, true) .build(); } - @Test public void testCustomScriptBoost() throws Exception { createIndex("test"); client().prepareIndex("test", "type1", "1") diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java similarity index 97% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java index e28554a44f5..c54510acd4e 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java @@ -37,7 +37,6 @@ import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetric; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; @@ -72,7 +71,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -116,7 +115,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { "{\"script\":\"newaggregation = []; sum = 0;for (agg in _aggs) { for (a in agg) { sum += a} }; newaggregation.add(sum); return newaggregation\"}") .get(); assertThat(indexScriptResponse.isCreated(), equalTo(true)); - + indexRandom(true, builders); ensureSearchable(); } @@ -130,7 +129,6 @@ public class ScriptedMetricTests extends ESIntegTestCase { return settings; } - @Test public void testMap() { SearchResponse response = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(scriptedMetric("scripted").mapScript(new Script("_agg['count'] = 1"))).execute().actionGet(); @@ -164,8 +162,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(numShardsRun, greaterThan(0)); } - @Test - public void testMap_withParams() { + public void testMapWithParams() { Map params = new HashMap<>(); params.put("_agg", new ArrayList<>()); @@ -199,8 +196,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(totalCount, equalTo(numDocs)); } - @Test - public void testInitMap_withParams() { + public void testInitMapWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -241,8 +237,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(totalCount, equalTo(numDocs * 3)); } - @Test - public void testMapCombine_withParams() { + public void testMapCombineWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -291,8 +286,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(totalCount, equalTo(numDocs)); } - @Test - public void testInitMapCombine_withParams() { + public void testInitMapCombineWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -342,8 +336,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(totalCount, equalTo(numDocs * 3)); } - @Test - public void testInitMapCombineReduce_withParams() { + public void testInitMapCombineReduceWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -383,9 +376,8 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); } - @SuppressWarnings({ "unchecked", "rawtypes" }) - @Test - public void testInitMapCombineReduce_getProperty() throws Exception { + @SuppressWarnings("rawtypes") + public void testInitMapCombineReduceGetProperty() throws Exception { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -436,8 +428,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { } - @Test - public void testMapCombineReduce_withParams() { + public void testMapCombineReduceWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -476,8 +467,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs)); } - @Test - public void testInitMapReduce_withParams() { + public void testInitMapReduceWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -514,8 +504,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); } - @Test - public void testMapReduce_withParams() { + public void testMapReduceWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -551,8 +540,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs)); } - @Test - public void testInitMapCombineReduce_withParamsAndReduceParams() { + public void testInitMapCombineReduceWithParamsAndReduceParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -595,8 +583,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs * 12)); } - @Test - public void testInitMapCombineReduce_withParams_Indexed() { + public void testInitMapCombineReduceWithParamsIndexed() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -630,9 +617,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); } - @Test - public void testInitMapCombineReduce_withParams_File() { - + public void testInitMapCombineReduceWithParamsFile() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -665,8 +650,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); } - @Test - public void testInitMapCombineReduce_withParams_asSubAgg() { + public void testInitMapCombineReduceWithParamsAsSubAgg() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); @@ -723,7 +707,6 @@ public class ScriptedMetricTests extends ESIntegTestCase { } } - @Test public void testEmptyAggregation() throws Exception { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java similarity index 93% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java index 7d9a080f666..8153d207b7c 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java @@ -19,9 +19,7 @@ package org.elasticsearch.messy.tests; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Base64; @@ -33,6 +31,7 @@ import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; @@ -40,11 +39,11 @@ import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; @@ -75,13 +74,11 @@ import static org.hamcrest.Matchers.nullValue; * */ public class SearchFieldsTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - @Test + public void testStoredFields() throws Exception { createIndex("test"); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); @@ -111,12 +108,12 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); - // field2 is not stored, check that it gets extracted from source + // field2 is not stored, check that it is not extracted from source. searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field2").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); - assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); - assertThat(searchResponse.getHits().getAt(0).fields().get("field2").value().toString(), equalTo("value2")); + assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(0)); + assertThat(searchResponse.getHits().getAt(0).fields().get("field2"), nullValue()); searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field3").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); @@ -124,6 +121,34 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").execute().actionGet(); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.getHits().hits().length, equalTo(1)); + assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); + assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); + + + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").addField("field1").addField("field2").execute().actionGet(); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.getHits().hits().length, equalTo(1)); + assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2)); + assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); + assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); + + + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field*").execute().actionGet(); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.getHits().hits().length, equalTo(1)); + assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2)); + assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); + assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); + + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("f*3").execute().actionGet(); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.getHits().hits().length, equalTo(1)); + assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); + assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); @@ -141,7 +166,6 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); } - @Test public void testScriptDocAndFields() throws Exception { createIndex("test"); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); @@ -226,7 +250,6 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0)); } - @Test public void testUidBasedScriptFields() throws Exception { prepareCreate("test").addMapping("type1", "num1", "type=long").execute().actionGet(); ensureYellow(); @@ -303,7 +326,6 @@ public class SearchFieldsTests extends ESIntegTestCase { } } - @Test public void testScriptFieldUsingSource() throws Exception { createIndex("test"); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); @@ -347,7 +369,6 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(((Map) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1")); } - @Test public void testPartialFields() throws Exception { createIndex("test"); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); @@ -367,7 +388,6 @@ public class SearchFieldsTests extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); } - @Test public void testStoredFieldsWithoutSource() throws Exception { createIndex("test"); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); @@ -434,8 +454,7 @@ public class SearchFieldsTests extends ESIntegTestCase { } - @Test - public void testSearchFields_metaData() throws Exception { + public void testSearchFieldsMetaData() throws Exception { client().prepareIndex("my-index", "my-type1", "1") .setRouting("1") .setSource(jsonBuilder().startObject().field("field1", "value").endObject()) @@ -448,14 +467,12 @@ public class SearchFieldsTests extends ESIntegTestCase { .get(); assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); - assertThat(searchResponse.getHits().getAt(0).field("field1").isMetadataField(), equalTo(false)); - assertThat(searchResponse.getHits().getAt(0).field("field1").getValue().toString(), equalTo("value")); + assertThat(searchResponse.getHits().getAt(0).field("field1"), nullValue()); assertThat(searchResponse.getHits().getAt(0).field("_routing").isMetadataField(), equalTo(true)); assertThat(searchResponse.getHits().getAt(0).field("_routing").getValue().toString(), equalTo("1")); } - @Test - public void testSearchFields_nonLeafField() throws Exception { + public void testSearchFieldsNonLeafField() throws Exception { client().prepareIndex("my-index", "my-type1", "1") .setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject()) .setRefresh(true) @@ -466,8 +483,7 @@ public class SearchFieldsTests extends ESIntegTestCase { containsString("field [field1] isn't a leaf field")); } - @Test - public void testGetFields_complexField() throws Exception { + public void testGetFieldsComplexField() throws Exception { client().admin().indices().prepareCreate("my-index") .setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1)) .addMapping("my-type2", jsonBuilder().startObject().startObject("my-type2").startObject("properties") @@ -524,28 +540,18 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(1).toString(), equalTo("value2")); } - @Test // see #8203 + // see #8203 public void testSingleValueFieldDatatField() throws ExecutionException, InterruptedException { createIndex("test"); indexRandom(true, client().prepareIndex("test", "type", "1").setSource("test_field", "foobar")); refresh(); - SearchResponse searchResponse = client().prepareSearch("test").setTypes("type").setSource(new BytesArray(new BytesRef("{\"query\":{\"match_all\":{}},\"fielddata_fields\": \"test_field\"}"))).get(); + SearchResponse searchResponse = client().prepareSearch("test").setTypes("type").setSource( + new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).fieldDataField("test_field")).get(); assertHitCount(searchResponse, 1); Map fields = searchResponse.getHits().getHits()[0].getFields(); assertThat((String)fields.get("test_field").value(), equalTo("foobar")); } - @Test(expected = SearchPhaseExecutionException.class) - public void testInvalidFieldDataField() throws ExecutionException, InterruptedException { - createIndex("test"); - if (randomBoolean()) { - client().prepareSearch("test").setTypes("type").setSource(new BytesArray(new BytesRef("{\"query\":{\"match_all\":{}},\"fielddata_fields\": {}}"))).get(); - } else { - client().prepareSearch("test").setTypes("type").setSource(new BytesArray(new BytesRef("{\"query\":{\"match_all\":{}},\"fielddata_fields\": 1.0}"))).get(); - } - } - - @Test public void testFieldsPulledFromFieldData() throws Exception { createIndex("test"); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); @@ -668,8 +674,7 @@ public class SearchFieldsTests extends ESIntegTestCase { Map fields = response.getHits().getAt(0).getFields(); - assertThat(fields.get("field1").isMetadataField(), equalTo(false)); - assertThat(fields.get("field1").getValue().toString(), equalTo("value")); + assertThat(fields.get("field1"), nullValue()); assertThat(fields.get("_routing").isMetadataField(), equalTo(true)); assertThat(fields.get("_routing").getValue().toString(), equalTo("1")); assertThat(fields.get("_timestamp").isMetadataField(), equalTo(true)); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java similarity index 98% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java index d9cfb7dc08e..c301f97ccc2 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java @@ -34,8 +34,8 @@ import org.elasticsearch.index.search.stats.SearchStats.Stats; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -60,18 +60,16 @@ import static org.hamcrest.Matchers.nullValue; */ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SearchStatsTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override protected int numberOfReplicas() { return 0; } - @Test public void testSimpleStats() throws Exception { // clear all stats first client().admin().indices().prepareStats().clear().execute().actionGet(); @@ -109,7 +107,7 @@ public class SearchStatsTests extends ESIntegTestCase { for (int i = 0; i < iters; i++) { SearchResponse searchResponse = internalCluster().clientNodeClient().prepareSearch() .setQuery(QueryBuilders.termQuery("field", "value")).setStats("group1", "group2") - .addHighlightedField("field") + .highlighter(new HighlightBuilder().field("field")) .addScriptField("scrip1", new Script("_source.field")) .setSize(100) .execute().actionGet(); @@ -146,9 +144,9 @@ public class SearchStatsTests extends ESIntegTestCase { assertThat(total.getQueryTimeInMillis(), equalTo(0l)); } } - + assertThat(num, greaterThan(0)); - + } private Set nodeIdsWithIndex(String... indices) { @@ -166,7 +164,6 @@ public class SearchStatsTests extends ESIntegTestCase { return nodes; } - @Test public void testOpenContexts() { String index = "test1"; createIndex(index); diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java similarity index 94% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java index 47bfb49bd5b..db8a13c5ab8 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java @@ -24,15 +24,16 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.StringAndBytesText; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.DistanceUnit; @@ -46,17 +47,19 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.*; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.junit.annotations.TestLogging; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutionException; +import static org.apache.lucene.util.GeoUtils.TOLERANCE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; @@ -64,12 +67,10 @@ import static org.elasticsearch.search.sort.SortBuilders.fieldSort; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.*; - /** * */ public class SimpleSortTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); @@ -178,7 +179,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertOrderedSearchHits(searchResponse, "data.activity.6", "data.activity.5"); } - @Test public void testTrackScores() throws Exception { createIndex("test"); ensureGreen(); @@ -295,8 +295,6 @@ public class SimpleSortTests extends ESIntegTestCase { } } - - @Test public void test3078() { createIndex("test"); ensureGreen(); @@ -327,8 +325,8 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); - // optimize - optimize(); + // force merge + forceMerge(); refresh(); client().prepareIndex("test", "type", Integer.toString(1)).setSource("field", Integer.toString(1)).execute().actionGet(); @@ -344,7 +342,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); } - @Test public void testScoreSortDirection() throws Exception { createIndex("test"); ensureGreen(); @@ -387,9 +384,7 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); } - - @Test - public void testScoreSortDirection_withFunctionScore() throws Exception { + public void testScoreSortDirectionWithFunctionScore() throws Exception { createIndex("test"); ensureGreen(); @@ -424,7 +419,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); } - @Test public void testIssue2986() { createIndex("test"); @@ -439,7 +433,6 @@ public class SimpleSortTests extends ESIntegTestCase { } } - @Test public void testIssue2991() { for (int i = 1; i < 4; i++) { try { @@ -475,9 +468,8 @@ public class SimpleSortTests extends ESIntegTestCase { } } - @Test public void testSimpleSorts() throws Exception { - Random random = getRandom(); + Random random = random(); assertAcked(prepareCreate("test") .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("str_value").field("type", "string").field("index", "not_analyzed").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() @@ -730,7 +722,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertNoFailures(searchResponse); } - @Test public void test2920() throws IOException { assertAcked(prepareCreate("test").addMapping( "test", @@ -747,7 +738,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertNoFailures(searchResponse); } - @Test public void testSortMinValueScript() throws IOException { String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties") .startObject("lvalue").field("type", "long").endObject() @@ -831,11 +821,10 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getTotalHits(), equalTo(20l)); for (int i = 0; i < 10; i++) { - assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), equalTo((double) i)); + assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo((double) i, TOLERANCE)); } } - @Test public void testDocumentsWithNullValue() throws Exception { // TODO: sort shouldn't fail when sort field is mapped dynamically // We have to specify mapping explicitly because by the time search is performed dynamic mapping might not @@ -929,7 +918,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("2")); } - @Test public void testSortMissingNumbers() throws Exception { assertAcked(prepareCreate("test").addMapping("type1", XContentFactory.jsonBuilder() @@ -1004,7 +992,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); } - @Test public void testSortMissingStrings() throws IOException { assertAcked(prepareCreate("test").addMapping("type1", XContentFactory.jsonBuilder() @@ -1092,7 +1079,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); } - @Test public void testIgnoreUnmapped() throws Exception { createIndex("test"); ensureYellow(); @@ -1124,7 +1110,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertNoFailures(searchResponse); } - @Test public void testSortMVField() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -1439,7 +1424,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03")); } - @Test public void testSortOnRareField() throws IOException { assertAcked(prepareCreate("test") .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") @@ -1605,7 +1589,6 @@ public class SimpleSortTests extends ESIntegTestCase { /** * Test case for issue 6150: https://github.com/elasticsearch/elasticsearch/issues/6150 */ - @Test public void testNestedSort() throws IOException, InterruptedException, ExecutionException { assertAcked(prepareCreate("test") .addMapping("type", @@ -1673,7 +1656,6 @@ public class SimpleSortTests extends ESIntegTestCase { } } - @Test public void testSortDuelBetweenSingleShardAndMultiShardIndex() throws Exception { String sortField = "sortField"; assertAcked(prepareCreate("test1") @@ -1720,7 +1702,9 @@ public class SimpleSortTests extends ESIntegTestCase { * |___________________________ * 1 2 3 4 5 6 7 */ - assertAcked(prepareCreate("index").addMapping("type", "location", "type=geo_point")); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point")); XContentBuilder d1Builder = jsonBuilder(); GeoPoint[] d1Points = {new GeoPoint(3, 2), new GeoPoint(4, 1)}; createShuffeldJSONArray(d1Builder, d1Points); @@ -1749,32 +1733,32 @@ public class SimpleSortTests extends ESIntegTestCase { .addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("min").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d1", "d2"); - assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS))); - assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS))); + assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS), 0.01d)); + assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS), 0.01d)); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("min").order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d2", "d1"); - assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS))); - assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS))); + assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS), 0.01d)); + assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS), 0.01d)); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("max").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d1", "d2"); - assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS))); - assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS))); + assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d)); + assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS), 0.01d)); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("max").order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d2", "d1"); - assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS))); - assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS))); + assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS), 0.01d)); + assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d)); } protected void createShuffeldJSONArray(XContentBuilder builder, GeoPoint[] pointsArray) throws IOException { @@ -1802,7 +1786,9 @@ public class SimpleSortTests extends ESIntegTestCase { * |______________________ * 1 2 3 4 5 6 */ - assertAcked(prepareCreate("index").addMapping("type", "location", "type=geo_point")); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point")); XContentBuilder d1Builder = jsonBuilder(); GeoPoint[] d1Points = {new GeoPoint(2.5, 1), new GeoPoint(2.75, 2), new GeoPoint(3, 3), new GeoPoint(3.25, 4)}; createShuffeldJSONArray(d1Builder, d1Points); @@ -1848,50 +1834,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(3.25, 4, 2, 1, DistanceUnit.KILOMETERS), 1.e-4)); assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(5.25, 4, 2, 1, DistanceUnit.KILOMETERS), 1.e-4)); - //test all the different formats in one - createQPoints(qHashes, qPoints); - XContentBuilder searchSourceBuilder = jsonBuilder(); - searchSourceBuilder.startObject().startArray("sort").startObject().startObject("_geo_distance").startArray("location"); - - for (int i = 0; i < 4; i++) { - int at = randomInt(qPoints.size() - 1); - int format = randomInt(3); - switch (format) { - case 0: { - searchSourceBuilder.value(qHashes.get(at)); - break; - } - case 1: { - searchSourceBuilder.value(qPoints.get(at).lat() + "," + qPoints.get(at).lon()); - break; - } - case 2: { - searchSourceBuilder.value(qPoints.get(at)); - break; - } - case 3: { - searchSourceBuilder.startArray().value(qPoints.get(at).lon()).value(qPoints.get(at).lat()).endArray(); - break; - } - } - qHashes.remove(at); - qPoints.remove(at); - } - - searchSourceBuilder.endArray(); - searchSourceBuilder.field("order", "asc"); - searchSourceBuilder.field("unit", "km"); - searchSourceBuilder.field("sort_mode", "min"); - searchSourceBuilder.field("distance_type", "plane"); - searchSourceBuilder.endObject(); - searchSourceBuilder.endObject(); - searchSourceBuilder.endArray(); - searchSourceBuilder.endObject(); - - searchResponse = client().prepareSearch().setSource(searchSourceBuilder.bytes()).execute().actionGet(); - assertOrderedSearchHits(searchResponse, "d1", "d2"); - assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2.5, 1, 2, 1, DistanceUnit.KILOMETERS), 1.e-4)); - assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(4.5, 1, 2, 1, DistanceUnit.KILOMETERS), 1.e-4)); } public void testSinglePointGeoDistanceSort() throws ExecutionException, InterruptedException, IOException { @@ -1930,40 +1872,25 @@ public class SimpleSortTests extends ESIntegTestCase { .execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); - String geoSortRequest = jsonBuilder().startObject().startArray("sort").startObject() - .startObject("_geo_distance") - .startArray("location").value(2f).value(2f).endArray() - .field("unit", "km") - .field("distance_type", "plane") - .endObject() - .endObject().endArray().string(); - searchResponse = client().prepareSearch().setSource(new BytesArray(geoSortRequest)) - .execute().actionGet(); + searchResponse = client() + .prepareSearch() + .setSource( + new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location").point(2.0, 2.0) + .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); - geoSortRequest = jsonBuilder().startObject().startArray("sort").startObject() - .startObject("_geo_distance") - .field("location", "s037ms06g7h0") - .field("unit", "km") - .field("distance_type", "plane") - .endObject() - .endObject().endArray().string(); - searchResponse = client().prepareSearch().setSource(new BytesArray(geoSortRequest)) - .execute().actionGet(); + searchResponse = client() + .prepareSearch() + .setSource( + new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location").geohashes("s037ms06g7h0") + .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); - geoSortRequest = jsonBuilder().startObject().startArray("sort").startObject() - .startObject("_geo_distance") - .startObject("location") - .field("lat", 2) - .field("lon", 2) - .endObject() - .field("unit", "km") - .field("distance_type", "plane") - .endObject() - .endObject().endArray().string(); - searchResponse = client().prepareSearch().setSource(new BytesArray(geoSortRequest)) - .execute().actionGet(); + searchResponse = client() + .prepareSearch() + .setSource( + new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location").point(2.0, 2.0) + .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java similarity index 86% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java index dfc37450b9d..f480ba4d2e0 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.stats.Stats; -import org.junit.Test; import java.util.Collection; import java.util.Collections; @@ -49,16 +48,13 @@ import static org.hamcrest.Matchers.sameInstance; * */ public class StatsTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - @Override - @Test - public void testEmptyAggregation() throws Exception { + @Override + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(stats("stats"))) @@ -83,7 +79,6 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test public void testUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx_unmapped") .setQuery(matchAllQuery()) @@ -105,7 +100,6 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -149,9 +143,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { - + public void testSingleValuedFieldGetProperty() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation(global("global").subAggregation(stats("stats").field("value"))).execute().actionGet(); @@ -188,8 +180,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") .setQuery(matchAllQuery()) .addAggregation(stats("stats").field("value")) @@ -210,8 +201,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(stats("stats").field("value").script(new Script("_value + 1"))) @@ -232,8 +222,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") @@ -256,7 +245,6 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -278,8 +266,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(stats("stats").field("values").script(new Script("_value - 1"))) @@ -300,8 +287,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); SearchResponse searchResponse = client().prepareSearch("idx") @@ -324,8 +310,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(stats("stats").script(new Script("doc['value'].value"))) @@ -346,8 +331,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") @@ -370,32 +354,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(stats("stats").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertShardExecutionState(searchResponse, 0); - - assertHitCount(searchResponse, 10); - - Stats stats = searchResponse.getAggregations().get("stats"); - assertThat(stats, notNullValue()); - assertThat(stats.getName(), equalTo("stats")); - assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); - assertThat(stats.getMin(), equalTo(2.0)); - assertThat(stats.getMax(), equalTo(11.0)); - assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11)); - assertThat(stats.getCount(), equalTo(10l)); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(stats("stats").script(new Script("doc['values'].values"))) @@ -416,30 +375,7 @@ public class StatsTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(stats("stats").script(new Script("doc['values'].values"))) - .execute().actionGet(); - - assertShardExecutionState(searchResponse, 0); - - assertHitCount(searchResponse, 10); - - Stats stats = searchResponse.getAggregations().get("stats"); - assertThat(stats, notNullValue()); - assertThat(stats.getName(), equalTo("stats")); - assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12) / 20)); - assertThat(stats.getMin(), equalTo(2.0)); - assertThat(stats.getMax(), equalTo(12.0)); - assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12)); - assertThat(stats.getCount(), equalTo(20l)); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); SearchResponse searchResponse = client().prepareSearch("idx") diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java similarity index 94% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java index be9e90f2c76..55672a0ce47 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; -import org.junit.Test; import java.io.IOException; import java.text.NumberFormat; @@ -79,7 +78,6 @@ import static org.hamcrest.core.IsNull.nullValue; */ @ESIntegTestCase.SuiteScopeTestCase public class StringTermsTests extends AbstractTermsTestCase { - private static final String SINGLE_VALUED_FIELD_NAME = "s_value"; private static final String MULTI_VALUED_FIELD_NAME = "s_values"; private static Map> expectedMultiSortBuckets; @@ -88,7 +86,7 @@ public class StringTermsTests extends AbstractTermsTestCase { protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -193,9 +191,8 @@ public class StringTermsTests extends AbstractTermsTestCase { return bucket.getKeyAsString(); } - @Test // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard - public void sizeIsZero() { + public void testSizeIsZero() { final int minDocCount = randomInt(1); SearchResponse response = client() .prepareSearch("idx") @@ -213,8 +210,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().size(), equalTo(minDocCount == 0 ? 105 : 100)); // 105 because of the other type } - @Test - public void singleValueField() throws Exception { + public void testSingleValueField() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -241,8 +237,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_withGlobalOrdinals() throws Exception { + public void testSingleValueFieldWithGlobalOrdinals() throws Exception { ExecutionMode[] executionModes = new ExecutionMode[] { null, ExecutionMode.GLOBAL_ORDINALS, ExecutionMode.GLOBAL_ORDINALS_HASH, ExecutionMode.GLOBAL_ORDINALS_LOW_CARDINALITY }; for (ExecutionMode executionMode : executionModes) { @@ -269,9 +264,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_WithRegexFiltering() throws Exception { - + public void testSingleValueFieldWithRegexFiltering() throws Exception { // include without exclude // we should be left with: val000, val001, val002, val003, val004, val005, val006, val007, val008, val009 @@ -346,8 +339,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_WithExactTermFiltering() throws Exception { + public void testSingleValueFieldWithExactTermFiltering() throws Exception { // include without exclude String incVals[] = { "val000", "val001", "val002", "val003", "val004", "val005", "val006", "val007", "val008", "val009" }; SearchResponse response = client() @@ -428,8 +420,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValueField_WithMaxSize() throws Exception { + public void testSingleValueFieldWithMaxSize() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("high_card_type") @@ -453,8 +444,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_OrderedByTermAsc() throws Exception { + public void testSingleValueFieldOrderedByTermAsc() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -479,8 +469,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValueField_OrderedByTermDesc() throws Exception { + public void testSingleValueFieldOrderedByTermDesc() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -505,8 +494,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithSubAggregation() throws Exception { + public void testSingleValuedFieldWithSubAggregation() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -539,8 +527,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithSubAggregation_Inherited() throws Exception { + public void testSingleValuedFieldWithSubAggregation_Inherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -567,8 +554,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -592,8 +578,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField_WithValueScript_NotUnique() throws Exception { + public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -615,8 +600,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(bucket.getDocCount(), equalTo(5l)); } - @Test - public void multiValuedField() throws Exception { + public void testMultiValuedField() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -643,8 +627,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedScript() throws Exception { + public void testMultiValuedScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -671,8 +654,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void multiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -712,8 +694,7 @@ public class StringTermsTests extends AbstractTermsTestCase { * doc_count: 1 - val_count: 2 */ - @Test - public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception { + public void testMultiValuedFieldWithValueScriptWithInheritedSubAggregator() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -747,8 +728,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_SingleValue() throws Exception { + public void testScriptSingleValue() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -771,8 +751,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_SingleValue_ExplicitSingleValue() throws Exception { + public void testScriptSingleValueExplicitSingleValue() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -795,8 +774,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_SingleValue_WithSubAggregator_Inherited() throws Exception { + public void testScriptSingleValueWithSubAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -823,8 +801,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -851,8 +828,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void script_MultiValued_WithAggregatorInherited() throws Exception { + public void testScriptMultiValuedWithAggregatorInherited() throws Exception { SearchResponse response = client() .prepareSearch("idx") .setTypes("type") @@ -886,8 +862,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void unmapped() throws Exception { + public void testUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx_unmapped") .setTypes("type") @@ -903,8 +878,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().size(), equalTo(0)); } - @Test - public void partiallyUnmapped() throws Exception { + public void testPartiallyUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx", "idx_unmapped") .setTypes("type") @@ -927,8 +901,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void stringTermsNestedIntoPerBucketAggregator() throws Exception { + public void testStringTermsNestedIntoPerBucketAggregator() throws Exception { // no execution hint so that the logic that decides whether or not to use ordinals is executed SearchResponse response = client() .prepareSearch("idx") @@ -955,8 +928,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void emptyAggregation() throws Exception { + public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client() .prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) @@ -976,8 +948,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(terms.getBuckets().isEmpty(), is(true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -1006,8 +977,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByIllegalAgg() throws Exception { + public void testSingleValuedFieldOrderedByIllegalAgg() throws Exception { boolean asc = true; try { client() @@ -1036,8 +1006,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleBucketSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleBucketSubAggregationAsc() throws Exception { boolean asc = randomBoolean(); SearchResponse response = client() .prepareSearch("idx") @@ -1073,8 +1042,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(filter.getDocCount(), equalTo(asc ? 3l : 2l)); } - @Test - public void singleValuedField_OrderedBySubAggregationAsc_MultiHierarchyLevels() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels() throws Exception { boolean asc = randomBoolean(); SearchResponse response = client() .prepareSearch("idx") @@ -1131,8 +1099,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(stats.getMax(), equalTo(asc ? 4.0 : 2.0)); } - @Test - public void singleValuedField_OrderedBySubAggregationAsc_MultiHierarchyLevels_specialChars() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsSpecialChars() throws Exception { StringBuilder filter2NameBuilder = new StringBuilder("filt.er2"); filter2NameBuilder.append(randomAsciiOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", "")); String filter2Name = filter2NameBuilder.toString(); @@ -1195,8 +1162,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(stats.getMax(), equalTo(asc ? 4.0 : 2.0)); } - @Test - public void singleValuedField_OrderedBySubAggregationAsc_MultiHierarchyLevels_specialCharsNoDotNotation() throws Exception { + public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsSpecialCharsNoDotNotation() throws Exception { StringBuilder filter2NameBuilder = new StringBuilder("filt.er2"); filter2NameBuilder.append(randomAsciiOfLengthBetween(3, 10).replace("[", "").replace("]", "").replace(">", "")); String filter2Name = filter2NameBuilder.toString(); @@ -1259,8 +1225,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(stats.getMax(), equalTo(asc ? 4.0 : 2.0)); } - @Test - public void singleValuedField_OrderedByMissingSubAggregation() throws Exception { + public void testSingleValuedFieldOrderedByMissingSubAggregation() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index) @@ -1278,8 +1243,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByNonMetricsOrMultiBucketSubAggregation() throws Exception { + public void testSingleValuedFieldOrderedByNonMetricsOrMultiBucketSubAggregation() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index) @@ -1299,8 +1263,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregation_WithUknownMetric() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithUknownMetric() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { SearchResponse response = client() @@ -1320,8 +1283,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedByMultiValuedSubAggregation_WithoutMetric() throws Exception { + public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithoutMetric() throws Exception { for (String index : Arrays.asList("idx", "idx_unmapped")) { try { client().prepareSearch(index) @@ -1341,8 +1303,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception { boolean asc = false; SearchResponse response = client() .prepareSearch("idx") @@ -1373,8 +1334,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedByMultiValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -1402,11 +1362,9 @@ public class StringTermsTests extends AbstractTermsTestCase { assertThat(stats.getMax(), equalTo((double) i)); i++; } - } - @Test - public void singleValuedField_OrderedByMultiValueSubAggregationDesc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueSubAggregationDesc() throws Exception { boolean asc = false; SearchResponse response = client() .prepareSearch("idx") @@ -1437,8 +1395,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedByMultiValueExtendedStatsAsc() throws Exception { + public void testSingleValuedFieldOrderedByMultiValueExtendedStatsAsc() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -1470,8 +1427,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedByStatsAggAscWithTermsSubAgg() throws Exception { + public void testSingleValuedFieldOrderedByStatsAggAscWithTermsSubAgg() throws Exception { boolean asc = true; SearchResponse response = client() .prepareSearch("idx") @@ -1516,45 +1472,38 @@ public class StringTermsTests extends AbstractTermsTestCase { } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAndTermsDesc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndTermsDesc() throws Exception { String[] expectedKeys = new String[] { "val1", "val2", "val4", "val3", "val7", "val6", "val5" }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true), Terms.Order.term(false)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAndTermsAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndTermsAsc() throws Exception { String[] expectedKeys = new String[] { "val1", "val2", "val3", "val4", "val5", "val6", "val7" }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true), Terms.Order.term(true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationDescAndTermsAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationDescAndTermsAsc() throws Exception { String[] expectedKeys = new String[] { "val5", "val6", "val7", "val3", "val4", "val2", "val1" }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true)); } - @Test - public void singleValuedField_OrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception { String[] expectedKeys = new String[] { "val6", "val7", "val3", "val4", "val5", "val1", "val2" }; assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception { String[] expectedKeys = new String[] { "val6", "val7", "val3", "val5", "val4", "val1", "val2" }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true)); } - @Test - public void singleValuedField_OrderedByThreeCriteria() throws Exception { + public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception { String[] expectedKeys = new String[] { "val2", "val1", "val4", "val5", "val3", "val6", "val7" }; assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false)); } - @Test - public void singleValuedField_OrderedBySingleValueSubAggregationAscAsCompound() throws Exception { + public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception { String[] expectedKeys = new String[] { "val1", "val2", "val3", "val4", "val5", "val6", "val7" }; assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", true)); } @@ -1590,8 +1539,7 @@ public class StringTermsTests extends AbstractTermsTestCase { } } - @Test - public void indexMetaField() throws Exception { + public void testIndexMetaField() throws Exception { SearchResponse response = client() .prepareSearch("idx", "empty_bucket_idx") .setTypes("type") @@ -1621,8 +1569,7 @@ public class StringTermsTests extends AbstractTermsTestCase { assertEquals(5L, terms.getBucketByKey("i").getDocCount()); } - @Test - public void otherDocCount() { + public void testOtherDocCount() { testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME); } } diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java similarity index 87% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java index 0c5aef40eb5..20916c33205 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks; import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksBuilder; -import org.junit.Test; import java.util.Arrays; import java.util.Collection; @@ -42,20 +41,25 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.percentileRanks; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; /** * */ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + private static double[] randomPercents(long minValue, long maxValue) { final int length = randomIntBetween(1, 20); @@ -108,9 +112,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testEmptyAggregation() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) @@ -132,7 +134,6 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx_unmapped") .setQuery(matchAllQuery()) @@ -153,7 +154,6 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { final double[] pcts = randomPercents(minValue, maxValue); SearchResponse searchResponse = client().prepareSearch("idx") @@ -170,8 +170,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { + public void testSingleValuedFieldGetProperty() throws Exception { final double[] pcts = randomPercents(minValue, maxValue); SearchResponse searchResponse = client() .prepareSearch("idx") @@ -197,7 +196,6 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } - @Test public void testSingleValuedFieldOutsideRange() throws Exception { final double[] pcts = new double[] {minValue - 1, maxValue + 1}; SearchResponse searchResponse = client().prepareSearch("idx") @@ -214,8 +212,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { final double[] pcts = randomPercents(minValue, maxValue); SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") .setQuery(matchAllQuery()) @@ -231,8 +228,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { final double[] pcts = randomPercents(minValue - 1, maxValue - 1); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -248,8 +244,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercents(minValue - 1, maxValue - 1); @@ -268,7 +263,6 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { final double[] pcts = randomPercents(minValues, maxValues); SearchResponse searchResponse = client().prepareSearch("idx") @@ -285,8 +279,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { final double[] pcts = randomPercents(minValues - 1, maxValues - 1); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -301,8 +294,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1); } - @Test - public void testMultiValuedField_WithValueScript_Reverse() throws Exception { + public void testMultiValuedFieldWithValueScriptReverse() throws Exception { final double[] pcts = randomPercents(-maxValues, -minValues); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -318,8 +310,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercents(minValues - 1, maxValues - 1); @@ -338,8 +329,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { final double[] pcts = randomPercents(minValue, maxValue); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -355,8 +345,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercents(minValue - 1, maxValue - 1); @@ -375,28 +364,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("dec", 1); - final double[] pcts = randomPercents(minValue -1 , maxValue - 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(randomCompression(percentileRanks("percentile_ranks")) -.script( - new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)) - .percentiles(pcts)) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { final double[] pcts = randomPercents(minValues, maxValues); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -412,25 +380,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - final double[] pcts = randomPercents(minValues, maxValues); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(randomCompression(percentileRanks("percentile_ranks")) -.script(new Script("doc['values'].values")) - .percentiles(pcts)) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final PercentileRanks percentiles = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, percentiles, minValues, maxValues); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercents(minValues - 1, maxValues - 1); @@ -449,7 +399,6 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1); } - @Test public void testOrderBySubAggregation() { boolean asc = randomBoolean(); SearchResponse searchResponse = client().prepareSearch("idx") diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java similarity index 86% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java index afa9f204da4..d14638a386b 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesBuilder; -import org.junit.Test; import java.util.Arrays; import java.util.Collection; @@ -42,20 +41,25 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.*; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; /** * */ public class TDigestPercentilesTests extends AbstractNumericTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - + private static double[] randomPercentiles() { final int length = randomIntBetween(1, 20); final double[] percentiles = new double[length]; @@ -108,9 +112,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testEmptyAggregation() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0) @@ -132,7 +134,6 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testUnmapped() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx_unmapped") .setQuery(matchAllQuery()) @@ -153,7 +154,6 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testSingleValuedField() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") @@ -170,8 +170,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { + public void testSingleValuedFieldGetProperty() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client() .prepareSearch("idx") @@ -197,8 +196,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_PartiallyUnmapped() throws Exception { + public void testSingleValuedFieldPartiallyUnmapped() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") .setQuery(matchAllQuery()) @@ -214,8 +212,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { + public void testSingleValuedFieldWithValueScript() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -231,8 +228,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { + public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); @@ -251,7 +247,6 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test public void testMultiValuedField() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") @@ -268,8 +263,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { + public void testMultiValuedFieldWithValueScript() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -284,13 +278,12 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1); } - @Test - public void testMultiValuedField_WithValueScript_Reverse() throws Exception { + public void testMultiValuedFieldWithValueScriptReverse() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(randomCompression(percentiles("percentiles")) -.field("values").script(new Script("_value * -1")) + .field("values").script(new Script("_value * -1")) .percentiles(pcts)) .execute().actionGet(); @@ -301,8 +294,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { + public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); @@ -321,8 +313,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued() throws Exception { + public void testScriptSingleValued() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -338,15 +329,14 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { + public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(randomCompression(percentiles("percentiles")) -.script( + .script( new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)) .percentiles(pcts)) .execute().actionGet(); @@ -358,28 +348,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("dec", 1); - final double[] pcts = randomPercentiles(); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(randomCompression(percentiles("percentiles")) -.script( - new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)) - .percentiles(pcts)) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final Percentiles percentiles = searchResponse.getAggregations().get("percentiles"); - assertConsistent(pcts, percentiles, minValue - 1, maxValue - 1); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { + public void testScriptMultiValued() throws Exception { final double[] pcts = randomPercentiles(); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -395,25 +364,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { } @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - final double[] pcts = randomPercentiles(); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(randomCompression(percentiles("percentiles")) -.script(new Script("doc['values'].values")) - .percentiles(pcts)) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - final Percentiles percentiles = searchResponse.getAggregations().get("percentiles"); - assertConsistent(pcts, percentiles, minValues, maxValues); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { + public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); final double[] pcts = randomPercentiles(); @@ -432,7 +383,6 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { assertConsistent(pcts, percentiles, minValues - 1, maxValues - 1); } - @Test public void testOrderBySubAggregation() { boolean asc = randomBoolean(); SearchResponse searchResponse = client().prepareSearch("idx") @@ -458,5 +408,4 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase { previous = p99; } } - } \ No newline at end of file diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java similarity index 98% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java index a0d2c785c5b..adf34927ba4 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java @@ -42,6 +42,7 @@ renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java renamed: core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ChildQuerySearchTests.java renamed: core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java + ^^^^^ note: the methods from this test using mustache were moved to the mustache module under its messy tests package. renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateHistogramTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java @@ -80,7 +81,6 @@ renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/TopHitsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TopHitsTests.java renamed: core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TransformOnIndexMapperTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ValueCountTests.java renamed: core/src/main/java/org/elasticsearch/script/groovy/GroovyScriptCompilationException.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java renamed: core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java renamed: core/src/test/java/org/elasticsearch/script/GroovySecurityIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java similarity index 85% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java index c9fabd72789..dcc3abf2e1c 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java @@ -22,23 +22,28 @@ package org.elasticsearch.script.groovy; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.lucene.search.function.CombineFunction; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyScriptEngineService; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.hamcrest.Matchers.equalTo; /** @@ -46,31 +51,28 @@ import static org.hamcrest.Matchers.equalTo; */ // TODO: refactor into unit test or proper rest tests public class GroovyScriptTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return Collections.singleton(GroovyPlugin.class); } - - @Test + public void testGroovyBigDecimalTransformation() { client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); // Test that something that would usually be a BigDecimal is transformed into a Double - assertScript("def n = 1.23; assert n instanceof Double;"); - assertScript("def n = 1.23G; assert n instanceof Double;"); - assertScript("def n = BigDecimal.ONE; assert n instanceof BigDecimal;"); + assertScript("def n = 1.23; assert n instanceof Double; return n;"); + assertScript("def n = 1.23G; assert n instanceof Double; return n;"); + assertScript("def n = BigDecimal.ONE; assert n instanceof BigDecimal; return n;"); } - public void assertScript(String script) { + public void assertScript(String scriptString) { + Script script = new Script(scriptString, ScriptType.INLINE, "groovy", null); SearchResponse resp = client().prepareSearch("test") - .setSource(new BytesArray("{\"query\": {\"match_all\": {}}," + - "\"sort\":{\"_script\": {\"script\": \""+ script + - "; 1\", \"type\": \"number\", \"lang\": \"groovy\"}}}")).get(); + .setSource(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).sort(SortBuilders.scriptSort(script, "number"))) + .get(); assertNoFailures(resp); } - @Test public void testGroovyExceptionSerialization() throws Exception { List reqs = new ArrayList<>(); for (int i = 0; i < randomIntBetween(50, 500); i++) { @@ -106,7 +108,6 @@ public class GroovyScriptTests extends ESIntegTestCase { } } - @Test public void testGroovyScriptAccess() { client().prepareIndex("test", "doc", "1").setSource("foo", "quick brow fox jumped over the lazy dog", "bar", 1).get(); client().prepareIndex("test", "doc", "2").setSource("foo", "fast jumping spiders", "bar", 2).get(); @@ -120,7 +121,7 @@ public class GroovyScriptTests extends ESIntegTestCase { assertNoFailures(resp); assertOrderedSearchHits(resp, "3", "2", "1"); } - + public void testScoreAccess() { client().prepareIndex("test", "doc", "1").setSource("foo", "quick brow fox jumped over the lazy dog", "bar", 1).get(); client().prepareIndex("test", "doc", "2").setSource("foo", "fast jumping spiders", "bar", 2).get(); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java new file mode 100644 index 00000000000..5f91631c021 --- /dev/null +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java @@ -0,0 +1,163 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.groovy; + +import org.apache.lucene.util.Constants; +import org.codehaus.groovy.control.MultipleCompilationErrorsException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ScriptException; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESTestCase; + +import groovy.lang.MissingPropertyException; + +import java.nio.file.Path; +import java.security.PrivilegedActionException; +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests for the Groovy security permissions + */ +public class GroovySecurityTests extends ESTestCase { + + private GroovyScriptEngineService se; + + static { + // ensure we load all the timezones in the parent classloader with all permissions + // relates to https://github.com/elastic/elasticsearch/issues/14524 + org.joda.time.DateTimeZone.getDefault(); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + se = new GroovyScriptEngineService(Settings.EMPTY); + // otherwise will exit your VM and other bad stuff + assumeTrue("test requires security manager to be enabled", System.getSecurityManager() != null); + } + + @Override + public void tearDown() throws Exception { + se.close(); + super.tearDown(); + } + + public void testEvilGroovyScripts() throws Exception { + // Plain test + assertSuccess(""); + // field access (via map) + assertSuccess("def foo = doc['foo'].value; if (foo == null) { return 5; }"); + // field access (via list) + assertSuccess("def foo = mylist[0]; if (foo == null) { return 5; }"); + // field access (via array) + assertSuccess("def foo = myarray[0]; if (foo == null) { return 5; }"); + // field access (via object) + assertSuccess("def foo = myobject.primitive.toString(); if (foo == null) { return 5; }"); + assertSuccess("def foo = myobject.object.toString(); if (foo == null) { return 5; }"); + assertSuccess("def foo = myobject.list[0].primitive.toString(); if (foo == null) { return 5; }"); + // List + assertSuccess("def list = [doc['foo'].value, 3, 4]; def v = list.get(1); list.add(10)"); + // Ranges + assertSuccess("def range = 1..doc['foo'].value; def v = range.get(0)"); + // Maps + assertSuccess("def v = doc['foo'].value; def m = [:]; m.put(\"value\", v)"); + // serialization to json (this is best effort considering the unsafe etc at play) + assertSuccess("def x = 5; groovy.json.JsonOutput.toJson(x)"); + // Times + assertSuccess("def t = Instant.now().getMillis()"); + // GroovyCollections + assertSuccess("def n = [1,2,3]; GroovyCollections.max(n)"); + + // Fail cases: + assertFailure("pr = Runtime.getRuntime().exec(\"touch /tmp/gotcha\"); pr.waitFor()", MissingPropertyException.class); + + // infamous: + assertFailure("java.lang.Math.class.forName(\"java.lang.Runtime\")", PrivilegedActionException.class); + // filtered directly by our classloader + assertFailure("getClass().getClassLoader().loadClass(\"java.lang.Runtime\").availableProcessors()", PrivilegedActionException.class); + // unfortunately, we have access to other classloaders (due to indy mechanism needing getClassLoader permission) + // but we can't do much with them directly at least. + assertFailure("myobject.getClass().getClassLoader().loadClass(\"java.lang.Runtime\").availableProcessors()", SecurityException.class); + assertFailure("d = new DateTime(); d.getClass().getDeclaredMethod(\"year\").setAccessible(true)", SecurityException.class); + assertFailure("d = new DateTime(); d.\"${'get' + 'Class'}\"()." + + "\"${'getDeclared' + 'Method'}\"(\"year\").\"${'set' + 'Accessible'}\"(false)", SecurityException.class); + assertFailure("Class.forName(\"org.joda.time.DateTime\").getDeclaredMethod(\"year\").setAccessible(true)", MissingPropertyException.class); + + assertFailure("Eval.me('2 + 2')", MissingPropertyException.class); + assertFailure("Eval.x(5, 'x + 2')", MissingPropertyException.class); + + assertFailure("d = new Date(); java.lang.reflect.Field f = Date.class.getDeclaredField(\"fastTime\");" + + " f.setAccessible(true); f.get(\"fastTime\")", MultipleCompilationErrorsException.class); + + assertFailure("def methodName = 'ex'; Runtime.\"${'get' + 'Runtime'}\"().\"${methodName}ec\"(\"touch /tmp/gotcha2\")", MissingPropertyException.class); + + assertFailure("t = new Thread({ println 3 });", MultipleCompilationErrorsException.class); + + // test a directory we normally have access to, but the groovy script does not. + Path dir = createTempDir(); + // TODO: figure out the necessary escaping for windows paths here :) + if (!Constants.WINDOWS) { + assertFailure("new File(\"" + dir + "\").exists()", MultipleCompilationErrorsException.class); + } + } + + /** runs a script */ + private void doTest(String script) { + Map vars = new HashMap(); + // we add a "mock document" containing a single field "foo" that returns 4 (abusing a jdk class with a getValue() method) + vars.put("doc", Collections.singletonMap("foo", new AbstractMap.SimpleEntry(null, 4))); + vars.put("mylist", Arrays.asList("foo")); + vars.put("myarray", Arrays.asList("foo")); + vars.put("myobject", new MyObject()); + + se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(script)), vars).run(); + } + + public static class MyObject { + public int getPrimitive() { return 0; } + public Object getObject() { return "value"; } + public List getList() { return Arrays.asList(new MyObject()); } + } + + /** asserts that a script runs without exception */ + private void assertSuccess(String script) { + doTest(script); + } + + /** asserts that a script triggers securityexception */ + private void assertFailure(String script, Class exceptionClass) { + try { + doTest(script); + fail("did not get expected exception"); + } catch (ScriptException expected) { + Throwable cause = expected.getCause(); + assertNotNull(cause); + if (exceptionClass.isAssignableFrom(cause.getClass()) == false) { + throw new AssertionError("unexpected exception: " + cause, expected); + } + } + } +} diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml similarity index 62% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml index 123b02fc7fa..c276bab6495 100644 --- a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml +++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml @@ -10,5 +10,5 @@ - do: nodes.info: {} - - match: { nodes.$master.plugins.0.name: lang-groovy } - - match: { nodes.$master.plugins.0.jvm: true } + - match: { nodes.$master.modules.0.name: lang-groovy } + - match: { nodes.$master.modules.0.jvm: true } diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamTests.java b/modules/lang-mustache/build.gradle similarity index 66% rename from core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamTests.java rename to modules/lang-mustache/build.gradle index 89ee148f4a8..4e8e9cc345d 100644 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamTests.java +++ b/modules/lang-mustache/build.gradle @@ -17,14 +17,20 @@ * under the License. */ -package org.elasticsearch.common.compress.lzf; - -import org.elasticsearch.common.compress.AbstractCompressedStreamTestCase; - -public class LZFCompressedStreamTests extends AbstractCompressedStreamTestCase { - - public LZFCompressedStreamTests() { - super(new LZFTestCompressor()); - } - +esplugin { + description 'Mustache scripting integration for Elasticsearch' + classname 'org.elasticsearch.script.mustache.MustachePlugin' +} + +dependencies { + compile "com.github.spullara.mustache.java:compiler:0.9.1" +} + +compileTestJava.options.compilerArgs << '-Xlint:-rawtypes,-unchecked' + +integTest { + cluster { + systemProperty 'es.script.inline', 'on' + systemProperty 'es.script.indexed', 'on' + } } diff --git a/distribution/licenses/compiler-0.9.1.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.1.jar.sha1 similarity index 100% rename from distribution/licenses/compiler-0.9.1.jar.sha1 rename to modules/lang-mustache/licenses/compiler-0.9.1.jar.sha1 diff --git a/distribution/licenses/compiler-LICENSE.txt b/modules/lang-mustache/licenses/compiler-LICENSE.txt similarity index 100% rename from distribution/licenses/compiler-LICENSE.txt rename to modules/lang-mustache/licenses/compiler-LICENSE.txt diff --git a/distribution/licenses/compiler-NOTICE.txt b/modules/lang-mustache/licenses/compiler-NOTICE.txt similarity index 100% rename from distribution/licenses/compiler-NOTICE.txt rename to modules/lang-mustache/licenses/compiler-NOTICE.txt diff --git a/core/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java similarity index 100% rename from core/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java diff --git a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScriptPlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java similarity index 65% rename from core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScriptPlugin.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index 92f19a7c358..3f6f6e00716 100644 --- a/core/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScriptPlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -17,27 +17,24 @@ * under the License. */ -package org.elasticsearch.benchmark.scripts.expression; +package org.elasticsearch.script.mustache; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; -public class NativeScriptPlugin extends Plugin { +public class MustachePlugin extends Plugin { @Override public String name() { - return "native-benchmark-scripts"; + return "lang-mustache"; } @Override public String description() { - return "Native benchmark script"; + return "Mustache scripting integration for Elasticsearch"; } public void onModule(ScriptModule module) { - module.registerScript(NativeScript1.NATIVE_SCRIPT_1, NativeScript1.Factory.class); - module.registerScript(NativeScript2.NATIVE_SCRIPT_2, NativeScript2.Factory.class); - module.registerScript(NativeScript3.NATIVE_SCRIPT_3, NativeScript3.Factory.class); - module.registerScript(NativeScript4.NATIVE_SCRIPT_4, NativeScript4.Factory.class); + module.addScriptEngine(MustacheScriptEngineService.class); } } diff --git a/core/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java similarity index 87% rename from core/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java index 3affd0c5a12..93172056071 100644 --- a/core/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java @@ -19,6 +19,8 @@ package org.elasticsearch.script.mustache; import com.github.mustachejava.Mustache; + +import org.elasticsearch.SpecialPermission; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -34,6 +36,8 @@ import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.lookup.SearchLookup; import java.lang.ref.SoftReference; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Collections; import java.util.Map; @@ -123,6 +127,9 @@ public class MustacheScriptEngineService extends AbstractComponent implements Sc // Nothing to do here } + // permission checked before doing crazy reflection + static final SpecialPermission SPECIAL_PERMISSION = new SpecialPermission(); + /** * Used at query execution time by script service in order to execute a query template. * */ @@ -148,9 +155,20 @@ public class MustacheScriptEngineService extends AbstractComponent implements Sc @Override public Object run() { - BytesStreamOutput result = new BytesStreamOutput(); + final BytesStreamOutput result = new BytesStreamOutput(); try (UTF8StreamWriter writer = utf8StreamWriter().setOutput(result)) { - ((Mustache) template.compiled()).execute(writer, vars); + // crazy reflection here + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(SPECIAL_PERMISSION); + } + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + ((Mustache) template.compiled()).execute(writer, vars); + return null; + } + }); } catch (Exception e) { logger.error("Error running " + template, e); throw new ScriptException("Error running " + template, e); diff --git a/core/src/main/java/org/elasticsearch/action/exists/package-info.java b/modules/lang-mustache/src/main/plugin-metadata/plugin-security.policy similarity index 87% rename from core/src/main/java/org/elasticsearch/action/exists/package-info.java rename to modules/lang-mustache/src/main/plugin-metadata/plugin-security.policy index 1b921187888..ea2db551912 100644 --- a/core/src/main/java/org/elasticsearch/action/exists/package-info.java +++ b/modules/lang-mustache/src/main/plugin-metadata/plugin-security.policy @@ -17,7 +17,7 @@ * under the License. */ -/** - * Exists action. - */ -package org.elasticsearch.action.exists; \ No newline at end of file +grant { + // needed to do crazy reflection + permission java.lang.RuntimePermission "accessDeclaredMembers"; +}; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java new file mode 100644 index 00000000000..92d15332780 --- /dev/null +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java @@ -0,0 +1,389 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.messy.tests; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionModule; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptRequest; +import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.ActionFilter; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.FilterClient; +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.script.Template; +import org.elasticsearch.script.mustache.MustachePlugin; +import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.junit.After; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.node.Node.HTTP_ENABLED; +import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion; +import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +@ClusterScope(scope = SUITE) +public class ContextAndHeaderTransportTests extends ESIntegTestCase { + private static final List requests = new CopyOnWriteArrayList<>(); + private String randomHeaderKey = randomAsciiOfLength(10); + private String randomHeaderValue = randomAsciiOfLength(20); + private String queryIndex = "query-" + randomAsciiOfLength(10).toLowerCase(Locale.ROOT); + private String lookupIndex = "lookup-" + randomAsciiOfLength(10).toLowerCase(Locale.ROOT); + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return settingsBuilder() + .put(super.nodeSettings(nodeOrdinal)) + .put("script.indexed", "on") + .put(HTTP_ENABLED, true) + .build(); + } + + @Override + protected Collection> nodePlugins() { + return pluginList(ActionLoggingPlugin.class, MustachePlugin.class); + } + + @Before + public void createIndices() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("location").field("type", "geo_shape").endObject() + .startObject("name").field("type", "string").endObject() + .endObject() + .endObject().endObject().string(); + + Settings settings = settingsBuilder() + .put(indexSettings()) + .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable. + .build(); + assertAcked(transportClient().admin().indices().prepareCreate(lookupIndex) + .setSettings(settings).addMapping("type", mapping)); + assertAcked(transportClient().admin().indices().prepareCreate(queryIndex) + .setSettings(settings).addMapping("type", mapping)); + ensureGreen(queryIndex, lookupIndex); + + requests.clear(); + } + + @After + public void checkAllRequestsContainHeaders() { + assertRequestsContainHeader(IndexRequest.class); + assertRequestsContainHeader(RefreshRequest.class); + } + + public void testThatIndexedScriptGetRequestInTemplateQueryContainsContextAndHeaders() throws Exception { + PutIndexedScriptResponse scriptResponse = transportClient() + .preparePutIndexedScript( + MustacheScriptEngineService.NAME, + "my_script", + jsonBuilder().startObject().field("script", "{ \"match\": { \"name\": \"Star Wars\" }}").endObject() + .string()).get(); + assertThat(scriptResponse.isCreated(), is(true)); + + transportClient().prepareIndex(queryIndex, "type", "1") + .setSource(jsonBuilder().startObject().field("name", "Star Wars - The new republic").endObject()).get(); + transportClient().admin().indices().prepareRefresh(queryIndex).get(); + + SearchResponse searchResponse = transportClient() + .prepareSearch(queryIndex) + .setQuery( + QueryBuilders.templateQuery(new Template("my_script", ScriptType.INDEXED, + MustacheScriptEngineService.NAME, null, null))).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 1); + + assertGetRequestsContainHeaders(".scripts"); + assertRequestsContainHeader(PutIndexedScriptRequest.class); + } + + public void testThatSearchTemplatesWithIndexedTemplatesGetRequestContainsContextAndHeaders() throws Exception { + PutIndexedScriptResponse scriptResponse = transportClient().preparePutIndexedScript(MustacheScriptEngineService.NAME, "the_template", + jsonBuilder().startObject().startObject("template").startObject("query").startObject("match") + .field("name", "{{query_string}}").endObject().endObject().endObject().endObject().string() + ).get(); + assertThat(scriptResponse.isCreated(), is(true)); + + transportClient().prepareIndex(queryIndex, "type", "1") + .setSource(jsonBuilder().startObject().field("name", "Star Wars - The new republic").endObject()) + .get(); + transportClient().admin().indices().prepareRefresh(queryIndex).get(); + + Map params = new HashMap<>(); + params.put("query_string", "star wars"); + + SearchResponse searchResponse = transportClient().prepareSearch(queryIndex).setTemplate(new Template("the_template", ScriptType.INDEXED, MustacheScriptEngineService.NAME, null, params)) + .get(); + + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 1); + + assertGetRequestsContainHeaders(".scripts"); + assertRequestsContainHeader(PutIndexedScriptRequest.class); + } + + public void testThatIndexedScriptGetRequestInPhraseSuggestContainsContextAndHeaders() throws Exception { + CreateIndexRequestBuilder builder = transportClient().admin().indices().prepareCreate("test").setSettings(settingsBuilder() + .put(indexSettings()) + .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable. + .put("index.analysis.analyzer.text.tokenizer", "standard") + .putArray("index.analysis.analyzer.text.filter", "lowercase", "my_shingle") + .put("index.analysis.filter.my_shingle.type", "shingle") + .put("index.analysis.filter.my_shingle.output_unigrams", true) + .put("index.analysis.filter.my_shingle.min_shingle_size", 2) + .put("index.analysis.filter.my_shingle.max_shingle_size", 3)); + + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("title") + .field("type", "string") + .field("analyzer", "text") + .endObject() + .endObject() + .endObject() + .endObject(); + assertAcked(builder.addMapping("type1", mapping)); + ensureGreen(); + + List titles = new ArrayList<>(); + + titles.add("United States House of Representatives Elections in Washington 2006"); + titles.add("United States House of Representatives Elections in Washington 2005"); + titles.add("State"); + titles.add("Houses of Parliament"); + titles.add("Representative Government"); + titles.add("Election"); + + List builders = new ArrayList<>(); + for (String title: titles) { + transportClient().prepareIndex("test", "type1").setSource("title", title).get(); + } + transportClient().admin().indices().prepareRefresh("test").get(); + + String filterStringAsFilter = XContentFactory.jsonBuilder() + .startObject() + .startObject("match_phrase") + .field("title", "{{suggestion}}") + .endObject() + .endObject() + .string(); + + PutIndexedScriptResponse scriptResponse = transportClient() + .preparePutIndexedScript( + MustacheScriptEngineService.NAME, + "my_script", + jsonBuilder().startObject().field("script", filterStringAsFilter).endObject() + .string()).get(); + assertThat(scriptResponse.isCreated(), is(true)); + + PhraseSuggestionBuilder suggest = phraseSuggestion("title") + .field("title") + .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title") + .suggestMode("always") + .maxTermFreq(.99f) + .size(10) + .maxInspections(200) + ) + .confidence(0f) + .maxErrors(2f) + .shardSize(30000) + .size(10); + + PhraseSuggestionBuilder filteredFilterSuggest = suggest.collateQuery(new Template("my_script", ScriptType.INDEXED, + MustacheScriptEngineService.NAME, null, null)); + + SearchRequestBuilder searchRequestBuilder = transportClient().prepareSearch("test").setSize(0); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + String suggestText = "united states house of representatives elections in washington 2006"; + if (suggestText != null) { + suggestBuilder.setText(suggestText); + } + suggestBuilder.addSuggestion(filteredFilterSuggest); + searchRequestBuilder.suggest(suggestBuilder); + SearchResponse actionGet = searchRequestBuilder.execute().actionGet(); + assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(0)); + Suggest searchSuggest = actionGet.getSuggest(); + + assertSuggestionSize(searchSuggest, 0, 2, "title"); + + assertGetRequestsContainHeaders(".scripts"); + assertRequestsContainHeader(PutIndexedScriptRequest.class); + } + + private List getRequests(Class clazz) { + List results = new ArrayList<>(); + for (ActionRequest request : requests) { + if (request.getClass().equals(clazz)) { + results.add((T) request); + } + } + + return results; + } + + private void assertRequestsContainHeader(Class clazz) { + List classRequests = getRequests(clazz); + for (ActionRequest request : classRequests) { + assertRequestContainsHeader(request); + } + } + + private void assertGetRequestsContainHeaders() { + assertGetRequestsContainHeaders(this.lookupIndex); + } + + private void assertGetRequestsContainHeaders(String index) { + List getRequests = getRequests(GetRequest.class); + assertThat(getRequests, hasSize(greaterThan(0))); + + for (GetRequest request : getRequests) { + if (!request.index().equals(index)) { + continue; + } + assertRequestContainsHeader(request); + } + } + + private void assertRequestContainsHeader(ActionRequest request) { + String msg = String.format(Locale.ROOT, "Expected header %s to be in request %s", randomHeaderKey, request.getClass().getName()); + if (request instanceof IndexRequest) { + IndexRequest indexRequest = (IndexRequest) request; + msg = String.format(Locale.ROOT, "Expected header %s to be in index request %s/%s/%s", randomHeaderKey, + indexRequest.index(), indexRequest.type(), indexRequest.id()); + } + assertThat(msg, request.hasHeader(randomHeaderKey), is(true)); + assertThat(request.getHeader(randomHeaderKey).toString(), is(randomHeaderValue)); + } + + /** + * a transport client that adds our random header + */ + private Client transportClient() { + Client transportClient = internalCluster().transportClient(); + FilterClient filterClient = new FilterClient(transportClient) { + @Override + protected > void doExecute(Action action, Request request, ActionListener listener) { + request.putHeader(randomHeaderKey, randomHeaderValue); + super.doExecute(action, request, listener); + } + }; + + return filterClient; + } + + public static class ActionLoggingPlugin extends Plugin { + + @Override + public String name() { + return "test-action-logging"; + } + + @Override + public String description() { + return "Test action logging"; + } + + @Override + public Collection nodeModules() { + return Collections.singletonList(new ActionLoggingModule()); + } + + public void onModule(ActionModule module) { + module.registerFilter(LoggingFilter.class); + } + } + + public static class ActionLoggingModule extends AbstractModule { + @Override + protected void configure() { + bind(LoggingFilter.class).asEagerSingleton(); + } + + } + + public static class LoggingFilter extends ActionFilter.Simple { + + @Inject + public LoggingFilter(Settings settings) { + super(settings); + } + + @Override + public int order() { + return 999; + } + + @Override + protected boolean apply(String action, ActionRequest request, ActionListener listener) { + requests.add(request); + return true; + } + + @Override + protected boolean apply(String action, ActionResponse response, ActionListener listener) { + return true; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/validate/RenderSearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java similarity index 86% rename from core/src/test/java/org/elasticsearch/validate/RenderSearchTemplateIT.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java index 10812c15555..87cc51c2ec2 100644 --- a/core/src/test/java/org/elasticsearch/validate/RenderSearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java @@ -17,20 +17,27 @@ * under the License. */ -package org.elasticsearch.validate; +package org.elasticsearch.messy.tests; import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.Template; +import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; +import org.elasticsearch.test.rest.support.FileUtils; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -39,24 +46,34 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.SuiteScopeTestCase -public class RenderSearchTemplateIT extends ESIntegTestCase { - +public class RenderSearchTemplateTests extends ESIntegTestCase { private static final String TEMPLATE_CONTENTS = "{\"size\":\"{{size}}\",\"query\":{\"match\":{\"foo\":\"{{value}}\"}},\"aggs\":{\"objects\":{\"terms\":{\"field\":\"{{value}}\",\"size\":\"{{size}}\"}}}}"; - + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MustachePlugin.class); + } + @Override protected void setupSuiteScopeCluster() throws Exception { client().preparePutIndexedScript(MustacheScriptEngineService.NAME, "index_template_1", "{ \"template\": " + TEMPLATE_CONTENTS + " }").get(); } - + @Override public Settings nodeSettings(int nodeOrdinal) { - //Set path so ScriptService will pick up the test scripts + Path configDir = createTempDir(); + Path scriptsDir = configDir.resolve("scripts"); + try { + Files.createDirectories(scriptsDir); + Files.write(scriptsDir.resolve("file_template_1.mustache"), TEMPLATE_CONTENTS.getBytes("UTF-8")); + } catch (Exception e) { + throw new RuntimeException(e); + } return settingsBuilder().put(super.nodeSettings(nodeOrdinal)) - .put("path.conf", this.getDataPath("config")).build(); + .put("path.conf", configDir).build(); } - - @Test - public void inlineTemplate() { + + public void testInlineTemplate() { Map params = new HashMap<>(); params.put("value", "bar"); params.put("size", 20); @@ -70,7 +87,7 @@ public class RenderSearchTemplateIT extends ESIntegTestCase { String expected = TEMPLATE_CONTENTS.replace("{{value}}", "bar").replace("{{size}}", "20"); Map expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); assertThat(sourceAsMap, equalTo(expectedMap)); - + params = new HashMap<>(); params.put("value", "baz"); params.put("size", 100); @@ -84,9 +101,8 @@ public class RenderSearchTemplateIT extends ESIntegTestCase { expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); assertThat(sourceAsMap, equalTo(expectedMap)); } - - @Test - public void indexedTemplate() { + + public void testIndexedTemplate() { Map params = new HashMap<>(); params.put("value", "bar"); params.put("size", 20); @@ -100,7 +116,7 @@ public class RenderSearchTemplateIT extends ESIntegTestCase { String expected = TEMPLATE_CONTENTS.replace("{{value}}", "bar").replace("{{size}}", "20"); Map expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); assertThat(sourceAsMap, equalTo(expectedMap)); - + params = new HashMap<>(); params.put("value", "baz"); params.put("size", 100); @@ -114,9 +130,8 @@ public class RenderSearchTemplateIT extends ESIntegTestCase { expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); assertThat(sourceAsMap, equalTo(expectedMap)); } - - @Test - public void fileTemplate() { + + public void testFileTemplate() { Map params = new HashMap<>(); params.put("value", "bar"); params.put("size", 20); @@ -130,7 +145,7 @@ public class RenderSearchTemplateIT extends ESIntegTestCase { String expected = TEMPLATE_CONTENTS.replace("{{value}}", "bar").replace("{{size}}", "20"); Map expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); assertThat(sourceAsMap, equalTo(expectedMap)); - + params = new HashMap<>(); params.put("value", "baz"); params.put("size", 100); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java similarity index 94% rename from core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java index 85993fdf812..a0699a35534 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java @@ -17,30 +17,44 @@ * under the License. */ -package org.elasticsearch.search.suggest; +package org.elasticsearch.messy.tests; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.*; +import org.elasticsearch.action.search.ReduceSearchPhaseException; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.suggest.SuggestRequestBuilder; import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.mustache.MustachePlugin; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.DirectCandidateGenerator; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.junit.Test; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; @@ -50,17 +64,31 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion; import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion; import static org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.candidateGenerator; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.nullValue; /** * Integration tests for term and phrase suggestions. Many of these tests many requests that vary only slightly from one another. Where * possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that * request, modify again, request again, etc. This makes it very obvious what changes between requests. */ -public class SuggestSearchIT extends ESIntegTestCase { - - @Test // see #3196 +public class SuggestSearchTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MustachePlugin.class); + } + + // see #3196 public void testSuggestAcrossMultipleIndices() throws IOException { createIndex("test"); ensureGreen(); @@ -151,7 +179,7 @@ public class SuggestSearchIT extends ESIntegTestCase { } } - @Test // see #3037 + // see #3037 public void testSuggestModes() throws IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 1) @@ -161,7 +189,7 @@ public class SuggestSearchIT extends ESIntegTestCase { .put("index.analysis.filter.shingler.type", "shingle") .put("index.analysis.filter.shingler.min_shingle_size", 2) .put("index.analysis.filter.shingler.max_shingle_size", 3)); - + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties") .startObject("name") @@ -181,7 +209,7 @@ public class SuggestSearchIT extends ESIntegTestCase { .endObject().endObject(); assertAcked(builder.addMapping("type1", mapping)); ensureGreen(); - + index("test", "type1", "1", "name", "I like iced tea"); index("test", "type1", "2", "name", "I like tea."); @@ -199,8 +227,8 @@ public class SuggestSearchIT extends ESIntegTestCase { searchSuggest = searchSuggest( "ice tea", phraseSuggestion); assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean"); } - - @Test // see #2729 + + // see #2729 public void testSizeOneShard() throws Exception { prepareCreate("test").setSettings( SETTING_NUMBER_OF_SHARDS, 1, @@ -214,7 +242,7 @@ public class SuggestSearchIT extends ESIntegTestCase { SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellchecker")).get(); assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue()); - + TermSuggestionBuilder termSuggestion = termSuggestion("test") .suggestMode("always") // Always, otherwise the results can vary between requests. .text("abcd") @@ -227,8 +255,7 @@ public class SuggestSearchIT extends ESIntegTestCase { suggest = searchSuggest( termSuggestion); assertSuggestion(suggest, 0, "test", 5, "abc0"); } - - @Test + public void testUnmappedField() throws IOException, InterruptedException, ExecutionException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) @@ -270,20 +297,17 @@ public class SuggestSearchIT extends ESIntegTestCase { phraseSuggestion.field("nosuchField"); { - SearchRequestBuilder suggestBuilder = client().prepareSearch().setSize(0); - suggestBuilder.setSuggestText("tetsting sugestion"); - suggestBuilder.addSuggestion(phraseSuggestion); - assertThrows(suggestBuilder, SearchPhaseExecutionException.class); + SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0); + searchBuilder.suggest(new SuggestBuilder().setText("tetsting sugestion").addSuggestion(phraseSuggestion)); + assertThrows(searchBuilder, SearchPhaseExecutionException.class); } { - SearchRequestBuilder suggestBuilder = client().prepareSearch().setSize(0); - suggestBuilder.setSuggestText("tetsting sugestion"); - suggestBuilder.addSuggestion(phraseSuggestion); - assertThrows(suggestBuilder, SearchPhaseExecutionException.class); + SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0); + searchBuilder.suggest(new SuggestBuilder().setText("tetsting sugestion").addSuggestion(phraseSuggestion)); + assertThrows(searchBuilder, SearchPhaseExecutionException.class); } } - @Test public void testSimple() throws Exception { createIndex("test"); ensureGreen(); @@ -293,10 +317,10 @@ public class SuggestSearchIT extends ESIntegTestCase { index("test", "type1", "3", "text", "abbd"); index("test", "type1", "4", "text", "abcc"); refresh(); - + SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellcecker")).get(); assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue()); - + TermSuggestionBuilder termSuggest = termSuggestion("test") .suggestMode("always") // Always, otherwise the results can vary between requests. .text("abcd") @@ -310,7 +334,6 @@ public class SuggestSearchIT extends ESIntegTestCase { assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd")); } - @Test public void testEmpty() throws Exception { createIndex("test"); ensureGreen(); @@ -331,7 +354,6 @@ public class SuggestSearchIT extends ESIntegTestCase { assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd")); } - @Test public void testWithMultipleCommands() throws Exception { createIndex("test"); ensureGreen(); @@ -360,7 +382,6 @@ public class SuggestSearchIT extends ESIntegTestCase { assertSuggestionSize(suggest, 0, 0, "accuracy"); } - @Test public void testSizeAndSort() throws Exception { createIndex("test"); ensureGreen(); @@ -408,8 +429,8 @@ public class SuggestSearchIT extends ESIntegTestCase { // assertThat(suggest.get(3).getSuggestedWords().get("prefix_abcd").get(4).getTerm(), equalTo("prefix_abcc")); // assertThat(suggest.get(3).getSuggestedWords().get("prefix_abcd").get(4).getTerm(), equalTo("prefix_accd")); } - - @Test // see #2817 + + // see #2817 public void testStopwordsOnlyPhraseSuggest() throws IOException { assertAcked(prepareCreate("test").addMapping("typ1", "body", "type=string,analyzer=stopwd").setSettings( settingsBuilder() @@ -426,9 +447,8 @@ public class SuggestSearchIT extends ESIntegTestCase { .size(1)); assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase"); } - - @Test - public void testPrefixLength() throws IOException { // Stopped here + + public void testPrefixLength() throws IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 1) .put("index.analysis.analyzer.reverse.tokenizer", "standard") @@ -462,15 +482,14 @@ public class SuggestSearchIT extends ESIntegTestCase { .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always")) .size(1).confidence(1.0f)); assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words"); - + searchSuggest = searchSuggest( "hello word", phraseSuggestion("simple_phrase").field("body") .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always")) .size(1).confidence(1.0f)); assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world"); } - - @Test + @Nightly public void testMarvelHerosPhraseSuggest() throws IOException, URISyntaxException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() @@ -574,7 +593,7 @@ public class SuggestSearchIT extends ESIntegTestCase { searchSuggest = searchSuggest( "american ame", phraseSuggest); assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace"); - + // try all smoothing methods phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2)); searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest); @@ -599,12 +618,11 @@ public class SuggestSearchIT extends ESIntegTestCase { // Check the name this time because we're repeating it which is funky assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel")); } - + private List readMarvelHeroNames() throws IOException, URISyntaxException { - return Files.readAllLines(PathUtils.get(SuggestSearchIT.class.getResource("/config/names.txt").toURI()), StandardCharsets.UTF_8); + return Files.readAllLines(PathUtils.get(Suggest.class.getResource("/config/names.txt").toURI()), StandardCharsets.UTF_8); } - @Test public void testSizePararm() throws IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 1) @@ -618,7 +636,7 @@ public class SuggestSearchIT extends ESIntegTestCase { .put("index.analysis.filter.my_shingle.output_unigrams", false) .put("index.analysis.filter.my_shingle.min_shingle_size", 2) .put("index.analysis.filter.my_shingle.max_shingle_size", 2)); - + XContentBuilder mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type1") @@ -670,7 +688,6 @@ public class SuggestSearchIT extends ESIntegTestCase { assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel"); } - @Test @Nightly public void testPhraseBoundaryCases() throws IOException, URISyntaxException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() @@ -691,7 +708,7 @@ public class SuggestSearchIT extends ESIntegTestCase { .put("index.analysis.filter.my_shingle2.output_unigrams", true) .put("index.analysis.filter.my_shingle2.min_shingle_size", 2) .put("index.analysis.filter.my_shingle2.max_shingle_size", 2)); - + XContentBuilder mapping = XContentFactory.jsonBuilder() .startObject().startObject("type1") .startObject("_all").field("store", "yes").field("termVector", "with_positions_offsets").endObject() @@ -764,7 +781,6 @@ public class SuggestSearchIT extends ESIntegTestCase { assertSuggestion(suggest, 0, "simple_phrase", "xorr the god jewel"); } - @Test public void testDifferentShardSize() throws Exception { createIndex("test"); ensureGreen(); @@ -778,7 +794,7 @@ public class SuggestSearchIT extends ESIntegTestCase { ElasticsearchAssertions.assertSuggestionSize(suggest, 0, 3, "simple"); } - @Test // see #3469 + // see #3469 public void testShardFailures() throws IOException, InterruptedException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) @@ -817,20 +833,22 @@ public class SuggestSearchIT extends ESIntegTestCase { // When searching on a shard with a non existing mapping, we should fail SearchRequestBuilder request = client().prepareSearch().setSize(0) - .setSuggestText("tetsting sugestion") - .addSuggestion(phraseSuggestion("did_you_mean").field("fielddoesnotexist").maxErrors(5.0f)); + .suggest( + new SuggestBuilder().setText("tetsting sugestion").addSuggestion( + phraseSuggestion("did_you_mean").field("fielddoesnotexist").maxErrors(5.0f))); assertThrows(request, SearchPhaseExecutionException.class); // When searching on a shard which does not hold yet any document of an existing type, we should not fail SearchResponse searchResponse = client().prepareSearch().setSize(0) - .setSuggestText("tetsting sugestion") - .addSuggestion(phraseSuggestion("did_you_mean").field("name").maxErrors(5.0f)) + .suggest( + new SuggestBuilder().setText("tetsting sugestion").addSuggestion( + phraseSuggestion("did_you_mean").field("name").maxErrors(5.0f))) .get(); ElasticsearchAssertions.assertNoFailures(searchResponse); ElasticsearchAssertions.assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions"); } - @Test // see #3469 + // see #3469 public void testEmptyShards() throws IOException, InterruptedException { XContentBuilder mappingBuilder = XContentFactory.jsonBuilder(). startObject(). @@ -866,8 +884,9 @@ public class SuggestSearchIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch() .setSize(0) - .setSuggestText("tetsting sugestion") - .addSuggestion(phraseSuggestion("did_you_mean").field("name").maxErrors(5.0f)) + .suggest( + new SuggestBuilder().setText("tetsting sugestion").addSuggestion( + phraseSuggestion("did_you_mean").field("name").maxErrors(5.0f))) .get(); assertNoFailures(searchResponse); @@ -878,7 +897,6 @@ public class SuggestSearchIT extends ESIntegTestCase { * Searching for a rare phrase shouldn't provide any suggestions if confidence > 1. This was possible before we rechecked the cutoff * score during the reduce phase. Failures don't occur every time - maybe two out of five tries but we don't repeat it to save time. */ - @Test public void testSearchForRarePhrase() throws IOException { // If there isn't enough chaf per shard then shards can become unbalanced, making the cutoff recheck this is testing do more harm then good. int chafPerShard = 100; @@ -942,12 +960,8 @@ public class SuggestSearchIT extends ESIntegTestCase { assertSuggestion(searchSuggest, 0, 0, "simple_phrase", "nobel prize"); } - /** - * If the suggester finds tons of options then picking the right one is slow without <<<INSERT SOLUTION HERE>>>. - */ - @Test @Nightly - public void suggestWithManyCandidates() throws InterruptedException, ExecutionException, IOException { + public void testSuggestWithManyCandidates() throws InterruptedException, ExecutionException, IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable. @@ -1092,7 +1106,6 @@ public class SuggestSearchIT extends ESIntegTestCase { // assertThat(total, lessThan(1000L)); // Takes many seconds without fix - just for debugging } - @Test public void testPhraseSuggesterCollate() throws InterruptedException, ExecutionException, IOException { CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(settingsBuilder() .put(indexSettings()) @@ -1258,12 +1271,14 @@ public class SuggestSearchIT extends ESIntegTestCase { protected Suggest searchSuggest(String suggestText, int expectShardsFailed, SuggestionBuilder... suggestions) { if (randomBoolean()) { SearchRequestBuilder builder = client().prepareSearch().setSize(0); + SuggestBuilder suggestBuilder = new SuggestBuilder(); if (suggestText != null) { - builder.setSuggestText(suggestText); + suggestBuilder.setText(suggestText); } for (SuggestionBuilder suggestion : suggestions) { - builder.addSuggestion(suggestion); + suggestBuilder.addSuggestion(suggestion); } + builder.suggest(suggestBuilder); SearchResponse actionGet = builder.execute().actionGet(); assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); return actionGet.getSuggest(); diff --git a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java similarity index 67% rename from core/src/test/java/org/elasticsearch/index/query/TemplateQueryParserTests.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java index 985fbfde894..29213f0ac0e 100644 --- a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryParserTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.query; +package org.elasticsearch.messy.tests; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.util.Accountable; import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; @@ -31,32 +32,46 @@ import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.inject.multibindings.Multibinder; import org.elasticsearch.common.inject.util.Providers; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.analysis.AnalysisModule; -import org.elasticsearch.index.cache.IndexCacheModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.TemplateQueryParser; import org.elasticsearch.index.query.functionscore.ScoreFunctionParser; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.index.similarity.SimilarityModule; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPoolModule; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.lang.reflect.Proxy; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; /** * Test parsing and executing a template request. @@ -77,27 +92,27 @@ public class TemplateQueryParserTests extends ESTestCase { .build(); final Client proxy = (Client) Proxy.newProxyInstance( Client.class.getClassLoader(), - new Class[]{Client.class}, (proxy1, method, args) -> { + new Class[]{Client.class}, (proxy1, method, args) -> { throw new UnsupportedOperationException("client is just a dummy"); }); Index index = new Index("test"); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, settings); + ScriptModule scriptModule = new ScriptModule(settings); + // TODO: make this use a mock engine instead of mustache and it will no longer be messy! + scriptModule.addScriptEngine(MustacheScriptEngineService.class); injector = new ModulesBuilder().add( new EnvironmentModule(new Environment(settings)), - new SettingsModule(settings), + new SettingsModule(settings, new SettingsFilter(settings)), new ThreadPoolModule(new ThreadPool(settings)), - new IndicesModule(settings) { + new IndicesModule() { @Override public void configure() { // skip services bindQueryParsersExtension(); } }, - new ScriptModule(settings), + scriptModule, new IndexSettingsModule(index, settings), - new IndexCacheModule(settings), - new AnalysisModule(settings, new IndicesAnalysisService(settings)), - new SimilarityModule(index, settings), - new IndexNameModule(index), new AbstractModule() { @Override protected void configure() { @@ -109,8 +124,25 @@ public class TemplateQueryParserTests extends ESTestCase { } ).createInjector(); - IndexQueryParserService queryParserService = injector.getInstance(IndexQueryParserService.class); - context = new QueryShardContext(index, queryParserService); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + ScriptService scriptService = injector.getInstance(ScriptService.class); + SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); + MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry(); + MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry); + IndexFieldDataService indexFieldDataService =new IndexFieldDataService(idxSettings, injector.getInstance(IndicesFieldDataCache.class), injector.getInstance(CircuitBreakerService.class), mapperService); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new IndicesWarmer(idxSettings.getNodeSettings(), null), new BitsetFilterCache.Listener() { + @Override + public void onCache(ShardId shardId, Accountable accountable) { + + } + + @Override + public void onRemoval(ShardId shardId, Accountable accountable) { + + } + }); + IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class); + context = new QueryShardContext(idxSettings, proxy, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry); } @Override @@ -120,7 +152,6 @@ public class TemplateQueryParserTests extends ESTestCase { terminate(injector.getInstance(ThreadPool.class)); } - @Test public void testParser() throws IOException { String templateString = "{" + "\"query\":{\"match_{{template}}\": {}}," + "\"params\":{\"template\":\"all\"}" + "}"; @@ -133,7 +164,6 @@ public class TemplateQueryParserTests extends ESTestCase { assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery); } - @Test public void testParseTemplateAsSingleStringWithConditionalClause() throws IOException { String templateString = "{" + " \"inline\" : \"{ \\\"match_{{#use_it}}{{template}}{{/use_it}}\\\":{} }\"," + " \"params\":{" + " \"template\":\"all\"," + " \"use_it\": true" + " }" + "}"; @@ -150,7 +180,6 @@ public class TemplateQueryParserTests extends ESTestCase { * expressed as a single string but still it expects only the query * specification (thus this test should fail with specific exception). */ - @Test(expected = ParsingException.class) public void testParseTemplateFailsToParseCompleteQueryAsSingleString() throws IOException { String templateString = "{" + " \"inline\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\"," + " \"params\":{" + " \"size\":2" + " }\n" + "}"; @@ -159,10 +188,14 @@ public class TemplateQueryParserTests extends ESTestCase { context.reset(templateSourceParser); TemplateQueryParser parser = injector.getInstance(TemplateQueryParser.class); - parser.fromXContent(context.parseContext()).toQuery(context); + try { + parser.fromXContent(context.parseContext()).toQuery(context); + fail("Expected ParsingException"); + } catch (ParsingException e) { + assertThat(e.getMessage(), containsString("query malformed, no field after start_object")); + } } - @Test public void testParserCanExtractTemplateNames() throws Exception { String templateString = "{ \"file\": \"storedTemplate\" ,\"params\":{\"template\":\"all\" } } "; diff --git a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java similarity index 83% rename from core/src/test/java/org/elasticsearch/index/query/TemplateQueryIT.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java index 0c9fc74cb5d..70298266df9 100644 --- a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.query; +package org.elasticsearch.messy.tests; import org.elasticsearch.action.index.IndexRequest.OpType; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -27,19 +27,27 @@ import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TemplateQueryBuilder; +import org.elasticsearch.index.query.TemplateQueryParser; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.Template; +import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -49,6 +57,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -56,7 +65,12 @@ import static org.hamcrest.Matchers.is; * Full integration test of the template query plugin. */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) -public class TemplateQueryIT extends ESIntegTestCase { +public class TemplateQueryTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MustachePlugin.class); + } @Before public void setup() throws IOException { @@ -74,7 +88,6 @@ public class TemplateQueryIT extends ESIntegTestCase { .put("path.conf", this.getDataPath("config")).build(); } - @Test public void testTemplateInBody() throws IOException { Map vars = new HashMap<>(); vars.put("template", "all"); @@ -86,42 +99,19 @@ public class TemplateQueryIT extends ESIntegTestCase { assertHitCount(sr, 2); } - @Test public void testTemplateInBodyWithSize() throws IOException { - String request = "{\n" + - " \"size\":0," + - " \"query\": {\n" + - " \"template\": {\n" + - " \"query\": {\"match_{{template}}\": {}},\n" + - " \"params\" : {\n" + - " \"template\" : \"all\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse sr = client().prepareSearch().setSource(new BytesArray(request)) - .execute().actionGet(); - assertNoFailures(sr); - assertThat(sr.getHits().hits().length, equalTo(0)); - request = "{\n" + - " \"query\": {\n" + - " \"template\": {\n" + - " \"query\": {\"match_{{template}}\": {}},\n" + - " \"params\" : {\n" + - " \"template\" : \"all\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"size\":0" + - "}"; - - sr = client().prepareSearch().setSource(new BytesArray(request)) - .execute().actionGet(); + Map params = new HashMap<>(); + params.put("template", "all"); + SearchResponse sr = client().prepareSearch() + .setSource( + new SearchSourceBuilder().size(0).query( + QueryBuilders.templateQuery(new Template("{ \"match_{{template}}\": {} }", + ScriptType.INLINE, null, null, params)))).execute() + .actionGet(); assertNoFailures(sr); assertThat(sr.getHits().hits().length, equalTo(0)); } - @Test public void testTemplateWOReplacementInBody() throws IOException { Map vars = new HashMap<>(); @@ -132,7 +122,6 @@ public class TemplateQueryIT extends ESIntegTestCase { assertHitCount(sr, 2); } - @Test public void testTemplateInFile() { Map vars = new HashMap<>(); vars.put("template", "all"); @@ -144,65 +133,50 @@ public class TemplateQueryIT extends ESIntegTestCase { assertHitCount(sr, 2); } - @Test - public void testRawEscapedTemplate() throws IOException { - String query = "{\"template\": {\"query\": \"{\\\"match_{{template}}\\\": {}}\\\"\",\"params\" : {\"template\" : \"all\"}}}"; - - SearchResponse sr = client().prepareSearch().setQuery(query).get(); - assertHitCount(sr, 2); - } - - @Test - public void testRawTemplate() throws IOException { - String query = "{\"template\": {\"query\": {\"match_{{template}}\": {}},\"params\" : {\"template\" : \"all\"}}}"; - SearchResponse sr = client().prepareSearch().setQuery(query).get(); - assertHitCount(sr, 2); - } - - @Test public void testRawFSTemplate() throws IOException { - String query = "{\"template\": {\"file\": \"storedTemplate\",\"params\" : {\"template\" : \"all\"}}}"; - - SearchResponse sr = client().prepareSearch().setQuery(query).get(); + Map params = new HashMap<>(); + params.put("template", "all"); + TemplateQueryBuilder builder = new TemplateQueryBuilder(new Template("storedTemplate", ScriptType.FILE, null, null, params)); + SearchResponse sr = client().prepareSearch().setQuery(builder).get(); assertHitCount(sr, 2); } - @Test public void testSearchRequestTemplateSource() throws Exception { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("_all"); String query = "{ \"template\" : { \"query\": {\"match_{{template}}\": {} } }, \"params\" : { \"template\":\"all\" } }"; - BytesReference bytesRef = new BytesArray(query); - searchRequest.templateSource(bytesRef); + searchRequest.template(parseTemplate(query)); SearchResponse searchResponse = client().search(searchRequest).get(); assertHitCount(searchResponse, 2); } - @Test + private Template parseTemplate(String template) throws IOException { + try (XContentParser parser = XContentFactory.xContent(template).createParser(template)) { + return TemplateQueryParser.parse(parser, ParseFieldMatcher.EMPTY, "params", "template"); + } + } + // Releates to #6318 public void testSearchRequestFail() throws Exception { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("_all"); try { String query = "{ \"template\" : { \"query\": {\"match_all\": {}}, \"size\" : \"{{my_size}}\" } }"; - BytesReference bytesRef = new BytesArray(query); - searchRequest.templateSource(bytesRef); + searchRequest.template(parseTemplate(query)); client().search(searchRequest).get(); fail("expected exception"); } catch (Exception ex) { // expected - no params } String query = "{ \"template\" : { \"query\": {\"match_all\": {}}, \"size\" : \"{{my_size}}\" }, \"params\" : { \"my_size\": 1 } }"; - BytesReference bytesRef = new BytesArray(query); - searchRequest.templateSource(bytesRef); + searchRequest.template(parseTemplate(query)); SearchResponse searchResponse = client().search(searchRequest).get(); assertThat(searchResponse.getHits().hits().length, equalTo(1)); } - @Test public void testThatParametersCanBeSet() throws Exception { index("test", "type", "1", jsonBuilder().startObject().field("theField", "foo").endObject()); index("test", "type", "2", jsonBuilder().startObject().field("theField", "foo 2").endObject()); @@ -230,14 +204,12 @@ public class TemplateQueryIT extends ESIntegTestCase { assertHitCount(searchResponse, 1); } - @Test public void testSearchTemplateQueryFromFile() throws Exception { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("_all"); - String templateString = "{" + " \"file\": \"full-query-template\"," + " \"params\":{" + " \"mySize\": 2," + String query = "{" + " \"file\": \"full-query-template\"," + " \"params\":{" + " \"mySize\": 2," + " \"myField\": \"text\"," + " \"myValue\": \"value1\"" + " }" + "}"; - BytesReference bytesRef = new BytesArray(templateString); - searchRequest.templateSource(bytesRef); + searchRequest.template(parseTemplate(query)); SearchResponse searchResponse = client().search(searchRequest).get(); assertThat(searchResponse.getHits().hits().length, equalTo(1)); } @@ -245,14 +217,12 @@ public class TemplateQueryIT extends ESIntegTestCase { /** * Test that template can be expressed as a single escaped string. */ - @Test public void testTemplateQueryAsEscapedString() throws Exception { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("_all"); - String templateString = "{" + " \"template\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\"," + String query = "{" + " \"template\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\"," + " \"params\":{" + " \"size\": 1" + " }" + "}"; - BytesReference bytesRef = new BytesArray(templateString); - searchRequest.templateSource(bytesRef); + searchRequest.template(parseTemplate(query)); SearchResponse searchResponse = client().search(searchRequest).get(); assertThat(searchResponse.getHits().hits().length, equalTo(1)); } @@ -261,15 +231,13 @@ public class TemplateQueryIT extends ESIntegTestCase { * Test that template can contain conditional clause. In this case it is at * the beginning of the string. */ - @Test public void testTemplateQueryAsEscapedStringStartingWithConditionalClause() throws Exception { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("_all"); String templateString = "{" + " \"template\" : \"{ {{#use_size}} \\\"size\\\": \\\"{{size}}\\\", {{/use_size}} \\\"query\\\":{\\\"match_all\\\":{}}}\"," + " \"params\":{" + " \"size\": 1," + " \"use_size\": true" + " }" + "}"; - BytesReference bytesRef = new BytesArray(templateString); - searchRequest.templateSource(bytesRef); + searchRequest.template(parseTemplate(templateString)); SearchResponse searchResponse = client().search(searchRequest).get(); assertThat(searchResponse.getHits().hits().length, equalTo(1)); } @@ -278,20 +246,17 @@ public class TemplateQueryIT extends ESIntegTestCase { * Test that template can contain conditional clause. In this case it is at * the end of the string. */ - @Test public void testTemplateQueryAsEscapedStringWithConditionalClauseAtEnd() throws Exception { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("_all"); String templateString = "{" + " \"inline\" : \"{ \\\"query\\\":{\\\"match_all\\\":{}} {{#use_size}}, \\\"size\\\": \\\"{{size}}\\\" {{/use_size}} }\"," + " \"params\":{" + " \"size\": 1," + " \"use_size\": true" + " }" + "}"; - BytesReference bytesRef = new BytesArray(templateString); - searchRequest.templateSource(bytesRef); + searchRequest.template(parseTemplate(templateString)); SearchResponse searchResponse = client().search(searchRequest).get(); assertThat(searchResponse.getHits().hits().length, equalTo(1)); } - @Test(expected = SearchPhaseExecutionException.class) public void testIndexedTemplateClient() throws Exception { createIndex(ScriptService.SCRIPT_INDEX); ensureGreen(ScriptService.SCRIPT_INDEX); @@ -346,14 +311,18 @@ public class TemplateQueryIT extends ESIntegTestCase { getResponse = client().prepareGetIndexedScript(MustacheScriptEngineService.NAME, "testTemplate").get(); assertFalse(getResponse.isExists()); - client().prepareSearch("test") - .setTypes("type") - .setTemplate( - new Template("/template_index/mustache/1000", ScriptType.INDEXED, MustacheScriptEngineService.NAME, null, - templateParams)).get(); + try { + client().prepareSearch("test") + .setTypes("type") + .setTemplate( + new Template("/template_index/mustache/1000", ScriptType.INDEXED, MustacheScriptEngineService.NAME, null, + templateParams)).get(); + fail("Expected SearchPhaseExecutionException"); + } catch (SearchPhaseExecutionException e) { + assertThat(e.toString(), containsString("Illegal index script format")); + } } - @Test public void testIndexedTemplate() throws Exception { createIndex(ScriptService.SCRIPT_INDEX); ensureGreen(ScriptService.SCRIPT_INDEX); @@ -451,17 +420,19 @@ public class TemplateQueryIT extends ESIntegTestCase { .execute().actionGet(); assertHitCount(sr, 1); - String query = "{\"template\": {\"id\": \"3\",\"params\" : {\"fieldParam\" : \"foo\"}}}"; - sr = client().prepareSearch().setQuery(query).get(); + // "{\"template\": {\"id\": \"3\",\"params\" : {\"fieldParam\" : \"foo\"}}}"; + Map params = new HashMap<>(); + params.put("fieldParam", "foo"); + TemplateQueryBuilder templateQuery = new TemplateQueryBuilder(new Template("3", ScriptType.INDEXED, null, null, params)); + sr = client().prepareSearch().setQuery(templateQuery).get(); assertHitCount(sr, 4); - query = "{\"template\": {\"id\": \"/mustache/3\",\"params\" : {\"fieldParam\" : \"foo\"}}}"; - sr = client().prepareSearch().setQuery(query).get(); + templateQuery = new TemplateQueryBuilder(new Template("/mustache/3", ScriptType.INDEXED, null, null, params)); + sr = client().prepareSearch().setQuery(templateQuery).get(); assertHitCount(sr, 4); } // Relates to #10397 - @Test public void testIndexedTemplateOverwrite() throws Exception { createIndex("testindex"); ensureGreen("testindex"); @@ -471,7 +442,7 @@ public class TemplateQueryIT extends ESIntegTestCase { int iterations = randomIntBetween(2, 11); for (int i = 1; i < iterations; i++) { - PutIndexedScriptResponse scriptResponse = client().preparePutIndexedScript(MustacheScriptEngineService.NAME, "git01", + PutIndexedScriptResponse scriptResponse = client().preparePutIndexedScript(MustacheScriptEngineService.NAME, "git01", "{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\",\"type\": \"ooophrase_prefix\"}}}}").get(); assertEquals(i * 2 - 1, scriptResponse.getVersion()); @@ -507,8 +478,6 @@ public class TemplateQueryIT extends ESIntegTestCase { } } - - @Test public void testIndexedTemplateWithArray() throws Exception { createIndex(ScriptService.SCRIPT_INDEX); ensureGreen(ScriptService.SCRIPT_INDEX); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java new file mode 100644 index 00000000000..a2325b2d511 --- /dev/null +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This package contains tests that use mustache to test what looks + * to be unrelated functionality, or functionality that should be + * tested with a mock instead. Instead of doing an epic battle + * with these tests, they are temporarily moved here to the mustache + * module's tests, but that is likely not where they belong. Please + * help by cleaning them up and we can remove this package! + * + *
      + *
    • If the test is actually testing mustache specifically, move to + * the org.elasticsearch.script.mustache tests package of this module
      • + *
    • If the test is testing templating integration with another core subsystem, + * fix it to use a mock instead, so it can be in the core tests again
      • + *
    • If the test is just being lazy, and does not really need templating to test + * something, clean it up!
      • + *
    + */ +/* List of renames that took place: +renamed: core/src/test/java/org/elasticsearch/validate/RenderSearchTemplateIT.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java +renamed: core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java +renamed: core/src/test/java/org/elasticsearch/index/query/TemplateQueryParserTests.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java +renamed: core/src/test/java/org/elasticsearch/index/query/TemplateQueryIT.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java +renamed: core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java -> module/lang-mustache/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java + ^^^^^ note: just the methods from this test using mustache were moved here, the others use groovy and are in the groovy module under its messy tests package. +renamed: rest-api-spec/test/msearch/10_basic.yaml -> module/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml + */ + +package org.elasticsearch.messy.tests; diff --git a/core/src/test/java/org/elasticsearch/test/rest/Rest1IT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheRestIT.java similarity index 68% rename from core/src/test/java/org/elasticsearch/test/rest/Rest1IT.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheRestIT.java index bc80123debc..0c489b3afb1 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/Rest1IT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheRestIT.java @@ -17,22 +17,32 @@ * under the License. */ -package org.elasticsearch.test.rest; +package org.elasticsearch.script.mustache; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; -/** Rest API tests subset 1 */ -public class Rest1IT extends ESRestTestCase { - public Rest1IT(@Name("yaml") RestTestCandidate testCandidate) { +public class MustacheRestIT extends ESRestTestCase { + + @Override + protected Collection> nodePlugins() { + return pluginList(MustachePlugin.class); + } + + public MustacheRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } + @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(1, 8); + return ESRestTestCase.createParameters(0, 1); } } + diff --git a/core/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java similarity index 96% rename from core/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index 28ae80809a1..ce29bf246be 100644 --- a/core/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import org.junit.Test; import java.io.IOException; import java.io.StringWriter; @@ -47,7 +46,6 @@ public class MustacheScriptEngineTests extends ESTestCase { escaper = new JsonEscapingMustacheFactory(); } - @Test public void testSimpleParameterReplace() { { String template = "GET _search {\"query\": " + "{\"boosting\": {" + "\"positive\": {\"match\": {\"body\": \"gift\"}}," @@ -72,7 +70,6 @@ public class MustacheScriptEngineTests extends ESTestCase { } } - @Test public void testEscapeJson() throws IOException { { StringWriter writer = new StringWriter(); @@ -86,9 +83,9 @@ public class MustacheScriptEngineTests extends ESTestCase { } Character[] specialChars = new Character[]{ - '\"', - '\\', - '\u0000', + '\"', + '\\', + '\u0000', '\u0001', '\u0002', '\u0003', @@ -104,9 +101,9 @@ public class MustacheScriptEngineTests extends ESTestCase { '\u000F', '\u001F'}; String[] escapedChars = new String[]{ - "\\\"", - "\\\\", - "\\u0000", + "\\\"", + "\\\\", + "\\u0000", "\\u0001", "\\u0002", "\\u0003", @@ -150,14 +147,14 @@ public class MustacheScriptEngineTests extends ESTestCase { } return string; } - + /** * From https://www.ietf.org/rfc/rfc4627.txt: - * + * * All Unicode characters may be placed within the * quotation marks except for the characters that must be escaped: * quotation mark, reverse solidus, and the control characters (U+0000 - * through U+001F). + * through U+001F). * */ private static boolean isEscapeChar(char c) { switch (c) { @@ -165,7 +162,7 @@ public class MustacheScriptEngineTests extends ESTestCase { case '\\': return true; } - + if (c < '\u002F') return true; return false; diff --git a/core/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java similarity index 98% rename from core/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index 9bda581b6d5..76c867802a9 100644 --- a/core/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -21,8 +21,8 @@ package org.elasticsearch.script.mustache; import com.github.mustachejava.DefaultMustacheFactory; import com.github.mustachejava.Mustache; import com.github.mustachejava.MustacheFactory; + import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.StringReader; import java.io.StringWriter; @@ -32,8 +32,6 @@ import java.util.HashMap; * Figure out how Mustache works for the simplest use case. Leaving in here for now for reference. * */ public class MustacheTests extends ESTestCase { - - @Test public void test() { HashMap scopes = new HashMap<>(); scopes.put("boost_val", "0.2"); diff --git a/core/src/test/resources/org/elasticsearch/index/query/config/scripts/full-query-template.mustache b/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/full-query-template.mustache similarity index 100% rename from core/src/test/resources/org/elasticsearch/index/query/config/scripts/full-query-template.mustache rename to modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/full-query-template.mustache diff --git a/core/src/test/resources/org/elasticsearch/index/query/config/scripts/storedTemplate.mustache b/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/storedTemplate.mustache similarity index 100% rename from core/src/test/resources/org/elasticsearch/index/query/config/scripts/storedTemplate.mustache rename to modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/storedTemplate.mustache diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/template/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml similarity index 80% rename from rest-api-spec/src/main/resources/rest-api-spec/test/template/10_basic.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml index bd1fd436648..9bfea28abfa 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/template/10_basic.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml @@ -1,3 +1,18 @@ +# Integration tests for Mustache scripts +# +"Mustache loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.modules.0.name: lang-mustache } + - match: { nodes.$master.modules.0.jvm: true } + --- "Indexed template": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/template/20_search.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_search.yaml similarity index 79% rename from rest-api-spec/src/main/resources/rest-api-spec/test/template/20_search.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_search.yaml index 5153f6cde1f..4da748aac80 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/template/20_search.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_search.yaml @@ -26,11 +26,6 @@ indices.refresh: {} - - do: - search_template: - body: { "template": { "id" : "1" }, "params" : { "my_value" : "value1_foo", "my_size" : 1 } } - - match: { hits.total: 1 } - - do: search_template: body: { "id" : "1", "params" : { "my_value" : "value1_foo", "my_size" : 1 } } @@ -39,5 +34,5 @@ - do: catch: /Unable.to.find.on.disk.file.script.\[simple1\].using.lang.\[mustache\]/ search_template: - body: { "template" : "simple1" } + body: { "file" : "simple1"} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/template/30_render_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_render_search_template.yaml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/template/30_render_search_template.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_render_search_template.yaml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_template_query_execution.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_template_query_execution.yaml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search/30_template_query_execution.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_template_query_execution.yaml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/40_search_request_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search/40_search_request_template.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml similarity index 99% rename from rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml index 49e34fb16cd..205070be13e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml @@ -51,3 +51,4 @@ - query: { "template": { "query": { "term": { "foo": { "value": "{{template}}" } } }, "params": { "template": "bar" } } } - match: { responses.0.hits.total: 1 } + diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle new file mode 100644 index 00000000000..a662f727eeb --- /dev/null +++ b/plugins/analysis-icu/build.gradle @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.' + classname 'org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin' +} + +dependencies { + compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}" + compile 'com.ibm.icu:icu4j:54.1' +} + +dependencyLicenses { + mapping from: /lucene-.*/, to: 'lucene' +} + +compileJava.options.compilerArgs << "-Xlint:-deprecation" + diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 8d39139d91c..00000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a71d7e2780ae064a6e6ea5b43357d070351620b8 diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.5.0-snapshot-1719088.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..4942bbc6af3 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +4e56ba76d6b23756b2bd4d9e42b2b00122cd4fa5 \ No newline at end of file diff --git a/plugins/analysis-icu/pom.xml b/plugins/analysis-icu/pom.xml deleted file mode 100644 index 6f204d6dd91..00000000000 --- a/plugins/analysis-icu/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - analysis-icu - Plugin: Analysis: ICU - The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components. - - - org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin - analysis_icu - false - -Xlint:-deprecation - - - - - org.apache.lucene - lucene-analyzers-icu - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - - - com.mycila - license-maven-plugin - - - - **/IndexableBinaryStringTools.java - **/ICUCollationKeyFilter.java - **/TestIndexableBinaryStringTools.java - - - - - - - - diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalysisBinderProcessor.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalysisBinderProcessor.java deleted file mode 100644 index 8db169b9318..00000000000 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalysisBinderProcessor.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis; - -/** - */ -public class IcuAnalysisBinderProcessor extends AnalysisModule.AnalysisBinderProcessor { - - @Override - public void processCharFilters(CharFiltersBindings charFiltersBindings) { - charFiltersBindings.processCharFilter("icu_normalizer", IcuNormalizerCharFilterFactory.class); - } - - @Override - public void processTokenizers(TokenizersBindings tokenizersBindings) { - tokenizersBindings.processTokenizer("icu_tokenizer", IcuTokenizerFactory.class); - } - - @Override - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - tokenFiltersBindings.processTokenFilter("icu_normalizer", IcuNormalizerTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("icu_folding", IcuFoldingTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("icu_collation", IcuCollationTokenFilterFactory.class); - tokenFiltersBindings.processTokenFilter("icu_transform", IcuTransformTokenFilterFactory.class); - } -} diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java index ca4be807278..1e7cd1b09f4 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java @@ -23,13 +23,10 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; import org.apache.lucene.analysis.TokenStream; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import java.io.IOException; import java.nio.charset.Charset; @@ -48,9 +45,8 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory { private final Collator collator; - @Inject - public IcuCollationTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, Environment environment, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public IcuCollationTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); Collator collator; String rules = settings.get("rules"); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java index 7abfd702ac3..51243856a1f 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java @@ -21,15 +21,13 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.icu.ICUFoldingFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; import com.ibm.icu.text.FilteredNormalizer2; import com.ibm.icu.text.Normalizer2; import com.ibm.icu.text.UnicodeSet; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** @@ -46,12 +44,13 @@ import com.ibm.icu.text.UnicodeSet; public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory { private final String unicodeSetFilter; - @Inject public IcuFoldingTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public IcuFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); this.unicodeSetFilter = settings.get("unicodeSetFilter"); } - @Override public TokenStream create(TokenStream tokenStream) { + @Override + public TokenStream create(TokenStream tokenStream) { // The ICUFoldingFilter is in fact implemented as a ICUNormalizer2Filter. // ICUFoldingFilter lacks a constructor for adding filtering so we implemement it here diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java index d8fec090a3f..02f9b5a3371 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java @@ -22,11 +22,9 @@ package org.elasticsearch.index.analysis; import com.ibm.icu.text.Normalizer2; import org.apache.lucene.analysis.icu.ICUNormalizer2CharFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; import java.io.Reader; @@ -42,10 +40,8 @@ public class IcuNormalizerCharFilterFactory extends AbstractCharFilterFactory { private final Normalizer2 normalizer; - - @Inject - public IcuNormalizerCharFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name); + public IcuNormalizerCharFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name); this.name = settings.get("name", "nfkc_cf"); String mode = settings.get("mode"); if (!"compose".equals(mode) && !"decompose".equals(mode)) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java index c27fc1d16a9..6f830b29d15 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java @@ -24,8 +24,8 @@ import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** @@ -38,9 +38,8 @@ public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory private final String name; - @Inject - public IcuNormalizerTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public IcuNormalizerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); this.name = settings.get("name", "nfkc_cf"); } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java index fe20d93069e..0d2a6cdeb22 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java @@ -21,21 +21,16 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; - -import java.io.Reader; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** */ public class IcuTokenizerFactory extends AbstractTokenizerFactory { - @Inject - public IcuTokenizerFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public IcuTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); } @Override diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java index 8ef48539daf..1d5136f60e1 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java @@ -25,8 +25,8 @@ import org.apache.lucene.analysis.icu.ICUTransformFilter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; /** @@ -38,8 +38,8 @@ public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory { private final Transliterator transliterator; @Inject - public IcuTransformTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public IcuTransformTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); this.id = settings.get("id", "Null"); String s = settings.get("dir", "forward"); this.dir = "forward".equals(s) ? Transliterator.FORWARD : Transliterator.REVERSE; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IndexableBinaryStringTools.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IndexableBinaryStringTools.java index a114a34d7dd..82be3c66159 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IndexableBinaryStringTools.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IndexableBinaryStringTools.java @@ -37,7 +37,6 @@ import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; // ja * final content char. *

    * - * @lucene.experimental * @deprecated Implement {@link TermToBytesRefAttribute} and store bytes directly * instead. This class WAS removed in Lucene 5.0 */ diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/indices/analysis/IcuIndicesAnalysis.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/indices/analysis/IcuIndicesAnalysis.java deleted file mode 100644 index cc7d56994da..00000000000 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/indices/analysis/IcuIndicesAnalysis.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.analysis; - -import com.ibm.icu.text.Collator; -import com.ibm.icu.text.Normalizer2; -import com.ibm.icu.text.Transliterator; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.icu.ICUFoldingFilter; -import org.apache.lucene.analysis.icu.ICUNormalizer2CharFilter; -import org.apache.lucene.analysis.icu.ICUTransformFilter; -import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.analysis.ICUCollationKeyFilter; -import org.elasticsearch.index.analysis.PreBuiltCharFilterFactoryFactory; -import org.elasticsearch.index.analysis.PreBuiltTokenFilterFactoryFactory; -import org.elasticsearch.index.analysis.PreBuiltTokenizerFactoryFactory; -import org.elasticsearch.index.analysis.CharFilterFactory; -import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.index.analysis.TokenizerFactory; - -import java.io.Reader; - -/** - * Registers indices level analysis components so, if not explicitly configured, will be shared - * among all indices. - */ -public class IcuIndicesAnalysis extends AbstractComponent { - - @Inject - public IcuIndicesAnalysis(Settings settings, IndicesAnalysisService indicesAnalysisService) { - super(settings); - - indicesAnalysisService.tokenizerFactories().put("icu_tokenizer", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() { - @Override - public String name() { - return "icu_tokenizer"; - } - - @Override - public Tokenizer create() { - return new ICUTokenizer(); - } - })); - - indicesAnalysisService.tokenFilterFactories().put("icu_normalizer", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "icu_normalizer"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE)); - } - })); - - - indicesAnalysisService.tokenFilterFactories().put("icu_folding", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "icu_folding"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new ICUFoldingFilter(tokenStream); - } - })); - - indicesAnalysisService.tokenFilterFactories().put("icu_collation", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "icu_collation"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new ICUCollationKeyFilter(tokenStream, Collator.getInstance()); - } - })); - - indicesAnalysisService.tokenFilterFactories().put("icu_transform", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "icu_transform"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new ICUTransformFilter(tokenStream, Transliterator.getInstance("Null", Transliterator.FORWARD)); - } - })); - - indicesAnalysisService.charFilterFactories().put("icu_normalizer", new PreBuiltCharFilterFactoryFactory(new CharFilterFactory() { - @Override - public String name() { - return "icu_normalizer"; - } - - @Override - public Reader create(Reader reader) { - return new ICUNormalizer2CharFilter(reader); - } - })); - } -} diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java index 6b9314cacff..47c2f8f051a 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java @@ -19,15 +19,10 @@ package org.elasticsearch.plugin.analysis.icu; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.index.analysis.AnalysisModule; -import org.elasticsearch.index.analysis.IcuAnalysisBinderProcessor; -import org.elasticsearch.indices.analysis.IcuIndicesAnalysisModule; +import org.elasticsearch.index.analysis.*; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - /** * */ @@ -43,15 +38,15 @@ public class AnalysisICUPlugin extends Plugin { return "UTF related ICU analysis support"; } - @Override - public Collection nodeModules() { - return Collections.singletonList(new IcuIndicesAnalysisModule()); - } - /** * Automatically called with the analysis module. */ public void onModule(AnalysisModule module) { - module.addProcessor(new IcuAnalysisBinderProcessor()); + module.registerCharFilter("icu_normalizer", IcuNormalizerCharFilterFactory::new); + module.registerTokenizer("icu_tokenizer", IcuTokenizerFactory::new); + module.registerTokenFilter("icu_normalizer", IcuNormalizerTokenFilterFactory::new); + module.registerTokenFilter("icu_folding", IcuFoldingTokenFilterFactory::new); + module.registerTokenFilter("icu_collation", IcuCollationTokenFilterFactory::new); + module.registerTokenFilter("icu_transform", IcuTransformTokenFilterFactory::new); } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisTestUtils.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisTestUtils.java index eceee3ef404..36a43e344d9 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisTestUtils.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisTestUtils.java @@ -24,30 +24,32 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; import static org.elasticsearch.common.settings.Settings.settingsBuilder; public class AnalysisTestUtils { - public static AnalysisService createAnalysisService(Settings settings) { + public static AnalysisService createAnalysisService(Settings settings) throws IOException { Index index = new Index("test"); Settings indexSettings = settingsBuilder().put(settings) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, indexSettings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)).addProcessor(new IcuAnalysisBinderProcessor())) - .createChildInjector(parentInjector); - - return injector.getInstance(AnalysisService.class); + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings)); + new AnalysisICUPlugin().onModule(analysisModule); + Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings, new SettingsFilter(settings)), + new EnvironmentModule(new Environment(settings)), analysisModule) + .createInjector(); + final AnalysisService analysisService = parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, indexSettings)); + return analysisService; } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java index 8369809aee2..d4b2530dbb6 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java @@ -21,7 +21,8 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; + +import java.io.IOException; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.analysis.AnalysisTestUtils.createAnalysisService; @@ -29,9 +30,7 @@ import static org.hamcrest.Matchers.instanceOf; /** */ public class SimpleIcuAnalysisTests extends ESTestCase { - - @Test - public void testDefaultsIcuAnalysis() { + public void testDefaultsIcuAnalysis() throws IOException { Settings settings = settingsBuilder() .put("path.home", createTempDir()).build(); AnalysisService analysisService = createAnalysisService(settings); diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java index 6be0b2f23af..9e59b8e42c3 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java @@ -22,13 +22,13 @@ package org.elasticsearch.index.analysis; import com.ibm.icu.text.Collator; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; + import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.io.StringReader; @@ -38,14 +38,12 @@ import static org.hamcrest.Matchers.equalTo; // Tests borrowed from Solr's Icu collation key filter factory test. public class SimpleIcuCollationTokenFilterTests extends ESTestCase { - /* * Turkish has some funny casing. * This test shows how you can solve this kind of thing easily with collation. * Instead of using LowerCaseFilter, use a turkish collator with primary strength. * Then things will sort and match correctly. */ - @Test public void testBasicUsage() throws Exception { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -62,7 +60,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { /* * Test usage of the decomposition option for unicode normalization. */ - @Test public void testNormalization() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -80,7 +77,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { /* * Test secondary strength, for english case is not significant. */ - @Test public void testSecondaryStrength() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -99,7 +95,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { * Setting alternate=shifted to shift whitespace, punctuation and symbols * to quaternary level */ - @Test public void testIgnorePunctuation() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -118,7 +113,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { * Setting alternate=shifted and variableTop to shift whitespace, but not * punctuation or symbols, to quaternary level */ - @Test public void testIgnoreWhitespace() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -140,7 +134,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { * Setting numeric to encode digits with numeric value, so that * foobar-9 sorts before foobar-10 */ - @Test public void testNumerics() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -158,7 +151,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { * Setting caseLevel=true to create an additional case level between * secondary and tertiary */ - @Test public void testIgnoreAccentsButNotCase() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -180,7 +172,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { * Setting caseFirst=upper to cause uppercase strings to sort * before lowercase ones. */ - @Test public void testUpperCaseFirst() throws IOException { Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) @@ -202,7 +193,6 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { * The default is DIN 5007-1, this shows how to tailor a collator to get DIN 5007-2 behavior. * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4423383 */ - @Test public void testCustomRules() throws Exception { RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new ULocale("de_DE")); String DIN5007_2_tailorings = @@ -224,20 +214,20 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator"); assertCollatesToSame(filterFactory, "Töne", "Toene"); } - + private void assertCollatesToSame(TokenFilterFactory factory, String string1, String string2) throws IOException { assertCollation(factory, string1, string2, 0); } - + private void assertCollation(TokenFilterFactory factory, String string1, String string2, int comparison) throws IOException { Tokenizer tokenizer = new KeywordTokenizer(); tokenizer.setReader(new StringReader(string1)); TokenStream stream1 = factory.create(tokenizer); - + tokenizer = new KeywordTokenizer(); tokenizer.setReader(new StringReader(string2)); TokenStream stream2 = factory.create(tokenizer); - + assertCollation(stream1, stream2, comparison); } @@ -253,10 +243,10 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase { assertThat(Integer.signum(term1.toString().compareTo(term2.toString())), equalTo(Integer.signum(comparison))); assertThat(stream1.incrementToken(), equalTo(false)); assertThat(stream2.incrementToken(), equalTo(false)); - + stream1.end(); stream2.end(); - + stream1.close(); stream2.close(); } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java index 748b439a9ec..bd2f959bf9c 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java @@ -20,10 +20,10 @@ package org.elasticsearch.index.analysis; import com.ibm.icu.text.Normalizer2; + import org.apache.lucene.analysis.CharFilter; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.StringReader; @@ -33,10 +33,7 @@ import static org.elasticsearch.index.analysis.AnalysisTestUtils.createAnalysisS * Test */ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase { - - @Test public void testDefaultSetting() throws Exception { - Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) .put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer") @@ -59,10 +56,7 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase { assertEquals(expectedOutput, output.toString()); } - - @Test public void testNameAndModeSetting() throws Exception { - Settings settings = Settings.settingsBuilder() .put("path.home", createTempDir()) .put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer") diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle new file mode 100644 index 00000000000..333818a8d61 --- /dev/null +++ b/plugins/analysis-kuromoji/build.gradle @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.' + classname 'org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin' +} + +dependencies { + compile "org.apache.lucene:lucene-analyzers-kuromoji:${versions.lucene}" +} + +dependencyLicenses { + mapping from: /lucene-.*/, to: 'lucene' +} + diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 69991e5f6ad..00000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a82b7a125e7cc16c6eb050b68bafc9f1e63eb646 diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.5.0-snapshot-1719088.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..1ba2a93066d --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +d6ccac802dc1e4c177be043a173377cf5e517cff \ No newline at end of file diff --git a/plugins/analysis-kuromoji/pom.xml b/plugins/analysis-kuromoji/pom.xml deleted file mode 100644 index 9b28307ac0f..00000000000 --- a/plugins/analysis-kuromoji/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - analysis-kuromoji - Plugin: Analysis: Japanese (kuromoji) - The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch. - - - org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin - analysis_kuromoji - false - - - - - org.apache.lucene - lucene-analyzers-kuromoji - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java index 951597111e5..7b760bc4f63 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java @@ -25,12 +25,9 @@ import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.ja.JapaneseAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.search.suggest.analyzing.SuggestStopFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import java.util.Map; import java.util.Set; @@ -46,9 +43,8 @@ public class JapaneseStopTokenFilterFactory extends AbstractTokenFilterFactory{ private final boolean removeTrailing; - @Inject - public JapaneseStopTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public JapaneseStopTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); this.ignoreCase = settings.getAsBoolean("ignore_case", false); this.removeTrailing = settings.getAsBoolean("remove_trailing", true); this.stopWords = Analysis.parseWords(env, settings, "stopwords", JapaneseAnalyzer.getDefaultStopSet(), NAMED_STOP_WORDS, ignoreCase); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java index 43fd2f75a73..8aa8ff3c1dd 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java @@ -24,11 +24,9 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer; import org.apache.lucene.analysis.ja.dict.UserDictionary; import org.apache.lucene.analysis.util.CharArraySet; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import java.util.Set; @@ -39,8 +37,8 @@ public class KuromojiAnalyzerProvider extends AbstractIndexAnalyzerProvider stopWords = Analysis.parseStopWords(env, settings, JapaneseAnalyzer.getDefaultStopSet()); final JapaneseTokenizer.Mode mode = KuromojiTokenizerFactory.getMode(settings); final UserDictionary userDictionary = KuromojiTokenizerFactory.getUserDictionary(env, settings); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java index 0db43bd429a..f363cf0c15c 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java @@ -24,14 +24,14 @@ import org.apache.lucene.analysis.ja.JapaneseBaseFormFilter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; public class KuromojiBaseFormFilterFactory extends AbstractTokenFilterFactory { @Inject - public KuromojiBaseFormFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KuromojiBaseFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); } @Override diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java index 74b63c32afd..a1220dba2be 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java @@ -20,11 +20,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.ja.JapaneseIterationMarkCharFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; import java.io.Reader; @@ -33,10 +31,8 @@ public class KuromojiIterationMarkCharFilterFactory extends AbstractCharFilterFa private final boolean normalizeKanji; private final boolean normalizeKana; - @Inject - public KuromojiIterationMarkCharFilterFactory(Index index, @IndexSettings Settings indexSettings, - @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name); + public KuromojiIterationMarkCharFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name); normalizeKanji = settings.getAsBoolean("normalize_kanji", JapaneseIterationMarkCharFilter.NORMALIZE_KANJI_DEFAULT); normalizeKana = settings.getAsBoolean("normalize_kana", JapaneseIterationMarkCharFilter.NORMALIZE_KANA_DEFAULT); } diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java index 93db459b865..7d25ca03fdb 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java @@ -24,16 +24,16 @@ import org.apache.lucene.analysis.ja.JapaneseKatakanaStemFilter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; public class KuromojiKatakanaStemmerFactory extends AbstractTokenFilterFactory { private final int minimumLength; @Inject - public KuromojiKatakanaStemmerFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KuromojiKatakanaStemmerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); minimumLength = settings.getAsInt("minimum_length", JapaneseKatakanaStemFilter.DEFAULT_MINIMUM_LENGTH); } diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java index 12a29a0741a..4598f6293ce 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiPartOfSpeechFilterFactory.java @@ -21,12 +21,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.ja.JapanesePartOfSpeechStopFilter; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import java.util.HashSet; import java.util.List; @@ -36,9 +33,8 @@ public class KuromojiPartOfSpeechFilterFactory extends AbstractTokenFilterFactor private final Set stopTags = new HashSet(); - @Inject - public KuromojiPartOfSpeechFilterFactory(Index index, @IndexSettings Settings indexSettings, Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KuromojiPartOfSpeechFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); List wordList = Analysis.getWordList(env, settings, "stoptags"); if (wordList != null) { stopTags.addAll(wordList); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java index a87ac8c8256..1d4ecc2c33d 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java @@ -24,16 +24,16 @@ import org.apache.lucene.analysis.ja.JapaneseReadingFormFilter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; public class KuromojiReadingFormFilterFactory extends AbstractTokenFilterFactory { private final boolean useRomaji; @Inject - public KuromojiReadingFormFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KuromojiReadingFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); useRomaji = settings.getAsBoolean("use_romaji", false); } diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java index 31b759c5e65..87e08c757b4 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java @@ -24,12 +24,9 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer; import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode; import org.apache.lucene.analysis.ja.dict.UserDictionary; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import java.io.IOException; import java.io.Reader; @@ -45,9 +42,8 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory { private boolean discartPunctuation; - @Inject - public KuromojiTokenizerFactory(Index index, @IndexSettings Settings indexSettings, Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); mode = getMode(settings); userDictionary = getUserDictionary(env, settings); discartPunctuation = settings.getAsBoolean("discard_punctuation", true); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/indices/analysis/KuromojiIndicesAnalysis.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/indices/analysis/KuromojiIndicesAnalysis.java deleted file mode 100644 index ba5d58073a3..00000000000 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/indices/analysis/KuromojiIndicesAnalysis.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.analysis; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.ja.*; -import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.analysis.*; - -import java.io.Reader; - -/** - * Registers indices level analysis components so, if not explicitly configured, - * will be shared among all indices. - */ -public class KuromojiIndicesAnalysis extends AbstractComponent { - - @Inject - public KuromojiIndicesAnalysis(Settings settings, - IndicesAnalysisService indicesAnalysisService) { - super(settings); - - indicesAnalysisService.analyzerProviderFactories().put("kuromoji", - new PreBuiltAnalyzerProviderFactory("kuromoji", AnalyzerScope.INDICES, - new JapaneseAnalyzer())); - - indicesAnalysisService.charFilterFactories().put("kuromoji_iteration_mark", - new PreBuiltCharFilterFactoryFactory(new CharFilterFactory() { - @Override - public String name() { - return "kuromoji_iteration_mark"; - } - - @Override - public Reader create(Reader reader) { - return new JapaneseIterationMarkCharFilter(reader, - JapaneseIterationMarkCharFilter.NORMALIZE_KANJI_DEFAULT, - JapaneseIterationMarkCharFilter.NORMALIZE_KANA_DEFAULT); - } - })); - - indicesAnalysisService.tokenizerFactories().put("kuromoji_tokenizer", - new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() { - @Override - public String name() { - return "kuromoji_tokenizer"; - } - - @Override - public Tokenizer create() { - return new JapaneseTokenizer(null, true, Mode.SEARCH); - } - })); - - indicesAnalysisService.tokenFilterFactories().put("kuromoji_baseform", - new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "kuromoji_baseform"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new JapaneseBaseFormFilter(tokenStream); - } - })); - - indicesAnalysisService.tokenFilterFactories().put( - "kuromoji_part_of_speech", - new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "kuromoji_part_of_speech"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new JapanesePartOfSpeechStopFilter(tokenStream, JapaneseAnalyzer - .getDefaultStopTags()); - } - })); - - indicesAnalysisService.tokenFilterFactories().put( - "kuromoji_readingform", - new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "kuromoji_readingform"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new JapaneseReadingFormFilter(tokenStream, true); - } - })); - - indicesAnalysisService.tokenFilterFactories().put("kuromoji_stemmer", - new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "kuromoji_stemmer"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new JapaneseKatakanaStemFilter(tokenStream); - } - })); - } -} diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java index 46dce43b307..fde7d3d5964 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java @@ -19,8 +19,7 @@ package org.elasticsearch.plugin.analysis.kuromoji; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.index.analysis.AnalysisModule; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory; import org.elasticsearch.index.analysis.KuromojiAnalyzerProvider; import org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory; @@ -29,12 +28,8 @@ import org.elasticsearch.index.analysis.KuromojiKatakanaStemmerFactory; import org.elasticsearch.index.analysis.KuromojiPartOfSpeechFilterFactory; import org.elasticsearch.index.analysis.KuromojiReadingFormFilterFactory; import org.elasticsearch.index.analysis.KuromojiTokenizerFactory; -import org.elasticsearch.indices.analysis.KuromojiIndicesAnalysisModule; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - /** * */ @@ -50,19 +45,15 @@ public class AnalysisKuromojiPlugin extends Plugin { return "Kuromoji analysis support"; } - @Override - public Collection nodeModules() { - return Collections.singletonList(new KuromojiIndicesAnalysisModule()); - } public void onModule(AnalysisModule module) { - module.addCharFilter("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory.class); - module.addAnalyzer("kuromoji", KuromojiAnalyzerProvider.class); - module.addTokenizer("kuromoji_tokenizer", KuromojiTokenizerFactory.class); - module.addTokenFilter("kuromoji_baseform", KuromojiBaseFormFilterFactory.class); - module.addTokenFilter("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory.class); - module.addTokenFilter("kuromoji_readingform", KuromojiReadingFormFilterFactory.class); - module.addTokenFilter("kuromoji_stemmer", KuromojiKatakanaStemmerFactory.class); - module.addTokenFilter("ja_stop", JapaneseStopTokenFilterFactory.class); + module.registerCharFilter("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new); + module.registerAnalyzer("kuromoji", KuromojiAnalyzerProvider::new); + module.registerTokenizer("kuromoji_tokenizer", KuromojiTokenizerFactory::new); + module.registerTokenFilter("kuromoji_baseform", KuromojiBaseFormFilterFactory::new); + module.registerTokenFilter("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory::new); + module.registerTokenFilter("kuromoji_readingform", KuromojiReadingFormFilterFactory::new); + module.registerTokenFilter("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new); + module.registerTokenFilter("ja_stop", JapaneseStopTokenFilterFactory::new); } } diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java index b39103bb239..0942be5399a 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java @@ -29,16 +29,15 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.io.InputStream; @@ -47,16 +46,11 @@ import java.io.StringReader; import java.nio.file.Files; import java.nio.file.Path; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.*; /** */ public class KuromojiAnalysisTests extends ESTestCase { - - @Test public void testDefaultsKuromojiAnalysis() throws IOException { AnalysisService analysisService = createAnalysisService(); @@ -90,7 +84,6 @@ public class KuromojiAnalysisTests extends ESTestCase { } - @Test public void testBaseFormFilterFactory() throws IOException { AnalysisService analysisService = createAnalysisService(); TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_pos"); @@ -102,7 +95,6 @@ public class KuromojiAnalysisTests extends ESTestCase { assertSimpleTSOutput(tokenFilter.create(tokenizer), expected); } - @Test public void testReadingFormFilterFactory() throws IOException { AnalysisService analysisService = createAnalysisService(); TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_rf"); @@ -123,7 +115,6 @@ public class KuromojiAnalysisTests extends ESTestCase { assertSimpleTSOutput(tokenFilter.create(tokenizer), expected_tokens_katakana); } - @Test public void testKatakanaStemFilter() throws IOException { AnalysisService analysisService = createAnalysisService(); TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_stemmer"); @@ -148,7 +139,7 @@ public class KuromojiAnalysisTests extends ESTestCase { expected_tokens_katakana = new String[]{"明後日", "パーティー", "に", "行く", "予定", "が", "ある", "図書館", "で", "資料", "を", "コピー", "し", "まし", "た"}; assertSimpleTSOutput(tokenFilter.create(tokenizer), expected_tokens_katakana); } - @Test + public void testIterationMarkCharFilter() throws IOException { AnalysisService analysisService = createAnalysisService(); // test only kanji @@ -182,7 +173,6 @@ public class KuromojiAnalysisTests extends ESTestCase { assertCharFilterEquals(charFilterFactory.create(new StringReader(source)), expected); } - @Test public void testJapaneseStopFilterFactory() throws IOException { AnalysisService analysisService = createAnalysisService(); TokenFilterFactory tokenFilter = analysisService.tokenFilter("ja_stop"); @@ -212,20 +202,13 @@ public class KuromojiAnalysisTests extends ESTestCase { Index index = new Index("test"); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), - new EnvironmentModule(new Environment(settings))) + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings)); + new AnalysisKuromojiPlugin().onModule(analysisModule); + Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings, new SettingsFilter(settings)), + new EnvironmentModule(new Environment(settings)), analysisModule) .createInjector(); - AnalysisModule analysisModule = new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)); - new AnalysisKuromojiPlugin().onModule(analysisModule); - - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - analysisModule) - .createChildInjector(parentInjector); - - return injector.getInstance(AnalysisService.class); + return parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings)); } public static void assertSimpleTSOutput(TokenStream stream, @@ -256,7 +239,6 @@ public class KuromojiAnalysisTests extends ESTestCase { return buffer.toString(); } - @Test public void testKuromojiUserDict() throws IOException { AnalysisService analysisService = createAnalysisService(); TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_user_dict"); @@ -269,11 +251,9 @@ public class KuromojiAnalysisTests extends ESTestCase { } // fix #59 - @Test public void testKuromojiEmptyUserDict() throws IOException { AnalysisService analysisService = createAnalysisService(); TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_empty_user_dict"); assertThat(tokenizerFactory, instanceOf(KuromojiTokenizerFactory.class)); } - } diff --git a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml index dfeac8b18ff..92c6527fe6a 100644 --- a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml +++ b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yaml @@ -42,7 +42,7 @@ tokenizer: kuromoji_tokenizer filters: kuromoji_readingform - length: { tokens: 1 } - - match: { tokens.0.token: sushi } + - match: { tokens.0.token: スシ } --- "Stemming filter": - do: diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle new file mode 100644 index 00000000000..13898be05a9 --- /dev/null +++ b/plugins/analysis-phonetic/build.gradle @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.' + classname 'org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin' +} + +dependencies { + compile "org.apache.lucene:lucene-analyzers-phonetic:${versions.lucene}" + compile "commons-codec:commons-codec:${versions.commonscodec}" +} + +dependencyLicenses { + mapping from: /lucene-.*/, to: 'lucene' +} + +compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked" + diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index b135301e7db..00000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64b4db89ab7612284b5c685769c3550fb3018bbc diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.5.0-snapshot-1719088.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..2b611862d41 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +70ad9f6c3738727229867419d949527cc7789f62 \ No newline at end of file diff --git a/plugins/analysis-phonetic/pom.xml b/plugins/analysis-phonetic/pom.xml deleted file mode 100644 index dfd6c166c14..00000000000 --- a/plugins/analysis-phonetic/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - analysis-phonetic - Plugin: Analysis: Phonetic - The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch. - - - org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin - analysis_phonetic - false - -Xlint:-rawtypes,-unchecked - - - - - org.apache.lucene - lucene-analyzers-phonetic - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticAnalysisBinderProcessor.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticAnalysisBinderProcessor.java deleted file mode 100644 index 45d7634081e..00000000000 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticAnalysisBinderProcessor.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis; - -/** - */ -public class PhoneticAnalysisBinderProcessor extends AnalysisModule.AnalysisBinderProcessor { - - @Override - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - tokenFiltersBindings.processTokenFilter("phonetic", PhoneticTokenFilterFactory.class); - } -} diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java index b23f311268a..37f7e0cd214 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java @@ -30,13 +30,12 @@ import org.apache.lucene.analysis.phonetic.BeiderMorseFilter; import org.apache.lucene.analysis.phonetic.DoubleMetaphoneFilter; import org.apache.lucene.analysis.phonetic.PhoneticFilter; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.phonetic.HaasePhonetik; import org.elasticsearch.index.analysis.phonetic.KoelnerPhonetik; import org.elasticsearch.index.analysis.phonetic.Nysiis; -import org.elasticsearch.index.settings.IndexSettings; import java.util.Arrays; import java.util.HashSet; @@ -54,8 +53,8 @@ public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { private RuleType ruletype; @Inject - public PhoneticTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public PhoneticTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); this.languageset = null; this.nametype = null; this.ruletype = null; @@ -105,6 +104,8 @@ public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { this.encoder = new HaasePhonetik(); } else if ("nysiis".equalsIgnoreCase(encodername)) { this.encoder = new Nysiis(); + } else if ("daitch_mokotoff".equalsIgnoreCase(encodername)) { + this.encoder = new DaitchMokotoffSoundex(); } else { throw new IllegalArgumentException("unknown encoder [" + encodername + "] for phonetic token filter"); } diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java index 1ef97c62e73..3156a6adfe3 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java @@ -19,8 +19,8 @@ package org.elasticsearch.plugin.analysis; -import org.elasticsearch.index.analysis.AnalysisModule; -import org.elasticsearch.index.analysis.PhoneticAnalysisBinderProcessor; +import org.elasticsearch.index.analysis.PhoneticTokenFilterFactory; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.Plugin; /** @@ -38,7 +38,7 @@ public class AnalysisPhoneticPlugin extends Plugin { } public void onModule(AnalysisModule module) { - module.addProcessor(new PhoneticAnalysisBinderProcessor()); + module.registerTokenFilter("phonetic", PhoneticTokenFilterFactory::new); } } diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java index 1467626dd7b..0b6a4027685 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java @@ -24,16 +24,18 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.MatcherAssert; -import org.junit.Test; + +import java.io.IOException; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.instanceOf; @@ -41,9 +43,7 @@ import static org.hamcrest.Matchers.instanceOf; /** */ public class SimplePhoneticAnalysisTests extends ESTestCase { - - @Test - public void testPhoneticTokenFilterFactory() { + public void testPhoneticTokenFilterFactory() throws IOException { String yaml = "/org/elasticsearch/index/analysis/phonetic-1.yml"; Settings settings = settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) @@ -54,18 +54,13 @@ public class SimplePhoneticAnalysisTests extends ESTestCase { MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class)); } - private AnalysisService testSimpleConfiguration(Settings settings) { + private AnalysisService testSimpleConfiguration(Settings settings) throws IOException { Index index = new Index("test"); - - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), - new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)) - .addProcessor(new PhoneticAnalysisBinderProcessor())).createChildInjector(parentInjector); - - AnalysisService analysisService = injector.getInstance(AnalysisService.class); - return analysisService; + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings)); + new AnalysisPhoneticPlugin().onModule(analysisModule); + Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings, new SettingsFilter(settings)), + new EnvironmentModule(new Environment(settings)), analysisModule) + .createInjector(); + return parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings)); } } diff --git a/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml b/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml index 41a4e3fc59f..6c0a0763881 100644 --- a/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml +++ b/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml @@ -28,3 +28,6 @@ index: nysiisfilter: type: phonetic encoder: nysiis + daitch_mokotoff: + type: phonetic + encoder: daitch_mokotoff diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yaml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yaml new file mode 100644 index 00000000000..b95138f2646 --- /dev/null +++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yaml @@ -0,0 +1,32 @@ +# Integration tests for Phonetic analysis components +# + + +"Daitch Mokotoff": + - do: + indices.create: + index: phonetic_sample + body: + settings: + index: + analysis: + analyzer: + my_analyzer: + tokenizer: standard + filter: ["standard", "lowercase", "daitch_mokotoff"] + filter: + daitch_mokotoff: + type: phonetic + encoder: daitch_mokotoff + - do: + cluster.health: + wait_for_status: yellow + - do: + indices.analyze: + index: phonetic_sample + analyzer: my_analyzer + text: Moskowitz + + - length: { tokens: 1 } + - match: { tokens.0.token: "645740" } + diff --git a/core/src/main/java/org/elasticsearch/index/IndexNameModule.java b/plugins/analysis-smartcn/build.gradle similarity index 68% rename from core/src/main/java/org/elasticsearch/index/IndexNameModule.java rename to plugins/analysis-smartcn/build.gradle index 0a2ee1c1e63..ebe44850d00 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexNameModule.java +++ b/plugins/analysis-smartcn/build.gradle @@ -17,23 +17,16 @@ * under the License. */ -package org.elasticsearch.index; - -import org.elasticsearch.common.inject.AbstractModule; - -/** - * - */ -public class IndexNameModule extends AbstractModule { - - private final Index index; - - public IndexNameModule(Index index) { - this.index = index; - } - - @Override - protected void configure() { - bind(Index.class).toInstance(index); - } +esplugin { + description 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.' + classname 'org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin' } + +dependencies { + compile "org.apache.lucene:lucene-analyzers-smartcn:${versions.lucene}" +} + +dependencyLicenses { + mapping from: /lucene-.*/, to: 'lucene' +} + diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index a32957d25e3..00000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a79e5ed1671d9f511c5aff273ba287557f5f9103 diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.5.0-snapshot-1719088.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..e28887afd56 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +75504fd906929700e7d11f9600e4a79de48e1090 \ No newline at end of file diff --git a/plugins/analysis-smartcn/pom.xml b/plugins/analysis-smartcn/pom.xml deleted file mode 100644 index 64e7b79f5cd..00000000000 --- a/plugins/analysis-smartcn/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - analysis-smartcn - Plugin: Analysis: Smart Chinese (smartcn) - Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch. - - - org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin - analysis_smartcn - false - - - - - org.apache.lucene - lucene-analyzers-smartcn - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalysisBinderProcessor.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalysisBinderProcessor.java deleted file mode 100644 index 4f498f116e5..00000000000 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalysisBinderProcessor.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.analysis; - -/** - */ -public class SmartChineseAnalysisBinderProcessor extends AnalysisModule.AnalysisBinderProcessor { - - @Override - public void processAnalyzers(AnalyzersBindings analyzersBindings) { - analyzersBindings.processAnalyzer("smartcn", SmartChineseAnalyzerProvider.class); - } - - @Override - public void processTokenizers(TokenizersBindings tokenizersBindings) { - tokenizersBindings.processTokenizer("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory.class); - // This is an alias to "smartcn_tokenizer"; it's here for backwards compat - tokenizersBindings.processTokenizer("smartcn_sentence", SmartChineseTokenizerTokenizerFactory.class); - } - - @Override - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer" - tokenFiltersBindings.processTokenFilter("smartcn_word", SmartChineseNoOpTokenFilterFactory.class); - } -} diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java index 1c6e23d6134..1daaa4b0a3d 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java @@ -24,8 +24,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; /** */ @@ -34,8 +33,8 @@ public class SmartChineseAnalyzerProvider extends AbstractIndexAnalyzerProvider< private final SmartChineseAnalyzer analyzer; @Inject - public SmartChineseAnalyzerProvider(Index index, @IndexSettings Settings indexSettings, Environment env, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public SmartChineseAnalyzerProvider(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); analyzer = new SmartChineseAnalyzer(SmartChineseAnalyzer.getDefaultStopSet()); } diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseNoOpTokenFilterFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseNoOpTokenFilterFactory.java index ddd30bd8542..e0f9f556896 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseNoOpTokenFilterFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseNoOpTokenFilterFactory.java @@ -23,14 +23,13 @@ import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; public class SmartChineseNoOpTokenFilterFactory extends AbstractTokenFilterFactory { - @Inject - public SmartChineseNoOpTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public SmartChineseNoOpTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); } @Override diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java index e73e72f2493..7dade32f0e8 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java @@ -24,16 +24,13 @@ import org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; - -import java.io.Reader; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; public class SmartChineseTokenizerTokenizerFactory extends AbstractTokenizerFactory { - @Inject - public SmartChineseTokenizerTokenizerFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { - super(index, indexSettings, name, settings); + public SmartChineseTokenizerTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); } @Override diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/indices/analysis/smartcn/SmartChineseIndicesAnalysis.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/indices/analysis/smartcn/SmartChineseIndicesAnalysis.java deleted file mode 100644 index 78a9934fc5a..00000000000 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/indices/analysis/smartcn/SmartChineseIndicesAnalysis.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.analysis.smartcn; - -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer; -import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.analysis.*; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; - -import java.io.Reader; - -/** - * Registers indices level analysis components so, if not explicitly configured, will be shared - * among all indices. - */ -public class SmartChineseIndicesAnalysis extends AbstractComponent { - - @Inject - public SmartChineseIndicesAnalysis(Settings settings, IndicesAnalysisService indicesAnalysisService) { - super(settings); - - // Register smartcn analyzer - indicesAnalysisService.analyzerProviderFactories().put("smartcn", new PreBuiltAnalyzerProviderFactory("smartcn", AnalyzerScope.INDICES, new SmartChineseAnalyzer())); - - // Register smartcn_tokenizer tokenizer - indicesAnalysisService.tokenizerFactories().put("smartcn_tokenizer", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() { - @Override - public String name() { - return "smartcn_tokenizer"; - } - - @Override - public Tokenizer create() { - return new HMMChineseTokenizer(); - } - })); - - // Register smartcn_sentence tokenizer -- for backwards compat an alias to smartcn_tokenizer - indicesAnalysisService.tokenizerFactories().put("smartcn_sentence", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() { - @Override - public String name() { - return "smartcn_sentence"; - } - - @Override - public Tokenizer create() { - return new HMMChineseTokenizer(); - } - })); - - // Register smartcn_word token filter -- noop - indicesAnalysisService.tokenFilterFactories().put("smartcn_word", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() { - @Override - public String name() { - return "smartcn_word"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return tokenStream; - } - })); - } -} diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/indices/analysis/smartcn/SmartChineseIndicesAnalysisModule.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/indices/analysis/smartcn/SmartChineseIndicesAnalysisModule.java deleted file mode 100644 index e02ae80dfb5..00000000000 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/indices/analysis/smartcn/SmartChineseIndicesAnalysisModule.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.analysis.smartcn; - -import org.elasticsearch.common.inject.AbstractModule; - -/** - */ -public class SmartChineseIndicesAnalysisModule extends AbstractModule { - - @Override - protected void configure() { - bind(SmartChineseIndicesAnalysis.class).asEagerSingleton(); - } -} diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java index 92b933fd725..801c51f8e81 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java @@ -19,15 +19,12 @@ package org.elasticsearch.plugin.analysis.smartcn; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.index.analysis.AnalysisModule; -import org.elasticsearch.index.analysis.SmartChineseAnalysisBinderProcessor; -import org.elasticsearch.indices.analysis.smartcn.SmartChineseIndicesAnalysisModule; +import org.elasticsearch.index.analysis.SmartChineseAnalyzerProvider; +import org.elasticsearch.index.analysis.SmartChineseNoOpTokenFilterFactory; +import org.elasticsearch.index.analysis.SmartChineseTokenizerTokenizerFactory; +import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - /** * */ @@ -43,12 +40,12 @@ public class AnalysisSmartChinesePlugin extends Plugin { return "Smart Chinese analysis support"; } - @Override - public Collection nodeModules() { - return Collections.singletonList(new SmartChineseIndicesAnalysisModule()); - } - public void onModule(AnalysisModule module) { - module.addProcessor(new SmartChineseAnalysisBinderProcessor()); + module.registerAnalyzer("smartcn", SmartChineseAnalyzerProvider::new); + module.registerTokenizer("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new); + // This is an alias to "smartcn_tokenizer"; it's here for backwards compat + module.registerTokenizer("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new); + // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer" + module.registerTokenFilter("smartcn_word", SmartChineseNoOpTokenFilterFactory::new); } } diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java index 0f5d300465a..cfc2b28ec6a 100644 --- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java +++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java @@ -24,41 +24,37 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.MatcherAssert; -import org.junit.Test; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; +import java.io.IOException; + import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.instanceOf; /** */ public class SimpleSmartChineseAnalysisTests extends ESTestCase { - - @Test - public void testDefaultsIcuAnalysis() { + public void testDefaultsIcuAnalysis() throws IOException { Index index = new Index("test"); Settings settings = settingsBuilder() .put("path.home", createTempDir()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(EMPTY_SETTINGS), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(EMPTY_SETTINGS, parentInjector.getInstance(IndicesAnalysisService.class)).addProcessor(new SmartChineseAnalysisBinderProcessor())) - .createChildInjector(parentInjector); - - AnalysisService analysisService = injector.getInstance(AnalysisService.class); - + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings)); + new AnalysisSmartChinesePlugin().onModule(analysisModule); + Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings, new SettingsFilter(settings)), + new EnvironmentModule(new Environment(settings)), analysisModule) + .createInjector(); + final AnalysisService analysisService = parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings)); TokenizerFactory tokenizerFactory = analysisService.tokenizer("smartcn_tokenizer"); MatcherAssert.assertThat(tokenizerFactory, instanceOf(SmartChineseTokenizerTokenizerFactory.class)); } diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle new file mode 100644 index 00000000000..488e99ec912 --- /dev/null +++ b/plugins/analysis-stempel/build.gradle @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.' + classname 'org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin' +} + +dependencies { + compile "org.apache.lucene:lucene-analyzers-stempel:${versions.lucene}" +} + +dependencyLicenses { + mapping from: /lucene-.*/, to: 'lucene' +} diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index 9421876bf2f..00000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7b0b74aebbbfdd2175d6ce045fc538261e016417 diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.5.0-snapshot-1719088.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.5.0-snapshot-1719088.jar.sha1 new file mode 100644 index 00000000000..739ecc4eb8f --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.5.0-snapshot-1719088.jar.sha1 @@ -0,0 +1 @@ +9eeeeabeab89ec305e831d80bdcc7e85a1140fbb \ No newline at end of file diff --git a/plugins/analysis-stempel/pom.xml b/plugins/analysis-stempel/pom.xml deleted file mode 100644 index 4b9b7c33985..00000000000 --- a/plugins/analysis-stempel/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - analysis-stempel - Plugin: Analysis: Polish (stempel) - The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch. - - - org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin - analysis_stempel - false - - - - - org.apache.lucene - lucene-analyzers-stempel - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java index 57207a80b6e..d80939cea04 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java +++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java @@ -20,13 +20,10 @@ package org.elasticsearch.index.analysis.pl; import org.apache.lucene.analysis.pl.PolishAnalyzer; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.assistedinject.Assisted; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider; -import org.elasticsearch.index.settings.IndexSettings; /** */ @@ -34,9 +31,8 @@ public class PolishAnalyzerProvider extends AbstractIndexAnalyzerProvider nodeModules() { - return Collections.singletonList(new PolishIndicesAnalysisModule()); - } - public void onModule(AnalysisModule module) { - module.addProcessor(new PolishAnalysisBinderProcessor()); + module.registerAnalyzer("polish", PolishAnalyzerProvider::new); + module.registerTokenFilter("polish_stem", PolishStemTokenFilterFactory::new); } } diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java index b17f3a12e3a..f3ce4326afb 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java @@ -26,44 +26,41 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.analysis.pl.PolishAnalysisBinderProcessor; import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.MatcherAssert; -import org.junit.Test; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; +import java.io.IOException; + import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.Matchers.instanceOf; /** */ public class PolishAnalysisTests extends ESTestCase { - - @Test - public void testDefaultsPolishAnalysis() { + public void testDefaultsPolishAnalysis() throws IOException { Index index = new Index("test"); Settings settings = settingsBuilder() .put("path.home", createTempDir()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build(); - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(EMPTY_SETTINGS), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(EMPTY_SETTINGS, parentInjector.getInstance(IndicesAnalysisService.class)).addProcessor(new PolishAnalysisBinderProcessor())) - .createChildInjector(parentInjector); - AnalysisService analysisService = injector.getInstance(AnalysisService.class); + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings)); + new AnalysisStempelPlugin().onModule(analysisModule); + Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings, new SettingsFilter(settings)), + new EnvironmentModule(new Environment(settings)), analysisModule) + .createInjector(); + final AnalysisService analysisService = parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings)); TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("polish_stem"); MatcherAssert.assertThat(tokenizerFactory, instanceOf(PolishStemTokenFilterFactory.class)); diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java index 70465417c6e..a68f958580e 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java @@ -29,16 +29,15 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.analysis.pl.PolishAnalysisBinderProcessor; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; +import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.io.StringReader; @@ -46,8 +45,6 @@ import java.io.StringReader; import static org.hamcrest.Matchers.equalTo; public class SimplePolishTokenFilterTests extends ESTestCase { - - @Test public void testBasicUsage() throws Exception { testToken("kwiaty", "kwć"); testToken("canona", "ć"); @@ -100,14 +97,12 @@ public class SimplePolishTokenFilterTests extends ESTestCase { } } - private AnalysisService createAnalysisService(Index index, Settings settings) { - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings))).createInjector(); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, settings), - new IndexNameModule(index), - new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)).addProcessor(new PolishAnalysisBinderProcessor())) - .createChildInjector(parentInjector); - - return injector.getInstance(AnalysisService.class); + private AnalysisService createAnalysisService(Index index, Settings settings) throws IOException { + AnalysisModule analysisModule = new AnalysisModule(new Environment(settings)); + new AnalysisStempelPlugin().onModule(analysisModule); + Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings, new SettingsFilter(settings)), + new EnvironmentModule(new Environment(settings)), analysisModule) + .createInjector(); + return parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings)); } } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CharFilterFactoryFactory.java b/plugins/build.gradle similarity index 76% rename from core/src/main/java/org/elasticsearch/index/analysis/CharFilterFactoryFactory.java rename to plugins/build.gradle index e7d3af08008..bdcc604a296 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/CharFilterFactoryFactory.java +++ b/plugins/build.gradle @@ -17,14 +17,14 @@ * under the License. */ -package org.elasticsearch.index.analysis; +subprojects { + group = 'org.elasticsearch.plugin' -import org.elasticsearch.common.settings.Settings; + apply plugin: 'elasticsearch.esplugin' + apply plugin: 'com.bmuschko.nexus' -/** - * - */ -public interface CharFilterFactoryFactory { - - CharFilterFactory create(String name, Settings settings); + esplugin { + // for local ES plugins, the name of the plugin is the same as the directory + name project.name + } } diff --git a/plugins/cloud-gce/.local-3.0.0-SNAPSHOT-integTest-execution-times.log b/plugins/cloud-gce/.local-3.0.0-SNAPSHOT-integTest-execution-times.log deleted file mode 100644 index 7636f46bb81..00000000000 --- a/plugins/cloud-gce/.local-3.0.0-SNAPSHOT-integTest-execution-times.log +++ /dev/null @@ -1 +0,0 @@ -org.elasticsearch.cloud.gce.CloudGCERestIT=1367 diff --git a/plugins/cloud-gce/.local-3.0.0-SNAPSHOT-test-execution-times.log b/plugins/cloud-gce/.local-3.0.0-SNAPSHOT-test-execution-times.log deleted file mode 100644 index 40b29398c55..00000000000 --- a/plugins/cloud-gce/.local-3.0.0-SNAPSHOT-test-execution-times.log +++ /dev/null @@ -1,2 +0,0 @@ -org.elasticsearch.discovery.gce.GceDiscoverySettingsTests=700 -org.elasticsearch.discovery.gce.GceDiscoveryTests=929 diff --git a/plugins/delete-by-query/.gitignore b/plugins/delete-by-query/.gitignore deleted file mode 100644 index ae3c1726048..00000000000 --- a/plugins/delete-by-query/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/bin/ diff --git a/plugins/delete-by-query/build.gradle b/plugins/delete-by-query/build.gradle new file mode 100644 index 00000000000..2a5d00519e2 --- /dev/null +++ b/plugins/delete-by-query/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Delete By Query plugin allows to delete documents in Elasticsearch with a single query.' + classname 'org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin' +} + diff --git a/plugins/delete-by-query/licenses/no_deps.txt b/plugins/delete-by-query/licenses/no_deps.txt deleted file mode 100644 index 8cce254d037..00000000000 --- a/plugins/delete-by-query/licenses/no_deps.txt +++ /dev/null @@ -1 +0,0 @@ -This plugin has no third party dependencies diff --git a/plugins/delete-by-query/pom.xml b/plugins/delete-by-query/pom.xml deleted file mode 100644 index 105c9f9ab4a..00000000000 --- a/plugins/delete-by-query/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - delete-by-query - Plugin: Delete By Query - The Delete By Query plugin allows to delete documents in Elasticsearch with a single query. - - - org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin - warn - delete_by_query - false - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java index 4c29e7c9ad8..e3faeb1badc 100644 --- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java +++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java @@ -19,27 +19,19 @@ package org.elasticsearch.action.deletebyquery; -import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; -import org.elasticsearch.client.Requests; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.Scroll; import java.io.IOException; import java.util.Arrays; -import java.util.Map; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.search.Scroll.readScroll; @@ -77,7 +69,7 @@ public class DeleteByQueryRequest extends ActionRequest im private String[] types = Strings.EMPTY_ARRAY; - private BytesReference source; + private QueryBuilder query; private String routing; @@ -101,7 +93,7 @@ public class DeleteByQueryRequest extends ActionRequest im @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (source == null) { + if (query == null) { validationException = addValidationError("source is missing", validationException); } return validationException; @@ -140,45 +132,12 @@ public class DeleteByQueryRequest extends ActionRequest im return this; } - public BytesReference source() { - return source; + public QueryBuilder query() { + return query; } - public DeleteByQueryRequest source(QuerySourceBuilder sourceBuilder) { - this.source = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE); - return this; - } - - public DeleteByQueryRequest source(Map querySource) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE); - builder.map(querySource); - return source(builder); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + querySource + "]", e); - } - } - - public DeleteByQueryRequest source(XContentBuilder builder) { - this.source = builder.bytes(); - return this; - } - - public DeleteByQueryRequest source(String querySource) { - this.source = new BytesArray(querySource); - return this; - } - - public DeleteByQueryRequest source(byte[] querySource) { - return source(querySource, 0, querySource.length); - } - - public DeleteByQueryRequest source(byte[] querySource, int offset, int length) { - return source(new BytesArray(querySource, offset, length)); - } - - public DeleteByQueryRequest source(BytesReference querySource) { - this.source = querySource; + public DeleteByQueryRequest query(QueryBuilder queryBuilder) { + this.query = queryBuilder; return this; } @@ -249,7 +208,7 @@ public class DeleteByQueryRequest extends ActionRequest im indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); types = in.readStringArray(); - source = in.readBytesReference(); + query = in.readQuery(); routing = in.readOptionalString(); size = in.readVInt(); if (in.readBoolean()) { @@ -266,7 +225,7 @@ public class DeleteByQueryRequest extends ActionRequest im out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); out.writeStringArray(types); - out.writeBytesReference(source); + out.writeQuery(query); out.writeOptionalString(routing); out.writeVInt(size); out.writeOptionalStreamable(scroll); @@ -275,12 +234,11 @@ public class DeleteByQueryRequest extends ActionRequest im @Override public String toString() { - String sSource = "_na_"; - try { - sSource = XContentHelper.convertToJson(source, false); - } catch (Exception e) { - // ignore - } - return "delete-by-query [" + Arrays.toString(indices) + "][" + Arrays.toString(types) + "], source[" + sSource + "]"; + return "delete-by-query indices:" + Arrays.toString(indices) + + ", types:" + Arrays.toString(types) + + ", size:" + size + + ", timeout:" + timeout + + ", routing:" + routing + + ", query:" + query.toString(); } } diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java index d30cfaa67f5..7560e1e8b11 100644 --- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java +++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java @@ -20,25 +20,17 @@ package org.elasticsearch.action.deletebyquery; import org.elasticsearch.action.ActionRequestBuilder; -import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; -import java.util.Map; - /** * Creates a new {@link DeleteByQueryRequestBuilder} * @see DeleteByQueryRequest */ public class DeleteByQueryRequestBuilder extends ActionRequestBuilder { - private QuerySourceBuilder sourceBuilder; - public DeleteByQueryRequestBuilder(ElasticsearchClient client, DeleteByQueryAction action) { super(client, action, new DeleteByQueryRequest()); } @@ -64,25 +56,10 @@ public class DeleteByQueryRequestBuilder extends ActionRequestBuilder queryBuilder) { - sourceBuilder().setQuery(queryBuilder); + request.query(queryBuilder); return this; } - /** - * The query binary used to delete documents. - */ - public DeleteByQueryRequestBuilder setQuery(BytesReference queryBinary) { - sourceBuilder().setQuery(queryBinary); - return this; - } - - /** - * Constructs a new builder with a raw search query. - */ - public DeleteByQueryRequestBuilder setQuery(XContentBuilder query) { - return setQuery(query.bytes()); - } - /** * A comma separated list of routing values to control the shards the action will be executed on. */ @@ -99,47 +76,6 @@ public class DeleteByQueryRequestBuilder extends ActionRequestBuilder source) { - request().source(source); - return this; - } - - /** - * The source to execute in the form of a builder. - */ - public DeleteByQueryRequestBuilder setSource(XContentBuilder builder) { - request().source(builder); - return this; - } - - /** - * The source to execute. - */ - public DeleteByQueryRequestBuilder setSource(byte[] source) { - request().source(source); - return this; - } - - /** - * The source to execute. - */ - public DeleteByQueryRequestBuilder setSource(BytesReference source) { - request().source(source); - return this; - } - /** * An optional timeout to control how long the delete by query is allowed to take. */ @@ -164,19 +100,4 @@ public class DeleteByQueryRequestBuilder extends ActionRequestBuilder execute() { - if (sourceBuilder != null) { - request.source(sourceBuilder); - } - return super.execute(); - } - - private QuerySourceBuilder sourceBuilder() { - if (sourceBuilder == null) { - sourceBuilder = new QuerySourceBuilder(); - } - return sourceBuilder; - } - } diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java index 252befd85b9..df57aca1668 100644 --- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java +++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java @@ -42,8 +42,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -101,15 +100,21 @@ public class TransportDeleteByQueryAction extends HandledTransportAction fields = new ArrayList<>(); + fields.add("_routing"); + fields.add("_parent"); SearchSourceBuilder source = new SearchSourceBuilder() - .query(request.source()) - .fields("_routing", "_parent") + .query(request.query()) + .fields(fields) .sort("_doc") // important for performance .fetchSource(false) .version(true); @@ -145,7 +150,7 @@ public class TransportDeleteByQueryAction extends HandledTransportAction() { + scrollAction.execute(new SearchScrollRequest(request).scrollId(scrollId).scroll(request.scroll()), new ActionListener() { @Override public void onResponse(SearchResponse scrollResponse) { deleteHits(scrollId, scrollResponse); @@ -187,9 +192,9 @@ public class TransportDeleteByQueryAction extends HandledTransportAction() { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(request); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, new ActionListener() { @Override public void onResponse(ClearScrollResponse clearScrollResponse) { logger.trace("scroll id [{}] cleared", scrollId); diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java index 251953da668..2b8dc02289c 100644 --- a/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java +++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java @@ -20,13 +20,13 @@ package org.elasticsearch.rest.action.deletebyquery; import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest; -import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.QuerySourceBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; @@ -34,6 +34,8 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestToXContentListener; +import java.io.IOException; + import static org.elasticsearch.action.deletebyquery.DeleteByQueryAction.INSTANCE; import static org.elasticsearch.rest.RestRequest.Method.DELETE; @@ -42,35 +44,34 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE; */ public class RestDeleteByQueryAction extends BaseRestHandler { + private IndicesQueriesRegistry indicesQueriesRegistry; + @Inject - public RestDeleteByQueryAction(Settings settings, RestController controller, Client client) { + public RestDeleteByQueryAction(Settings settings, RestController controller, Client client, + IndicesQueriesRegistry indicesQueriesRegistry) { super(settings, controller, client); + this.indicesQueriesRegistry = indicesQueriesRegistry; controller.registerHandler(DELETE, "/{index}/_query", this); controller.registerHandler(DELETE, "/{index}/{type}/_query", this); } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { DeleteByQueryRequest delete = new DeleteByQueryRequest(Strings.splitStringByCommaToArray(request.param("index"))); delete.indicesOptions(IndicesOptions.fromRequest(request, delete.indicesOptions())); delete.routing(request.param("routing")); if (request.hasParam("timeout")) { delete.timeout(request.paramAsTime("timeout", null)); } - if (request.hasContent()) { - delete.source(request.content()); + if (RestActions.hasBodyContent(request)) { + delete.query(RestActions.getQueryContent(RestActions.getRestContent(request), indicesQueriesRegistry, parseFieldMatcher)); } else { - String source = request.param("source"); - if (source != null) { - delete.source(source); - } else { - QuerySourceBuilder querySourceBuilder = RestActions.parseQuerySource(request); - if (querySourceBuilder != null) { - delete.source(querySourceBuilder); - } + QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request); + if (queryBuilder != null) { + delete.query(queryBuilder); } } delete.types(Strings.splitStringByCommaToArray(request.param("type"))); - client.execute(INSTANCE, delete, new RestToXContentListener(channel)); + client.execute(INSTANCE, delete, new RestToXContentListener<>(channel)); } } diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java index f11a0043c38..ea814b44f5d 100644 --- a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java +++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java @@ -24,13 +24,10 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; public class IndexDeleteByQueryResponseTests extends ESTestCase { - - @Test public void testIncrements() { String indexName = randomAsciiOfLength(5); @@ -70,7 +67,6 @@ public class IndexDeleteByQueryResponseTests extends ESTestCase { assertThat(response.getFailed(), equalTo(failed + 1 + inc)); } - @Test public void testNegativeCounters() { assumeTrue("assertions must be enable for this test to pass", assertionsEnabled()); try { @@ -106,7 +102,6 @@ public class IndexDeleteByQueryResponseTests extends ESTestCase { } } - @Test public void testNegativeIncrements() { assumeTrue("assertions must be enable for this test to pass", assertionsEnabled()); try { @@ -146,7 +141,6 @@ public class IndexDeleteByQueryResponseTests extends ESTestCase { } } - @Test public void testSerialization() throws Exception { IndexDeleteByQueryResponse response = new IndexDeleteByQueryResponse(randomAsciiOfLength(5), Math.abs(randomLong()), Math.abs(randomLong()), Math.abs(randomLong()), Math.abs(randomLong())); Version testVersion = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT); @@ -165,5 +159,4 @@ public class IndexDeleteByQueryResponseTests extends ESTestCase { assertThat(deserializedResponse.getMissing(), equalTo(response.getMissing())); assertThat(deserializedResponse.getFailed(), equalTo(response.getFailed())); } - } diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java index c9d3f447b82..c44608c4e4b 100644 --- a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java +++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.deletebyquery; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -33,20 +32,19 @@ import org.elasticsearch.common.text.StringText; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Test; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { - - @Test public void testExecuteScanFailsOnMissingIndex() { DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"none"}); TestActionListener listener = new TestActionListener(); @@ -58,21 +56,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test - public void testExecuteScanFailsOnMalformedQuery() { - createIndex("test"); - - DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).source("{...}"); - TestActionListener listener = new TestActionListener(); - - newAsyncAction(delete, listener).executeScan(); - waitForCompletion("scan request should fail on malformed query", listener); - - assertFailure(listener, "all shards failed"); - assertSearchContextsClosed(); - } - - @Test public void testExecuteScan() { createIndex("test"); final int numDocs = randomIntBetween(1, 200); @@ -80,10 +63,10 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { client().prepareIndex("test", "type").setSource("num", i).get(); } client().admin().indices().prepareRefresh("test").get(); - assertHitCount(client().prepareCount("test").get(), numDocs); + assertHitCount(client().prepareSearch("test").setSize(0).get(), numDocs); final long limit = randomIntBetween(0, numDocs); - DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).source(boolQuery().must(rangeQuery("num").lte(limit)).buildAsBytes()); + DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).query(boolQuery().must(rangeQuery("num").lte(limit))); TestActionListener listener = new TestActionListener(); newAsyncAction(delete, listener).executeScan(); @@ -97,7 +80,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test public void testExecuteScrollFailsOnMissingScrollId() { DeleteByQueryRequest delete = new DeleteByQueryRequest(); TestActionListener listener = new TestActionListener(); @@ -109,7 +91,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test public void testExecuteScrollFailsOnMalformedScrollId() { DeleteByQueryRequest delete = new DeleteByQueryRequest(); TestActionListener listener = new TestActionListener(); @@ -121,14 +102,13 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test public void testExecuteScrollFailsOnExpiredScrollId() { final long numDocs = randomIntBetween(1, 100); for (int i = 1; i <= numDocs; i++) { client().prepareIndex("test", "type").setSource("num", i).get(); } client().admin().indices().prepareRefresh("test").get(); - assertHitCount(client().prepareCount("test").get(), numDocs); + assertHitCount(client().prepareSearch("test").setSize(0).get(), numDocs); SearchResponse searchResponse = client().prepareSearch("test").setScroll(TimeValue.timeValueSeconds(10)).get(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocs)); @@ -150,7 +130,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test public void testExecuteScrollTimedOut() throws InterruptedException { client().prepareIndex("test", "type", "1").setSource("num", "1").get(); client().prepareIndex("test", "type", "2").setSource("num", "1").get(); @@ -176,7 +155,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test public void testExecuteScrollNoDocuments() { createIndex("test"); SearchResponse searchResponse = client().prepareSearch("test").setScroll(TimeValue.timeValueSeconds(10)).get(); @@ -196,21 +174,20 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test public void testExecuteScroll() { final int numDocs = randomIntBetween(1, 100); for (int i = 1; i <= numDocs; i++) { client().prepareIndex("test", "type").setSource("num", i).get(); } client().admin().indices().prepareRefresh("test").get(); - assertHitCount(client().prepareCount("test").get(), numDocs); + assertHitCount(client().prepareSearch("test").setSize(0).get(), numDocs); final long limit = randomIntBetween(0, numDocs); SearchResponse searchResponse = client().prepareSearch("test") .setScroll(TimeValue.timeValueSeconds(10)) .setQuery(boolQuery().must(rangeQuery("num").lte(limit))) - .addFields("_routing", "_parent") + .fields("_routing", "_parent") .setFetchSource(false) .setVersion(true) .get(); @@ -219,7 +196,7 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertTrue(Strings.hasText(scrollId)); assertThat(searchResponse.getHits().getTotalHits(), equalTo(limit)); - DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).size(100).source(boolQuery().must(rangeQuery("num").lte(limit)).buildAsBytes()); + DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).size(100).query(boolQuery().must(rangeQuery("num").lte(limit))); TestActionListener listener = new TestActionListener(); newAsyncAction(delete, listener).executeScroll(searchResponse.getScrollId()); @@ -233,7 +210,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertSearchContextsClosed(); } - @Test public void testOnBulkResponse() { final int nbItems = randomIntBetween(0, 20); long deleted = 0; @@ -250,7 +226,7 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { } else { deleted++; } - items[i] = new BulkItemResponse(i, "delete", new DeleteResponse("test", "type", String.valueOf(i), 1, delete)); + items[i] = new BulkItemResponse(i, "delete", new DeleteResponse(new ShardId("test", 0), "type", String.valueOf(i), 1, delete)); } else { items[i] = new BulkItemResponse(i, "delete", new BulkItemResponse.Failure("test", "type", String.valueOf(i), new Throwable("item failed"))); failed++; @@ -279,7 +255,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { } } - @Test public void testOnBulkResponseMultipleIndices() { final int nbIndices = randomIntBetween(2, 5); @@ -307,7 +282,7 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { deleted[0] = deleted[0] + 1; deleted[index] = deleted[index] + 1; } - items[i] = new BulkItemResponse(i, "delete", new DeleteResponse("test-" + index, "type", String.valueOf(i), 1, delete)); + items[i] = new BulkItemResponse(i, "delete", new DeleteResponse(new ShardId("test-" + index, 0), "type", String.valueOf(i), 1, delete)); } else { items[i] = new BulkItemResponse(i, "delete", new BulkItemResponse.Failure("test-" + index, "type", String.valueOf(i), new Throwable("item failed"))); failed[0] = failed[0] + 1; @@ -350,7 +325,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { } } - @Test public void testOnBulkFailureNoDocuments() { DeleteByQueryRequest delete = new DeleteByQueryRequest(); TestActionListener listener = new TestActionListener(); @@ -361,7 +335,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertFailure(listener, "This is a bulk failure"); } - @Test public void testOnBulkFailure() { final int nbDocs = randomIntBetween(0, 20); SearchHit[] docs = new SearchHit[nbDocs]; @@ -384,7 +357,6 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertThat(response.getTotalDeleted(), equalTo(0L)); } - @Test public void testFinishHim() { TestActionListener listener = new TestActionListener(); newAsyncAction(new DeleteByQueryRequest(), listener).finishHim(null, false, null); @@ -446,19 +418,7 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase { assertThat(t.toString(), containsString(expectedFailure)); } - private void assertShardFailuresContains(ShardOperationFailedException[] shardFailures, String expectedFailure) { - assertNotNull(shardFailures); - for (ShardOperationFailedException failure : shardFailures) { - Throwable t = failure.getCause(); - if (t.toString().contains(expectedFailure)) { - return; - } - } - fail("failed to find shard failure [" + expectedFailure + "]"); - } - private class TestActionListener implements ActionListener { - private final CountDown count = new CountDown(1); private DeleteByQueryResponse response; diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java index f4ae08d5e3d..a4aa334e399 100644 --- a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java +++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; import java.util.Collection; import java.util.concurrent.CountDownLatch; @@ -47,25 +46,29 @@ import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope = SUITE, transportClientRatio = 0) public class DeleteByQueryTests extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(DeleteByQueryPlugin.class); } - @Test(expected = ActionRequestValidationException.class) public void testDeleteByQueryWithNoSource() { - newDeleteByQuery().get(); - fail("should have thrown a validation exception because of the missing source"); + try { + newDeleteByQuery().get(); + fail("should have thrown a validation exception because of the missing source"); + } catch (ActionRequestValidationException e) { + assertThat(e.getMessage(), containsString("source is missing")); + } } - @Test public void testDeleteByQueryWithNoIndices() throws Exception { DeleteByQueryRequestBuilder delete = newDeleteByQuery().setQuery(QueryBuilders.matchAllQuery()); delete.setIndicesOptions(IndicesOptions.fromOptions(false, true, true, false)); @@ -73,23 +76,21 @@ public class DeleteByQueryTests extends ESIntegTestCase { assertSearchContextsClosed(); } - @Test public void testDeleteByQueryWithOneIndex() throws Exception { final long docs = randomIntBetween(1, 50); for (int i = 0; i < docs; i++) { index("test", "test", String.valueOf(i), "fields1", 1); } refresh(); - assertHitCount(client().prepareCount("test").get(), docs); + assertHitCount(client().prepareSearch("test").setSize(0).get(), docs); DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("t*").setQuery(QueryBuilders.matchAllQuery()); assertDBQResponse(delete.get(), docs, docs, 0l, 0l); refresh(); - assertHitCount(client().prepareCount("test").get(), 0); + assertHitCount(client().prepareSearch("test").setSize(0).get(), 0); assertSearchContextsClosed(); } - @Test public void testDeleteByQueryWithMultipleIndices() throws Exception { final int indices = randomIntBetween(2, 5); final int docs = randomIntBetween(2, 10) * 2; @@ -113,9 +114,9 @@ public class DeleteByQueryTests extends ESIntegTestCase { } refresh(); - assertHitCount(client().prepareCount().get(), docs * indices); + assertHitCount(client().prepareSearch().setSize(0).get(), docs * indices); for (int i = 0; i < indices; i++) { - assertHitCount(client().prepareCount("test-" + i).get(), docs); + assertHitCount(client().prepareSearch("test-" + i).setSize(0).get(), docs); } // Deletes all the documents with candidate=true @@ -136,20 +137,19 @@ public class DeleteByQueryTests extends ESIntegTestCase { assertThat(indexResponse.getMissing(), equalTo(0L)); assertThat(indexResponse.getIndex(), equalTo(indexName)); long remaining = docs - candidates[i]; - assertHitCount(client().prepareCount(indexName).get(), remaining); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), remaining); } - assertHitCount(client().prepareCount().get(), (indices * docs) - deletions); + assertHitCount(client().prepareSearch().setSize(0).get(), (indices * docs) - deletions); assertSearchContextsClosed(); } - @Test public void testDeleteByQueryWithMissingIndex() throws Exception { client().prepareIndex("test", "test") .setSource(jsonBuilder().startObject().field("field1", 1).endObject()) .setRefresh(true) .get(); - assertHitCount(client().prepareCount().get(), 1); + assertHitCount(client().prepareSearch().setSize(0).get(), 1); DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("test", "missing").setQuery(QueryBuilders.matchAllQuery()); try { @@ -162,11 +162,10 @@ public class DeleteByQueryTests extends ESIntegTestCase { delete.setIndicesOptions(IndicesOptions.lenientExpandOpen()); assertDBQResponse(delete.get(), 1L, 1L, 0l, 0l); refresh(); - assertHitCount(client().prepareCount("test").get(), 0); + assertHitCount(client().prepareSearch("test").setSize(0).get(), 0); assertSearchContextsClosed(); } - @Test public void testDeleteByQueryWithTypes() throws Exception { final long docs = randomIntBetween(1, 50); for (int i = 0; i < docs; i++) { @@ -174,21 +173,20 @@ public class DeleteByQueryTests extends ESIntegTestCase { index(randomFrom("test1", "test2", "test3"), "type2", String.valueOf(i), "foo", "bar"); } refresh(); - assertHitCount(client().prepareCount().get(), docs * 2); - assertHitCount(client().prepareCount().setTypes("type1").get(), docs); - assertHitCount(client().prepareCount().setTypes("type2").get(), docs); + assertHitCount(client().prepareSearch().setSize(0).get(), docs * 2); + assertHitCount(client().prepareSearch().setSize(0).setTypes("type1").get(), docs); + assertHitCount(client().prepareSearch().setSize(0).setTypes("type2").get(), docs); DeleteByQueryRequestBuilder delete = newDeleteByQuery().setTypes("type1").setQuery(QueryBuilders.matchAllQuery()); assertDBQResponse(delete.get(), docs, docs, 0l, 0l); refresh(); - assertHitCount(client().prepareCount().get(), docs); - assertHitCount(client().prepareCount().setTypes("type1").get(), 0); - assertHitCount(client().prepareCount().setTypes("type2").get(), docs); + assertHitCount(client().prepareSearch().setSize(0).get(), docs); + assertHitCount(client().prepareSearch().setSize(0).setTypes("type1").get(), 0); + assertHitCount(client().prepareSearch().setSize(0).setTypes("type2").get(), docs); assertSearchContextsClosed(); } - @Test public void testDeleteByQueryWithRouting() throws Exception { assertAcked(prepareCreate("test").setSettings("number_of_shards", 2)); ensureGreen("test"); @@ -201,23 +199,22 @@ public class DeleteByQueryTests extends ESIntegTestCase { refresh(); logger.info("--> counting documents with no routing, should be equal to [{}]", docs); - assertHitCount(client().prepareCount().get(), docs); + assertHitCount(client().prepareSearch().setSize(0).get(), docs); String routing = String.valueOf(randomIntBetween(2, docs)); logger.info("--> counting documents with routing [{}]", routing); - long expected = client().prepareCount().setRouting(routing).get().getCount(); + long expected = client().prepareSearch().setSize(0).setRouting(routing).get().getHits().totalHits(); logger.info("--> delete all documents with routing [{}] with a delete-by-query", routing); DeleteByQueryRequestBuilder delete = newDeleteByQuery().setRouting(routing).setQuery(QueryBuilders.matchAllQuery()); assertDBQResponse(delete.get(), expected, expected, 0l, 0l); refresh(); - assertHitCount(client().prepareCount().get(), docs - expected); + assertHitCount(client().prepareSearch().setSize(0).get(), docs - expected); assertSearchContextsClosed(); } - @Test public void testDeleteByFieldQuery() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); @@ -230,31 +227,29 @@ public class DeleteByQueryTests extends ESIntegTestCase { refresh(); int n = between(0, numDocs - 1); - assertHitCount(client().prepareCount("test").setQuery(QueryBuilders.matchQuery("_id", Integer.toString(n))).get(), 1); - assertHitCount(client().prepareCount("test").setQuery(QueryBuilders.matchAllQuery()).get(), numDocs); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchQuery("_id", Integer.toString(n))).get(), 1); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get(), numDocs); DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("alias").setQuery(QueryBuilders.matchQuery("_id", Integer.toString(n))); assertDBQResponse(delete.get(), 1L, 1L, 0l, 0l); refresh(); - assertHitCount(client().prepareCount("test").setQuery(QueryBuilders.matchAllQuery()).get(), numDocs - 1); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get(), numDocs - 1); assertSearchContextsClosed(); } - @Test public void testDeleteByQueryWithDateMath() throws Exception { index("test", "type", "1", "d", "2013-01-01"); ensureGreen(); refresh(); - assertHitCount(client().prepareCount("test").get(), 1); + assertHitCount(client().prepareSearch("test").setSize(0).get(), 1); DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("test").setQuery(QueryBuilders.rangeQuery("d").to("now-1h")); assertDBQResponse(delete.get(), 1L, 1L, 0l, 0l); refresh(); - assertHitCount(client().prepareCount("test").get(), 0); + assertHitCount(client().prepareSearch("test").setSize(0).get(), 0); assertSearchContextsClosed(); } - @Test public void testDeleteByTermQuery() throws Exception { createIndex("test"); ensureGreen(); @@ -281,8 +276,6 @@ public class DeleteByQueryTests extends ESIntegTestCase { assertSearchContextsClosed(); } - @Test - public void testConcurrentDeleteByQueriesOnDifferentDocs() throws Exception { createIndex("test"); ensureGreen(); @@ -295,14 +288,14 @@ public class DeleteByQueryTests extends ESIntegTestCase { } } refresh(); - assertHitCount(client().prepareCount("test").get(), docs * threads.length); + assertHitCount(client().prepareSearch("test").setSize(0).get(), docs * threads.length); final CountDownLatch start = new CountDownLatch(1); final AtomicReference exceptionHolder = new AtomicReference<>(); for (int i = 0; i < threads.length; i++) { final int threadNum = i; - assertHitCount(client().prepareCount("test").setQuery(QueryBuilders.termQuery("field", threadNum)).get(), docs); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", threadNum)).get(), docs); Runnable r = new Runnable() { @Override @@ -337,12 +330,11 @@ public class DeleteByQueryTests extends ESIntegTestCase { refresh(); for (int i = 0; i < threads.length; i++) { - assertHitCount(client().prepareCount("test").setQuery(QueryBuilders.termQuery("field", i)).get(), 0); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", i)).get(), 0); } assertSearchContextsClosed(); } - @Test public void testConcurrentDeleteByQueriesOnSameDocs() throws Exception { assertAcked(prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1))); ensureGreen(); @@ -352,7 +344,7 @@ public class DeleteByQueryTests extends ESIntegTestCase { index("test", "test", String.valueOf(i), "foo", "bar"); } refresh(); - assertHitCount(client().prepareCount("test").get(), docs); + assertHitCount(client().prepareSearch("test").setSize(0).get(), docs); final Thread[] threads = new Thread[scaledRandomIntBetween(2, 9)]; @@ -363,7 +355,7 @@ public class DeleteByQueryTests extends ESIntegTestCase { final AtomicLong deleted = new AtomicLong(0); for (int i = 0; i < threads.length; i++) { - assertHitCount(client().prepareCount("test").setQuery(query).get(), docs); + assertHitCount(client().prepareSearch("test").setSize(0).setQuery(query).get(), docs); Runnable r = new Runnable() { @Override @@ -397,12 +389,11 @@ public class DeleteByQueryTests extends ESIntegTestCase { assertionError.printStackTrace(); } assertThat(assertionError + " should be null", assertionError, nullValue()); - assertHitCount(client().prepareCount("test").get(), 0L); + assertHitCount(client().prepareSearch("test").setSize(0).get(), 0L); assertThat(deleted.get(), equalTo(docs)); assertSearchContextsClosed(); } - @Test public void testDeleteByQueryOnReadOnlyIndex() throws Exception { createIndex("test"); ensureGreen(); @@ -412,7 +403,7 @@ public class DeleteByQueryTests extends ESIntegTestCase { index("test", "test", String.valueOf(i), "field", 1); } refresh(); - assertHitCount(client().prepareCount("test").get(), docs); + assertHitCount(client().prepareSearch("test").setSize(0).get(), docs); try { enableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY); @@ -422,7 +413,7 @@ public class DeleteByQueryTests extends ESIntegTestCase { disableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY); } - assertHitCount(client().prepareCount("test").get(), docs); + assertHitCount(client().prepareSearch("test").setSize(0).get(), docs); assertSearchContextsClosed(); } diff --git a/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml index c253ad8d276..063e959a807 100644 --- a/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml +++ b/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml @@ -1,5 +1,4 @@ ---- -"Basic delete_by_query": +setup: - do: index: index: test_1 @@ -24,6 +23,8 @@ - do: indices.refresh: {} +--- +"Basic delete_by_query": - do: delete_by_query: index: test_1 @@ -40,3 +41,14 @@ index: test_1 - match: { count: 2 } + +--- +"Delete_by_query body without query element": + - do: + catch: request + delete_by_query: + index: test_1 + body: + match: + foo: bar + diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure/build.gradle new file mode 100644 index 00000000000..5042824eb07 --- /dev/null +++ b/plugins/discovery-azure/build.gradle @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Azure Discovery plugin allows to use Azure API for the unicast discovery mechanism.' + classname 'org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin' +} + +versions << [ + 'azure': '0.9.0', + 'jersey': '1.13' +] + +dependencies { + compile "com.microsoft.azure:azure-svc-mgmt-compute:${versions.azure}" + compile "com.microsoft.azure:azure-core:${versions.azure}" + compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "commons-lang:commons-lang:2.6" + compile "commons-io:commons-io:2.4" + compile 'javax.mail:mail:1.4.5' + compile 'javax.activation:activation:1.1' + compile 'javax.inject:javax.inject:1' + compile "com.sun.jersey:jersey-client:${versions.jersey}" + compile "com.sun.jersey:jersey-core:${versions.jersey}" + compile "com.sun.jersey:jersey-json:${versions.jersey}" + compile 'org.codehaus.jettison:jettison:1.1' + compile 'com.sun.xml.bind:jaxb-impl:2.2.3-1' + compile 'javax.xml.bind:jaxb-api:2.2.2' + compile 'javax.xml.stream:stax-api:1.0-2' + compile 'org.codehaus.jackson:jackson-core-asl:1.9.2' + compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.2' + compile 'org.codehaus.jackson:jackson-jaxrs:1.9.2' + compile 'org.codehaus.jackson:jackson-xc:1.9.2' +} + +dependencyLicenses { + mapping from: /azure-.*/, to: 'azure' + mapping from: /jackson-.*/, to: 'jackson' + mapping from: /jersey-.*/, to: 'jersey' + mapping from: /jaxb-.*/, to: 'jaxb' + mapping from: /stax-.*/, to: 'stax' +} + +compileJava.options.compilerArgs << '-Xlint:-path,-serial,-static,-unchecked' +// TODO: why is deprecation needed here but not in maven....? +compileJava.options.compilerArgs << '-Xlint:-deprecation' +// TODO: and why does this static not show up in maven... +compileTestJava.options.compilerArgs << '-Xlint:-static' + diff --git a/plugins/discovery-azure/licenses/azure-core-0.7.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-core-0.7.0.jar.sha1 deleted file mode 100644 index f7d0b7caabc..00000000000 --- a/plugins/discovery-azure/licenses/azure-core-0.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -feed802efe8a7a83d15962d11c6780c63997c528 diff --git a/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1 new file mode 100644 index 00000000000..f9696307afe --- /dev/null +++ b/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1 @@ -0,0 +1 @@ +050719f91deceed1be1aaf87e85099a861295fa2 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/azure-management-0.7.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-management-0.7.0.jar.sha1 deleted file mode 100644 index f69856a386e..00000000000 --- a/plugins/discovery-azure/licenses/azure-management-0.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0dfdd1c3a9bd783b087050e979f6ba34f06a68f3 diff --git a/plugins/discovery-azure/licenses/azure-management-compute-0.7.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-management-compute-0.7.0.jar.sha1 deleted file mode 100644 index bcab189bc14..00000000000 --- a/plugins/discovery-azure/licenses/azure-management-compute-0.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b945fc3968a4e5a64bbde419c14d92a4a53fa7a1 diff --git a/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1 new file mode 100644 index 00000000000..c971d7c5724 --- /dev/null +++ b/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1 @@ -0,0 +1 @@ +887ca8ee5564e8ba2351e6b5db2a1293a8d04674 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-codec-1.6.jar.sha1 b/plugins/discovery-azure/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/discovery-azure/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1 b/plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1 new file mode 100644 index 00000000000..2f5b30d0edb --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1 @@ -0,0 +1 @@ +b1b6ea3b7e4aa4f492509a4952029cd8e48019ad \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-io-LICENSE.txt b/plugins/discovery-azure/licenses/commons-io-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-io-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/discovery-azure/licenses/commons-io-NOTICE.txt b/plugins/discovery-azure/licenses/commons-io-NOTICE.txt new file mode 100644 index 00000000000..a6b77d1eb60 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-io-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons IO +Copyright 2002-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1 b/plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1 new file mode 100644 index 00000000000..4ee9249d2b7 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1 @@ -0,0 +1 @@ +0ce1edb914c94ebc388f086c6827e8bdeec71ac2 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-lang-LICENSE.txt b/plugins/discovery-azure/licenses/commons-lang-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-lang-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/discovery-azure/licenses/commons-lang-NOTICE.txt b/plugins/discovery-azure/licenses/commons-lang-NOTICE.txt new file mode 100644 index 00000000000..592023af76b --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-lang-NOTICE.txt @@ -0,0 +1,8 @@ +Apache Commons Lang +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) diff --git a/plugins/discovery-azure/pom.xml b/plugins/discovery-azure/pom.xml deleted file mode 100644 index 1ce0dc482a1..00000000000 --- a/plugins/discovery-azure/pom.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - discovery-azure - Plugin: Discovery: Azure - The Azure Discovery plugin allows to use Azure API for the unicast discovery mechanism. - - - org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin - 1 - discovery_azure - false - - -Xlint:-path,-serial,-static,-unchecked - - - - - - com.microsoft.azure - azure-management-compute - 0.7.0 - - - stax - stax-api - - - - - com.microsoft.azure - azure-management - 0.7.0 - - - - - org.apache.httpcomponents - httpclient - compile - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java index c10e86b9219..35bb20bc8a7 100644 --- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java +++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java @@ -107,5 +107,4 @@ public class AzureDiscoveryModule extends AbstractModule { } return false; } - } diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java index b2e6821e30d..9f58b0bbb18 100644 --- a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java +++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java @@ -177,7 +177,7 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic InetAddress ipAddress = null; try { - ipAddress = networkService.resolvePublishHostAddress(null); + ipAddress = networkService.resolvePublishHostAddresses(null); logger.trace("ip of current node: [{}]", ipAddress); } catch (IOException e) { // We can't find the publish host address... Hmmm. Too bad :-( diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTests.java index 75ae011e750..19d6d038e8d 100644 --- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTests.java +++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTests.java @@ -19,14 +19,13 @@ package org.elasticsearch.discovery.azure; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.cloud.azure.AbstractAzureComputeServiceTestCase; import org.elasticsearch.cloud.azure.AzureComputeServiceTwoNodesMock; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import java.io.IOException; @@ -60,8 +59,7 @@ public class AzureMinimumMasterNodesTests extends AbstractAzureComputeServiceTes return builder.build(); } - @Test - public void simpleOnlyMasterNodeElection() throws IOException { + public void testSimpleOnlyMasterNodeElection() throws IOException { logger.info("--> start data node / non master node"); internalCluster().startNode(); try { diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java index 74daf1a75e7..cc4021fb78c 100644 --- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java +++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java @@ -20,12 +20,11 @@ package org.elasticsearch.discovery.azure; import org.elasticsearch.cloud.azure.AbstractAzureComputeServiceTestCase; +import org.elasticsearch.cloud.azure.AzureComputeServiceSimpleMock; import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery; import org.elasticsearch.cloud.azure.management.AzureComputeService.Management; -import org.elasticsearch.cloud.azure.AzureComputeServiceSimpleMock; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.notNullValue; @@ -34,13 +33,11 @@ import static org.hamcrest.Matchers.notNullValue; transportClientRatio = 0.0, numClientNodes = 0) public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase { - public AzureSimpleTests() { super(AzureComputeServiceSimpleMock.TestPlugin.class); } - @Test - public void one_node_should_run_using_private_ip() { + public void testOneNodeDhouldRunUsingPrivateIp() { Settings.Builder settings = Settings.settingsBuilder() .put(Management.SERVICE_NAME, "dummy") .put(Discovery.HOST_TYPE, "private_ip"); @@ -53,8 +50,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase { checkNumberOfNodes(1); } - @Test - public void one_node_should_run_using_public_ip() { + public void testOneNodeShouldRunUsingPublicIp() { Settings.Builder settings = Settings.settingsBuilder() .put(Management.SERVICE_NAME, "dummy") .put(Discovery.HOST_TYPE, "public_ip"); @@ -67,8 +63,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase { checkNumberOfNodes(1); } - @Test - public void one_node_should_run_using_wrong_settings() { + public void testOneNodeShouldRunUsingWrongSettings() { Settings.Builder settings = Settings.settingsBuilder() .put(Management.SERVICE_NAME, "dummy") .put(Discovery.HOST_TYPE, "do_not_exist"); diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java index dbccc4ac452..2d134d0cc83 100644 --- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java +++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java @@ -20,12 +20,11 @@ package org.elasticsearch.discovery.azure; import org.elasticsearch.cloud.azure.AbstractAzureComputeServiceTestCase; +import org.elasticsearch.cloud.azure.AzureComputeServiceTwoNodesMock; import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery; import org.elasticsearch.cloud.azure.management.AzureComputeService.Management; -import org.elasticsearch.cloud.azure.AzureComputeServiceTwoNodesMock; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.notNullValue; @@ -39,9 +38,8 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa super(AzureComputeServiceTwoNodesMock.TestPlugin.class); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11533") - public void two_nodes_should_run_using_private_ip() { + public void testTwoNodesShouldRunUsingPrivateIp() { Settings.Builder settings = Settings.settingsBuilder() .put(Management.SERVICE_NAME, "dummy") .put(Discovery.HOST_TYPE, "private_ip"); @@ -58,9 +56,8 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa checkNumberOfNodes(2); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11533") - public void two_nodes_should_run_using_public_ip() { + public void testTwoNodesShouldRunUsingPublicIp() { Settings.Builder settings = Settings.settingsBuilder() .put(Management.SERVICE_NAME, "dummy") .put(Discovery.HOST_TYPE, "public_ip"); diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle new file mode 100644 index 00000000000..77cfd6626d5 --- /dev/null +++ b/plugins/discovery-ec2/build.gradle @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.' + classname 'org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin' +} + +versions << [ + 'aws': '1.10.33' +] + +dependencies { + compile "com.amazonaws:aws-java-sdk-ec2:${versions.aws}" + compile "com.amazonaws:aws-java-sdk-core:${versions.aws}" + compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile 'com.fasterxml.jackson.core:jackson-databind:2.5.3' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.5.0' +} + +dependencyLicenses { + mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk' + mapping from: /jackson-.*/, to: 'jackson' +} + +compileJava.options.compilerArgs << '-Xlint:-rawtypes,-deprecation' + +test { + // this is needed for insecure plugins, remove if possible! + systemProperty 'tests.artifact', project.name +} diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.19.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.19.jar.sha1 deleted file mode 100644 index 66e418e6fb2..00000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b53f650323b7242dcced25b679f3e9aa4b494da5 diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 new file mode 100644 index 00000000000..332a8f01035 --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 @@ -0,0 +1 @@ +fabedbbe2b834b1add150b6a38395c5ef7380168 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.19.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.19.jar.sha1 deleted file mode 100644 index 26fa78d2fd4..00000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -50ba7eb31719be1260bdae51cf69340df2d91ec4 diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 new file mode 100644 index 00000000000..4737b80b3f2 --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 @@ -0,0 +1 @@ +202f6b5dbc196e355d50c131b0fd34969bfd89e6 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.6.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/discovery-ec2/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/discovery-ec2/pom.xml b/plugins/discovery-ec2/pom.xml deleted file mode 100644 index 3a4f674db31..00000000000 --- a/plugins/discovery-ec2/pom.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - discovery-ec2 - Plugin: Discovery: EC2 - The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism. - - - org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin - 1 - discovery_ec2 - false - -Xlint:-rawtypes - - - - - - com.amazonaws - aws-java-sdk-ec2 - ${amazonaws.version} - - - - - org.apache.httpcomponents - httpclient - compile - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index ab2b54633f4..d71d9dfb0af 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -27,20 +27,32 @@ public interface AwsEc2Service extends LifecycleComponent { public static final String KEY = "cloud.aws.access_key"; public static final String SECRET = "cloud.aws.secret_key"; public static final String PROTOCOL = "cloud.aws.protocol"; - public static final String PROXY_HOST = "cloud.aws.proxy_host"; - public static final String PROXY_PORT = "cloud.aws.proxy_port"; + public static final String PROXY_HOST = "cloud.aws.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.proxy.password"; public static final String SIGNER = "cloud.aws.signer"; public static final String REGION = "cloud.aws.region"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.proxy_port"; } final class CLOUD_EC2 { public static final String KEY = "cloud.aws.ec2.access_key"; public static final String SECRET = "cloud.aws.ec2.secret_key"; public static final String PROTOCOL = "cloud.aws.ec2.protocol"; - public static final String PROXY_HOST = "cloud.aws.ec2.proxy_host"; - public static final String PROXY_PORT = "cloud.aws.ec2.proxy_port"; + public static final String PROXY_HOST = "cloud.aws.ec2.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.ec2.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.ec2.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.ec2.proxy.password"; public static final String SIGNER = "cloud.aws.ec2.signer"; public static final String ENDPOINT = "cloud.aws.ec2.endpoint"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.ec2.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.ec2.proxy_port"; } final class DISCOVERY_EC2 { @@ -49,6 +61,7 @@ public interface AwsEc2Service extends LifecycleComponent { public static final String GROUPS = "discovery.ec2.groups"; public static final String TAG_PREFIX = "discovery.ec2.tag."; public static final String AVAILABILITY_ZONES = "discovery.ec2.availability_zones"; + public static final String NODE_CACHE_TIME = "discovery.ec2.node_cache_time"; } AmazonEC2 client(); diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java index 26e001c2666..b6306e6209c 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java @@ -19,16 +19,20 @@ package org.elasticsearch.cloud.aws; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.*; import com.amazonaws.internal.StaticCredentialsProvider; +import com.amazonaws.retry.RetryPolicy; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cloud.aws.network.Ec2NameResolver; import org.elasticsearch.cloud.aws.node.Ec2CustomNodeAttributes; import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.network.NetworkService; @@ -36,6 +40,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import java.util.Locale; +import java.util.Random; /** * @@ -52,8 +57,10 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent // Filter global settings settingsFilter.addFilter(CLOUD_AWS.KEY); settingsFilter.addFilter(CLOUD_AWS.SECRET); + settingsFilter.addFilter(CLOUD_AWS.PROXY_PASSWORD); settingsFilter.addFilter(CLOUD_EC2.KEY); settingsFilter.addFilter(CLOUD_EC2.SECRET); + settingsFilter.addFilter(CLOUD_EC2.PROXY_PASSWORD); // add specific ec2 name resolver networkService.addCustomNameResolver(new Ec2NameResolver(settings)); discoveryNodeService.addCustomAttributeProvider(new Ec2CustomNodeAttributes(settings)); @@ -79,16 +86,25 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent String account = settings.get(CLOUD_EC2.KEY, settings.get(CLOUD_AWS.KEY)); String key = settings.get(CLOUD_EC2.SECRET, settings.get(CLOUD_AWS.SECRET)); - String proxyHost = settings.get(CLOUD_EC2.PROXY_HOST, settings.get(CLOUD_AWS.PROXY_HOST)); + String proxyHost = settings.get(CLOUD_AWS.PROXY_HOST, settings.get(CLOUD_AWS.DEPRECATED_PROXY_HOST)); + proxyHost = settings.get(CLOUD_EC2.PROXY_HOST, settings.get(CLOUD_EC2.DEPRECATED_PROXY_HOST, proxyHost)); if (proxyHost != null) { - String portString = settings.get(CLOUD_EC2.PROXY_PORT, settings.get(CLOUD_AWS.PROXY_PORT, "80")); + String portString = settings.get(CLOUD_AWS.PROXY_PORT, settings.get(CLOUD_AWS.DEPRECATED_PROXY_PORT, "80")); + portString = settings.get(CLOUD_EC2.PROXY_PORT, settings.get(CLOUD_EC2.DEPRECATED_PROXY_PORT, portString)); Integer proxyPort; try { proxyPort = Integer.parseInt(portString, 10); } catch (NumberFormatException ex) { throw new IllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex); } - clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); + String proxyUsername = settings.get(CLOUD_EC2.PROXY_USERNAME, settings.get(CLOUD_AWS.PROXY_USERNAME)); + String proxyPassword = settings.get(CLOUD_EC2.PROXY_PASSWORD, settings.get(CLOUD_AWS.PROXY_PASSWORD)); + + clientConfiguration + .withProxyHost(proxyHost) + .withProxyPort(proxyPort) + .withProxyUsername(proxyUsername) + .withProxyPassword(proxyPassword); } // #155: we might have 3rd party users using older EC2 API version @@ -103,6 +119,24 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent } } + // Increase the number of retries in case of 5xx API responses + final Random rand = Randomness.get(); + RetryPolicy retryPolicy = new RetryPolicy( + RetryPolicy.RetryCondition.NO_RETRY_CONDITION, + new RetryPolicy.BackoffStrategy() { + @Override + public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest, + AmazonClientException exception, + int retriesAttempted) { + // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000) + logger.warn("EC2 API request failed, retry again. Reason was:", exception); + return 1000L * (long) (10d * Math.pow(2, ((double) retriesAttempted) / 2.0d) * (1.0d + rand.nextDouble())); + } + }, + 10, + false); + clientConfiguration.setRetryPolicy(retryPolicy); + AWSCredentialsProvider credentials; if (account == null && key == null) { @@ -134,6 +168,8 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent endpoint = "ec2.us-west-2.amazonaws.com"; } else if (region.equals("ap-southeast") || region.equals("ap-southeast-1")) { endpoint = "ec2.ap-southeast-1.amazonaws.com"; + } else if (region.equals("us-gov-west") || region.equals("us-gov-west-1")) { + endpoint = "ec2.us-gov-west-1.amazonaws.com"; } else if (region.equals("ap-southeast-2")) { endpoint = "ec2.ap-southeast-2.amazonaws.com"; } else if (region.equals("ap-northeast") || region.equals("ap-northeast-1")) { diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java index 94c65047847..f7e70281a3d 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java @@ -31,6 +31,8 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.SingleObjectCache; import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider; import org.elasticsearch.transport.TransportService; @@ -64,6 +66,8 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni private final HostType hostType; + private final DiscoNodesCache discoNodes; + @Inject public AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AwsEc2Service awsEc2Service, Version version) { super(settings); @@ -74,6 +78,9 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni this.hostType = HostType.valueOf(settings.get(DISCOVERY_EC2.HOST_TYPE, "private_ip") .toUpperCase(Locale.ROOT)); + this.discoNodes = new DiscoNodesCache(this.settings.getAsTime(DISCOVERY_EC2.NODE_CACHE_TIME, + TimeValue.timeValueMillis(10_000L))); + this.bindAnyGroup = settings.getAsBoolean(DISCOVERY_EC2.ANY_GROUP, true); this.groups = new HashSet<>(); groups.addAll(Arrays.asList(settings.getAsArray(DISCOVERY_EC2.GROUPS))); @@ -94,6 +101,11 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni @Override public List buildDynamicNodes() { + return discoNodes.getOrRefresh(); + } + + protected List fetchDynamicNodes() { + List discoNodes = new ArrayList<>(); DescribeInstancesResult descInstances; @@ -199,4 +211,25 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni return describeInstancesRequest; } + + private final class DiscoNodesCache extends SingleObjectCache> { + + private boolean empty = true; + + protected DiscoNodesCache(TimeValue refreshInterval) { + super(refreshInterval, new ArrayList<>()); + } + + @Override + protected boolean needsRefresh() { + return (empty || super.needsRefresh()); + } + + @Override + protected List refresh() { + List nodes = fetchDynamicNodes(); + empty = nodes.isEmpty(); + return nodes; + } + } } diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java index 6b73a71e0be..ffa76c6b9b3 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java @@ -41,11 +41,10 @@ import java.util.Collection; * */ public class Ec2DiscoveryPlugin extends Plugin { - + + // ClientConfiguration clinit has some classloader problems + // TODO: fix that static { - // This internal config is deserialized but with wrong access modifiers, - // cannot work without suppressAccessChecks permission right now. We force - // a one time load with elevated privileges as a workaround. SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new SpecialPermission()); @@ -54,9 +53,9 @@ public class Ec2DiscoveryPlugin extends Plugin { @Override public Void run() { try { - Class.forName("com.amazonaws.internal.config.InternalConfig$Factory"); + Class.forName("com.amazonaws.ClientConfiguration"); } catch (ClassNotFoundException e) { - throw new RuntimeException("Unable to initialize internal aws config", e); + throw new RuntimeException(e); } return null; } diff --git a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy index 66810451688..d5c92a9d67b 100644 --- a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy @@ -18,6 +18,10 @@ */ grant { - // needed because of problems in aws-sdk - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + // needed because of problems in ClientConfiguration + // TODO: get these fixed in aws sdk + permission java.lang.RuntimePermission "accessDeclaredMembers"; + // NOTE: no tests fail without this, but we know the problem + // exists in AWS sdk, and tests here are not thorough + permission java.lang.RuntimePermission "getClassLoader"; }; diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java index c1d5daf07f9..d69d939e5b4 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java @@ -20,14 +20,12 @@ package org.elasticsearch.cloud.aws; import com.amazonaws.ClientConfiguration; + import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import static org.hamcrest.CoreMatchers.is; public class AWSSignersTests extends ESTestCase { - - @Test public void testSigners() { assertThat(signerTester(null), is(false)); assertThat(signerTester("QueryStringSignerType"), is(true)); diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java index 47647e17333..ec9155c51b3 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java @@ -26,11 +26,6 @@ import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ThirdParty; -import org.junit.After; -import org.junit.Before; - -import java.util.HashMap; -import java.util.Map; /** * Base class for AWS tests that require credentials. @@ -41,35 +36,6 @@ import java.util.Map; @ThirdParty public abstract class AbstractAwsTestCase extends ESIntegTestCase { - /** - * Those properties are set by the AWS SDK v1.9.4 and if not ignored, - * lead to tests failure (see AbstractRandomizedTest#IGNORED_INVARIANT_PROPERTIES) - */ - private static final String[] AWS_INVARIANT_PROPERTIES = { - "com.sun.org.apache.xml.internal.dtm.DTMManager", - "javax.xml.parsers.DocumentBuilderFactory" - }; - - private Map properties = new HashMap<>(); - - @Before - public void saveProperties() { - for (String p : AWS_INVARIANT_PROPERTIES) { - properties.put(p, System.getProperty(p)); - } - } - - @After - public void restoreProperties() { - for (String p : AWS_INVARIANT_PROPERTIES) { - if (properties.get(p) != null) { - System.setProperty(p, properties.get(p)); - } else { - System.clearProperty(p); - } - } - } - @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder settings = Settings.builder() diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java index 7213a3ace6d..47e2554dcd4 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java @@ -1355,4 +1355,9 @@ public class AmazonEC2Mock implements AmazonEC2 { public ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest request) { throw new UnsupportedOperationException("Not supported in mock"); } + + @Override + public ModifySpotFleetRequestResult modifySpotFleetRequest(ModifySpotFleetRequestRequest modifySpotFleetRequestRequest) throws AmazonServiceException, AmazonClientException { + throw new UnsupportedOperationException("Not supported in mock"); + } } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java index 1d7b525ed8f..6f88be2be5a 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.discovery.ec2; import com.amazonaws.services.ec2.model.Tag; + import org.elasticsearch.Version; import org.elasticsearch.cloud.aws.AwsEc2Service; import org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2; @@ -31,15 +32,16 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.local.LocalTransport; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -86,8 +88,7 @@ public class Ec2DiscoveryTests extends ESTestCase { return discoveryNodes; } - @Test - public void defaultSettings() throws InterruptedException { + public void testDefaultSettings() throws InterruptedException { int nodes = randomInt(10); Settings nodeSettings = Settings.builder() .build(); @@ -95,8 +96,7 @@ public class Ec2DiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(nodes)); } - @Test - public void privateIp() throws InterruptedException { + public void testPrivateIp() throws InterruptedException { int nodes = randomInt(10); Settings nodeSettings = Settings.builder() .put(DISCOVERY_EC2.HOST_TYPE, "private_ip") @@ -112,8 +112,7 @@ public class Ec2DiscoveryTests extends ESTestCase { } } - @Test - public void publicIp() throws InterruptedException { + public void testPublicIp() throws InterruptedException { int nodes = randomInt(10); Settings nodeSettings = Settings.builder() .put(DISCOVERY_EC2.HOST_TYPE, "public_ip") @@ -129,8 +128,7 @@ public class Ec2DiscoveryTests extends ESTestCase { } } - @Test - public void privateDns() throws InterruptedException { + public void testPrivateDns() throws InterruptedException { int nodes = randomInt(10); Settings nodeSettings = Settings.builder() .put(DISCOVERY_EC2.HOST_TYPE, "private_dns") @@ -148,8 +146,7 @@ public class Ec2DiscoveryTests extends ESTestCase { } } - @Test - public void publicDns() throws InterruptedException { + public void testPublicDns() throws InterruptedException { int nodes = randomInt(10); Settings nodeSettings = Settings.builder() .put(DISCOVERY_EC2.HOST_TYPE, "public_dns") @@ -167,16 +164,19 @@ public class Ec2DiscoveryTests extends ESTestCase { } } - @Test(expected = IllegalArgumentException.class) - public void invalidHostType() throws InterruptedException { + public void testInvalidHostType() throws InterruptedException { Settings nodeSettings = Settings.builder() .put(DISCOVERY_EC2.HOST_TYPE, "does_not_exist") .build(); - buildDynamicNodes(nodeSettings, 1); + try { + buildDynamicNodes(nodeSettings, 1); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("No enum constant")); + } } - @Test - public void filterByTags() throws InterruptedException { + public void testFilterByTags() throws InterruptedException { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder() .put(DISCOVERY_EC2.TAG_PREFIX + "stage", "prod") @@ -201,8 +201,7 @@ public class Ec2DiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(prodInstances)); } - @Test - public void filterByMultipleTags() throws InterruptedException { + public void testFilterByMultipleTags() throws InterruptedException { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder() .putArray(DISCOVERY_EC2.TAG_PREFIX + "stage", "prod", "preprod") @@ -233,4 +232,47 @@ public class Ec2DiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(prodInstances)); } + abstract class DummyEc2HostProvider extends AwsEc2UnicastHostsProvider { + public int fetchCount = 0; + public DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service, Version version) { + super(settings, transportService, service, version); + } + } + + public void testGetNodeListEmptyCache() throws Exception { + AwsEc2Service awsEc2Service = new AwsEc2ServiceMock(Settings.EMPTY, 1, null); + DummyEc2HostProvider provider = new DummyEc2HostProvider(Settings.EMPTY, transportService, awsEc2Service, Version.CURRENT) { + @Override + protected List fetchDynamicNodes() { + fetchCount++; + return new ArrayList<>(); + } + }; + for (int i=0; i<3; i++) { + provider.buildDynamicNodes(); + } + assertThat(provider.fetchCount, is(3)); + } + + public void testGetNodeListCached() throws Exception { + Settings.Builder builder = Settings.settingsBuilder() + .put(DISCOVERY_EC2.NODE_CACHE_TIME, "500ms"); + AwsEc2Service awsEc2Service = new AwsEc2ServiceMock(Settings.EMPTY, 1, null); + DummyEc2HostProvider provider = new DummyEc2HostProvider(builder.build(), transportService, awsEc2Service, Version.CURRENT) { + @Override + protected List fetchDynamicNodes() { + fetchCount++; + return Ec2DiscoveryTests.this.buildDynamicNodes(Settings.EMPTY, 1); + } + }; + for (int i=0; i<3; i++) { + provider.buildDynamicNodes(); + } + assertThat(provider.fetchCount, is(1)); + Thread.sleep(1_000L); // wait for cache to expire + for (int i=0; i<3; i++) { + provider.buildDynamicNodes(); + } + assertThat(provider.fetchCount, is(2)); + } } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java index de3efcf1342..68596ce2ace 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Test; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.hamcrest.CoreMatchers.is; @@ -38,8 +37,6 @@ import static org.hamcrest.CoreMatchers.is; */ @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class Ec2DiscoveryUpdateSettingsTests extends AbstractAwsTestCase { - - @Test public void testMinimumMasterNodesStart() { Settings nodeSettings = settingsBuilder() .put("plugin.types", Ec2DiscoveryPlugin.class.getName()) @@ -57,5 +54,4 @@ public class Ec2DiscoveryUpdateSettingsTests extends AbstractAwsTestCase { Integer min = response.getPersistentSettings().getAsInt("discovery.zen.minimum_master_nodes", null); assertThat(min, is(1)); } - } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java index 8aa9ca56a3f..b69ebd369a4 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.cloud.aws.network.Ec2NameResolver; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.net.InetAddress; @@ -35,12 +34,10 @@ import static org.hamcrest.Matchers.containsString; * Test for EC2 network.host settings. */ public class Ec2NetworkTests extends ESTestCase { - /** * Test for network.host: _ec2_ */ - @Test - public void networkHostEc2() throws IOException { + public void testNetworkHostEc2() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_ec2_") .build(); @@ -49,7 +46,7 @@ public class Ec2NetworkTests extends ESTestCase { networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. try { - networkService.resolveBindHostAddress(null); + networkService.resolveBindHostAddresses(null); } catch (IOException e) { assertThat(e.getMessage(), containsString("local-ipv4")); } @@ -58,8 +55,7 @@ public class Ec2NetworkTests extends ESTestCase { /** * Test for network.host: _ec2:publicIp_ */ - @Test - public void networkHostEc2PublicIp() throws IOException { + public void testNetworkHostEc2PublicIp() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_ec2:publicIp_") .build(); @@ -68,7 +64,7 @@ public class Ec2NetworkTests extends ESTestCase { networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. try { - networkService.resolveBindHostAddress(null); + networkService.resolveBindHostAddresses(null); } catch (IOException e) { assertThat(e.getMessage(), containsString("public-ipv4")); } @@ -77,8 +73,7 @@ public class Ec2NetworkTests extends ESTestCase { /** * Test for network.host: _ec2:privateIp_ */ - @Test - public void networkHostEc2PrivateIp() throws IOException { + public void testNetworkHostEc2PrivateIp() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_ec2:privateIp_") .build(); @@ -87,7 +82,7 @@ public class Ec2NetworkTests extends ESTestCase { networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. try { - networkService.resolveBindHostAddress(null); + networkService.resolveBindHostAddresses(null); } catch (IOException e) { assertThat(e.getMessage(), containsString("local-ipv4")); } @@ -96,8 +91,7 @@ public class Ec2NetworkTests extends ESTestCase { /** * Test for network.host: _ec2:privateIpv4_ */ - @Test - public void networkHostEc2PrivateIpv4() throws IOException { + public void testNetworkHostEc2PrivateIpv4() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_ec2:privateIpv4_") .build(); @@ -106,7 +100,7 @@ public class Ec2NetworkTests extends ESTestCase { networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. try { - networkService.resolveBindHostAddress(null); + networkService.resolveBindHostAddresses(null); } catch (IOException e) { assertThat(e.getMessage(), containsString("local-ipv4")); } @@ -115,8 +109,7 @@ public class Ec2NetworkTests extends ESTestCase { /** * Test for network.host: _ec2:privateDns_ */ - @Test - public void networkHostEc2PrivateDns() throws IOException { + public void testNetworkHostEc2PrivateDns() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_ec2:privateDns_") .build(); @@ -125,7 +118,7 @@ public class Ec2NetworkTests extends ESTestCase { networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. try { - networkService.resolveBindHostAddress(null); + networkService.resolveBindHostAddresses(null); } catch (IOException e) { assertThat(e.getMessage(), containsString("local-hostname")); } @@ -134,8 +127,7 @@ public class Ec2NetworkTests extends ESTestCase { /** * Test for network.host: _ec2:publicIpv4_ */ - @Test - public void networkHostEc2PublicIpv4() throws IOException { + public void testNetworkHostEc2PublicIpv4() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_ec2:publicIpv4_") .build(); @@ -144,7 +136,7 @@ public class Ec2NetworkTests extends ESTestCase { networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. try { - networkService.resolveBindHostAddress(null); + networkService.resolveBindHostAddresses(null); } catch (IOException e) { assertThat(e.getMessage(), containsString("public-ipv4")); } @@ -153,8 +145,7 @@ public class Ec2NetworkTests extends ESTestCase { /** * Test for network.host: _ec2:publicDns_ */ - @Test - public void networkHostEc2PublicDns() throws IOException { + public void testNetworkHostEc2PublicDns() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_ec2:publicDns_") .build(); @@ -163,7 +154,7 @@ public class Ec2NetworkTests extends ESTestCase { networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. try { - networkService.resolveBindHostAddress(null); + networkService.resolveBindHostAddresses(null); } catch (IOException e) { assertThat(e.getMessage(), containsString("public-hostname")); } @@ -173,15 +164,14 @@ public class Ec2NetworkTests extends ESTestCase { * Test that we don't have any regression with network host core settings such as * network.host: _local_ */ - @Test - public void networkHostCoreLocal() throws IOException { + public void testNetworkHostCoreLocal() throws IOException { Settings nodeSettings = Settings.builder() .put("network.host", "_local_") .build(); NetworkService networkService = new NetworkService(nodeSettings); networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); - InetAddress[] addresses = networkService.resolveBindHostAddress(null); - assertThat(addresses, arrayContaining(networkService.resolveBindHostAddress("_local_"))); + InetAddress[] addresses = networkService.resolveBindHostAddresses(null); + assertThat(addresses, arrayContaining(networkService.resolveBindHostAddresses(new String[] { "_local_" }))); } } diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle new file mode 100644 index 00000000000..4e6ade8788f --- /dev/null +++ b/plugins/discovery-gce/build.gradle @@ -0,0 +1,33 @@ + +esplugin { + description 'The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.' + classname 'org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin' +} + +versions << [ + 'google': '1.20.0' +] + +dependencies { + compile "com.google.apis:google-api-services-compute:v1-rev71-${versions.google}" + compile "com.google.api-client:google-api-client:${versions.google}" + compile "com.google.oauth-client:google-oauth-client:${versions.google}" + compile "com.google.http-client:google-http-client:${versions.google}" + compile "com.google.http-client:google-http-client-jackson2:${versions.google}" + compile 'com.google.code.findbugs:jsr305:1.3.9' + compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" +} + +dependencyLicenses { + mapping from: /google-.*/, to: 'google' +} + +compileJava.options.compilerArgs << '-Xlint:-rawtypes,-unchecked' + +test { + // this is needed for insecure plugins, remove if possible! + systemProperty 'tests.artifact', project.name +} diff --git a/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.6.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/discovery-gce/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/discovery-gce/pom.xml b/plugins/discovery-gce/pom.xml deleted file mode 100644 index b7c1c0a7a61..00000000000 --- a/plugins/discovery-gce/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - discovery-gce - Plugin: Discovery: Google Compute Engine - The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism. - - - org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin - v1-rev71-1.20.0 - - discovery_gce - false - -Xlint:-rawtypes,-unchecked - - - - - - com.google.apis - google-api-services-compute - ${google.gce.version} - - - com.google.guava - guava-jdk5 - - - - - - - org.apache.httpcomponents - httpclient - compile - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index c7f45980b9b..a1e5424a37e 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -32,6 +32,9 @@ public interface GceComputeService extends LifecycleComponent public static final String REFRESH = "cloud.gce.refresh_interval"; public static final String TAGS = "discovery.gce.tags"; public static final String VERSION = "Elasticsearch/GceCloud/1.0"; + + public static final String RETRY = "cloud.gce.retry"; + public static final String MAXWAIT = "cloud.gce.max_wait"; } /** diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index a29c21ec526..07e05f06c6d 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -39,11 +39,13 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.gce.RetryHttpInitializerWrapper; import java.io.IOException; import java.net.URL; import java.security.AccessController; import java.security.GeneralSecurityException; +import java.security.PrivilegedAction; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.*; @@ -78,14 +80,13 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponentemptyList() : instanceList.getItems(); } catch (PrivilegedActionException e) { logger.warn("Problem fetching instance list for zone {}", zoneId); logger.debug("Full exception:", e); - return Collections.EMPTY_LIST; + // assist type inference + return Collections.emptyList(); } }).reduce(new ArrayList<>(), (a, b) -> { a.addAll(b); @@ -103,23 +104,33 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent() { @Override public HttpHeaders run() throws IOException { return new HttpHeaders(); } }); + GenericUrl genericUrl = AccessController.doPrivileged(new PrivilegedAction() { + @Override + public GenericUrl run() { + return new GenericUrl(url); + } + }); // This is needed to query meta data: https://cloud.google.com/compute/docs/metadata headers.put("Metadata-Flavor", "Google"); HttpResponse response; response = getGceHttpTransport().createRequestFactory() - .buildGetRequest(new GenericUrl(url)) + .buildGetRequest(genericUrl) .setHeaders(headers) .execute(); String metadata = response.parseAsString(); @@ -171,7 +182,7 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent 0) { + retryHttpInitializerWrapper = new RetryHttpInitializerWrapper(credential, maxWait); + } else { + retryHttpInitializerWrapper = new RetryHttpInitializerWrapper(credential); + } + builder.setHttpRequestInitializer(retryHttpInitializerWrapper); + + } else { + builder.setHttpRequestInitializer(credential); + } + + this.client = builder.build(); } catch (Exception e) { logger.warn("unable to start GCE discovery service", e); throw new IllegalArgumentException("unable to start GCE discovery service", e); diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 8feb9b8697c..476773dcc73 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -110,7 +110,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas cachedDiscoNodes = new ArrayList<>(); String ipAddress = null; try { - InetAddress inetAddress = networkService.resolvePublishHostAddress(null); + InetAddress inetAddress = networkService.resolvePublishHostAddresses(null); if (inetAddress != null) { ipAddress = NetworkAddress.formatAddress(inetAddress); } diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java new file mode 100644 index 00000000000..1d73e1d540e --- /dev/null +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential; +import com.google.api.client.http.*; +import com.google.api.client.util.ExponentialBackOff; +import com.google.api.client.util.Sleeper; + +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Objects; + +public class RetryHttpInitializerWrapper implements HttpRequestInitializer { + + private int maxWait; + + private static final ESLogger logger = + ESLoggerFactory.getLogger(RetryHttpInitializerWrapper.class.getName()); + + // Intercepts the request for filling in the "Authorization" + // header field, as well as recovering from certain unsuccessful + // error codes wherein the Credential must refresh its token for a + // retry. + private final Credential wrappedCredential; + + // A sleeper; you can replace it with a mock in your test. + private final Sleeper sleeper; + + public RetryHttpInitializerWrapper(Credential wrappedCredential) { + this(wrappedCredential, Sleeper.DEFAULT, ExponentialBackOff.DEFAULT_MAX_ELAPSED_TIME_MILLIS); + } + + public RetryHttpInitializerWrapper(Credential wrappedCredential, int maxWait) { + this(wrappedCredential, Sleeper.DEFAULT, maxWait); + } + + // Use only for testing. + RetryHttpInitializerWrapper( + Credential wrappedCredential, Sleeper sleeper, int maxWait) { + this.wrappedCredential = Objects.requireNonNull(wrappedCredential); + this.sleeper = sleeper; + this.maxWait = maxWait; + } + + // Use only for testing + static MockGoogleCredential.Builder newMockCredentialBuilder() { + // TODO: figure out why GCE is so bad like this + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public MockGoogleCredential.Builder run() { + return new MockGoogleCredential.Builder(); + } + }); + } + + @Override + public void initialize(HttpRequest httpRequest) { + final HttpUnsuccessfulResponseHandler backoffHandler = + new HttpBackOffUnsuccessfulResponseHandler( + new ExponentialBackOff.Builder() + .setMaxElapsedTimeMillis(maxWait) + .build()) + .setSleeper(sleeper); + + httpRequest.setInterceptor(wrappedCredential); + httpRequest.setUnsuccessfulResponseHandler( + new HttpUnsuccessfulResponseHandler() { + int retry = 0; + + @Override + public boolean handleResponse(HttpRequest request, HttpResponse response, boolean supportsRetry) throws IOException { + if (wrappedCredential.handleResponse( + request, response, supportsRetry)) { + // If credential decides it can handle it, + // the return code or message indicated + // something specific to authentication, + // and no backoff is desired. + return true; + } else if (backoffHandler.handleResponse( + request, response, supportsRetry)) { + // Otherwise, we defer to the judgement of + // our internal backoff handler. + logger.debug("Retrying [{}] times : [{}]", retry, request.getUrl()); + return true; + } else { + return false; + } + } + }); + httpRequest.setIOExceptionHandler( + new HttpBackOffIOExceptionHandler( + new ExponentialBackOff.Builder() + .setMaxElapsedTimeMillis(maxWait) + .build()) + .setSleeper(sleeper) + ); + } +} + diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java index a17c3962797..5f01a98a5f2 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java @@ -19,6 +19,10 @@ package org.elasticsearch.plugin.discovery.gce; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.util.ClassInfo; + +import org.elasticsearch.SpecialPermission; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cloud.gce.GceModule; import org.elasticsearch.common.Strings; @@ -32,11 +36,34 @@ import org.elasticsearch.discovery.gce.GceDiscovery; import org.elasticsearch.discovery.gce.GceUnicastHostsProvider; import org.elasticsearch.plugins.Plugin; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; import java.util.List; public class GceDiscoveryPlugin extends Plugin { + static { + /* + * GCE's http client changes access levels because its silly and we + * can't allow that on any old stack stack so we pull it here, up front, + * so we can cleanly check the permissions for it. Without this changing + * the permission can fail if any part of core is on the stack because + * our plugin permissions don't allow core to "reach through" plugins to + * change the permission. Because that'd be silly. + */ + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + ClassInfo.of(HttpHeaders.class, true); + return null; + } + }); + } private final Settings settings; protected final ESLogger logger = Loggers.getLogger(GceDiscoveryPlugin.class); diff --git a/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy index 80a99785e45..429c47287b7 100644 --- a/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy @@ -19,5 +19,6 @@ grant { // needed because of problems in gce + permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java index 450ff72ca47..eafd3997b55 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java @@ -29,7 +29,10 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.local.LocalTransport; -import org.junit.*; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; import java.util.List; import java.util.Locale; @@ -45,7 +48,7 @@ import static org.hamcrest.Matchers.is; * * compute/v1/projects/[project-id]/zones/[zone] * - * By default, project-id is the test method name, lowercase. + * By default, project-id is the test method name, lowercase and missing the "test" prefix. * * For example, if you create a test `myNewAwesomeTest` with following settings: * @@ -83,6 +86,10 @@ public class GceDiscoveryTests extends ESTestCase { @Before public void setProjectName() { projectName = getTestName().toLowerCase(Locale.ROOT); + // Slice off the "test" part of the method names so the project names + if (projectName.startsWith("test")) { + projectName = projectName.substring("test".length()); + } } @Before @@ -113,8 +120,7 @@ public class GceDiscoveryTests extends ESTestCase { return discoveryNodes; } - @Test - public void nodesWithDifferentTagsAndNoTagSet() { + public void testNodesWithDifferentTagsAndNoTagSet() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") @@ -124,8 +130,7 @@ public class GceDiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(2)); } - @Test - public void nodesWithDifferentTagsAndOneTagSet() { + public void testNodesWithDifferentTagsAndOneTagSet() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") @@ -137,8 +142,7 @@ public class GceDiscoveryTests extends ESTestCase { assertThat(discoveryNodes.get(0).getId(), is("#cloud-test2-0")); } - @Test - public void nodesWithDifferentTagsAndTwoTagSet() { + public void testNodesWithDifferentTagsAndTwoTagSet() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") @@ -150,8 +154,7 @@ public class GceDiscoveryTests extends ESTestCase { assertThat(discoveryNodes.get(0).getId(), is("#cloud-test2-0")); } - @Test - public void nodesWithSameTagsAndNoTagSet() { + public void testNodesWithSameTagsAndNoTagSet() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") @@ -161,8 +164,7 @@ public class GceDiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(2)); } - @Test - public void nodesWithSameTagsAndOneTagSet() { + public void testNodesWithSameTagsAndOneTagSet() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") @@ -173,8 +175,7 @@ public class GceDiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(2)); } - @Test - public void nodesWithSameTagsAndTwoTagsSet() { + public void testNodesWithSameTagsAndTwoTagsSet() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") @@ -185,8 +186,7 @@ public class GceDiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(2)); } - @Test - public void multipleZonesAndTwoNodesInSameZone() { + public void testMultipleZonesAndTwoNodesInSameZone() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .putArray(GceComputeService.Fields.ZONE, "us-central1-a", "europe-west1-b") @@ -196,8 +196,7 @@ public class GceDiscoveryTests extends ESTestCase { assertThat(discoveryNodes, hasSize(2)); } - @Test - public void multipleZonesAndTwoNodesInDifferentZones() { + public void testMultipleZonesAndTwoNodesInDifferentZones() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .putArray(GceComputeService.Fields.ZONE, "us-central1-a", "europe-west1-b") @@ -210,8 +209,7 @@ public class GceDiscoveryTests extends ESTestCase { /** * For issue https://github.com/elastic/elasticsearch-cloud-gce/issues/43 */ - @Test - public void zeroNode43() { + public void testZeroNode43() { Settings nodeSettings = Settings.builder() .put(GceComputeService.Fields.PROJECT, projectName) .putArray(GceComputeService.Fields.ZONE, "us-central1-a", "us-central1-b") diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java index 7550cdce7e4..c09e51fe1ef 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.cloud.gce.network.GceNameResolver; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; import java.net.InetAddress; @@ -36,28 +35,24 @@ import static org.hamcrest.Matchers.containsString; * Related to https://github.com/elastic/elasticsearch/issues/13605 */ public class GceNetworkTests extends ESTestCase { - /** * Test for network.host: _gce_ */ - @Test - public void networkHostGceDefault() throws IOException { + public void testNetworkHostGceDefault() throws IOException { resolveGce("_gce_", InetAddress.getByName("10.240.0.2")); } /** * Test for network.host: _gce:privateIp_ */ - @Test - public void networkHostPrivateIp() throws IOException { + public void testNetworkHostPrivateIp() throws IOException { resolveGce("_gce:privateIp_", InetAddress.getByName("10.240.0.2")); } /** * Test for network.host: _gce:hostname_ */ - @Test - public void networkHostPrivateDns() throws IOException { + public void testNetworkHostPrivateDns() throws IOException { resolveGce("_gce:hostname_", InetAddress.getByName("localhost")); } @@ -65,8 +60,7 @@ public class GceNetworkTests extends ESTestCase { * Test for network.host: _gce:doesnotexist_ * This should raise an IllegalArgumentException as this setting does not exist */ - @Test - public void networkHostWrongSetting() throws IOException { + public void testNetworkHostWrongSetting() throws IOException { resolveGce("_gce:doesnotexist_", (InetAddress) null); } @@ -75,8 +69,7 @@ public class GceNetworkTests extends ESTestCase { * network.host: _gce:privateIp:0_ * network.host: _gce:privateIp:1_ */ - @Test - public void networkHostPrivateIpInterface() throws IOException { + public void testNetworkHostPrivateIpInterface() throws IOException { resolveGce("_gce:privateIp:0_", InetAddress.getByName("10.240.0.2")); resolveGce("_gce:privateIp:1_", InetAddress.getByName("10.150.0.1")); } @@ -85,9 +78,8 @@ public class GceNetworkTests extends ESTestCase { * Test that we don't have any regression with network host core settings such as * network.host: _local_ */ - @Test public void networkHostCoreLocal() throws IOException { - resolveGce("_local_", new NetworkService(Settings.EMPTY).resolveBindHostAddress(NetworkService.DEFAULT_NETWORK_HOST)); + resolveGce("_local_", new NetworkService(Settings.EMPTY).resolveBindHostAddresses(new String[] { NetworkService.DEFAULT_NETWORK_HOST })); } /** @@ -115,7 +107,7 @@ public class GceNetworkTests extends ESTestCase { GceComputeServiceMock mock = new GceComputeServiceMock(nodeSettings, networkService); networkService.addCustomNameResolver(new GceNameResolver(nodeSettings, mock)); try { - InetAddress[] addresses = networkService.resolveBindHostAddress(null); + InetAddress[] addresses = networkService.resolveBindHostAddresses(null); if (expected == null) { fail("We should get a IllegalArgumentException when setting network.host: _gce:doesnotexist_"); } diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java new file mode 100644 index 00000000000..ef92bd74305 --- /dev/null +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java @@ -0,0 +1,174 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpResponseException; +import com.google.api.client.http.HttpStatusCodes; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.client.testing.util.MockSleeper; +import com.google.api.services.compute.Compute; + +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; + +public class RetryHttpInitializerWrapperTests extends ESTestCase { + + static private class FailThenSuccessBackoffTransport extends MockHttpTransport { + + public int lowLevelExecCalls; + int errorStatusCode; + int callsBeforeSuccess; + boolean throwException; + + protected FailThenSuccessBackoffTransport(int errorStatusCode, int callsBeforeSuccess) { + this.errorStatusCode = errorStatusCode; + this.callsBeforeSuccess = callsBeforeSuccess; + this.throwException = false; + } + + protected FailThenSuccessBackoffTransport(int errorStatusCode, int callsBeforeSuccess, boolean throwException) { + this.errorStatusCode = errorStatusCode; + this.callsBeforeSuccess = callsBeforeSuccess; + this.throwException = throwException; + } + + public LowLevelHttpRequest retryableGetRequest = new MockLowLevelHttpRequest() { + + @Override + public LowLevelHttpResponse execute() throws IOException { + lowLevelExecCalls++; + + if (lowLevelExecCalls <= callsBeforeSuccess) { + if (throwException) { + throw new IOException("Test IOException"); + } + + // Return failure on the first call + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setContent("Request should fail"); + response.setStatusCode(errorStatusCode); + return response; + } + // Return success on the second + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setStatusCode(200); + return response; + } + }; + + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return retryableGetRequest; + } + } + + public void testSimpleRetry() throws Exception { + FailThenSuccessBackoffTransport fakeTransport = + new FailThenSuccessBackoffTransport(HttpStatusCodes.STATUS_CODE_SERVER_ERROR, 3); + + MockGoogleCredential credential = RetryHttpInitializerWrapper.newMockCredentialBuilder() + .build(); + MockSleeper mockSleeper = new MockSleeper(); + + RetryHttpInitializerWrapper retryHttpInitializerWrapper = new RetryHttpInitializerWrapper(credential, mockSleeper, 5000); + + Compute client = new Compute.Builder(fakeTransport, new JacksonFactory(), null) + .setHttpRequestInitializer(retryHttpInitializerWrapper) + .setApplicationName("test") + .build(); + + HttpRequest request = client.getRequestFactory().buildRequest("Get", new GenericUrl("http://elasticsearch.com"), null); + HttpResponse response = request.execute(); + + assertThat(mockSleeper.getCount(), equalTo(3)); + assertThat(response.getStatusCode(), equalTo(200)); + } + + public void testRetryWaitTooLong() throws Exception { + int maxWaitTime = 10; + int maxRetryTimes = 50; + + FailThenSuccessBackoffTransport fakeTransport = + new FailThenSuccessBackoffTransport(HttpStatusCodes.STATUS_CODE_SERVER_ERROR, maxRetryTimes); + JsonFactory jsonFactory = new JacksonFactory(); + MockGoogleCredential credential = RetryHttpInitializerWrapper.newMockCredentialBuilder() + .build(); + + MockSleeper oneTimeSleeper = new MockSleeper() { + @Override + public void sleep(long millis) throws InterruptedException { + Thread.sleep(maxWaitTime); + super.sleep(0); // important number, use this to get count + } + }; + + RetryHttpInitializerWrapper retryHttpInitializerWrapper = new RetryHttpInitializerWrapper(credential, oneTimeSleeper, maxWaitTime); + + Compute client = new Compute.Builder(fakeTransport, jsonFactory, null) + .setHttpRequestInitializer(retryHttpInitializerWrapper) + .setApplicationName("test") + .build(); + + HttpRequest request1 = client.getRequestFactory().buildRequest("Get", new GenericUrl("http://elasticsearch.com"), null); + try { + request1.execute(); + fail("Request should fail if wait too long"); + } catch (HttpResponseException e) { + assertThat(e.getStatusCode(), equalTo(HttpStatusCodes.STATUS_CODE_SERVER_ERROR)); + // should only retry once. + assertThat(oneTimeSleeper.getCount(), lessThan(maxRetryTimes)); + } + } + + public void testIOExceptionRetry() throws Exception { + FailThenSuccessBackoffTransport fakeTransport = + new FailThenSuccessBackoffTransport(HttpStatusCodes.STATUS_CODE_SERVER_ERROR, 1, true); + + MockGoogleCredential credential = RetryHttpInitializerWrapper.newMockCredentialBuilder() + .build(); + MockSleeper mockSleeper = new MockSleeper(); + RetryHttpInitializerWrapper retryHttpInitializerWrapper = new RetryHttpInitializerWrapper(credential, mockSleeper, 500); + + Compute client = new Compute.Builder(fakeTransport, new JacksonFactory(), null) + .setHttpRequestInitializer(retryHttpInitializerWrapper) + .setApplicationName("test") + .build(); + + HttpRequest request = client.getRequestFactory().buildRequest("Get", new GenericUrl("http://elasticsearch.com"), null); + HttpResponse response = request.execute(); + + assertThat(mockSleeper.getCount(), equalTo(1)); + assertThat(response.getStatusCode(), equalTo(200)); + } +} diff --git a/core/src/main/java/org/elasticsearch/index/analysis/TokenizerFactoryFactory.java b/plugins/discovery-multicast/build.gradle similarity index 75% rename from core/src/main/java/org/elasticsearch/index/analysis/TokenizerFactoryFactory.java rename to plugins/discovery-multicast/build.gradle index e1d135dd4e1..f48f62841b7 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/TokenizerFactoryFactory.java +++ b/plugins/discovery-multicast/build.gradle @@ -17,14 +17,9 @@ * under the License. */ -package org.elasticsearch.index.analysis; - -import org.elasticsearch.common.settings.Settings; - -/** - * - */ -public interface TokenizerFactoryFactory { - - TokenizerFactory create(String name, Settings settings); +esplugin { + description 'The Multicast Discovery plugin allows discovery other nodes using multicast requests' + classname 'org.elasticsearch.plugin.discovery.multicast.MulticastDiscoveryPlugin' } + +compileJava.options.compilerArgs << "-Xlint:-deprecation" diff --git a/plugins/discovery-multicast/pom.xml b/plugins/discovery-multicast/pom.xml deleted file mode 100644 index eaa1e112f34..00000000000 --- a/plugins/discovery-multicast/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - discovery-multicast - Plugin: Discovery: Multicast - The Multicast Discovery plugin allows discovery other nodes using multicast requests - - - org.elasticsearch.plugin.discovery.multicast.MulticastDiscoveryPlugin - 1 - discovery_multicast - false - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastChannel.java b/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastChannel.java index ba620ddb3ca..dee74b9ddce 100644 --- a/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastChannel.java +++ b/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastChannel.java @@ -45,7 +45,7 @@ import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadF * A multicast channel that supports registering for receive events, and sending datagram packets. Allows * to easily share the same multicast socket if it holds the same config. */ -public abstract class MulticastChannel implements Closeable { +abstract class MulticastChannel implements Closeable { /** * Builds a channel based on the provided config, allowing to control if sharing a channel that uses diff --git a/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPing.java b/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPing.java index e05fc319955..f28bc08e9a6 100644 --- a/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPing.java +++ b/plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPing.java @@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.util.Constants; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.SpecialPermission; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -35,6 +36,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.network.NetworkUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -55,7 +57,12 @@ import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.net.InetAddress; import java.net.SocketAddress; +import java.security.AccessController; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -138,20 +145,26 @@ public class MulticastZenPing extends AbstractLifecycleComponent implem boolean shared = settings.getAsBoolean("discovery.zen.ping.multicast.shared", Constants.MAC_OS_X); // OSX does not correctly send multicasts FROM the right interface boolean deferToInterface = settings.getAsBoolean("discovery.zen.ping.multicast.defer_group_to_set_interface", Constants.MAC_OS_X); - multicastChannel = MulticastChannel.getChannel(nodeName(), shared, - new MulticastChannel.Config(port, group, bufferSize, ttl, - // don't use publish address, the use case for that is e.g. a firewall or proxy and - // may not even be bound to an interface on this machine! use the first bound address. - networkService.resolveBindHostAddress(address)[0], - deferToInterface), - new Receiver()); + // don't use publish address, the use case for that is e.g. a firewall or proxy and + // may not even be bound to an interface on this machine! use the first bound address. + List addresses = Arrays.asList(networkService.resolveBindHostAddresses(address == null ? null : new String[] { address })); + NetworkUtils.sortAddresses(addresses); + + final MulticastChannel.Config config = new MulticastChannel.Config(port, group, bufferSize, ttl, + addresses.get(0), deferToInterface); + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + multicastChannel = AccessController.doPrivileged(new PrivilegedExceptionAction() { + @Override + public MulticastChannel run() throws Exception { + return MulticastChannel.getChannel(nodeName(), shared, config, new Receiver()); + } + }); } catch (Throwable t) { String msg = "multicast failed to start [{}], disabling. Consider using IPv4 only (by defining env. variable `ES_USE_IPV4`)"; - if (logger.isDebugEnabled()) { - logger.debug(msg, t, ExceptionsHelper.detailedMessage(t)); - } else { - logger.info(msg, ExceptionsHelper.detailedMessage(t)); - } + logger.info(msg, t, ExceptionsHelper.detailedMessage(t)); } } diff --git a/plugins/discovery-multicast/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-multicast/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..5752c86bb4f --- /dev/null +++ b/plugins/discovery-multicast/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grant { + // needed to bind multicast to arbitrary port + permission java.net.SocketPermission "localhost:1024-", "listen,resolve"; +}; diff --git a/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPingTests.java b/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPingTests.java index 7ab0aa7ee98..ba673127f4f 100644 --- a/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPingTests.java +++ b/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPingTests.java @@ -39,14 +39,12 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.local.LocalTransport; import org.hamcrest.Matchers; import org.junit.Assert; -import org.junit.Test; import java.net.DatagramPacket; import java.net.InetAddress; import java.net.MulticastSocket; public class MulticastZenPingTests extends ESTestCase { - private Settings buildRandomMulticast(Settings settings) { Settings.Builder builder = Settings.builder().put(settings); builder.put("discovery.zen.ping.multicast.group", "224.2.3." + randomIntBetween(0, 255)); @@ -58,7 +56,6 @@ public class MulticastZenPingTests extends ESTestCase { return builder.build(); } - @Test public void testSimplePings() throws InterruptedException { Settings settings = Settings.EMPTY; settings = buildRandomMulticast(settings); @@ -132,7 +129,7 @@ public class MulticastZenPingTests extends ESTestCase { } } - @Test @SuppressForbidden(reason = "I bind to wildcard addresses. I am a total nightmare") + @SuppressForbidden(reason = "I bind to wildcard addresses. I am a total nightmare") public void testExternalPing() throws Exception { Settings settings = Settings.EMPTY; settings = buildRandomMulticast(settings); @@ -164,7 +161,7 @@ public class MulticastZenPingTests extends ESTestCase { MulticastSocket multicastSocket = null; try { Loggers.getLogger(MulticastZenPing.class).setLevel("TRACE"); - multicastSocket = new MulticastSocket(54328); + multicastSocket = new MulticastSocket(); multicastSocket.setReceiveBufferSize(2048); multicastSocket.setSendBufferSize(2048); multicastSocket.setSoTimeout(60000); diff --git a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactoryFactory.java b/plugins/jvm-example/build.gradle similarity index 76% rename from core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactoryFactory.java rename to plugins/jvm-example/build.gradle index d374187de9e..d8440eaecad 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactoryFactory.java +++ b/plugins/jvm-example/build.gradle @@ -17,14 +17,13 @@ * under the License. */ -package org.elasticsearch.index.analysis; - -import org.elasticsearch.common.settings.Settings; - -/** - * - */ -public interface TokenFilterFactoryFactory { - - TokenFilterFactory create(String name, Settings settings); +esplugin { + description 'Demonstrates all the pluggable Java entry points in Elasticsearch' + classname 'org.elasticsearch.plugin.example.JvmExamplePlugin' } + +// no unit tests +test.enabled = false + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" + diff --git a/plugins/jvm-example/licenses/no_deps.txt b/plugins/jvm-example/licenses/no_deps.txt deleted file mode 100644 index 8cce254d037..00000000000 --- a/plugins/jvm-example/licenses/no_deps.txt +++ /dev/null @@ -1 +0,0 @@ -This plugin has no third party dependencies diff --git a/plugins/jvm-example/pom.xml b/plugins/jvm-example/pom.xml deleted file mode 100644 index 96c9a0316c1..00000000000 --- a/plugins/jvm-example/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - jvm-example - Plugin: JVM example - Demonstrates all the pluggable Java entry points in Elasticsearch - - - org.elasticsearch.plugin.example.JvmExamplePlugin - jvm_example - false - true - -Xlint:-rawtypes - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - src/main/assemblies/plugin.xml - - - - - - - diff --git a/plugins/jvm-example/src/main/assemblies/plugin.xml b/plugins/jvm-example/src/main/assemblies/plugin.xml deleted file mode 100644 index 999ae36f4ca..00000000000 --- a/plugins/jvm-example/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - plugin - - zip - - false - - - ${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties - - true - - - - - src/main/config - config - - - src/main/bin - bin - - - - - / - true - true - true - - - diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java index 9c4ec733a9f..9dd9cb740ed 100644 --- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java +++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java @@ -65,12 +65,6 @@ public class JvmExamplePlugin extends Plugin { return services; } - @Override - public Collection indexModules(Settings indexSettings) { return Collections.emptyList();} - - @Override - public Collection> indexServices() { return Collections.emptyList();} - @Override public Settings additionalSettings() { return Settings.EMPTY; diff --git a/plugins/lang-expression/licenses/antlr4-runtime-4.5.jar.sha1 b/plugins/lang-expression/licenses/antlr4-runtime-4.5.jar.sha1 deleted file mode 100644 index 5299c19c73b..00000000000 --- a/plugins/lang-expression/licenses/antlr4-runtime-4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -29e48af049f17dd89153b83a7ad5d01b3b4bcdda diff --git a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1702855.jar.sha1 b/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1702855.jar.sha1 deleted file mode 100644 index d04bca5b6e4..00000000000 --- a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1702855.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ecfb9a923b19fac61b4e9a79275f6bd242b1f091 diff --git a/plugins/lang-expression/pom.xml b/plugins/lang-expression/pom.xml deleted file mode 100644 index cf503d4fb01..00000000000 --- a/plugins/lang-expression/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - lang-expression - Plugin: Language: Expression - Lucene expressions integration for Elasticsearch - - - org.elasticsearch.script.expression.ExpressionPlugin - lang_expression - false - -Xlint:-rawtypes - - - - - org.apache.lucene - lucene-expressions - ${lucene.maven.version} - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/lang-groovy/pom.xml b/plugins/lang-groovy/pom.xml deleted file mode 100644 index eeb5244b9a0..00000000000 --- a/plugins/lang-groovy/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - lang-groovy - Plugin: Language: Groovy - Groovy scripting integration for Elasticsearch - - - org.elasticsearch.script.groovy.GroovyPlugin - lang_groovy - false - -Xlint:-rawtypes,-unchecked,-cast,-deprecation - - - - - org.codehaus.groovy - groovy-all - indy - 2.4.4 - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AvgTests.java b/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AvgTests.java deleted file mode 100644 index 8ee84f1bcff..00000000000 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AvgTests.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.messy.tests; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.search.aggregations.bucket.global.Global; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; -import org.elasticsearch.search.aggregations.metrics.avg.Avg; -import org.junit.Test; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; -import static org.elasticsearch.search.aggregations.AggregationBuilders.global; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; - -/** - * - */ -public class AvgTests extends AbstractNumericTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - - @Override - @Test - public void testEmptyAggregation() throws Exception { - - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(avg("avg"))) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l)); - Histogram histo = searchResponse.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - Histogram.Bucket bucket = histo.getBuckets().get(1); - assertThat(bucket, notNullValue()); - - Avg avg = bucket.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(Double.isNaN(avg.getValue()), is(true)); - } - - @Override - @Test - public void testUnmapped() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx_unmapped") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("value")) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(0l)); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo(Double.NaN)); - } - - @Override - @Test - public void testSingleValuedField() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("value")) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10)); - } - - @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { - - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(global("global").subAggregation(avg("avg").field("value"))).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Global global = searchResponse.getAggregations().get("global"); - assertThat(global, notNullValue()); - assertThat(global.getName(), equalTo("global")); - assertThat(global.getDocCount(), equalTo(10l)); - assertThat(global.getAggregations(), notNullValue()); - assertThat(global.getAggregations().asMap().size(), equalTo(1)); - - Avg avg = global.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - double expectedAvgValue = (double) (1+2+3+4+5+6+7+8+9+10) / 10; - assertThat(avg.getValue(), equalTo(expectedAvgValue)); - assertThat((Avg) global.getProperty("avg"), equalTo(avg)); - assertThat((double) global.getProperty("avg.value"), equalTo(expectedAvgValue)); - assertThat((double) avg.getProperty("value"), equalTo(expectedAvgValue)); - } - - @Override - public void testSingleValuedField_PartiallyUnmapped() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("value")) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10)); - } - - @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("value").script(new Script("_value + 1"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); - } - - @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("value").script(new Script("_value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); - } - - public void testSingleValuedField_WithFormatter() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(avg("avg").format("#").field("value")).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10) / 10)); - assertThat(avg.getValueAsString(), equalTo("6")); - } - - @Override - @Test - public void testMultiValuedField() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("values")) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12) / 20)); - } - - @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("values").script(new Script("_value + 1"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12+12+13) / 20)); - } - - @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").field("values").script(new Script("_value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12+12+13) / 20)); - } - - @Override - @Test - public void testScript_SingleValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").script(new Script("doc['value'].value"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10)); - } - - @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); - } - - @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10)); - } - - @Override - @Test - public void testScript_MultiValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").script(new Script("[ doc['value'].value, doc['value'].value + 1 ]"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (1+2+2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11) / 20)); - } - - @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(avg("avg").script(new Script("[ doc['value'].value, doc['value'].value + 1 ]"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (1+2+2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11) / 20)); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - avg("avg").script(new Script("[ doc['value'].value, doc['value'].value + inc ]", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Avg avg = searchResponse.getAggregations().get("avg"); - assertThat(avg, notNullValue()); - assertThat(avg.getName(), equalTo("avg")); - assertThat(avg.getValue(), equalTo((double) (1+2+2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11) / 20)); - } -} \ No newline at end of file diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchTimeoutTests.java b/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchTimeoutTests.java deleted file mode 100644 index 2a982df00a8..00000000000 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchTimeoutTests.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.messy.tests; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; - -import java.util.Collection; -import java.util.Collections; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; -import static org.hamcrest.Matchers.equalTo; - -/** - */ -@ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE) -public class SearchTimeoutTests extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)).build(); - } - - @Test - public void simpleTimeoutTest() throws Exception { - client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet(); - - SearchResponse searchResponse = client().prepareSearch("test") - .setTimeout("10ms") - .setQuery(scriptQuery(new Script("Thread.sleep(500); return true;"))) - .execute().actionGet(); - assertThat(searchResponse.isTimedOut(), equalTo(true)); - } -} diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SumTests.java b/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SumTests.java deleted file mode 100644 index d36d833e093..00000000000 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SumTests.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.messy.tests; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.search.aggregations.bucket.global.Global; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.junit.Test; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.global; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; - -/** - * - */ -public class SumTests extends AbstractNumericTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - - @Override - @Test - public void testEmptyAggregation() throws Exception { - - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field("value").interval(1l).minDocCount(0).subAggregation(sum("sum"))) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l)); - Histogram histo = searchResponse.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - Histogram.Bucket bucket = histo.getBuckets().get(1); - assertThat(bucket, notNullValue()); - - Sum sum = bucket.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo(0.0)); - } - - @Override - @Test - public void testUnmapped() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx_unmapped") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("value")) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(0l)); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo(0.0)); - } - - @Override - @Test - public void testSingleValuedField() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("value")) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); - } - - @Test - public void testSingleValuedField_WithFormatter() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(sum("sum").format("0000.0").field("value")).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10)); - assertThat(sum.getValueAsString(), equalTo("0055.0")); - } - - @Override - @Test - public void testSingleValuedField_getProperty() throws Exception { - - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(global("global").subAggregation(sum("sum").field("value"))).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Global global = searchResponse.getAggregations().get("global"); - assertThat(global, notNullValue()); - assertThat(global.getName(), equalTo("global")); - assertThat(global.getDocCount(), equalTo(10l)); - assertThat(global.getAggregations(), notNullValue()); - assertThat(global.getAggregations().asMap().size(), equalTo(1)); - - Sum sum = global.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - double expectedSumValue = (double) 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10; - assertThat(sum.getValue(), equalTo(expectedSumValue)); - assertThat((Sum) global.getProperty("sum"), equalTo(sum)); - assertThat((double) global.getProperty("sum.value"), equalTo(expectedSumValue)); - assertThat((double) sum.getProperty("value"), equalTo(expectedSumValue)); - } - - @Override - public void testSingleValuedField_PartiallyUnmapped() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("value")) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); - } - - @Override - @Test - public void testSingleValuedField_WithValueScript() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("value").script(new Script("_value + 1"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 2+3+4+5+6+7+8+9+10+11)); - } - - @Override - @Test - public void testSingleValuedField_WithValueScript_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("increment", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("value").script(new Script("_value + increment", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 2+3+4+5+6+7+8+9+10+11)); - } - - @Override - @Test - public void testScript_SingleValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").script(new Script("doc['value'].value"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10)); - } - - @Override - @Test - public void testScript_SingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 2+3+4+5+6+7+8+9+10+11)); - } - - @Override - @Test - public void testScript_ExplicitSingleValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 2+3+4+5+6+7+8+9+10+11)); - } - - - @Override - @Test - public void testScript_MultiValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").script(new Script("[ doc['value'].value, doc['value'].value + 1 ]"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 1+2+2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11)); - } - - @Override - @Test - public void testScript_ExplicitMultiValued() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").script(new Script("[ doc['value'].value, doc['value'].value + 1 ]"))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 1+2+2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11)); - } - - @Override - @Test - public void testScript_MultiValued_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("inc", 1); - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - sum("sum").script(new Script("[ doc['value'].value, doc['value'].value + inc ]", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 1+2+2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11)); - } - - @Override - @Test - public void testMultiValuedField() throws Exception { - - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("values")) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12)); - } - - @Override - @Test - public void testMultiValuedField_WithValueScript() throws Exception { - - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("values").script(new Script("_value + 1"))).execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 3 + 4 + 4 + 5 + 5 + 6 + 6 + 7 + 7 + 8 + 8 + 9 + 9 + 10 + 10 + 11 + 11 + 12 + 12 + 13)); - } - - @Override - @Test - public void testMultiValuedField_WithValueScript_WithParams() throws Exception { - Map params = new HashMap<>(); - params.put("increment", 1); - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(sum("sum").field("values").script(new Script("_value + increment", ScriptType.INLINE, null, params))) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - - Sum sum = searchResponse.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat(sum.getName(), equalTo("sum")); - assertThat(sum.getValue(), equalTo((double) 3 + 4 + 4 + 5 + 5 + 6 + 6 + 7 + 7 + 8 + 8 + 9 + 9 + 10 + 10 + 11 + 11 + 12 + 12 + 13)); - } -} \ No newline at end of file diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TransformOnIndexMapperTests.java b/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TransformOnIndexMapperTests.java deleted file mode 100644 index 69da9c7d65e..00000000000 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TransformOnIndexMapperTests.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.messy.tests; - -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.suggest.SuggestResponse; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.script.groovy.GroovyScriptEngineService; -import org.elasticsearch.search.suggest.SuggestBuilders; -import org.elasticsearch.search.suggest.completion.CompletionSuggestion; -import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -import static java.util.Collections.singletonMap; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion; -import static org.hamcrest.Matchers.both; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.not; - -/** - * Tests for transforming the source document before indexing. - */ -@SuppressCodecs("*") // requires custom completion format -public class TransformOnIndexMapperTests extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - - @Test - public void searchOnTransformed() throws Exception { - setup(true); - - // Searching by the field created in the transport finds the entry - SearchResponse response = client().prepareSearch("test").setQuery(termQuery("destination", "findme")).get(); - assertSearchHits(response, "righttitle"); - // The field built in the transform isn't in the source but source is, - // even though we didn't index it! - assertRightTitleSourceUntransformed(response.getHits().getAt(0).sourceAsMap()); - - // Can't find by a field removed from the document by the transform - response = client().prepareSearch("test").setQuery(termQuery("content", "findme")).get(); - assertHitCount(response, 0); - } - - @Test - public void getTransformed() throws Exception { - setup(getRandom().nextBoolean()); - GetResponse response = client().prepareGet("test", "test", "righttitle").get(); - assertExists(response); - assertRightTitleSourceUntransformed(response.getSource()); - - response = client().prepareGet("test", "test", "righttitle").setTransformSource(true).get(); - assertExists(response); - assertRightTitleSourceTransformed(response.getSource()); - } - - // TODO: the completion suggester currently returns payloads with no reencoding so this test - // exists to make sure that _source transformation and completion work well together. If we - // ever fix the completion suggester to reencode the payloads then we can remove this test. - @Test - public void contextSuggestPayloadTransformed() throws Exception { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - builder.startObject("properties"); - builder.startObject("suggest").field("type", "completion").field("payloads", true).endObject(); - builder.endObject(); - builder.startObject("transform"); - builder.field("script", "ctx._source.suggest = ['input': ctx._source.text];ctx._source.suggest.payload = ['display': ctx._source.text, 'display_detail': 'on the fly']"); - builder.field("lang", GroovyScriptEngineService.NAME); - builder.endObject(); - assertAcked(client().admin().indices().prepareCreate("test").addMapping("test", builder)); - // Payload is stored using original source format (json, smile, yaml, whatever) - XContentType type = XContentType.values()[between(0, XContentType.values().length - 1)]; - XContentBuilder source = XContentFactory.contentBuilder(type); - source.startObject().field("text", "findme").endObject(); - indexRandom(true, client().prepareIndex("test", "test", "findme").setSource(source)); - SuggestResponse response = client().prepareSuggest("test").addSuggestion( - SuggestBuilders.completionSuggestion("test").field("suggest").text("findme")).get(); - assertSuggestion(response.getSuggest(), 0, 0, "test", "findme"); - CompletionSuggestion.Entry.Option option = (CompletionSuggestion.Entry.Option)response.getSuggest().getSuggestion("test").getEntries().get(0).getOptions().get(0); - // And it comes back in exactly that way. - XContentBuilder expected = XContentFactory.contentBuilder(type); - expected.startObject().field("display", "findme").field("display_detail", "on the fly").endObject(); - assertEquals(expected.string(), option.getPayloadAsString()); - } - - /** - * Setup an index with some source transforms. Randomly picks the number of - * transforms but all but one of the transforms is a noop. The other is a - * script that fills the 'destination' field with the 'content' field only - * if the 'title' field starts with 't' and then always removes the - * 'content' field regarless of the contents of 't'. The actual script - * randomly uses parameters or not. - * - * @param forceRefresh - * should the data be flushed to disk? Set to false to test real - * time fetching - */ - private void setup(boolean forceRefresh) throws IOException, InterruptedException, ExecutionException { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - builder.field("transform"); - if (getRandom().nextBoolean()) { - // Single transform - builder.startObject(); - buildTransformScript(builder); - builder.field("lang", randomFrom(null, GroovyScriptEngineService.NAME)); - builder.endObject(); - } else { - // Multiple transforms - int total = between(1, 10); - int actual = between(0, total - 1); - builder.startArray(); - for (int s = 0; s < total; s++) { - builder.startObject(); - if (s == actual) { - buildTransformScript(builder); - } else { - builder.field("script", "true"); - } - builder.field("lang", randomFrom(null, GroovyScriptEngineService.NAME)); - builder.endObject(); - } - builder.endArray(); - } - assertAcked(client().admin().indices().prepareCreate("test").addMapping("test", builder)); - - indexRandom(forceRefresh, client().prepareIndex("test", "test", "notitle").setSource("content", "findme"), - client().prepareIndex("test", "test", "badtitle").setSource("content", "findme", "title", "cat"), - client().prepareIndex("test", "test", "righttitle").setSource("content", "findme", "title", "table")); - } - - private void buildTransformScript(XContentBuilder builder) throws IOException { - String script = "if (ctx._source['title']?.startsWith('t')) { ctx._source['destination'] = ctx._source[sourceField] }; ctx._source.remove(sourceField);"; - if (getRandom().nextBoolean()) { - script = script.replace("sourceField", "'content'"); - } else { - builder.field("params", singletonMap("sourceField", "content")); - } - builder.field("script", script); - } - - private void assertRightTitleSourceUntransformed(Map source) { - assertThat(source, both(hasEntry("content", (Object) "findme")).and(not(hasKey("destination")))); - } - - private void assertRightTitleSourceTransformed(Map source) { - assertThat(source, both(hasEntry("destination", (Object) "findme")).and(not(hasKey("content")))); - } - -} diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java deleted file mode 100644 index 043a5d1d2f7..00000000000 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script.groovy; - -import org.apache.lucene.util.Constants; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptException; -import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; - -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.Locale; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; - -/** - * Tests for the Groovy security permissions - */ -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) -// TODO: refactor into unit test, or, proper REST test -public class GroovySecurityTests extends ESIntegTestCase { - - @Override - public void setUp() throws Exception { - super.setUp(); - assumeTrue("test requires security manager to be enabled", System.getSecurityManager() != null); - } - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - - @Test - public void testEvilGroovyScripts() throws Exception { - int nodes = randomIntBetween(1, 3); - Settings nodeSettings = Settings.builder() - .put("script.inline", true) - .put("script.indexed", true) - .build(); - internalCluster().startNodesAsync(nodes, nodeSettings).get(); - client().admin().cluster().prepareHealth().setWaitForNodes(nodes + "").get(); - - client().prepareIndex("test", "doc", "1").setSource("foo", 5, "bar", "baz").setRefresh(true).get(); - - // Plain test - assertSuccess(""); - // numeric field access - assertSuccess("def foo = doc['foo'].value; if (foo == null) { return 5; }"); - // string field access - assertSuccess("def bar = doc['bar'].value; if (bar == null) { return 5; }"); - // List - assertSuccess("def list = [doc['foo'].value, 3, 4]; def v = list.get(1); list.add(10)"); - // Ranges - assertSuccess("def range = 1..doc['foo'].value; def v = range.get(0)"); - // Maps - assertSuccess("def v = doc['foo'].value; def m = [:]; m.put(\\\"value\\\", v)"); - // Times - assertSuccess("def t = Instant.now().getMillis()"); - // GroovyCollections - assertSuccess("def n = [1,2,3]; GroovyCollections.max(n)"); - - // Fail cases: - // AccessControlException[access denied ("java.io.FilePermission" "<>" "execute")] - assertFailure("pr = Runtime.getRuntime().exec(\\\"touch /tmp/gotcha\\\"); pr.waitFor()"); - - // AccessControlException[access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.reflect")] - assertFailure("d = new DateTime(); d.getClass().getDeclaredMethod(\\\"year\\\").setAccessible(true)"); - assertFailure("d = new DateTime(); d.\\\"${'get' + 'Class'}\\\"()." + - "\\\"${'getDeclared' + 'Method'}\\\"(\\\"year\\\").\\\"${'set' + 'Accessible'}\\\"(false)"); - assertFailure("Class.forName(\\\"org.joda.time.DateTime\\\").getDeclaredMethod(\\\"year\\\").setAccessible(true)"); - - // AccessControlException[access denied ("groovy.security.GroovyCodeSourcePermission" "/groovy/shell")] - assertFailure("Eval.me('2 + 2')"); - assertFailure("Eval.x(5, 'x + 2')"); - - // AccessControlException[access denied ("java.lang.RuntimePermission" "accessDeclaredMembers")] - assertFailure("d = new Date(); java.lang.reflect.Field f = Date.class.getDeclaredField(\\\"fastTime\\\");" + - " f.setAccessible(true); f.get(\\\"fastTime\\\")"); - - // AccessControlException[access denied ("java.io.FilePermission" "<>" "execute")] - assertFailure("def methodName = 'ex'; Runtime.\\\"${'get' + 'Runtime'}\\\"().\\\"${methodName}ec\\\"(\\\"touch /tmp/gotcha2\\\")"); - - // test a directory we normally have access to, but the groovy script does not. - Path dir = createTempDir(); - // TODO: figure out the necessary escaping for windows paths here :) - if (!Constants.WINDOWS) { - // access denied ("java.io.FilePermission" ".../tempDir-00N" "read") - assertFailure("new File(\\\"" + dir + "\\\").exists()"); - } - } - - private void assertSuccess(String script) { - logger.info("--> script: " + script); - SearchResponse resp = client().prepareSearch("test") - .setSource(new BytesArray("{\"query\": {\"match_all\": {}}," + - "\"sort\":{\"_script\": {\"script\": \"" + script + - "; doc['foo'].value + 2\", \"type\": \"number\", \"lang\": \"groovy\"}}}")).get(); - assertNoFailures(resp); - assertEquals(1, resp.getHits().getTotalHits()); - assertThat(resp.getHits().getAt(0).getSortValues(), equalTo(new Object[]{7.0})); - } - - private void assertFailure(String script) { - logger.info("--> script: " + script); - SearchResponse resp = client().prepareSearch("test") - .setSource(new BytesArray("{\"query\": {\"match_all\": {}}," + - "\"sort\":{\"_script\": {\"script\": \"" + script + - "; doc['foo'].value + 2\", \"type\": \"number\", \"lang\": \"groovy\"}}}")).get(); - assertEquals(0, resp.getHits().getTotalHits()); - ShardSearchFailure fails[] = resp.getShardFailures(); - // TODO: GroovyScriptExecutionException needs work: - // fix it to preserve cause so we don't do this flaky string-check stuff - for (ShardSearchFailure fail : fails) { - assertThat(fail.getCause(), instanceOf(ScriptException.class)); - assertTrue("unexpected exception" + fail.getCause(), - // different casing, depending on jvm impl... - fail.getCause().toString().toLowerCase(Locale.ROOT).contains("[access denied")); - } - } -} diff --git a/plugins/lang-javascript/build.gradle b/plugins/lang-javascript/build.gradle new file mode 100644 index 00000000000..ead459f29d1 --- /dev/null +++ b/plugins/lang-javascript/build.gradle @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The JavaScript language plugin allows to have javascript as the language of scripts to execute.' + classname 'org.elasticsearch.plugin.javascript.JavaScriptPlugin' +} + +dependencies { + compile 'org.mozilla:rhino:1.7.7' +} + +compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked" + +integTest { + cluster { + systemProperty 'es.script.inline', 'on' + systemProperty 'es.script.indexed', 'on' + } +} + diff --git a/plugins/lang-javascript/licenses/rhino-1.7.7.jar.sha1 b/plugins/lang-javascript/licenses/rhino-1.7.7.jar.sha1 new file mode 100644 index 00000000000..8c997d41c2b --- /dev/null +++ b/plugins/lang-javascript/licenses/rhino-1.7.7.jar.sha1 @@ -0,0 +1 @@ +3a9ea863b86126b0ed8f2fe2230412747cd3c254 \ No newline at end of file diff --git a/plugins/lang-javascript/licenses/rhino-1.7R4.jar.sha1 b/plugins/lang-javascript/licenses/rhino-1.7R4.jar.sha1 deleted file mode 100644 index 3432f18a5de..00000000000 --- a/plugins/lang-javascript/licenses/rhino-1.7R4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e982f2136574b9a423186fbaeaaa98dc3e5a5288 diff --git a/plugins/lang-javascript/pom.xml b/plugins/lang-javascript/pom.xml deleted file mode 100644 index 69da06ec4d2..00000000000 --- a/plugins/lang-javascript/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - lang-javascript - Plugin: Language: JavaScript - The JavaScript language plugin allows to have javascript as the language of scripts to execute. - - - org.elasticsearch.plugin.javascript.JavaScriptPlugin - lang_javascript - false - -Xlint:-rawtypes,-unchecked - - - - - org.mozilla - rhino - 1.7R4 - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java index a6832fe7afe..9ca36bd9f86 100644 --- a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java @@ -28,6 +28,11 @@ import org.elasticsearch.script.javascript.JavaScriptScriptEngineService; */ public class JavaScriptPlugin extends Plugin { + static { + // install rhino policy on plugin init + JavaScriptScriptEngineService.init(); + } + @Override public String name() { return "lang-javascript"; diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java index 70fc5f46d63..33a4e55801b 100644 --- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java @@ -39,7 +39,10 @@ import org.mozilla.javascript.Script; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.security.AccessControlContext; +import java.security.AccessController; import java.security.CodeSource; +import java.security.PrivilegedAction; import java.security.cert.Certificate; import java.util.List; import java.util.Map; @@ -54,19 +57,74 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements private static WrapFactory wrapFactory = new CustomWrapFactory(); - private final int optimizationLevel; - private Scriptable globalScope; + // one time initialization of rhino security manager integration + private static final CodeSource DOMAIN; + private static final int OPTIMIZATION_LEVEL = 1; + + static { + try { + DOMAIN = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + ContextFactory factory = new ContextFactory() { + @Override + protected void onContextCreated(Context cx) { + cx.setWrapFactory(wrapFactory); + cx.setOptimizationLevel(OPTIMIZATION_LEVEL); + } + }; + if (System.getSecurityManager() != null) { + factory.initApplicationClassLoader(AccessController.doPrivileged(new PrivilegedAction() { + @Override + public ClassLoader run() { + // snapshot our context (which has permissions for classes), since the script has none + final AccessControlContext engineContext = AccessController.getContext(); + return new ClassLoader(JavaScriptScriptEngineService.class.getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + return super.loadClass(name, resolve); + } + }; + } + })); + } + factory.seal(); + ContextFactory.initGlobal(factory); + SecurityController.initGlobal(new PolicySecurityController() { + @Override + public GeneratedClassLoader createClassLoader(ClassLoader parent, Object securityDomain) { + // don't let scripts compile other scripts + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + // check the domain, this is all we allow + if (securityDomain != DOMAIN) { + throw new SecurityException("illegal securityDomain: " + securityDomain); + } + + return super.createClassLoader(parent, securityDomain); + } + }); + } + + /** ensures this engine is initialized */ + public static void init() {} + @Inject public JavaScriptScriptEngineService(Settings settings) { super(settings); - this.optimizationLevel = settings.getAsInt("script.javascript.optimization_level", 1); - Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); globalScope = ctx.initStandardObjects(null, true); } finally { Context.exit(); @@ -100,21 +158,9 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements @Override public Object compile(String script) { - // we don't know why kind of safeguards rhino has, - // but just be safe - SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPermission(new SpecialPermission()); - } Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); - ctx.setOptimizationLevel(optimizationLevel); - ctx.setSecurityController(new PolicySecurityController()); - return ctx.compileString(script, generateScriptName(), 1, - new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null)); - } catch (MalformedURLException e) { - throw new RuntimeException(e); + return ctx.compileString(script, generateScriptName(), 1, DOMAIN); } finally { Context.exit(); } @@ -124,8 +170,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public ExecutableScript executable(CompiledScript compiledScript, Map vars) { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); - Scriptable scope = ctx.newObject(globalScope); scope.setPrototype(globalScope); scope.setParentScope(null); @@ -143,8 +187,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public SearchScript search(final CompiledScript compiledScript, final SearchLookup lookup, @Nullable final Map vars) { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); - final Scriptable scope = ctx.newObject(globalScope); scope.setPrototype(globalScope); scope.setParentScope(null); @@ -197,7 +239,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public Object run() { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); return ScriptValueConverter.unwrapValue(script.exec(ctx, scope)); } finally { Context.exit(); @@ -258,7 +299,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public Object run() { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); return ScriptValueConverter.unwrapValue(script.exec(ctx, scope)); } finally { Context.exit(); diff --git a/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy b/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy index e45c1b86ceb..739a2531d2f 100644 --- a/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy @@ -20,4 +20,15 @@ grant { // needed to generate runtime classes permission java.lang.RuntimePermission "createClassLoader"; + + // Standard set of classes + permission org.elasticsearch.script.ClassPermission "<>"; + // rhino runtime (TODO: clean these up if possible) + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.ContextFactory"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.Callable"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.NativeFunction"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.Script"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.ScriptRuntime"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.Undefined"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.optimizer.OptRuntime"; }; diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java index f0a31810c4c..fe9cc324f1c 100644 --- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.Arrays; import java.util.HashMap; @@ -41,7 +40,6 @@ import static org.hamcrest.Matchers.instanceOf; * */ public class JavaScriptScriptEngineTests extends ESTestCase { - private JavaScriptScriptEngineService se; @Before @@ -54,14 +52,12 @@ public class JavaScriptScriptEngineTests extends ESTestCase { se.close(); } - @Test public void testSimpleEquation() { Map vars = new HashMap(); Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "js", se.compile("1 + 2")), vars).run(); assertThat(((Number) o).intValue(), equalTo(3)); } - @Test public void testMapAccess() { Map vars = new HashMap(); @@ -78,7 +74,6 @@ public class JavaScriptScriptEngineTests extends ESTestCase { assertThat(((String) o), equalTo("2")); } - @Test public void testJavaScriptObjectToMap() { Map vars = new HashMap(); Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptObjectToMap", "js", @@ -88,7 +83,6 @@ public class JavaScriptScriptEngineTests extends ESTestCase { assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); } - @Test public void testJavaScriptObjectMapInter() { Map vars = new HashMap(); Map ctx = new HashMap(); @@ -107,7 +101,6 @@ public class JavaScriptScriptEngineTests extends ESTestCase { assertThat((String) ((Map) ctx.get("obj2")).get("prop2"), equalTo("value2")); } - @Test public void testJavaScriptInnerArrayCreation() { Map ctx = new HashMap(); Map doc = new HashMap(); @@ -124,7 +117,6 @@ public class JavaScriptScriptEngineTests extends ESTestCase { assertThat(((Map) unwrap.get("doc")).get("field1"), instanceOf(List.class)); } - @Test public void testAccessListInScript() { Map vars = new HashMap(); Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); @@ -150,7 +142,6 @@ public class JavaScriptScriptEngineTests extends ESTestCase { assertThat(((String) o), equalTo("value1")); } - @Test public void testChangingVarsCrossExecution1() { Map vars = new HashMap(); Map ctx = new HashMap(); @@ -168,7 +159,6 @@ public class JavaScriptScriptEngineTests extends ESTestCase { assertThat(((Number) o).intValue(), equalTo(2)); } - @Test public void testChangingVarsCrossExecution2() { Map vars = new HashMap(); Object compiledScript = se.compile("value"); diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java index b639ed99f6b..2308e666c51 100644 --- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -39,8 +38,6 @@ import static org.hamcrest.Matchers.equalTo; * */ public class JavaScriptScriptMultiThreadedTests extends ESTestCase { - - @Test public void testExecutableNoRuntimeParams() throws Exception { final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); final Object compiled = se.compile("x + y"); @@ -83,8 +80,6 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase { assertThat(failed.get(), equalTo(false)); } - - @Test public void testExecutableWithRuntimeParams() throws Exception { final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); final Object compiled = se.compile("x + y"); @@ -127,7 +122,6 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase { assertThat(failed.get(), equalTo(false)); } - @Test public void testExecute() throws Exception { final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); final Object compiled = se.compile("x + y"); diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java index 887317fb744..c6f9805f818 100644 --- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java @@ -23,8 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; -import org.junit.After; -import org.junit.Before; +import org.mozilla.javascript.EcmaError; import org.mozilla.javascript.WrappedException; import java.util.HashMap; @@ -37,14 +36,18 @@ public class JavaScriptSecurityTests extends ESTestCase { private JavaScriptScriptEngineService se; - @Before - public void setup() { + @Override + public void setUp() throws Exception { + super.setUp(); se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + // otherwise will exit your VM and other bad stuff + assumeTrue("test requires security manager to be enabled", System.getSecurityManager() != null); } - @After - public void close() { + @Override + public void tearDown() throws Exception { se.close(); + super.tearDown(); } /** runs a script */ @@ -59,14 +62,20 @@ public class JavaScriptSecurityTests extends ESTestCase { } /** assert that a security exception is hit */ - private void assertFailure(String script) { + private void assertFailure(String script, Class exceptionClass) { try { doTest(script); fail("did not get expected exception"); } catch (WrappedException expected) { Throwable cause = expected.getCause(); assertNotNull(cause); - assertTrue("unexpected exception: " + cause, cause instanceof SecurityException); + if (exceptionClass.isAssignableFrom(cause.getClass()) == false) { + throw new AssertionError("unexpected exception: " + expected, expected); + } + } catch (EcmaError expected) { + if (exceptionClass.isAssignableFrom(expected.getClass()) == false) { + throw new AssertionError("unexpected exception: " + expected, expected); + } } } @@ -77,13 +86,22 @@ public class JavaScriptSecurityTests extends ESTestCase { } /** Test some javascripts that should hit security exception */ - public void testNotOK() { + public void testNotOK() throws Exception { // sanity check :) - assertFailure("java.lang.Runtime.getRuntime().halt(0)"); + assertFailure("java.lang.Runtime.getRuntime().halt(0)", EcmaError.class); // check a few things more restrictive than the ordinary policy // no network - assertFailure("new java.net.Socket(\"localhost\", 1024)"); + assertFailure("new java.net.Socket(\"localhost\", 1024)", EcmaError.class); // no files - assertFailure("java.io.File.createTempFile(\"test\", \"tmp\")"); + assertFailure("java.io.File.createTempFile(\"test\", \"tmp\")", EcmaError.class); + } + + public void testDefinitelyNotOK() { + // no mucking with security controller + assertFailure("var ctx = org.mozilla.javascript.Context.getCurrentContext(); " + + "ctx.setSecurityController(new org.mozilla.javascript.PolicySecurityController());", EcmaError.class); + // no compiling scripts from scripts + assertFailure("var ctx = org.mozilla.javascript.Context.getCurrentContext(); " + + "ctx.compileString(\"1 + 1\", \"foobar\", 1, null); ", EcmaError.class); } } diff --git a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml index ee77a848c44..6d1625a9494 100644 --- a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml +++ b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/10_basic.yaml @@ -18,9 +18,10 @@ setup: body: script_fields: bar: - lang: javascript - script: "doc['foo'].value + x" - params: - x: "bbb" + script: + inline: "doc['foo'].value + x" + lang: javascript + params: + x: "bbb" - match: { hits.hits.0.fields.bar.0: "aaabbb"} diff --git a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml index 24a6c8b9807..742c8f0204d 100644 --- a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml +++ b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml @@ -33,8 +33,9 @@ lang: js script_fields: sNum1: - lang: js - script: "doc['num1'].value" + script: + inline: "doc['num1'].value" + lang: js sort: num1: order: asc @@ -57,8 +58,9 @@ script_fields: sNum1: - lang: js - script: "doc['num1'].value" + script: + inline: "doc['num1'].value" + lang: js sort: num1: order: asc @@ -81,8 +83,9 @@ script_fields: sNum1: - lang: js - script: "doc['num1'].value" + script: + inline: "doc['num1'].value" + lang: js sort: num1: order: asc @@ -118,17 +121,21 @@ body: script_fields: s_obj1: - lang: js - script: "_source.obj1" + script: + inline: "_source.obj1" + lang: js s_obj1_test: - lang: js - script: "_source.obj1.test" + script: + inline: "_source.obj1.test" + lang: js s_obj2: - lang: js - script: "_source.obj2" + script: + inline: "_source.obj2" + lang: js s_obj2_arr2: - lang: js - script: "_source.obj2.arr2" + script: + inline: "_source.obj2.arr2" + lang: js - match: { hits.total: 1 } - match: { hits.hits.0.fields.s_obj1.0.test: something } @@ -399,8 +406,9 @@ body: script_fields: foobar: - lang: js - script: "doc['f'].values.length" + script: + inline: "doc['f'].values.length" + lang: js - match: { hits.total: 1 } diff --git a/plugins/lang-plan-a/ant.xml b/plugins/lang-plan-a/ant.xml new file mode 100644 index 00000000000..bf1c9b93757 --- /dev/null +++ b/plugins/lang-plan-a/ant.xml @@ -0,0 +1,145 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/lang-plan-a/build.gradle b/plugins/lang-plan-a/build.gradle new file mode 100644 index 00000000000..618c094f683 --- /dev/null +++ b/plugins/lang-plan-a/build.gradle @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.tools.ant.types.Path + +esplugin { + description 'An easy, safe and fast scripting language for Elasticsearch' + classname 'org.elasticsearch.plan.a.PlanAPlugin' +} + +dependencies { + compile 'org.antlr:antlr4-runtime:4.5.1-1' + compile 'org.ow2.asm:asm:5.0.4' + compile 'org.ow2.asm:asm-commons:5.0.4' +} + +compileJava.options.compilerArgs << '-Xlint:-cast,-fallthrough,-rawtypes' +compileTestJava.options.compilerArgs << '-Xlint:-unchecked' + +// regeneration logic, comes in via ant right now +// don't port it to gradle, it works fine. + +configurations { + regenerate +} + +dependencies { + regenerate 'org.antlr:antlr4:4.5.1-1' +} + +ant.references['regenerate.classpath'] = new Path(ant.project, configurations.regenerate.asPath) +ant.importBuild 'ant.xml' diff --git a/plugins/lang-plan-a/licenses/antlr4-runtime-4.5.1-1.jar.sha1 b/plugins/lang-plan-a/licenses/antlr4-runtime-4.5.1-1.jar.sha1 new file mode 100644 index 00000000000..37f80b91724 --- /dev/null +++ b/plugins/lang-plan-a/licenses/antlr4-runtime-4.5.1-1.jar.sha1 @@ -0,0 +1 @@ +66144204f9d6d7d3f3f775622c2dd7e9bd511d97 \ No newline at end of file diff --git a/plugins/lang-plan-a/licenses/antlr4-runtime-LICENSE.txt b/plugins/lang-plan-a/licenses/antlr4-runtime-LICENSE.txt new file mode 100644 index 00000000000..95d0a2554f6 --- /dev/null +++ b/plugins/lang-plan-a/licenses/antlr4-runtime-LICENSE.txt @@ -0,0 +1,26 @@ +[The "BSD license"] +Copyright (c) 2015 Terence Parr, Sam Harwell +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plugins/discovery-multicast/licenses/no_deps.txt b/plugins/lang-plan-a/licenses/antlr4-runtime-NOTICE.txt similarity index 100% rename from plugins/discovery-multicast/licenses/no_deps.txt rename to plugins/lang-plan-a/licenses/antlr4-runtime-NOTICE.txt diff --git a/plugins/lang-plan-a/licenses/asm-5.0.4.jar.sha1 b/plugins/lang-plan-a/licenses/asm-5.0.4.jar.sha1 new file mode 100644 index 00000000000..9223dba380f --- /dev/null +++ b/plugins/lang-plan-a/licenses/asm-5.0.4.jar.sha1 @@ -0,0 +1 @@ +0da08b8cce7bbf903602a25a3a163ae252435795 diff --git a/plugins/lang-plan-a/licenses/asm-LICENSE.txt b/plugins/lang-plan-a/licenses/asm-LICENSE.txt new file mode 100644 index 00000000000..afb064f2f26 --- /dev/null +++ b/plugins/lang-plan-a/licenses/asm-LICENSE.txt @@ -0,0 +1,26 @@ +Copyright (c) 2012 France Télécom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/distribution/licenses/snakeyaml-NOTICE.txt b/plugins/lang-plan-a/licenses/asm-NOTICE.txt similarity index 100% rename from distribution/licenses/snakeyaml-NOTICE.txt rename to plugins/lang-plan-a/licenses/asm-NOTICE.txt diff --git a/plugins/lang-plan-a/licenses/asm-commons-5.0.4.jar.sha1 b/plugins/lang-plan-a/licenses/asm-commons-5.0.4.jar.sha1 new file mode 100644 index 00000000000..94fe0cd92c9 --- /dev/null +++ b/plugins/lang-plan-a/licenses/asm-commons-5.0.4.jar.sha1 @@ -0,0 +1 @@ +5a556786086c23cd689a0328f8519db93821c04c diff --git a/plugins/lang-plan-a/licenses/asm-commons-LICENSE.txt b/plugins/lang-plan-a/licenses/asm-commons-LICENSE.txt new file mode 100644 index 00000000000..afb064f2f26 --- /dev/null +++ b/plugins/lang-plan-a/licenses/asm-commons-LICENSE.txt @@ -0,0 +1,26 @@ +Copyright (c) 2012 France Télécom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plugins/lang-plan-a/licenses/asm-commons-NOTICE.txt b/plugins/lang-plan-a/licenses/asm-commons-NOTICE.txt new file mode 100644 index 00000000000..8d1c8b69c3f --- /dev/null +++ b/plugins/lang-plan-a/licenses/asm-commons-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/plugins/lang-plan-a/src/main/antlr/PlanALexer.g4 b/plugins/lang-plan-a/src/main/antlr/PlanALexer.g4 new file mode 100644 index 00000000000..5110a73e8ca --- /dev/null +++ b/plugins/lang-plan-a/src/main/antlr/PlanALexer.g4 @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +lexer grammar PlanALexer; + +@header { + import java.util.Set; +} + +@members { + private Set types = null; + + void setTypes(Set types) { + this.types = types; + } +} + +WS: [ \t\n\r]+ -> skip; +COMMENT: ( '//' .*? [\n\r] | '/*' .*? '*/' ) -> skip; + +LBRACK: '{'; +RBRACK: '}'; +LBRACE: '['; +RBRACE: ']'; +LP: '('; +RP: ')'; +DOT: '.' -> mode(EXT); +COMMA: ','; +SEMICOLON: ';'; +IF: 'if'; +ELSE: 'else'; +WHILE: 'while'; +DO: 'do'; +FOR: 'for'; +CONTINUE: 'continue'; +BREAK: 'break'; +RETURN: 'return'; +NEW: 'new'; +TRY: 'try'; +CATCH: 'catch'; +THROW: 'throw'; + +BOOLNOT: '!'; +BWNOT: '~'; +MUL: '*'; +DIV: '/'; +REM: '%'; +ADD: '+'; +SUB: '-'; +LSH: '<<'; +RSH: '>>'; +USH: '>>>'; +LT: '<'; +LTE: '<='; +GT: '>'; +GTE: '>='; +EQ: '=='; +EQR: '==='; +NE: '!='; +NER: '!=='; +BWAND: '&'; +BWXOR: '^'; +BWOR: '|'; +BOOLAND: '&&'; +BOOLOR: '||'; +COND: '?'; +COLON: ':'; +INCR: '++'; +DECR: '--'; + +ASSIGN: '='; +AADD: '+='; +ASUB: '-='; +AMUL: '*='; +ADIV: '/='; +AREM: '%='; +AAND: '&='; +AXOR: '^='; +AOR: '|='; +ALSH: '<<='; +ARSH: '>>='; +AUSH: '>>>='; +ACAT: '..='; + +OCTAL: '0' [0-7]+ [lL]?; +HEX: '0' [xX] [0-9a-fA-F]+ [lL]?; +INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?; +DECIMAL: ( '0' | [1-9] [0-9]* ) DOT [0-9]* ( [eE] [+\-]? [0-9]+ )? [fF]?; + +STRING: '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' {setText(getText().substring(1, getText().length() - 1));}; +CHAR: '\'' . '\'' {setText(getText().substring(1, getText().length() - 1));}; + +TRUE: 'true'; +FALSE: 'false'; + +NULL: 'null'; + +TYPE: ID GENERIC? {types.contains(getText().replace(" ", ""))}? {setText(getText().replace(" ", ""));}; +fragment GENERIC: ' '* '<' ' '* ( ID GENERIC? ) ' '* ( COMMA ' '* ( ID GENERIC? ) ' '* )* '>'; +ID: [_a-zA-Z] [_a-zA-Z0-9]*; + +mode EXT; +EXTINTEGER: ( '0' | [1-9] [0-9]* ) -> mode(DEFAULT_MODE); +EXTID: [_a-zA-Z] [_a-zA-Z0-9]* -> mode(DEFAULT_MODE); diff --git a/plugins/lang-plan-a/src/main/antlr/PlanAParser.g4 b/plugins/lang-plan-a/src/main/antlr/PlanAParser.g4 new file mode 100644 index 00000000000..1b177a43381 --- /dev/null +++ b/plugins/lang-plan-a/src/main/antlr/PlanAParser.g4 @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +parser grammar PlanAParser; + +options { tokenVocab=PlanALexer; } + +source + : statement+ EOF + ; + +statement + : IF LP expression RP block ( ELSE block )? # if + | WHILE LP expression RP ( block | empty ) # while + | DO block WHILE LP expression RP SEMICOLON? # do + | FOR LP initializer? SEMICOLON expression? SEMICOLON afterthought? RP ( block | empty ) # for + | declaration SEMICOLON? # decl + | CONTINUE SEMICOLON? # continue + | BREAK SEMICOLON? # break + | RETURN expression SEMICOLON? # return + | TRY block ( CATCH LP ( TYPE ID ) RP block )+ # try + | THROW expression SEMICOLON? # throw + | expression SEMICOLON? # expr + ; + +block + : LBRACK statement* RBRACK # multiple + | statement # single + ; + +empty + : SEMICOLON + ; + +initializer + : declaration + | expression + ; + +afterthought + : expression + ; + +declaration + : decltype declvar ( COMMA declvar )* + ; + +decltype + : TYPE (LBRACE RBRACE)* + ; + +declvar + : ID ( ASSIGN expression )? + ; + +expression + : LP expression RP # precedence + | ( OCTAL | HEX | INTEGER | DECIMAL ) # numeric + | CHAR # char + | TRUE # true + | FALSE # false + | NULL # null + | extstart increment # postinc + | increment extstart # preinc + | extstart # external + | ( BOOLNOT | BWNOT | ADD | SUB ) expression # unary + | LP decltype RP expression # cast + | expression ( MUL | DIV | REM ) expression # binary + | expression ( ADD | SUB ) expression # binary + | expression ( LSH | RSH | USH ) expression # binary + | expression ( LT | LTE | GT | GTE ) expression # comp + | expression ( EQ | EQR | NE | NER ) expression # comp + | expression BWAND expression # binary + | expression BWXOR expression # binary + | expression BWOR expression # binary + | expression BOOLAND expression # bool + | expression BOOLOR expression # bool + | expression COND expression COLON expression # conditional + | extstart ( ASSIGN | AADD | ASUB | AMUL | ADIV + | AREM | AAND | AXOR | AOR + | ALSH | ARSH | AUSH ) expression # assignment + ; + +extstart + : extprec + | extcast + | exttype + | extvar + | extnew + | extstring + ; + +extprec: LP ( extprec | extcast | exttype | extvar | extnew | extstring ) RP ( extdot | extbrace )?; +extcast: LP decltype RP ( extprec | extcast | exttype | extvar | extnew | extstring ); +extbrace: LBRACE expression RBRACE ( extdot | extbrace )?; +extdot: DOT ( extcall | extfield ); +exttype: TYPE extdot; +extcall: EXTID arguments ( extdot | extbrace )?; +extvar: ID ( extdot | extbrace )?; +extfield: ( EXTID | EXTINTEGER ) ( extdot | extbrace )?; +extnew: NEW TYPE ( ( arguments ( extdot | extbrace)? ) | ( ( LBRACE expression RBRACE )+ extdot? ) ); +extstring: STRING (extdot | extbrace )?; + +arguments + : ( LP ( expression ( COMMA expression )* )? RP ) + ; + +increment + : INCR + | DECR + ; diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Adapter.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Adapter.java new file mode 100644 index 00000000000..baa06f45ff8 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Adapter.java @@ -0,0 +1,276 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.util.HashMap; +import java.util.Map; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ParseTree; + +import static org.elasticsearch.plan.a.Definition.*; +import static org.elasticsearch.plan.a.PlanAParser.*; + +class Adapter { + static class StatementMetadata { + final ParserRuleContext source; + + boolean last; + + boolean allExit; + boolean allReturn; + boolean anyReturn; + boolean allBreak; + boolean anyBreak; + boolean allContinue; + boolean anyContinue; + + private StatementMetadata(final ParserRuleContext source) { + this.source = source; + + last = false; + + allExit = false; + allReturn = false; + anyReturn = false; + allBreak = false; + anyBreak = false; + allContinue = false; + anyContinue = false; + } + } + + static class ExpressionMetadata { + final ParserRuleContext source; + + boolean read; + boolean statement; + + Object preConst; + Object postConst; + boolean isNull; + + Type to; + Type from; + boolean explicit; + boolean typesafe; + + Cast cast; + + private ExpressionMetadata(final ParserRuleContext source) { + this.source = source; + + read = true; + statement = false; + + preConst = null; + postConst = null; + isNull = false; + + to = null; + from = null; + explicit = false; + typesafe = true; + + cast = null; + } + } + + static class ExternalMetadata { + final ParserRuleContext source; + + boolean read; + ParserRuleContext storeExpr; + int token; + boolean pre; + boolean post; + + int scope; + Type current; + boolean statik; + boolean statement; + Object constant; + + private ExternalMetadata(final ParserRuleContext source) { + this.source = source; + + read = false; + storeExpr = null; + token = 0; + pre = false; + post = false; + + scope = 0; + current = null; + statik = false; + statement = false; + constant = null; + } + } + + static class ExtNodeMetadata { + final ParserRuleContext parent; + final ParserRuleContext source; + + Object target; + boolean last; + + Type type; + Type promote; + + Cast castFrom; + Cast castTo; + + private ExtNodeMetadata(final ParserRuleContext parent, final ParserRuleContext source) { + this.parent = parent; + this.source = source; + + target = null; + last = false; + + type = null; + promote = null; + + castFrom = null; + castTo = null; + } + } + + static String error(final ParserRuleContext ctx) { + return "Error [" + ctx.getStart().getLine() + ":" + ctx.getStart().getCharPositionInLine() + "]: "; + } + + final Definition definition; + final String source; + final ParserRuleContext root; + final CompilerSettings settings; + + private final Map statementMetadata; + private final Map expressionMetadata; + private final Map externalMetadata; + private final Map extNodeMetadata; + + Adapter(final Definition definition, final String source, final ParserRuleContext root, final CompilerSettings settings) { + this.definition = definition; + this.source = source; + this.root = root; + this.settings = settings; + + statementMetadata = new HashMap<>(); + expressionMetadata = new HashMap<>(); + externalMetadata = new HashMap<>(); + extNodeMetadata = new HashMap<>(); + } + + StatementMetadata createStatementMetadata(final ParserRuleContext source) { + final StatementMetadata sourcesmd = new StatementMetadata(source); + statementMetadata.put(source, sourcesmd); + + return sourcesmd; + } + + StatementMetadata getStatementMetadata(final ParserRuleContext source) { + final StatementMetadata sourcesmd = statementMetadata.get(source); + + if (sourcesmd == null) { + throw new IllegalStateException(error(source) + "Statement metadata does not exist at" + + " the parse node with text [" + source.getText() + "]."); + } + + return sourcesmd; + } + + ExpressionContext updateExpressionTree(ExpressionContext source) { + if (source instanceof PrecedenceContext) { + final ParserRuleContext parent = source.getParent(); + int index = 0; + + for (final ParseTree child : parent.children) { + if (child == source) { + break; + } + + ++index; + } + + while (source instanceof PrecedenceContext) { + source = ((PrecedenceContext)source).expression(); + } + + parent.children.set(index, source); + } + + return source; + } + + ExpressionMetadata createExpressionMetadata(ParserRuleContext source) { + final ExpressionMetadata sourceemd = new ExpressionMetadata(source); + expressionMetadata.put(source, sourceemd); + + return sourceemd; + } + + ExpressionMetadata getExpressionMetadata(final ParserRuleContext source) { + final ExpressionMetadata sourceemd = expressionMetadata.get(source); + + if (sourceemd == null) { + throw new IllegalStateException(error(source) + "Expression metadata does not exist at" + + " the parse node with text [" + source.getText() + "]."); + } + + return sourceemd; + } + + ExternalMetadata createExternalMetadata(final ParserRuleContext source) { + final ExternalMetadata sourceemd = new ExternalMetadata(source); + externalMetadata.put(source, sourceemd); + + return sourceemd; + } + + ExternalMetadata getExternalMetadata(final ParserRuleContext source) { + final ExternalMetadata sourceemd = externalMetadata.get(source); + + if (sourceemd == null) { + throw new IllegalStateException(error(source) + "External metadata does not exist at" + + " the parse node with text [" + source.getText() + "]."); + } + + return sourceemd; + } + + ExtNodeMetadata createExtNodeMetadata(final ParserRuleContext parent, final ParserRuleContext source) { + final ExtNodeMetadata sourceemd = new ExtNodeMetadata(parent, source); + extNodeMetadata.put(source, sourceemd); + + return sourceemd; + } + + ExtNodeMetadata getExtNodeMetadata(final ParserRuleContext source) { + final ExtNodeMetadata sourceemd = extNodeMetadata.get(source); + + if (sourceemd == null) { + throw new IllegalStateException(error(source) + "External metadata does not exist at" + + " the parse node with text [" + source.getText() + "]."); + } + + return sourceemd; + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Analyzer.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Analyzer.java new file mode 100644 index 00000000000..a7e2986d633 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Analyzer.java @@ -0,0 +1,2983 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.antlr.v4.runtime.ParserRuleContext; + +import static org.elasticsearch.plan.a.Adapter.*; +import static org.elasticsearch.plan.a.Definition.*; +import static org.elasticsearch.plan.a.PlanAParser.*; + +class Analyzer extends PlanAParserBaseVisitor { + private static class Variable { + final String name; + final Type type; + final int slot; + + private Variable(final String name, final Type type, final int slot) { + this.name = name; + this.type = type; + this.slot = slot; + } + } + + static void analyze(final Adapter adapter) { + new Analyzer(adapter); + } + + private final Adapter adapter; + private final Definition definition; + private final CompilerSettings settings; + + private final Deque scopes; + private final Deque variables; + + private Analyzer(final Adapter adapter) { + this.adapter = adapter; + definition = adapter.definition; + settings = adapter.settings; + + scopes = new ArrayDeque<>(); + variables = new ArrayDeque<>(); + + incrementScope(); + addVariable(null, "this", definition.execType); + addVariable(null, "input", definition.smapType); + + adapter.createStatementMetadata(adapter.root); + visit(adapter.root); + + decrementScope(); + } + + void incrementScope() { + scopes.push(0); + } + + void decrementScope() { + int remove = scopes.pop(); + + while (remove > 0) { + variables.pop(); + --remove; + } + } + + Variable getVariable(final String name) { + final Iterator itr = variables.iterator(); + + while (itr.hasNext()) { + final Variable variable = itr.next(); + + if (variable.name.equals(name)) { + return variable; + } + } + + return null; + } + + Variable addVariable(final ParserRuleContext source, final String name, final Type type) { + if (getVariable(name) != null) { + if (source == null) { + throw new IllegalArgumentException("Argument name [" + name + "] already defined within the scope."); + } else { + throw new IllegalArgumentException( + error(source) + "Variable name [" + name + "] already defined within the scope."); + } + } + + final Variable previous = variables.peekFirst(); + int slot = 0; + + if (previous != null) { + slot += previous.slot + previous.type.type.getSize(); + } + + final Variable variable = new Variable(name, type, slot); + variables.push(variable); + + final int update = scopes.pop() + 1; + scopes.push(update); + + return variable; + } + + @Override + public Void visitSource(final SourceContext ctx) { + final StatementMetadata sourcesmd = adapter.getStatementMetadata(ctx); + final List statectxs = ctx.statement(); + final StatementContext lastctx = statectxs.get(statectxs.size() - 1); + + incrementScope(); + + for (final StatementContext statectx : statectxs) { + if (sourcesmd.allExit) { + throw new IllegalArgumentException(error(statectx) + + "Statement will never be executed because all prior paths exit."); + } + + final StatementMetadata statesmd = adapter.createStatementMetadata(statectx); + statesmd.last = statectx == lastctx; + visit(statectx); + + if (statesmd.anyContinue) { + throw new IllegalArgumentException(error(statectx) + + "Cannot have a continue statement outside of a loop."); + } + + if (statesmd.anyBreak) { + throw new IllegalArgumentException(error(statectx) + + "Cannot have a break statement outside of a loop."); + } + + sourcesmd.allExit = statesmd.allExit; + sourcesmd.allReturn = statesmd.allReturn; + } + + decrementScope(); + + return null; + } + + @Override + public Void visitIf(final IfContext ctx) { + final StatementMetadata ifsmd = adapter.getStatementMetadata(ctx); + + incrementScope(); + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = definition.booleanType; + visit(exprctx); + markCast(expremd); + + if (expremd.postConst != null) { + throw new IllegalArgumentException(error(ctx) + "If statement is not necessary."); + } + + final BlockContext blockctx0 = ctx.block(0); + final StatementMetadata blocksmd0 = adapter.createStatementMetadata(blockctx0); + blocksmd0.last = ifsmd.last; + visit(blockctx0); + + ifsmd.anyReturn = blocksmd0.anyReturn; + ifsmd.anyBreak = blocksmd0.anyBreak; + ifsmd.anyContinue = blocksmd0.anyContinue; + + if (ctx.ELSE() != null) { + final BlockContext blockctx1 = ctx.block(1); + final StatementMetadata blocksmd1 = adapter.createStatementMetadata(blockctx1); + blocksmd1.last = ifsmd.last; + visit(blockctx1); + + ifsmd.allExit = blocksmd0.allExit && blocksmd1.allExit; + ifsmd.allReturn = blocksmd0.allReturn && blocksmd1.allReturn; + ifsmd.anyReturn |= blocksmd1.anyReturn; + ifsmd.allBreak = blocksmd0.allBreak && blocksmd1.allBreak; + ifsmd.anyBreak |= blocksmd1.anyBreak; + ifsmd.allContinue = blocksmd0.allContinue && blocksmd1.allContinue; + ifsmd.anyContinue |= blocksmd1.anyContinue; + } + + decrementScope(); + + return null; + } + + @Override + public Void visitWhile(final WhileContext ctx) { + final StatementMetadata whilesmd = adapter.getStatementMetadata(ctx); + + incrementScope(); + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = definition.booleanType; + visit(exprctx); + markCast(expremd); + + boolean exitrequired = false; + + if (expremd.postConst != null) { + boolean constant = (boolean)expremd.postConst; + + if (!constant) { + throw new IllegalArgumentException(error(ctx) + "The loop will never be executed."); + } + + exitrequired = true; + } + + final BlockContext blockctx = ctx.block(); + + if (blockctx != null) { + final StatementMetadata blocksmd = adapter.createStatementMetadata(blockctx); + visit(blockctx); + + if (blocksmd.allReturn) { + throw new IllegalArgumentException(error(ctx) + "All paths return so the loop is not necessary."); + } + + if (blocksmd.allBreak) { + throw new IllegalArgumentException(error(ctx) + "All paths break so the loop is not necessary."); + } + + if (exitrequired && !blocksmd.anyReturn && !blocksmd.anyBreak) { + throw new IllegalArgumentException(error(ctx) + "The loop will never exit."); + } + + if (exitrequired && blocksmd.anyReturn && !blocksmd.anyBreak) { + whilesmd.allExit = true; + whilesmd.allReturn = true; + } + } else if (exitrequired) { + throw new IllegalArgumentException(error(ctx) + "The loop will never exit."); + } + + decrementScope(); + + return null; + } + + @Override + public Void visitDo(final DoContext ctx) { + final StatementMetadata dosmd = adapter.getStatementMetadata(ctx); + + incrementScope(); + + final BlockContext blockctx = ctx.block(); + final StatementMetadata blocksmd = adapter.createStatementMetadata(blockctx); + visit(blockctx); + + if (blocksmd.allReturn) { + throw new IllegalArgumentException(error(ctx) + "All paths return so the loop is not necessary."); + } + + if (blocksmd.allBreak) { + throw new IllegalArgumentException(error(ctx) + "All paths break so the loop is not necessary."); + } + + if (blocksmd.allContinue) { + throw new IllegalArgumentException(error(ctx) + "The loop will never exit."); + } + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = definition.booleanType; + visit(exprctx); + markCast(expremd); + + if (expremd.postConst != null) { + final boolean exitrequired = (boolean)expremd.postConst; + + if (exitrequired && !blocksmd.anyReturn && !blocksmd.anyBreak) { + throw new IllegalArgumentException(error(ctx) + "The loop will never exit."); + } + + if (exitrequired && blocksmd.anyReturn && !blocksmd.anyBreak) { + dosmd.allExit = true; + dosmd.allReturn = true; + } + + if (!exitrequired && !blocksmd.anyContinue) { + throw new IllegalArgumentException(error(ctx) + "All paths exit so the loop is not necessary."); + } + } + + decrementScope(); + + return null; + } + + @Override + public Void visitFor(final ForContext ctx) { + final StatementMetadata forsmd = adapter.getStatementMetadata(ctx); + boolean exitrequired = false; + + incrementScope(); + + final InitializerContext initctx = ctx.initializer(); + + if (initctx != null) { + adapter.createStatementMetadata(initctx); + visit(initctx); + } + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + + if (exprctx != null) { + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = definition.booleanType; + visit(exprctx); + markCast(expremd); + + if (expremd.postConst != null) { + boolean constant = (boolean)expremd.postConst; + + if (!constant) { + throw new IllegalArgumentException(error(ctx) + "The loop will never be executed."); + } + + exitrequired = true; + } + } else { + exitrequired = true; + } + + final AfterthoughtContext atctx = ctx.afterthought(); + + if (atctx != null) { + adapter.createStatementMetadata(atctx); + visit(atctx); + } + + final BlockContext blockctx = ctx.block(); + + if (blockctx != null) { + final StatementMetadata blocksmd = adapter.createStatementMetadata(blockctx); + visit(blockctx); + + if (blocksmd.allReturn) { + throw new IllegalArgumentException(error(ctx) + "All paths return so the loop is not necessary."); + } + + if (blocksmd.allBreak) { + throw new IllegalArgumentException(error(ctx) + "All paths break so the loop is not necessary."); + } + + if (exitrequired && !blocksmd.anyReturn && !blocksmd.anyBreak) { + throw new IllegalArgumentException(error(ctx) + "The loop will never exit."); + } + + if (exitrequired && blocksmd.anyReturn && !blocksmd.anyBreak) { + forsmd.allExit = true; + forsmd.allReturn = true; + } + } else if (exitrequired) { + throw new IllegalArgumentException(error(ctx) + "The loop will never exit."); + } + + decrementScope(); + + return null; + } + + @Override + public Void visitDecl(final DeclContext ctx) { + final DeclarationContext declctx = ctx.declaration(); + adapter.createStatementMetadata(declctx); + visit(declctx); + + return null; + } + + @Override + public Void visitContinue(final ContinueContext ctx) { + final StatementMetadata continuesmd = adapter.getStatementMetadata(ctx); + + continuesmd.allExit = true; + continuesmd.allContinue = true; + continuesmd.anyContinue = true; + + return null; + } + + @Override + public Void visitBreak(final BreakContext ctx) { + final StatementMetadata breaksmd = adapter.getStatementMetadata(ctx); + + breaksmd.allExit = true; + breaksmd.allBreak = true; + breaksmd.anyBreak = true; + + return null; + } + + @Override + public Void visitReturn(final ReturnContext ctx) { + final StatementMetadata returnsmd = adapter.getStatementMetadata(ctx); + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = definition.objectType; + visit(exprctx); + markCast(expremd); + + returnsmd.allExit = true; + returnsmd.allReturn = true; + returnsmd.anyReturn = true; + + return null; + } + + @Override + public Void visitExpr(final ExprContext ctx) { + final StatementMetadata exprsmd = adapter.getStatementMetadata(ctx); + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.read = exprsmd.last; + visit(exprctx); + + if (!expremd.statement && !exprsmd.last) { + throw new IllegalArgumentException(error(ctx) + "Not a statement."); + } + + final boolean rtn = exprsmd.last && expremd.from.sort != Sort.VOID; + exprsmd.allExit = rtn; + exprsmd.allReturn = rtn; + exprsmd.anyReturn = rtn; + expremd.to = rtn ? definition.objectType : expremd.from; + markCast(expremd); + + return null; + } + + @Override + public Void visitMultiple(final MultipleContext ctx) { + final StatementMetadata multiplesmd = adapter.getStatementMetadata(ctx); + final List statectxs = ctx.statement(); + final StatementContext lastctx = statectxs.get(statectxs.size() - 1); + + for (StatementContext statectx : statectxs) { + if (multiplesmd.allExit) { + throw new IllegalArgumentException(error(statectx) + + "Statement will never be executed because all prior paths exit."); + } + + final StatementMetadata statesmd = adapter.createStatementMetadata(statectx); + statesmd.last = multiplesmd.last && statectx == lastctx; + visit(statectx); + + multiplesmd.allExit = statesmd.allExit; + multiplesmd.allReturn = statesmd.allReturn && !statesmd.anyBreak && !statesmd.anyContinue; + multiplesmd.anyReturn |= statesmd.anyReturn; + multiplesmd.allBreak = !statesmd.anyReturn && statesmd.allBreak && !statesmd.anyContinue; + multiplesmd.anyBreak |= statesmd.anyBreak; + multiplesmd.allContinue = !statesmd.anyReturn && !statesmd.anyBreak && statesmd.allContinue; + multiplesmd.anyContinue |= statesmd.anyContinue; + } + + return null; + } + + @Override + public Void visitSingle(final SingleContext ctx) { + final StatementMetadata singlesmd = adapter.getStatementMetadata(ctx); + + final StatementContext statectx = ctx.statement(); + final StatementMetadata statesmd = adapter.createStatementMetadata(statectx); + statesmd.last = singlesmd.last; + visit(statectx); + + singlesmd.allExit = statesmd.allExit; + singlesmd.allReturn = statesmd.allReturn; + singlesmd.anyReturn = statesmd.anyReturn; + singlesmd.allBreak = statesmd.allBreak; + singlesmd.anyBreak = statesmd.anyBreak; + singlesmd.allContinue = statesmd.allContinue; + singlesmd.anyContinue = statesmd.anyContinue; + + return null; + } + + @Override + public Void visitEmpty(final EmptyContext ctx) { + throw new UnsupportedOperationException(error(ctx) + "Unexpected parser state."); + } + + @Override + public Void visitInitializer(InitializerContext ctx) { + final DeclarationContext declctx = ctx.declaration(); + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + + if (declctx != null) { + adapter.createStatementMetadata(declctx); + visit(declctx); + } else if (exprctx != null) { + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.read = false; + visit(exprctx); + + expremd.to = expremd.from; + markCast(expremd); + + if (!expremd.statement) { + throw new IllegalArgumentException(error(exprctx) + + "The intializer of a for loop must be a statement."); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + return null; + } + + @Override + public Void visitAfterthought(AfterthoughtContext ctx) { + ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + + if (exprctx != null) { + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.read = false; + visit(exprctx); + + expremd.to = expremd.from; + markCast(expremd); + + if (!expremd.statement) { + throw new IllegalArgumentException(error(exprctx) + + "The afterthought of a for loop must be a statement."); + } + } + + return null; + } + + @Override + public Void visitDeclaration(final DeclarationContext ctx) { + final DecltypeContext decltypectx = ctx.decltype(); + final ExpressionMetadata decltypeemd = adapter.createExpressionMetadata(decltypectx); + visit(decltypectx); + + for (final DeclvarContext declvarctx : ctx.declvar()) { + final ExpressionMetadata declvaremd = adapter.createExpressionMetadata(declvarctx); + declvaremd.to = decltypeemd.from; + visit(declvarctx); + } + + return null; + } + + @Override + public Void visitDecltype(final DecltypeContext ctx) { + final ExpressionMetadata decltypeemd = adapter.getExpressionMetadata(ctx); + + final String name = ctx.getText(); + decltypeemd.from = definition.getType(name); + + return null; + } + + @Override + public Void visitDeclvar(final DeclvarContext ctx) { + final ExpressionMetadata declvaremd = adapter.getExpressionMetadata(ctx); + + final String name = ctx.ID().getText(); + declvaremd.postConst = addVariable(ctx, name, declvaremd.to).slot; + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + + if (exprctx != null) { + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = declvaremd.to; + visit(exprctx); + markCast(expremd); + } + + return null; + } + + @Override + public Void visitPrecedence(final PrecedenceContext ctx) { + throw new UnsupportedOperationException(error(ctx) + "Unexpected parser state."); + } + + @Override + public Void visitNumeric(final NumericContext ctx) { + final ExpressionMetadata numericemd = adapter.getExpressionMetadata(ctx); + final boolean negate = ctx.parent instanceof UnaryContext && ((UnaryContext)ctx.parent).SUB() != null; + + if (ctx.DECIMAL() != null) { + final String svalue = (negate ? "-" : "") + ctx.DECIMAL().getText(); + + if (svalue.endsWith("f") || svalue.endsWith("F")) { + try { + numericemd.from = definition.floatType; + numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1)); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException(error(ctx) + "Invalid float constant [" + svalue + "]."); + } + } else { + try { + numericemd.from = definition.doubleType; + numericemd.preConst = Double.parseDouble(svalue); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException(error(ctx) + "Invalid double constant [" + svalue + "]."); + } + } + } else { + String svalue = negate ? "-" : ""; + int radix; + + if (ctx.OCTAL() != null) { + svalue += ctx.OCTAL().getText(); + radix = 8; + } else if (ctx.INTEGER() != null) { + svalue += ctx.INTEGER().getText(); + radix = 10; + } else if (ctx.HEX() != null) { + svalue += ctx.HEX().getText(); + radix = 16; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + if (svalue.endsWith("d") || svalue.endsWith("D")) { + try { + numericemd.from = definition.doubleType; + numericemd.preConst = Double.parseDouble(svalue.substring(0, svalue.length() - 1)); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException(error(ctx) + "Invalid float constant [" + svalue + "]."); + } + } else if (svalue.endsWith("f") || svalue.endsWith("F")) { + try { + numericemd.from = definition.floatType; + numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1)); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException(error(ctx) + "Invalid float constant [" + svalue + "]."); + } + } else if (svalue.endsWith("l") || svalue.endsWith("L")) { + try { + numericemd.from = definition.longType; + numericemd.preConst = Long.parseLong(svalue.substring(0, svalue.length() - 1), radix); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException(error(ctx) + "Invalid long constant [" + svalue + "]."); + } + } else { + try { + final Type type = numericemd.to; + final Sort sort = type == null ? Sort.INT : type.sort; + final int value = Integer.parseInt(svalue, radix); + + if (sort == Sort.BYTE && value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE) { + numericemd.from = definition.byteType; + numericemd.preConst = (byte)value; + } else if (sort == Sort.CHAR && value >= Character.MIN_VALUE && value <= Character.MAX_VALUE) { + numericemd.from = definition.charType; + numericemd.preConst = (char)value; + } else if (sort == Sort.SHORT && value >= Short.MIN_VALUE && value <= Short.MAX_VALUE) { + numericemd.from = definition.shortType; + numericemd.preConst = (short)value; + } else { + numericemd.from = definition.intType; + numericemd.preConst = value; + } + } catch (NumberFormatException exception) { + throw new IllegalArgumentException(error(ctx) + "Invalid int constant [" + svalue + "]."); + } + } + } + + return null; + } + + @Override + public Void visitChar(final CharContext ctx) { + final ExpressionMetadata charemd = adapter.getExpressionMetadata(ctx); + + if (ctx.CHAR() == null) { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + charemd.preConst = ctx.CHAR().getText().charAt(0); + charemd.from = definition.charType; + + return null; + } + + @Override + public Void visitTrue(final TrueContext ctx) { + final ExpressionMetadata trueemd = adapter.getExpressionMetadata(ctx); + + if (ctx.TRUE() == null) { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + trueemd.preConst = true; + trueemd.from = definition.booleanType; + + return null; + } + + @Override + public Void visitFalse(final FalseContext ctx) { + final ExpressionMetadata falseemd = adapter.getExpressionMetadata(ctx); + + if (ctx.FALSE() == null) { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + falseemd.preConst = false; + falseemd.from = definition.booleanType; + + return null; + } + + @Override + public Void visitNull(final NullContext ctx) { + final ExpressionMetadata nullemd = adapter.getExpressionMetadata(ctx); + + if (ctx.NULL() == null) { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + nullemd.isNull = true; + + if (nullemd.to != null) { + if (nullemd.to.sort.primitive) { + throw new IllegalArgumentException("Cannot cast null to a primitive type [" + nullemd.to.name + "]."); + } + + nullemd.from = nullemd.to; + } else { + nullemd.from = definition.objectType; + } + + return null; + } + + @Override + public Void visitExternal(final ExternalContext ctx) { + final ExpressionMetadata extemd = adapter.getExpressionMetadata(ctx); + + final ExtstartContext extstartctx = ctx.extstart(); + final ExternalMetadata extstartemd = adapter.createExternalMetadata(extstartctx); + extstartemd.read = extemd.read; + visit(extstartctx); + + extemd.statement = extstartemd.statement; + extemd.preConst = extstartemd.constant; + extemd.from = extstartemd.current; + extemd.typesafe = extstartemd.current.sort != Sort.DEF; + + return null; + } + + @Override + public Void visitPostinc(final PostincContext ctx) { + final ExpressionMetadata postincemd = adapter.getExpressionMetadata(ctx); + + final ExtstartContext extstartctx = ctx.extstart(); + final ExternalMetadata extstartemd = adapter.createExternalMetadata(extstartctx); + extstartemd.read = postincemd.read; + extstartemd.storeExpr = ctx.increment(); + extstartemd.token = ADD; + extstartemd.post = true; + visit(extstartctx); + + postincemd.statement = true; + postincemd.from = extstartemd.read ? extstartemd.current : definition.voidType; + postincemd.typesafe = extstartemd.current.sort != Sort.DEF; + + return null; + } + + @Override + public Void visitPreinc(final PreincContext ctx) { + final ExpressionMetadata preincemd = adapter.getExpressionMetadata(ctx); + + final ExtstartContext extstartctx = ctx.extstart(); + final ExternalMetadata extstartemd = adapter.createExternalMetadata(extstartctx); + extstartemd.read = preincemd.read; + extstartemd.storeExpr = ctx.increment(); + extstartemd.token = ADD; + extstartemd.pre = true; + visit(extstartctx); + + preincemd.statement = true; + preincemd.from = extstartemd.read ? extstartemd.current : definition.voidType; + preincemd.typesafe = extstartemd.current.sort != Sort.DEF; + + return null; + } + + @Override + public Void visitUnary(final UnaryContext ctx) { + final ExpressionMetadata unaryemd = adapter.getExpressionMetadata(ctx); + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + + if (ctx.BOOLNOT() != null) { + expremd.to = definition.booleanType; + visit(exprctx); + markCast(expremd); + + if (expremd.postConst != null) { + unaryemd.preConst = !(boolean)expremd.postConst; + } + + unaryemd.from = definition.booleanType; + } else if (ctx.BWNOT() != null || ctx.ADD() != null || ctx.SUB() != null) { + visit(exprctx); + + final Type promote = promoteNumeric(expremd.from, ctx.BWNOT() == null, true); + + if (promote == null) { + throw new ClassCastException("Cannot apply [" + ctx.getChild(0).getText() + "] " + + "operation to type [" + expremd.from.name + "]."); + } + + expremd.to = promote; + markCast(expremd); + + if (expremd.postConst != null) { + final Sort sort = promote.sort; + + if (ctx.BWNOT() != null) { + if (sort == Sort.INT) { + unaryemd.preConst = ~(int)expremd.postConst; + } else if (sort == Sort.LONG) { + unaryemd.preConst = ~(long)expremd.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.SUB() != null) { + if (exprctx instanceof NumericContext) { + unaryemd.preConst = expremd.postConst; + } else { + if (sort == Sort.INT) { + if (settings.getNumericOverflow()) { + unaryemd.preConst = -(int)expremd.postConst; + } else { + unaryemd.preConst = Math.negateExact((int)expremd.postConst); + } + } else if (sort == Sort.LONG) { + if (settings.getNumericOverflow()) { + unaryemd.preConst = -(long)expremd.postConst; + } else { + unaryemd.preConst = Math.negateExact((long)expremd.postConst); + } + } else if (sort == Sort.FLOAT) { + unaryemd.preConst = -(float)expremd.postConst; + } else if (sort == Sort.DOUBLE) { + unaryemd.preConst = -(double)expremd.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } + } else if (ctx.ADD() != null) { + if (sort == Sort.INT) { + unaryemd.preConst = +(int)expremd.postConst; + } else if (sort == Sort.LONG) { + unaryemd.preConst = +(long)expremd.postConst; + } else if (sort == Sort.FLOAT) { + unaryemd.preConst = +(float)expremd.postConst; + } else if (sort == Sort.DOUBLE) { + unaryemd.preConst = +(double)expremd.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } + + unaryemd.from = promote; + unaryemd.typesafe = expremd.typesafe; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + return null; + } + + @Override + public Void visitCast(final CastContext ctx) { + final ExpressionMetadata castemd = adapter.getExpressionMetadata(ctx); + + final DecltypeContext decltypectx = ctx.decltype(); + final ExpressionMetadata decltypemd = adapter.createExpressionMetadata(decltypectx); + visit(decltypectx); + + final Type type = decltypemd.from; + castemd.from = type; + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = type; + expremd.explicit = true; + visit(exprctx); + markCast(expremd); + + if (expremd.postConst != null) { + castemd.preConst = expremd.postConst; + } + + castemd.typesafe = expremd.typesafe && castemd.from.sort != Sort.DEF; + + return null; + } + + @Override + public Void visitBinary(final BinaryContext ctx) { + final ExpressionMetadata binaryemd = adapter.getExpressionMetadata(ctx); + + final ExpressionContext exprctx0 = adapter.updateExpressionTree(ctx.expression(0)); + final ExpressionMetadata expremd0 = adapter.createExpressionMetadata(exprctx0); + visit(exprctx0); + + final ExpressionContext exprctx1 = adapter.updateExpressionTree(ctx.expression(1)); + final ExpressionMetadata expremd1 = adapter.createExpressionMetadata(exprctx1); + visit(exprctx1); + + final boolean decimal = ctx.MUL() != null || ctx.DIV() != null || ctx.REM() != null || ctx.SUB() != null; + final boolean add = ctx.ADD() != null; + final boolean xor = ctx.BWXOR() != null; + final Type promote = add ? promoteAdd(expremd0.from, expremd1.from) : + xor ? promoteXor(expremd0.from, expremd1.from) : + promoteNumeric(expremd0.from, expremd1.from, decimal, true); + + if (promote == null) { + throw new ClassCastException("Cannot apply [" + ctx.getChild(1).getText() + "] " + + "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "]."); + } + + final Sort sort = promote.sort; + expremd0.to = add && sort == Sort.STRING ? expremd0.from : promote; + expremd1.to = add && sort == Sort.STRING ? expremd1.from : promote; + markCast(expremd0); + markCast(expremd1); + + if (expremd0.postConst != null && expremd1.postConst != null) { + if (ctx.MUL() != null) { + if (sort == Sort.INT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (int)expremd0.postConst * (int)expremd1.postConst; + } else { + binaryemd.preConst = Math.multiplyExact((int)expremd0.postConst, (int)expremd1.postConst); + } + } else if (sort == Sort.LONG) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (long)expremd0.postConst * (long)expremd1.postConst; + } else { + binaryemd.preConst = Math.multiplyExact((long)expremd0.postConst, (long)expremd1.postConst); + } + } else if (sort == Sort.FLOAT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (float)expremd0.postConst * (float)expremd1.postConst; + } else { + binaryemd.preConst = Utility.multiplyWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst); + } + } else if (sort == Sort.DOUBLE) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (double)expremd0.postConst * (double)expremd1.postConst; + } else { + binaryemd.preConst = Utility.multiplyWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.DIV() != null) { + if (sort == Sort.INT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (int)expremd0.postConst / (int)expremd1.postConst; + } else { + binaryemd.preConst = Utility.divideWithoutOverflow((int)expremd0.postConst, (int)expremd1.postConst); + } + } else if (sort == Sort.LONG) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (long)expremd0.postConst / (long)expremd1.postConst; + } else { + binaryemd.preConst = Utility.divideWithoutOverflow((long)expremd0.postConst, (long)expremd1.postConst); + } + } else if (sort == Sort.FLOAT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (float)expremd0.postConst / (float)expremd1.postConst; + } else { + binaryemd.preConst = Utility.divideWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst); + } + } else if (sort == Sort.DOUBLE) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (double)expremd0.postConst / (double)expremd1.postConst; + } else { + binaryemd.preConst = Utility.divideWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.REM() != null) { + if (sort == Sort.INT) { + binaryemd.preConst = (int)expremd0.postConst % (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + binaryemd.preConst = (long)expremd0.postConst % (long)expremd1.postConst; + } else if (sort == Sort.FLOAT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (float)expremd0.postConst % (float)expremd1.postConst; + } else { + binaryemd.preConst = Utility.remainderWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst); + } + } else if (sort == Sort.DOUBLE) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (double)expremd0.postConst % (double)expremd1.postConst; + } else { + binaryemd.preConst = Utility.remainderWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.ADD() != null) { + if (sort == Sort.INT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (int)expremd0.postConst + (int)expremd1.postConst; + } else { + binaryemd.preConst = Math.addExact((int)expremd0.postConst, (int)expremd1.postConst); + } + } else if (sort == Sort.LONG) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (long)expremd0.postConst + (long)expremd1.postConst; + } else { + binaryemd.preConst = Math.addExact((long)expremd0.postConst, (long)expremd1.postConst); + } + } else if (sort == Sort.FLOAT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (float)expremd0.postConst + (float)expremd1.postConst; + } else { + binaryemd.preConst = Utility.addWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst); + } + } else if (sort == Sort.DOUBLE) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (double)expremd0.postConst + (double)expremd1.postConst; + } else { + binaryemd.preConst = Utility.addWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst); + } + } else if (sort == Sort.STRING) { + binaryemd.preConst = "" + expremd0.postConst + expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.SUB() != null) { + if (sort == Sort.INT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (int)expremd0.postConst - (int)expremd1.postConst; + } else { + binaryemd.preConst = Math.subtractExact((int)expremd0.postConst, (int)expremd1.postConst); + } + } else if (sort == Sort.LONG) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (long)expremd0.postConst - (long)expremd1.postConst; + } else { + binaryemd.preConst = Math.subtractExact((long)expremd0.postConst, (long)expremd1.postConst); + } + } else if (sort == Sort.FLOAT) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (float)expremd0.postConst - (float)expremd1.postConst; + } else { + binaryemd.preConst = Utility.subtractWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst); + } + } else if (sort == Sort.DOUBLE) { + if (settings.getNumericOverflow()) { + binaryemd.preConst = (double)expremd0.postConst - (double)expremd1.postConst; + } else { + binaryemd.preConst = Utility.subtractWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.LSH() != null) { + if (sort == Sort.INT) { + binaryemd.preConst = (int)expremd0.postConst << (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + binaryemd.preConst = (long)expremd0.postConst << (long)expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.RSH() != null) { + if (sort == Sort.INT) { + binaryemd.preConst = (int)expremd0.postConst >> (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + binaryemd.preConst = (long)expremd0.postConst >> (long)expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.USH() != null) { + if (sort == Sort.INT) { + binaryemd.preConst = (int)expremd0.postConst >>> (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + binaryemd.preConst = (long)expremd0.postConst >>> (long)expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.BWAND() != null) { + if (sort == Sort.INT) { + binaryemd.preConst = (int)expremd0.postConst & (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + binaryemd.preConst = (long)expremd0.postConst & (long)expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.BWXOR() != null) { + if (sort == Sort.BOOL) { + binaryemd.preConst = (boolean)expremd0.postConst ^ (boolean)expremd1.postConst; + } else if (sort == Sort.INT) { + binaryemd.preConst = (int)expremd0.postConst ^ (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + binaryemd.preConst = (long)expremd0.postConst ^ (long)expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else if (ctx.BWOR() != null) { + if (sort == Sort.INT) { + binaryemd.preConst = (int)expremd0.postConst | (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + binaryemd.preConst = (long)expremd0.postConst | (long)expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } + + binaryemd.from = promote; + binaryemd.typesafe = expremd0.typesafe && expremd1.typesafe; + + return null; + } + + @Override + public Void visitComp(final CompContext ctx) { + final ExpressionMetadata compemd = adapter.getExpressionMetadata(ctx); + final boolean equality = ctx.EQ() != null || ctx.NE() != null; + final boolean reference = ctx.EQR() != null || ctx.NER() != null; + + final ExpressionContext exprctx0 = adapter.updateExpressionTree(ctx.expression(0)); + final ExpressionMetadata expremd0 = adapter.createExpressionMetadata(exprctx0); + visit(exprctx0); + + final ExpressionContext exprctx1 = adapter.updateExpressionTree(ctx.expression(1)); + final ExpressionMetadata expremd1 = adapter.createExpressionMetadata(exprctx1); + visit(exprctx1); + + if (expremd0.isNull && expremd1.isNull) { + throw new IllegalArgumentException(error(ctx) + "Unnecessary comparison of null constants."); + } + + final Type promote = equality ? promoteEquality(expremd0.from, expremd1.from) : + reference ? promoteReference(expremd0.from, expremd1.from) : + promoteNumeric(expremd0.from, expremd1.from, true, true); + + if (promote == null) { + throw new ClassCastException("Cannot apply [" + ctx.getChild(1).getText() + "] " + + "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "]."); + } + + expremd0.to = promote; + expremd1.to = promote; + markCast(expremd0); + markCast(expremd1); + + if (expremd0.postConst != null && expremd1.postConst != null) { + final Sort sort = promote.sort; + + if (ctx.EQ() != null || ctx.EQR() != null) { + if (sort == Sort.BOOL) { + compemd.preConst = (boolean)expremd0.postConst == (boolean)expremd1.postConst; + } else if (sort == Sort.INT) { + compemd.preConst = (int)expremd0.postConst == (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + compemd.preConst = (long)expremd0.postConst == (long)expremd1.postConst; + } else if (sort == Sort.FLOAT) { + compemd.preConst = (float)expremd0.postConst == (float)expremd1.postConst; + } else if (sort == Sort.DOUBLE) { + compemd.preConst = (double)expremd0.postConst == (double)expremd1.postConst; + } else { + if (ctx.EQ() != null && !expremd0.isNull && !expremd1.isNull) { + compemd.preConst = expremd0.postConst.equals(expremd1.postConst); + } else if (ctx.EQR() != null) { + compemd.preConst = expremd0.postConst == expremd1.postConst; + } + } + } else if (ctx.NE() != null || ctx.NER() != null) { + if (sort == Sort.BOOL) { + compemd.preConst = (boolean)expremd0.postConst != (boolean)expremd1.postConst; + } else if (sort == Sort.INT) { + compemd.preConst = (int)expremd0.postConst != (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + compemd.preConst = (long)expremd0.postConst != (long)expremd1.postConst; + } else if (sort == Sort.FLOAT) { + compemd.preConst = (float)expremd0.postConst != (float)expremd1.postConst; + } else if (sort == Sort.DOUBLE) { + compemd.preConst = (double)expremd0.postConst != (double)expremd1.postConst; + } else { + if (ctx.NE() != null && !expremd0.isNull && !expremd1.isNull) { + compemd.preConst = expremd0.postConst.equals(expremd1.postConst); + } else if (ctx.NER() != null) { + compemd.preConst = expremd0.postConst == expremd1.postConst; + } + } + } else if (ctx.GTE() != null) { + if (sort == Sort.INT) { + compemd.preConst = (int)expremd0.postConst >= (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + compemd.preConst = (long)expremd0.postConst >= (long)expremd1.postConst; + } else if (sort == Sort.FLOAT) { + compemd.preConst = (float)expremd0.postConst >= (float)expremd1.postConst; + } else if (sort == Sort.DOUBLE) { + compemd.preConst = (double)expremd0.postConst >= (double)expremd1.postConst; + } + } else if (ctx.GT() != null) { + if (sort == Sort.INT) { + compemd.preConst = (int)expremd0.postConst > (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + compemd.preConst = (long)expremd0.postConst > (long)expremd1.postConst; + } else if (sort == Sort.FLOAT) { + compemd.preConst = (float)expremd0.postConst > (float)expremd1.postConst; + } else if (sort == Sort.DOUBLE) { + compemd.preConst = (double)expremd0.postConst > (double)expremd1.postConst; + } + } else if (ctx.LTE() != null) { + if (sort == Sort.INT) { + compemd.preConst = (int)expremd0.postConst <= (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + compemd.preConst = (long)expremd0.postConst <= (long)expremd1.postConst; + } else if (sort == Sort.FLOAT) { + compemd.preConst = (float)expremd0.postConst <= (float)expremd1.postConst; + } else if (sort == Sort.DOUBLE) { + compemd.preConst = (double)expremd0.postConst <= (double)expremd1.postConst; + } + } else if (ctx.LT() != null) { + if (sort == Sort.INT) { + compemd.preConst = (int)expremd0.postConst < (int)expremd1.postConst; + } else if (sort == Sort.LONG) { + compemd.preConst = (long)expremd0.postConst < (long)expremd1.postConst; + } else if (sort == Sort.FLOAT) { + compemd.preConst = (float)expremd0.postConst < (float)expremd1.postConst; + } else if (sort == Sort.DOUBLE) { + compemd.preConst = (double)expremd0.postConst < (double)expremd1.postConst; + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } + + compemd.from = definition.booleanType; + compemd.typesafe = expremd0.typesafe && expremd1.typesafe; + + return null; + } + + @Override + public Void visitBool(final BoolContext ctx) { + final ExpressionMetadata boolemd = adapter.getExpressionMetadata(ctx); + + final ExpressionContext exprctx0 = adapter.updateExpressionTree(ctx.expression(0)); + final ExpressionMetadata expremd0 = adapter.createExpressionMetadata(exprctx0); + expremd0.to = definition.booleanType; + visit(exprctx0); + markCast(expremd0); + + final ExpressionContext exprctx1 = adapter.updateExpressionTree(ctx.expression(1)); + final ExpressionMetadata expremd1 = adapter.createExpressionMetadata(exprctx1); + expremd1.to = definition.booleanType; + visit(exprctx1); + markCast(expremd1); + + if (expremd0.postConst != null && expremd1.postConst != null) { + if (ctx.BOOLAND() != null) { + boolemd.preConst = (boolean)expremd0.postConst && (boolean)expremd1.postConst; + } else if (ctx.BOOLOR() != null) { + boolemd.preConst = (boolean)expremd0.postConst || (boolean)expremd1.postConst; + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } + + boolemd.from = definition.booleanType; + boolemd.typesafe = expremd0.typesafe && expremd1.typesafe; + + return null; + } + + @Override + public Void visitConditional(final ConditionalContext ctx) { + final ExpressionMetadata condemd = adapter.getExpressionMetadata(ctx); + + final ExpressionContext exprctx0 = adapter.updateExpressionTree(ctx.expression(0)); + final ExpressionMetadata expremd0 = adapter.createExpressionMetadata(exprctx0); + expremd0.to = definition.booleanType; + visit(exprctx0); + markCast(expremd0); + + if (expremd0.postConst != null) { + throw new IllegalArgumentException(error(ctx) + "Unnecessary conditional statement."); + } + + final ExpressionContext exprctx1 = adapter.updateExpressionTree(ctx.expression(1)); + final ExpressionMetadata expremd1 = adapter.createExpressionMetadata(exprctx1); + expremd1.to = condemd.to; + expremd1.explicit = condemd.explicit; + visit(exprctx1); + + final ExpressionContext exprctx2 = adapter.updateExpressionTree(ctx.expression(2)); + final ExpressionMetadata expremd2 = adapter.createExpressionMetadata(exprctx2); + expremd2.to = condemd.to; + expremd2.explicit = condemd.explicit; + visit(exprctx2); + + if (condemd.to == null) { + final Type promote = promoteConditional(expremd1.from, expremd2.from, expremd1.preConst, expremd2.preConst); + + expremd1.to = promote; + expremd2.to = promote; + condemd.from = promote; + } else { + condemd.from = condemd.to; + } + + markCast(expremd1); + markCast(expremd2); + + condemd.typesafe = expremd0.typesafe && expremd1.typesafe; + + return null; + } + + @Override + public Void visitAssignment(final AssignmentContext ctx) { + final ExpressionMetadata assignemd = adapter.getExpressionMetadata(ctx); + + final ExtstartContext extstartctx = ctx.extstart(); + final ExternalMetadata extstartemd = adapter.createExternalMetadata(extstartctx); + + extstartemd.read = assignemd.read; + extstartemd.storeExpr = adapter.updateExpressionTree(ctx.expression()); + + if (ctx.AMUL() != null) { + extstartemd.token = MUL; + } else if (ctx.ADIV() != null) { + extstartemd.token = DIV; + } else if (ctx.AREM() != null) { + extstartemd.token = REM; + } else if (ctx.AADD() != null) { + extstartemd.token = ADD; + } else if (ctx.ASUB() != null) { + extstartemd.token = SUB; + } else if (ctx.ALSH() != null) { + extstartemd.token = LSH; + } else if (ctx.AUSH() != null) { + extstartemd.token = USH; + } else if (ctx.ARSH() != null) { + extstartemd.token = RSH; + } else if (ctx.AAND() != null) { + extstartemd.token = BWAND; + } else if (ctx.AXOR() != null) { + extstartemd.token = BWXOR; + } else if (ctx.AOR() != null) { + extstartemd.token = BWOR; + } + + visit(extstartctx); + + assignemd.statement = true; + assignemd.from = extstartemd.read ? extstartemd.current : definition.voidType; + assignemd.typesafe = extstartemd.current.sort != Sort.DEF; + + return null; + } + + @Override + public Void visitExtstart(final ExtstartContext ctx) { + final ExtprecContext precctx = ctx.extprec(); + final ExtcastContext castctx = ctx.extcast(); + final ExttypeContext typectx = ctx.exttype(); + final ExtvarContext varctx = ctx.extvar(); + final ExtnewContext newctx = ctx.extnew(); + final ExtstringContext stringctx = ctx.extstring(); + + if (precctx != null) { + adapter.createExtNodeMetadata(ctx, precctx); + visit(precctx); + } else if (castctx != null) { + adapter.createExtNodeMetadata(ctx, castctx); + visit(castctx); + } else if (typectx != null) { + adapter.createExtNodeMetadata(ctx, typectx); + visit(typectx); + } else if (varctx != null) { + adapter.createExtNodeMetadata(ctx, varctx); + visit(varctx); + } else if (newctx != null) { + adapter.createExtNodeMetadata(ctx, newctx); + visit(newctx); + } else if (stringctx != null) { + adapter.createExtNodeMetadata(ctx, stringctx); + visit(stringctx); + } else { + throw new IllegalStateException(); + } + + return null; + } + + @Override + public Void visitExtprec(final ExtprecContext ctx) { + final ExtNodeMetadata precenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = precenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + final ExtprecContext precctx = ctx.extprec(); + final ExtcastContext castctx = ctx.extcast(); + final ExttypeContext typectx = ctx.exttype(); + final ExtvarContext varctx = ctx.extvar(); + final ExtnewContext newctx = ctx.extnew(); + final ExtstringContext stringctx = ctx.extstring(); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null || bracectx != null) { + ++parentemd.scope; + } + + if (precctx != null) { + adapter.createExtNodeMetadata(parent, precctx); + visit(precctx); + } else if (castctx != null) { + adapter.createExtNodeMetadata(parent, castctx); + visit(castctx); + } else if (typectx != null) { + adapter.createExtNodeMetadata(parent, typectx); + visit(typectx); + } else if (varctx != null) { + adapter.createExtNodeMetadata(parent, varctx); + visit(varctx); + } else if (newctx != null) { + adapter.createExtNodeMetadata(parent, newctx); + visit(newctx); + } else if (stringctx != null) { + adapter.createExtNodeMetadata(ctx, stringctx); + visit(stringctx); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + parentemd.statement = false; + + if (dotctx != null) { + --parentemd.scope; + + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + } else if (bracectx != null) { + --parentemd.scope; + + adapter.createExtNodeMetadata(parent, bracectx); + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtcast(final ExtcastContext ctx) { + final ExtNodeMetadata castenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = castenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + final ExtprecContext precctx = ctx.extprec(); + final ExtcastContext castctx = ctx.extcast(); + final ExttypeContext typectx = ctx.exttype(); + final ExtvarContext varctx = ctx.extvar(); + final ExtnewContext newctx = ctx.extnew(); + final ExtstringContext stringctx = ctx.extstring(); + + if (precctx != null) { + adapter.createExtNodeMetadata(parent, precctx); + visit(precctx); + } else if (castctx != null) { + adapter.createExtNodeMetadata(parent, castctx); + visit(castctx); + } else if (typectx != null) { + adapter.createExtNodeMetadata(parent, typectx); + visit(typectx); + } else if (varctx != null) { + adapter.createExtNodeMetadata(parent, varctx); + visit(varctx); + } else if (newctx != null) { + adapter.createExtNodeMetadata(parent, newctx); + visit(newctx); + } else if (stringctx != null) { + adapter.createExtNodeMetadata(ctx, stringctx); + visit(stringctx); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + final DecltypeContext declctx = ctx.decltype(); + final ExpressionMetadata declemd = adapter.createExpressionMetadata(declctx); + visit(declctx); + + castenmd.castTo = getLegalCast(ctx, parentemd.current, declemd.from, true); + castenmd.type = declemd.from; + parentemd.current = declemd.from; + parentemd.statement = false; + + return null; + } + + @Override + public Void visitExtbrace(final ExtbraceContext ctx) { + final ExtNodeMetadata braceenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = braceenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + final boolean array = parentemd.current.sort == Sort.ARRAY; + final boolean def = parentemd.current.sort == Sort.DEF; + boolean map = false; + boolean list = false; + + try { + parentemd.current.clazz.asSubclass(Map.class); + map = true; + } catch (ClassCastException exception) { + // Do nothing. + } + + try { + parentemd.current.clazz.asSubclass(List.class); + list = true; + } catch (ClassCastException exception) { + // Do nothing. + } + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + braceenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null; + + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + + if (array || def) { + expremd.to = array ? definition.intType : definition.objectType; + visit(exprctx); + markCast(expremd); + + braceenmd.target = "#brace"; + braceenmd.type = def ? definition.defType : + definition.getType(parentemd.current.struct, parentemd.current.type.getDimensions() - 1); + analyzeLoadStoreExternal(ctx); + parentemd.current = braceenmd.type; + + if (dotctx != null) { + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + } else if (bracectx != null) { + adapter.createExtNodeMetadata(parent, bracectx); + visit(bracectx); + } + } else { + final boolean store = braceenmd.last && parentemd.storeExpr != null; + final boolean get = parentemd.read || parentemd.token > 0 || !braceenmd.last; + final boolean set = braceenmd.last && store; + + Method getter; + Method setter; + Type valuetype; + Type settype; + + if (map) { + getter = parentemd.current.struct.methods.get("get"); + setter = parentemd.current.struct.methods.get("put"); + + if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal map get shortcut for type [" + parentemd.current.name + "]."); + } + + if (setter != null && setter.arguments.size() != 2) { + throw new IllegalArgumentException(error(ctx) + + "Illegal map set shortcut for type [" + parentemd.current.name + "]."); + } + + if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0)) + || !getter.rtn.equals(setter.arguments.get(1)))) { + throw new IllegalArgumentException(error(ctx) + "Shortcut argument types must match."); + } + + valuetype = setter != null ? setter.arguments.get(0) : getter != null ? getter.arguments.get(0) : null; + settype = setter == null ? null : setter.arguments.get(1); + } else if (list) { + getter = parentemd.current.struct.methods.get("get"); + setter = parentemd.current.struct.methods.get("add"); + + if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 || + getter.arguments.get(0).sort != Sort.INT)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal list get shortcut for type [" + parentemd.current.name + "]."); + } + + if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal list set shortcut for type [" + parentemd.current.name + "]."); + } + + if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0)) + || !getter.rtn.equals(setter.arguments.get(1)))) { + throw new IllegalArgumentException(error(ctx) + "Shortcut argument types must match."); + } + + valuetype = definition.intType; + settype = setter == null ? null : setter.arguments.get(1); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + + if ((get || set) && (!get || getter != null) && (!set || setter != null)) { + expremd.to = valuetype; + visit(exprctx); + markCast(expremd); + + braceenmd.target = new Object[] {getter, setter, true, null}; + braceenmd.type = get ? getter.rtn : settype; + analyzeLoadStoreExternal(ctx); + parentemd.current = get ? getter.rtn : setter.rtn; + } + } + + if (braceenmd.target == null) { + throw new IllegalArgumentException(error(ctx) + + "Attempting to address a non-array type [" + parentemd.current.name + "] as an array."); + } + + return null; + } + + @Override + public Void visitExtdot(final ExtdotContext ctx) { + final ExtNodeMetadata dotemnd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = dotemnd.parent; + + final ExtcallContext callctx = ctx.extcall(); + final ExtfieldContext fieldctx = ctx.extfield(); + + if (callctx != null) { + adapter.createExtNodeMetadata(parent, callctx); + visit(callctx); + } else if (fieldctx != null) { + adapter.createExtNodeMetadata(parent, fieldctx); + visit(fieldctx); + } + + return null; + } + + @Override + public Void visitExttype(final ExttypeContext ctx) { + final ExtNodeMetadata typeenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = typeenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + if (parentemd.current != null) { + throw new IllegalArgumentException(error(ctx) + "Unexpected static type."); + } + + final String typestr = ctx.TYPE().getText(); + typeenmd.type = definition.getType(typestr); + parentemd.current = typeenmd.type; + parentemd.statik = true; + + final ExtdotContext dotctx = ctx.extdot(); + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + + return null; + } + + @Override + public Void visitExtcall(final ExtcallContext ctx) { + final ExtNodeMetadata callenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = callenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + callenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null; + + final String name = ctx.EXTID().getText(); + + if (parentemd.current.sort == Sort.ARRAY) { + throw new IllegalArgumentException(error(ctx) + "Unexpected call [" + name + "] on an array."); + } else if (callenmd.last && parentemd.storeExpr != null) { + throw new IllegalArgumentException(error(ctx) + "Cannot assign a value to a call [" + name + "]."); + } + + final Struct struct = parentemd.current.struct; + final List arguments = ctx.arguments().expression(); + final int size = arguments.size(); + Type[] types; + + final Method method = parentemd.statik ? struct.functions.get(name) : struct.methods.get(name); + final boolean def = parentemd.current.sort == Sort.DEF; + + if (method == null && !def) { + throw new IllegalArgumentException( + error(ctx) + "Unknown call [" + name + "] on type [" + struct.name + "]."); + } else if (method != null) { + types = new Type[method.arguments.size()]; + method.arguments.toArray(types); + + callenmd.target = method; + callenmd.type = method.rtn; + parentemd.statement = !parentemd.read && callenmd.last; + parentemd.current = method.rtn; + + if (size != types.length) { + throw new IllegalArgumentException(error(ctx) + "When calling [" + name + "] on type " + + "[" + struct.name + "] expected [" + types.length + "] arguments," + + " but found [" + arguments.size() + "]."); + } + } else { + types = new Type[arguments.size()]; + Arrays.fill(types, definition.defType); + + callenmd.target = name; + callenmd.type = definition.defType; + parentemd.statement = !parentemd.read && callenmd.last; + parentemd.current = callenmd.type; + } + + for (int argument = 0; argument < size; ++argument) { + final ExpressionContext exprctx = adapter.updateExpressionTree(arguments.get(argument)); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = types[argument]; + visit(exprctx); + markCast(expremd); + } + + parentemd.statik = false; + + if (dotctx != null) { + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + } else if (bracectx != null) { + adapter.createExtNodeMetadata(parent, bracectx); + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtvar(final ExtvarContext ctx) { + final ExtNodeMetadata varenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = varenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + final String name = ctx.ID().getText(); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (parentemd.current != null) { + throw new IllegalStateException(error(ctx) + "Unexpected variable [" + name + "] load."); + } + + varenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null; + + final Variable variable = getVariable(name); + + if (variable == null) { + throw new IllegalArgumentException(error(ctx) + "Unknown variable [" + name + "]."); + } + + varenmd.target = variable.slot; + varenmd.type = variable.type; + analyzeLoadStoreExternal(ctx); + parentemd.current = varenmd.type; + + if (dotctx != null) { + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + } else if (bracectx != null) { + adapter.createExtNodeMetadata(parent, bracectx); + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtfield(final ExtfieldContext ctx) { + final ExtNodeMetadata memberenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = memberenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + if (ctx.EXTID() == null && ctx.EXTINTEGER() == null) { + throw new IllegalArgumentException(error(ctx) + "Unexpected parser state."); + } + + final String value = ctx.EXTID() == null ? ctx.EXTINTEGER().getText() : ctx.EXTID().getText(); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null; + final boolean store = memberenmd.last && parentemd.storeExpr != null; + + if (parentemd.current == null) { + throw new IllegalStateException(error(ctx) + "Unexpected field [" + value + "] load."); + } + + if (parentemd.current.sort == Sort.ARRAY) { + if ("length".equals(value)) { + if (!parentemd.read) { + throw new IllegalArgumentException(error(ctx) + "Must read array field [length]."); + } else if (store) { + throw new IllegalArgumentException( + error(ctx) + "Cannot write to read-only array field [length]."); + } + + memberenmd.target = "#length"; + memberenmd.type = definition.intType; + parentemd.current = definition.intType; + } else { + throw new IllegalArgumentException(error(ctx) + "Unexpected array field [" + value + "]."); + } + } else if (parentemd.current.sort == Sort.DEF) { + memberenmd.target = value; + memberenmd.type = definition.defType; + analyzeLoadStoreExternal(ctx); + parentemd.current = memberenmd.type; + } else { + final Struct struct = parentemd.current.struct; + final Field field = parentemd.statik ? struct.statics.get(value) : struct.members.get(value); + + if (field != null) { + if (store && java.lang.reflect.Modifier.isFinal(field.reflect.getModifiers())) { + throw new IllegalArgumentException(error(ctx) + "Cannot write to read-only" + + " field [" + value + "] for type [" + struct.name + "]."); + } + + memberenmd.target = field; + memberenmd.type = field.type; + analyzeLoadStoreExternal(ctx); + parentemd.current = memberenmd.type; + } else { + final boolean get = parentemd.read || parentemd.token > 0 || !memberenmd.last; + final boolean set = memberenmd.last && store; + + Method getter = struct.methods.get("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1)); + Method setter = struct.methods.get("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1)); + Object constant = null; + + if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) { + throw new IllegalArgumentException(error(ctx) + + "Illegal get shortcut on field [" + value + "] for type [" + struct.name + "]."); + } + + if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 1)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal set shortcut on field [" + value + "] for type [" + struct.name + "]."); + } + + Type settype = setter == null ? null : setter.arguments.get(0); + + if (getter == null && setter == null) { + if (ctx.EXTID() != null) { + try { + parentemd.current.clazz.asSubclass(Map.class); + + getter = parentemd.current.struct.methods.get("get"); + setter = parentemd.current.struct.methods.get("put"); + + if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 || + getter.arguments.get(0).sort != Sort.STRING)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal map get shortcut [" + value + "] for type [" + struct.name + "]."); + } + + if (setter != null && (setter.arguments.size() != 2 || + setter.arguments.get(0).sort != Sort.STRING)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal map set shortcut [" + value + "] for type [" + struct.name + "]."); + } + + if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) { + throw new IllegalArgumentException(error(ctx) + "Shortcut argument types must match."); + } + + settype = setter == null ? null : setter.arguments.get(1); + constant = value; + } catch (ClassCastException exception) { + //Do nothing. + } + } else if (ctx.EXTINTEGER() != null) { + try { + parentemd.current.clazz.asSubclass(List.class); + + getter = parentemd.current.struct.methods.get("get"); + setter = parentemd.current.struct.methods.get("add"); + + if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 || + getter.arguments.get(0).sort != Sort.INT)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal list get shortcut [" + value + "] for type [" + struct.name + "]."); + } + + if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 2 || + setter.arguments.get(0).sort != Sort.INT)) { + throw new IllegalArgumentException(error(ctx) + + "Illegal list add shortcut [" + value + "] for type [" + struct.name + "]."); + } + + if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) { + throw new IllegalArgumentException(error(ctx) + "Shortcut argument types must match."); + } + + settype = setter == null ? null : setter.arguments.get(1); + + try { + constant = Integer.parseInt(value); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException(error(ctx) + + "Illegal list shortcut value [" + value + "]."); + } + } catch (ClassCastException exception) { + //Do nothing. + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected parser state."); + } + } + + if ((get || set) && (!get || getter != null) && (!set || setter != null)) { + memberenmd.target = new Object[] {getter, setter, constant != null, constant}; + memberenmd.type = get ? getter.rtn : settype; + analyzeLoadStoreExternal(ctx); + parentemd.current = get ? getter.rtn : setter.rtn; + } + } + + if (memberenmd.target == null) { + throw new IllegalArgumentException( + error(ctx) + "Unknown field [" + value + "] for type [" + struct.name + "]."); + } + } + + parentemd.statik = false; + + if (dotctx != null) { + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + } else if (bracectx != null) { + adapter.createExtNodeMetadata(parent, bracectx); + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtnew(ExtnewContext ctx) { + final ExtNodeMetadata newenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = newenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + newenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null; + + final String name = ctx.TYPE().getText(); + final Struct struct = definition.structs.get(name); + + if (parentemd.current != null) { + throw new IllegalArgumentException(error(ctx) + "Unexpected new call."); + } else if (struct == null) { + throw new IllegalArgumentException(error(ctx) + "Specified type [" + name + "] not found."); + } else if (newenmd.last && parentemd.storeExpr != null) { + throw new IllegalArgumentException(error(ctx) + "Cannot assign a value to a new call."); + } + + final boolean newclass = ctx.arguments() != null; + final boolean newarray = !ctx.expression().isEmpty(); + + final List arguments = newclass ? ctx.arguments().expression() : ctx.expression(); + final int size = arguments.size(); + + Type[] types; + + if (newarray) { + if (!parentemd.read) { + throw new IllegalArgumentException(error(ctx) + "A newly created array must be assigned."); + } + + types = new Type[size]; + Arrays.fill(types, definition.intType); + + newenmd.target = "#makearray"; + + if (size > 1) { + newenmd.type = definition.getType(struct, size); + parentemd.current = newenmd.type; + } else if (size == 1) { + newenmd.type = definition.getType(struct, 0); + parentemd.current = definition.getType(struct, 1); + } else { + throw new IllegalArgumentException(error(ctx) + "A newly created array cannot have zero dimensions."); + } + } else if (newclass) { + final Constructor constructor = struct.constructors.get("new"); + + if (constructor != null) { + types = new Type[constructor.arguments.size()]; + constructor.arguments.toArray(types); + + newenmd.target = constructor; + newenmd.type = definition.getType(struct, 0); + parentemd.statement = !parentemd.read && newenmd.last; + parentemd.current = newenmd.type; + } else { + throw new IllegalArgumentException( + error(ctx) + "Unknown new call on type [" + struct.name + "]."); + } + } else { + throw new IllegalArgumentException(error(ctx) + "Unknown parser state."); + } + + if (size != types.length) { + throw new IllegalArgumentException(error(ctx) + "When calling [" + name + "] on type " + + "[" + struct.name + "] expected [" + types.length + "] arguments," + + " but found [" + arguments.size() + "]."); + } + + for (int argument = 0; argument < size; ++argument) { + final ExpressionContext exprctx = adapter.updateExpressionTree(arguments.get(argument)); + final ExpressionMetadata expremd = adapter.createExpressionMetadata(exprctx); + expremd.to = types[argument]; + visit(exprctx); + markCast(expremd); + } + + if (dotctx != null) { + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + } else if (bracectx != null) { + adapter.createExtNodeMetadata(parent, bracectx); + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtstring(final ExtstringContext ctx) { + final ExtNodeMetadata memberenmd = adapter.getExtNodeMetadata(ctx); + final ParserRuleContext parent = memberenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + final String string = ctx.STRING().getText(); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null; + final boolean store = memberenmd.last && parentemd.storeExpr != null; + + if (parentemd.current != null) { + throw new IllegalStateException(error(ctx) + "Unexpected String constant [" + string + "]."); + } + + if (!parentemd.read) { + throw new IllegalArgumentException(error(ctx) + "Must read String constant [" + string + "]."); + } else if (store) { + throw new IllegalArgumentException( + error(ctx) + "Cannot write to read-only String constant [" + string + "]."); + } + + memberenmd.target = string; + memberenmd.type = definition.stringType; + parentemd.current = definition.stringType; + + if (memberenmd.last) { + parentemd.constant = string; + } + + if (dotctx != null) { + adapter.createExtNodeMetadata(parent, dotctx); + visit(dotctx); + } else if (bracectx != null) { + adapter.createExtNodeMetadata(parent, bracectx); + visit(bracectx); + } + + return null; + } + + @Override + public Void visitArguments(final ArgumentsContext ctx) { + throw new UnsupportedOperationException(error(ctx) + "Unexpected parser state."); + } + + @Override + public Void visitIncrement(IncrementContext ctx) { + final ExpressionMetadata incremd = adapter.getExpressionMetadata(ctx); + final Sort sort = incremd.to == null ? null : incremd.to.sort; + final boolean positive = ctx.INCR() != null; + + if (incremd.to == null) { + incremd.preConst = positive ? 1 : -1; + incremd.from = definition.intType; + } else { + switch (sort) { + case LONG: + incremd.preConst = positive ? 1L : -1L; + incremd.from = definition.longType; + case FLOAT: + incremd.preConst = positive ? 1.0F : -1.0F; + incremd.from = definition.floatType; + case DOUBLE: + incremd.preConst = positive ? 1.0 : -1.0; + incremd.from = definition.doubleType; + default: + incremd.preConst = positive ? 1 : -1; + incremd.from = definition.intType; + } + } + + return null; + } + + private void analyzeLoadStoreExternal(final ParserRuleContext source) { + final ExtNodeMetadata extenmd = adapter.getExtNodeMetadata(source); + final ParserRuleContext parent = extenmd.parent; + final ExternalMetadata parentemd = adapter.getExternalMetadata(parent); + + if (extenmd.last && parentemd.storeExpr != null) { + final ParserRuleContext store = parentemd.storeExpr; + final ExpressionMetadata storeemd = adapter.createExpressionMetadata(parentemd.storeExpr); + final int token = parentemd.token; + + if (token > 0) { + visit(store); + + final boolean add = token == ADD; + final boolean xor = token == BWAND || token == BWXOR || token == BWOR; + final boolean decimal = token == MUL || token == DIV || token == REM || token == SUB; + + extenmd.promote = add ? promoteAdd(extenmd.type, storeemd.from) : + xor ? promoteXor(extenmd.type, storeemd.from) : + promoteNumeric(extenmd.type, storeemd.from, decimal, true); + + if (extenmd.promote == null) { + throw new IllegalArgumentException("Cannot apply compound assignment to " + + " types [" + extenmd.type.name + "] and [" + storeemd.from.name + "]."); + } + + extenmd.castFrom = getLegalCast(source, extenmd.type, extenmd.promote, false); + extenmd.castTo = getLegalCast(source, extenmd.promote, extenmd.type, true); + + storeemd.to = add && extenmd.promote.sort == Sort.STRING ? storeemd.from : extenmd.promote; + markCast(storeemd); + } else { + storeemd.to = extenmd.type; + visit(store); + markCast(storeemd); + } + } + } + + private void markCast(final ExpressionMetadata emd) { + if (emd.from == null) { + throw new IllegalStateException(error(emd.source) + "From cast type should never be null."); + } + + if (emd.to != null) { + emd.cast = getLegalCast(emd.source, emd.from, emd.to, emd.explicit || !emd.typesafe); + + if (emd.preConst != null && emd.to.sort.constant) { + emd.postConst = constCast(emd.source, emd.preConst, emd.cast); + } + } else { + throw new IllegalStateException(error(emd.source) + "To cast type should never be null."); + } + } + + private Cast getLegalCast(final ParserRuleContext source, final Type from, final Type to, final boolean explicit) { + final Cast cast = new Cast(from, to); + + if (from.equals(to)) { + return cast; + } + + if (from.sort == Sort.DEF && to.sort != Sort.VOID || from.sort != Sort.VOID && to.sort == Sort.DEF) { + final Transform transform = definition.transforms.get(cast); + + if (transform != null) { + return transform; + } + + return cast; + } + + switch (from.sort) { + case BOOL: + switch (to.sort) { + case OBJECT: + case BOOL_OBJ: + return checkTransform(source, cast); + } + + break; + case BYTE: + switch (to.sort) { + case SHORT: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + return cast; + case CHAR: + if (explicit) + return cast; + + break; + case OBJECT: + case NUMBER: + case BYTE_OBJ: + case SHORT_OBJ: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case CHAR_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case SHORT: + switch (to.sort) { + case INT: + case LONG: + case FLOAT: + case DOUBLE: + return cast; + case BYTE: + case CHAR: + if (explicit) + return cast; + + break; + case OBJECT: + case NUMBER: + case SHORT_OBJ: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE_OBJ: + case CHAR_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case CHAR: + switch (to.sort) { + case INT: + case LONG: + case FLOAT: + case DOUBLE: + return cast; + case BYTE: + case SHORT: + if (explicit) + return cast; + + break; + case OBJECT: + case NUMBER: + case CHAR_OBJ: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE_OBJ: + case SHORT_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case INT: + switch (to.sort) { + case LONG: + case FLOAT: + case DOUBLE: + return cast; + case BYTE: + case SHORT: + case CHAR: + if (explicit) + return cast; + + break; + case OBJECT: + case NUMBER: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case LONG: + switch (to.sort) { + case FLOAT: + case DOUBLE: + return cast; + case BYTE: + case SHORT: + case CHAR: + case INT: + if (explicit) + return cast; + + break; + case OBJECT: + case NUMBER: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + case INT_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case FLOAT: + switch (to.sort) { + case DOUBLE: + return cast; + case BYTE: + case SHORT: + case CHAR: + case INT: + case LONG: + if (explicit) + return cast; + + break; + case OBJECT: + case NUMBER: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + case INT_OBJ: + case LONG_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case DOUBLE: + switch (to.sort) { + case BYTE: + case SHORT: + case CHAR: + case INT: + case LONG: + case FLOAT: + if (explicit) + return cast; + + break; + case OBJECT: + case NUMBER: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case OBJECT: + case NUMBER: + switch (to.sort) { + case BYTE: + case SHORT: + case CHAR: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case BOOL_OBJ: + switch (to.sort) { + case BOOL: + return checkTransform(source, cast); + } + + break; + case BYTE_OBJ: + switch (to.sort) { + case BYTE: + case SHORT: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case SHORT_OBJ: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case CHAR: + case CHAR_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case SHORT_OBJ: + switch (to.sort) { + case SHORT: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE: + case CHAR: + case BYTE_OBJ: + case CHAR_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case CHAR_OBJ: + switch (to.sort) { + case CHAR: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE: + case SHORT: + case BYTE_OBJ: + case SHORT_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case INT_OBJ: + switch (to.sort) { + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case LONG_OBJ: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE: + case SHORT: + case CHAR: + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case LONG_OBJ: + switch (to.sort) { + case LONG: + case FLOAT: + case DOUBLE: + case FLOAT_OBJ: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE: + case SHORT: + case CHAR: + case INT: + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + case INT_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case FLOAT_OBJ: + switch (to.sort) { + case FLOAT: + case DOUBLE: + case DOUBLE_OBJ: + return checkTransform(source, cast); + case BYTE: + case SHORT: + case CHAR: + case INT: + case LONG: + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + case INT_OBJ: + case LONG_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + case DOUBLE_OBJ: + switch (to.sort) { + case DOUBLE: + return checkTransform(source, cast); + case BYTE: + case SHORT: + case CHAR: + case INT: + case LONG: + case FLOAT: + case BYTE_OBJ: + case SHORT_OBJ: + case CHAR_OBJ: + case INT_OBJ: + case LONG_OBJ: + case FLOAT_OBJ: + if (explicit) + return checkTransform(source, cast); + + break; + } + + break; + } + + try { + from.clazz.asSubclass(to.clazz); + + return cast; + } catch (ClassCastException cce0) { + try { + if (explicit) { + to.clazz.asSubclass(from.clazz); + + return cast; + } else { + throw new ClassCastException( + error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "]."); + } + } catch (ClassCastException cce1) { + throw new ClassCastException( + error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "]."); + } + } + } + + private Transform checkTransform(final ParserRuleContext source, final Cast cast) { + final Transform transform = definition.transforms.get(cast); + + if (transform == null) { + throw new ClassCastException( + error(source) + "Cannot cast from [" + cast.from.name + "] to [" + cast.to.name + "]."); + } + + return transform; + } + + private Object constCast(final ParserRuleContext source, final Object constant, final Cast cast) { + if (cast instanceof Transform) { + final Transform transform = (Transform)cast; + return invokeTransform(source, transform, constant); + } else { + final Sort fsort = cast.from.sort; + final Sort tsort = cast.to.sort; + + if (fsort == tsort) { + return constant; + } else if (fsort.numeric && tsort.numeric) { + Number number; + + if (fsort == Sort.CHAR) { + number = (int)(char)constant; + } else { + number = (Number)constant; + } + + switch (tsort) { + case BYTE: return number.byteValue(); + case SHORT: return number.shortValue(); + case CHAR: return (char)number.intValue(); + case INT: return number.intValue(); + case LONG: return number.longValue(); + case FLOAT: return number.floatValue(); + case DOUBLE: return number.doubleValue(); + default: + throw new IllegalStateException(error(source) + "Expected numeric type for cast."); + } + } else { + throw new IllegalStateException(error(source) + "No valid constant cast from " + + "[" + cast.from.clazz.getCanonicalName() + "] to " + + "[" + cast.to.clazz.getCanonicalName() + "]."); + } + } + } + + private Object invokeTransform(final ParserRuleContext source, final Transform transform, final Object object) { + final Method method = transform.method; + final java.lang.reflect.Method jmethod = method.reflect; + final int modifiers = jmethod.getModifiers(); + + try { + if (java.lang.reflect.Modifier.isStatic(modifiers)) { + return jmethod.invoke(null, object); + } else { + return jmethod.invoke(object); + } + } catch (IllegalAccessException | IllegalArgumentException | + java.lang.reflect.InvocationTargetException | NullPointerException | + ExceptionInInitializerError exception) { + throw new IllegalStateException(error(source) + "Unable to invoke transform to cast constant from " + + "[" + transform.from.name + "] to [" + transform.to.name + "]."); + } + } + + private Type promoteNumeric(final Type from, boolean decimal, boolean primitive) { + final Sort sort = from.sort; + + if (sort == Sort.DEF) { + return definition.defType; + } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ || sort == Sort.NUMBER) && decimal) { + return primitive ? definition.doubleType : definition.doubleobjType; + } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) { + return primitive ? definition.floatType : definition.floatobjType; + } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ || sort == Sort.NUMBER) { + return primitive ? definition.longType : definition.longobjType; + } else if (sort.numeric) { + return primitive ? definition.intType : definition.intobjType; + } + + return null; + } + + private Type promoteNumeric(final Type from0, final Type from1, boolean decimal, boolean primitive) { + final Sort sort0 = from0.sort; + final Sort sort1 = from1.sort; + + if (sort0 == Sort.DEF || sort1 == Sort.DEF) { + return definition.defType; + } + + if (decimal) { + if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort0 == Sort.NUMBER || + sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ || sort1 == Sort.NUMBER) { + return primitive ? definition.doubleType : definition.doubleobjType; + } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) { + return primitive ? definition.floatType : definition.floatobjType; + } + } + + if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort0 == Sort.NUMBER || + sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ || sort1 == Sort.NUMBER) { + return primitive ? definition.longType : definition.longobjType; + } else if (sort0.numeric && sort1.numeric) { + return primitive ? definition.intType : definition.intobjType; + } + + return null; + } + + private Type promoteAdd(final Type from0, final Type from1) { + final Sort sort0 = from0.sort; + final Sort sort1 = from1.sort; + + if (sort0 == Sort.STRING || sort1 == Sort.STRING) { + return definition.stringType; + } + + return promoteNumeric(from0, from1, true, true); + } + + private Type promoteXor(final Type from0, final Type from1) { + final Sort sort0 = from0.sort; + final Sort sort1 = from1.sort; + + if (sort0.bool || sort1.bool) { + return definition.booleanType; + } + + return promoteNumeric(from0, from1, false, true); + } + + private Type promoteEquality(final Type from0, final Type from1) { + final Sort sort0 = from0.sort; + final Sort sort1 = from1.sort; + + if (sort0 == Sort.DEF || sort1 == Sort.DEF) { + return definition.defType; + } + + final boolean primitive = sort0.primitive && sort1.primitive; + + if (sort0.bool && sort1.bool) { + return primitive ? definition.booleanType : definition.byteobjType; + } + + if (sort0.numeric && sort1.numeric) { + return promoteNumeric(from0, from1, true, primitive); + } + + return definition.objectType; + } + + private Type promoteReference(final Type from0, final Type from1) { + final Sort sort0 = from0.sort; + final Sort sort1 = from1.sort; + + if (sort0 == Sort.DEF || sort1 == Sort.DEF) { + return definition.defType; + } + + if (sort0.primitive && sort1.primitive) { + if (sort0.bool && sort1.bool) { + return definition.booleanType; + } + + if (sort0.numeric && sort1.numeric) { + return promoteNumeric(from0, from1, true, true); + } + } + + return definition.objectType; + } + + private Type promoteConditional(final Type from0, final Type from1, final Object const0, final Object const1) { + if (from0.equals(from1)) { + return from0; + } + + final Sort sort0 = from0.sort; + final Sort sort1 = from1.sort; + + if (sort0 == Sort.DEF || sort1 == Sort.DEF) { + return definition.defType; + } + + final boolean primitive = sort0.primitive && sort1.primitive; + + if (sort0.bool && sort1.bool) { + return primitive ? definition.booleanType : definition.booleanobjType; + } + + if (sort0.numeric && sort1.numeric) { + if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) { + return primitive ? definition.doubleType : definition.doubleobjType; + } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) { + return primitive ? definition.floatType : definition.floatobjType; + } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) { + return sort0.primitive && sort1.primitive ? definition.longType : definition.longobjType; + } else { + if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) { + if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) { + return primitive ? definition.byteType : definition.byteobjType; + } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) { + if (const1 != null) { + final short constant = (short)const1; + + if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { + return primitive ? definition.byteType : definition.byteobjType; + } + } + + return primitive ? definition.shortType : definition.shortobjType; + } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) { + return primitive ? definition.intType : definition.intobjType; + } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) { + if (const1 != null) { + final int constant = (int)const1; + + if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { + return primitive ? definition.byteType : definition.byteobjType; + } + } + + return primitive ? definition.intType : definition.intobjType; + } + } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) { + if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) { + if (const0 != null) { + final short constant = (short)const0; + + if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { + return primitive ? definition.byteType : definition.byteobjType; + } + } + + return primitive ? definition.shortType : definition.shortobjType; + } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) { + return primitive ? definition.shortType : definition.shortobjType; + } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) { + return primitive ? definition.intType : definition.intobjType; + } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) { + if (const1 != null) { + final int constant = (int)const1; + + if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) { + return primitive ? definition.shortType : definition.shortobjType; + } + } + + return primitive ? definition.intType : definition.intobjType; + } + } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) { + if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) { + return primitive ? definition.intType : definition.intobjType; + } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) { + return primitive ? definition.intType : definition.intobjType; + } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) { + return primitive ? definition.charType : definition.charobjType; + } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) { + if (const1 != null) { + final int constant = (int)const1; + + if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) { + return primitive ? definition.byteType : definition.byteobjType; + } + } + + return primitive ? definition.intType : definition.intobjType; + } + } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) { + if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) { + if (const0 != null) { + final int constant = (int)const0; + + if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) { + return primitive ? definition.byteType : definition.byteobjType; + } + } + + return primitive ? definition.intType : definition.intobjType; + } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) { + if (const0 != null) { + final int constant = (int)const0; + + if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) { + return primitive ? definition.byteType : definition.byteobjType; + } + } + + return primitive ? definition.intType : definition.intobjType; + } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) { + if (const0 != null) { + final int constant = (int)const0; + + if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) { + return primitive ? definition.byteType : definition.byteobjType; + } + } + + return primitive ? definition.intType : definition.intobjType; + } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) { + return primitive ? definition.intType : definition.intobjType; + } + } + } + } + + final Pair pair = new Pair(from0, from1); + final Type bound = definition.bounds.get(pair); + + return bound == null ? definition.objectType : bound; + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Compiler.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Compiler.java new file mode 100644 index 00000000000..6f4a23765b5 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Compiler.java @@ -0,0 +1,154 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.net.MalformedURLException; +import java.net.URL; +import java.security.CodeSource; +import java.security.SecureClassLoader; +import java.security.cert.Certificate; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.elasticsearch.bootstrap.BootstrapInfo; + +final class Compiler { + private static Definition DEFAULT_DEFINITION = new Definition(new Definition()); + + /** we define the class with lowest privileges */ + private static final CodeSource CODESOURCE; + + static { + try { + CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null); + } catch (MalformedURLException impossible) { + throw new RuntimeException(impossible); + } + } + + static class Loader extends SecureClassLoader { + Loader(ClassLoader parent) { + super(parent); + } + + Class define(String name, byte[] bytes) { + return defineClass(name, bytes, 0, bytes.length, CODESOURCE).asSubclass(Executable.class); + } + } + + static Executable compile(Loader loader, final String name, final String source, final Definition custom, CompilerSettings settings) { + long start = System.currentTimeMillis(); + + final Definition definition = custom == null ? DEFAULT_DEFINITION : new Definition(custom); + + //long end = System.currentTimeMillis() - start; + //System.out.println("types: " + end); + //start = System.currentTimeMillis(); + + //final ParserRuleContext root = createParseTree(source, types); + final ANTLRInputStream stream = new ANTLRInputStream(source); + final ErrorHandlingLexer lexer = new ErrorHandlingLexer(stream); + final PlanAParser parser = new PlanAParser(new CommonTokenStream(lexer)); + final ParserErrorStrategy strategy = new ParserErrorStrategy(); + + lexer.removeErrorListeners(); + lexer.setTypes(definition.structs.keySet()); + + //List tokens = lexer.getAllTokens(); + + //for (final Token token : tokens) { + // System.out.println(token.getType() + " " + token.getText()); + //} + + parser.removeErrorListeners(); + parser.setErrorHandler(strategy); + + ParserRuleContext root = parser.source(); + + //end = System.currentTimeMillis() - start; + //System.out.println("tree: " + end); + + final Adapter adapter = new Adapter(definition, source, root, settings); + + start = System.currentTimeMillis(); + + Analyzer.analyze(adapter); + //System.out.println(root.toStringTree(parser)); + + //end = System.currentTimeMillis() - start; + //System.out.println("analyze: " + end); + //start = System.currentTimeMillis(); + + final byte[] bytes = Writer.write(adapter); + + //end = System.currentTimeMillis() - start; + //System.out.println("write: " + end); + //start = System.currentTimeMillis(); + + final Executable executable = createExecutable(loader, definition, name, source, bytes); + + //end = System.currentTimeMillis() - start; + //System.out.println("create: " + end); + + return executable; + } + + private static ParserRuleContext createParseTree(String source, Definition definition) { + final ANTLRInputStream stream = new ANTLRInputStream(source); + final ErrorHandlingLexer lexer = new ErrorHandlingLexer(stream); + final PlanAParser parser = new PlanAParser(new CommonTokenStream(lexer)); + final ParserErrorStrategy strategy = new ParserErrorStrategy(); + + lexer.removeErrorListeners(); + lexer.setTypes(definition.structs.keySet()); + + parser.removeErrorListeners(); + parser.setErrorHandler(strategy); + + ParserRuleContext root = parser.source(); + // System.out.println(root.toStringTree(parser)); + return root; + } + + private static Executable createExecutable(Loader loader, Definition definition, String name, String source, byte[] bytes) { + try { + // for debugging: + //try { + // FileOutputStream f = new FileOutputStream(new File("/Users/jdconrad/lang/generated/out.class"), false); + // f.write(bytes); + // f.close(); + //} catch (Exception e) { + // throw new RuntimeException(e); + //} + + final Class clazz = loader.define(Writer.CLASS_NAME, bytes); + final java.lang.reflect.Constructor constructor = + clazz.getConstructor(Definition.class, String.class, String.class); + + return constructor.newInstance(definition, name, source); + } catch (Exception exception) { + throw new IllegalStateException( + "An internal error occurred attempting to define the script [" + name + "].", exception); + } + } + + private Compiler() {} +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/CompilerSettings.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/CompilerSettings.java new file mode 100644 index 00000000000..f66b65d0612 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/CompilerSettings.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** + * Settings to use when compiling a script + */ +final class CompilerSettings { + + private boolean numericOverflow = true; + + /** + * Returns {@code true} if numeric operations should overflow, {@code false} + * if they should signal an exception. + *

    + * If this value is {@code true} (default), then things behave like java: + * overflow for integer types can result in unexpected values / unexpected + * signs, and overflow for floating point types can result in infinite or + * {@code NaN} values. + */ + public boolean getNumericOverflow() { + return numericOverflow; + } + + /** + * Set {@code true} for numerics to overflow, false to deliver exceptions. + * @see #getNumericOverflow + */ + public void setNumericOverflow(boolean allow) { + this.numericOverflow = allow; + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Def.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Def.java new file mode 100644 index 00000000000..2a1eb13408c --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Def.java @@ -0,0 +1,1250 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.lang.invoke.MethodHandle; +import java.lang.reflect.Array; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.plan.a.Definition.*; + +public class Def { + public static Object methodCall(final Object owner, final String name, final Definition definition, + final Object[] arguments, final boolean[] typesafe) { + final Method method = getMethod(owner, name, definition); + + if (method == null) { + throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] " + + "for class [" + owner.getClass().getCanonicalName() + "]."); + } + + final MethodHandle handle = method.handle; + final List types = method.arguments; + final Object[] parameters = new Object[arguments.length + 1]; + + parameters[0] = owner; + + if (types.size() != arguments.length) { + throw new IllegalArgumentException("When dynamically calling [" + name + "] from class " + + "[" + owner.getClass() + "] expected [" + types.size() + "] arguments," + + " but found [" + arguments.length + "]."); + } + + try { + for (int count = 0; count < arguments.length; ++count) { + if (typesafe[count]) { + parameters[count + 1] = arguments[count]; + } else { + final Transform transform = getTransform(arguments[count].getClass(), types.get(count).clazz, definition); + parameters[count + 1] = transform == null ? arguments[count] : transform.method.handle.invoke(arguments[count]); + } + } + + return handle.invokeWithArguments(parameters); + } catch (Throwable throwable) { + throw new IllegalArgumentException("Error invoking method [" + name + "] " + + "with owner class [" + owner.getClass().getCanonicalName() + "].", throwable); + } + } + + @SuppressWarnings("unchecked") + public static void fieldStore(final Object owner, Object value, final String name, + final Definition definition, final boolean typesafe) { + final Field field = getField(owner, name, definition); + MethodHandle handle = null; + + if (field == null) { + final String set = "set" + Character.toUpperCase(name.charAt(0)) + name.substring(1); + final Method method = getMethod(owner, set, definition); + + if (method != null) { + handle = method.handle; + } + } else { + handle = field.setter; + } + + if (handle != null) { + try { + if (!typesafe) { + final Transform transform = getTransform(value.getClass(), handle.type().parameterType(1), definition); + + if (transform != null) { + value = transform.method.handle.invoke(value); + } + } + + handle.invoke(owner, value); + } catch (Throwable throwable) { + throw new IllegalArgumentException("Error storing value [" + value + "] " + + "in field [" + name + "] with owner class [" + owner.getClass() + "].", throwable); + } + } else if (owner instanceof Map) { + ((Map)owner).put(name, value); + } else if (owner instanceof List) { + try { + final int index = Integer.parseInt(name); + ((List)owner).add(index, value); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "]."); + } + } else { + throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " + + "for class [" + owner.getClass().getCanonicalName() + "]."); + } + } + + @SuppressWarnings("unchecked") + public static Object fieldLoad(final Object owner, final String name, final Definition definition) { + if (owner.getClass().isArray() && "length".equals(name)) { + return Array.getLength(owner); + } else { + final Field field = getField(owner, name, definition); + MethodHandle handle; + + if (field == null) { + final String get = "get" + Character.toUpperCase(name.charAt(0)) + name.substring(1); + final Method method = getMethod(owner, get, definition); + + if (method != null) { + handle = method.handle; + } else if (owner instanceof Map) { + return ((Map)owner).get(name); + } else if (owner instanceof List) { + try { + final int index = Integer.parseInt(name); + + return ((List)owner).get(index); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "]."); + } + } else { + throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " + + "for class [" + owner.getClass().getCanonicalName() + "]."); + } + } else { + handle = field.getter; + } + + if (handle == null) { + throw new IllegalArgumentException( + "Unable to read from field [" + name + "] with owner class [" + owner.getClass() + "]."); + } else { + try { + return handle.invoke(owner); + } catch (final Throwable throwable) { + throw new IllegalArgumentException("Error loading value from " + + "field [" + name + "] with owner class [" + owner.getClass() + "].", throwable); + } + } + } + } + + @SuppressWarnings("unchecked") + public static void arrayStore(final Object array, Object index, Object value, final Definition definition, + final boolean indexsafe, final boolean valuesafe) { + if (array instanceof Map) { + ((Map)array).put(index, value); + } else { + try { + if (!indexsafe) { + final Transform transform = getTransform(index.getClass(), Integer.class, definition); + + if (transform != null) { + index = transform.method.handle.invoke(index); + } + } + } catch (final Throwable throwable) { + throw new IllegalArgumentException( + "Error storing value [" + value + "] in list using index [" + index + "].", throwable); + } + + if (array.getClass().isArray()) { + try { + if (!valuesafe) { + final Transform transform = getTransform(value.getClass(), array.getClass().getComponentType(), definition); + + if (transform != null) { + value = transform.method.handle.invoke(value); + } + } + + Array.set(array, (int)index, value); + } catch (final Throwable throwable) { + throw new IllegalArgumentException("Error storing value [" + value + "] " + + "in array class [" + array.getClass().getCanonicalName() + "].", throwable); + } + } else if (array instanceof List) { + ((List)array).add((int)index, value); + } else { + throw new IllegalArgumentException("Attempting to address a non-array type " + + "[" + array.getClass().getCanonicalName() + "] as an array."); + } + } + } + + @SuppressWarnings("unchecked") + public static Object arrayLoad(final Object array, Object index, + final Definition definition, final boolean indexsafe) { + if (array instanceof Map) { + return ((Map)array).get(index); + } else { + try { + if (!indexsafe) { + final Transform transform = getTransform(index.getClass(), Integer.class, definition); + + if (transform != null) { + index = transform.method.handle.invoke(index); + } + } + } catch (final Throwable throwable) { + throw new IllegalArgumentException( + "Error loading value using index [" + index + "].", throwable); + } + + if (array.getClass().isArray()) { + try { + return Array.get(array, (int)index); + } catch (final Throwable throwable) { + throw new IllegalArgumentException("Error loading value from " + + "array class [" + array.getClass().getCanonicalName() + "].", throwable); + } + } else if (array instanceof List) { + return ((List)array).get((int)index); + } else { + throw new IllegalArgumentException("Attempting to address a non-array type " + + "[" + array.getClass().getCanonicalName() + "] as an array."); + } + } + } + + public static Method getMethod(final Object owner, final String name, final Definition definition) { + Struct struct = null; + Class clazz = owner.getClass(); + Method method = null; + + while (clazz != null) { + struct = definition.classes.get(clazz); + + if (struct != null) { + method = struct.methods.get(name); + + if (method != null) { + break; + } + } + + for (final Class iface : clazz.getInterfaces()) { + struct = definition.classes.get(iface); + + if (struct != null) { + method = struct.methods.get(name); + + if (method != null) { + break; + } + } + } + + if (struct != null) { + method = struct.methods.get(name); + + if (method != null) { + break; + } + } + + clazz = clazz.getSuperclass(); + } + + if (struct == null) { + throw new IllegalArgumentException("Unable to find a dynamic struct for class [" + owner.getClass() + "]."); + } + + return method; + } + + public static Field getField(final Object owner, final String name, final Definition definition) { + Struct struct = null; + Class clazz = owner.getClass(); + Field field = null; + + while (clazz != null) { + struct = definition.classes.get(clazz); + + if (struct != null) { + field = struct.members.get(name); + + if (field != null) { + break; + } + } + + for (final Class iface : clazz.getInterfaces()) { + struct = definition.classes.get(iface); + + if (struct != null) { + field = struct.members.get(name); + + if (field != null) { + break; + } + } + } + + if (struct != null) { + field = struct.members.get(name); + + if (field != null) { + break; + } + } + + clazz = clazz.getSuperclass(); + } + + if (struct == null) { + throw new IllegalArgumentException("Unable to find a dynamic struct for class [" + owner.getClass() + "]."); + } + + return field; + } + + public static Transform getTransform(Class fromClass, Class toClass, final Definition definition) { + Struct fromStruct = null; + Struct toStruct = null; + + if (fromClass.equals(toClass)) { + return null; + } + + while (fromClass != null) { + fromStruct = definition.classes.get(fromClass); + + if (fromStruct != null) { + break; + } + + for (final Class iface : fromClass.getInterfaces()) { + fromStruct = definition.classes.get(iface); + + if (fromStruct != null) { + break; + } + } + + if (fromStruct != null) { + break; + } + + fromClass = fromClass.getSuperclass(); + } + + if (fromStruct != null) { + while (toClass != null) { + toStruct = definition.classes.get(toClass); + + if (toStruct != null) { + break; + } + + for (final Class iface : toClass.getInterfaces()) { + toStruct = definition.classes.get(iface); + + if (toStruct != null) { + break; + } + } + + if (toStruct != null) { + break; + } + + toClass = toClass.getSuperclass(); + } + } + + if (toStruct != null) { + final Type fromType = definition.getType(fromStruct.name); + final Type toType = definition.getType(toStruct.name); + final Cast cast = new Cast(fromType, toType); + + return definition.transforms.get(cast); + } + + return null; + } + + public static Object not(final Object unary) { + if (unary instanceof Double || unary instanceof Float || unary instanceof Long) { + return ~((Number)unary).longValue(); + } else if (unary instanceof Number) { + return ~((Number)unary).intValue(); + } else if (unary instanceof Character) { + return ~(int)(char)unary; + } + + throw new ClassCastException("Cannot apply [~] operation to type " + + "[" + unary.getClass().getCanonicalName() + "]."); + } + + public static Object neg(final Object unary) { + if (unary instanceof Double) { + return -(double)unary; + } else if (unary instanceof Float) { + return -(float)unary; + } else if (unary instanceof Long) { + return -(long)unary; + } else if (unary instanceof Number) { + return -((Number)unary).intValue(); + } else if (unary instanceof Character) { + return -(char)unary; + } + + throw new ClassCastException("Cannot apply [-] operation to type " + + "[" + unary.getClass().getCanonicalName() + "]."); + } + + public static Object mul(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() * ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() * ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() * ((Number)right).longValue(); + } else { + return ((Number)left).intValue() * ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() * (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() * (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() * (long)(char)right; + } else { + return ((Number)left).intValue() * (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left * ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left * ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left * ((Number)right).longValue(); + } else { + return (int)(char)left * ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left * (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [*] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object div(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() / ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() / ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() / ((Number)right).longValue(); + } else { + return ((Number)left).intValue() / ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() / (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() / (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() / (long)(char)right; + } else { + return ((Number)left).intValue() / (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left / ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left / ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left / ((Number)right).longValue(); + } else { + return (int)(char)left / ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left / (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [/] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object rem(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() % ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() % ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() % ((Number)right).longValue(); + } else { + return ((Number)left).intValue() % ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() % (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() % (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() % (long)(char)right; + } else { + return ((Number)left).intValue() % (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left % ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left % ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left % ((Number)right).longValue(); + } else { + return (int)(char)left % ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left % (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [%] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object add(final Object left, final Object right) { + if (left instanceof String || right instanceof String) { + return "" + left + right; + } else if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() + ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() + ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() + ((Number)right).longValue(); + } else { + return ((Number)left).intValue() + ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() + (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() + (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() + (long)(char)right; + } else { + return ((Number)left).intValue() + (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left + ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left + ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left + ((Number)right).longValue(); + } else { + return (int)(char)left + ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left + (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [+] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object sub(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() - ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() - ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() - ((Number)right).longValue(); + } else { + return ((Number)left).intValue() - ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() - (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() - (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() - (long)(char)right; + } else { + return ((Number)left).intValue() - (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left - ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left - ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left - ((Number)right).longValue(); + } else { + return (int)(char)left - ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left - (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [-] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object lsh(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double || + left instanceof Float || right instanceof Float || + left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() << ((Number)right).longValue(); + } else { + return ((Number)left).intValue() << ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double || left instanceof Float || left instanceof Long) { + return ((Number)left).longValue() << (long)(char)right; + } else { + return ((Number)left).intValue() << (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double || right instanceof Float || right instanceof Long) { + return (long)(char)left << ((Number)right).longValue(); + } else { + return (int)(char)left << ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left << (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [<<] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object rsh(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double || + left instanceof Float || right instanceof Float || + left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() >> ((Number)right).longValue(); + } else { + return ((Number)left).intValue() >> ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double || left instanceof Float || left instanceof Long) { + return ((Number)left).longValue() >> (long)(char)right; + } else { + return ((Number)left).intValue() >> (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double || right instanceof Float || right instanceof Long) { + return (long)(char)left >> ((Number)right).longValue(); + } else { + return (int)(char)left >> ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left >> (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [>>] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object ush(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double || + left instanceof Float || right instanceof Float || + left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() >>> ((Number)right).longValue(); + } else { + return ((Number)left).intValue() >>> ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double || left instanceof Float || left instanceof Long) { + return ((Number)left).longValue() >>> (long)(char)right; + } else { + return ((Number)left).intValue() >>> (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double || right instanceof Float || right instanceof Long) { + return (long)(char)left >>> ((Number)right).longValue(); + } else { + return (int)(char)left >>> ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left >>> (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [>>>] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object and(final Object left, final Object right) { + if (left instanceof Boolean && right instanceof Boolean) { + return (boolean)left && (boolean)right; + } else if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double || + left instanceof Float || right instanceof Float || + left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() & ((Number)right).longValue(); + } else { + return ((Number)left).intValue() & ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double || left instanceof Float || left instanceof Long) { + return ((Number)left).longValue() & (long)(char)right; + } else { + return ((Number)left).intValue() & (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double || right instanceof Float || right instanceof Long) { + return (long)(char)left & ((Number)right).longValue(); + } else { + return (int)(char)left & ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left & (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [&] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object xor(final Object left, final Object right) { + if (left instanceof Boolean && right instanceof Boolean) { + return (boolean)left ^ (boolean)right; + } else if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double || + left instanceof Float || right instanceof Float || + left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() ^ ((Number)right).longValue(); + } else { + return ((Number)left).intValue() ^ ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double || left instanceof Float || left instanceof Long) { + return ((Number)left).longValue() ^ (long)(char)right; + } else { + return ((Number)left).intValue() ^ (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double || right instanceof Float || right instanceof Long) { + return (long)(char)left ^ ((Number)right).longValue(); + } else { + return (int)(char)left ^ ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left ^ (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [^] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static Object or(final Object left, final Object right) { + if (left instanceof Boolean && right instanceof Boolean) { + return (boolean)left || (boolean)right; + } else if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double || + left instanceof Float || right instanceof Float || + left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() | ((Number)right).longValue(); + } else { + return ((Number)left).intValue() | ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double || left instanceof Float || left instanceof Long) { + return ((Number)left).longValue() | (long)(char)right; + } else { + return ((Number)left).intValue() | (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double || right instanceof Float || right instanceof Long) { + return (long)(char)left | ((Number)right).longValue(); + } else { + return (int)(char)left | ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left | (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [|] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static boolean eq(final Object left, final Object right) { + if (left != null && right != null) { + if (left instanceof Double) { + if (right instanceof Number) { + return (double)left == ((Number)right).doubleValue(); + } else if (right instanceof Character) { + return (double)left == (double)(char)right; + } + } else if (right instanceof Double) { + if (left instanceof Number) { + return ((Number)left).doubleValue() == (double)right; + } else if (left instanceof Character) { + return (double)(char)left == ((Number)right).doubleValue(); + } + } else if (left instanceof Float) { + if (right instanceof Number) { + return (float)left == ((Number)right).floatValue(); + } else if (right instanceof Character) { + return (float)left == (float)(char)right; + } + } else if (right instanceof Float) { + if (left instanceof Number) { + return ((Number)left).floatValue() == (float)right; + } else if (left instanceof Character) { + return (float)(char)left == ((Number)right).floatValue(); + } + } else if (left instanceof Long) { + if (right instanceof Number) { + return (long)left == ((Number)right).longValue(); + } else if (right instanceof Character) { + return (long)left == (long)(char)right; + } + } else if (right instanceof Long) { + if (left instanceof Number) { + return ((Number)left).longValue() == (long)right; + } else if (left instanceof Character) { + return (long)(char)left == ((Number)right).longValue(); + } + } else if (left instanceof Number) { + if (right instanceof Number) { + return ((Number)left).intValue() == ((Number)right).intValue(); + } else if (right instanceof Character) { + return ((Number)left).intValue() == (int)(char)right; + } + } else if (right instanceof Number && left instanceof Character) { + return (int)(char)left == ((Number)right).intValue(); + } else if (left instanceof Character && right instanceof Character) { + return (int)(char)left == (int)(char)right; + } + + return left.equals(right); + } + + return left == null && right == null; + } + + public static boolean lt(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() < ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() < ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() < ((Number)right).longValue(); + } else { + return ((Number)left).intValue() < ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() < (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() < (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() < (long)(char)right; + } else { + return ((Number)left).intValue() < (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left < ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left < ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left < ((Number)right).longValue(); + } else { + return (int)(char)left < ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left < (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [<] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static boolean lte(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() <= ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() <= ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() <= ((Number)right).longValue(); + } else { + return ((Number)left).intValue() <= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() <= (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() <= (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() <= (long)(char)right; + } else { + return ((Number)left).intValue() <= (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left <= ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left <= ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left <= ((Number)right).longValue(); + } else { + return (int)(char)left <= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left <= (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [<=] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static boolean gt(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() > ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() > ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() > ((Number)right).longValue(); + } else { + return ((Number)left).intValue() > ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() > (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() > (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() > (long)(char)right; + } else { + return ((Number)left).intValue() > (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left > ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left > ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left > ((Number)right).longValue(); + } else { + return (int)(char)left > ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left > (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [>] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static boolean gte(final Object left, final Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() >= ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() >= ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() >= ((Number)right).longValue(); + } else { + return ((Number)left).intValue() >= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() >= (double)(char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() >= (float)(char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() >= (long)(char)right; + } else { + return ((Number)left).intValue() >= (int)(char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (double)(char)left >= ((Number)right).doubleValue(); + } else if (right instanceof Float) { + return (float)(char)left >= ((Number)right).floatValue(); + } else if (right instanceof Long) { + return (long)(char)left >= ((Number)right).longValue(); + } else { + return (int)(char)left >= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (int)(char)left >= (int)(char)right; + } + } + + throw new ClassCastException("Cannot apply [>] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + public static boolean DefToboolean(final Object value) { + if (value instanceof Boolean) { + return (boolean)value; + } else if (value instanceof Character) { + return ((char)value) != 0; + } else { + return ((Number)value).intValue() != 0; + } + } + + public static byte DefTobyte(final Object value) { + if (value instanceof Boolean) { + return ((Boolean)value) ? (byte)1 : 0; + } else if (value instanceof Character) { + return (byte)(char)value; + } else { + return ((Number)value).byteValue(); + } + } + + public static short DefToshort(final Object value) { + if (value instanceof Boolean) { + return ((Boolean)value) ? (short)1 : 0; + } else if (value instanceof Character) { + return (short)(char)value; + } else { + return ((Number)value).shortValue(); + } + } + + public static char DefTochar(final Object value) { + if (value instanceof Boolean) { + return ((Boolean)value) ? (char)1 : 0; + } else if (value instanceof Character) { + return ((Character)value); + } else { + return (char)((Number)value).intValue(); + } + } + + public static int DefToint(final Object value) { + if (value instanceof Boolean) { + return ((Boolean)value) ? 1 : 0; + } else if (value instanceof Character) { + return (int)(char)value; + } else { + return ((Number)value).intValue(); + } + } + + public static long DefTolong(final Object value) { + if (value instanceof Boolean) { + return ((Boolean)value) ? 1L : 0; + } else if (value instanceof Character) { + return (long)(char)value; + } else { + return ((Number)value).longValue(); + } + } + + public static float DefTofloat(final Object value) { + if (value instanceof Boolean) { + return ((Boolean)value) ? (float)1 : 0; + } else if (value instanceof Character) { + return (float)(char)value; + } else { + return ((Number)value).floatValue(); + } + } + + public static double DefTodouble(final Object value) { + if (value instanceof Boolean) { + return ((Boolean)value) ? (double)1 : 0; + } else if (value instanceof Character) { + return (double)(char)value; + } else { + return ((Number)value).doubleValue(); + } + } + + public static Boolean DefToBoolean(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return (boolean)value; + } else if (value instanceof Character) { + return ((char)value) != 0; + } else { + return ((Number)value).intValue() != 0; + } + } + + public static Byte DefToByte(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return ((Boolean)value) ? (byte)1 : 0; + } else if (value instanceof Character) { + return (byte)(char)value; + } else { + return ((Number)value).byteValue(); + } + } + + public static Short DefToShort(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return ((Boolean)value) ? (short)1 : 0; + } else if (value instanceof Character) { + return (short)(char)value; + } else { + return ((Number)value).shortValue(); + } + } + + public static Character DefToCharacter(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return ((Boolean)value) ? (char)1 : 0; + } else if (value instanceof Character) { + return ((Character)value); + } else { + return (char)((Number)value).intValue(); + } + } + + public static Integer DefToInteger(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return ((Boolean)value) ? 1 : 0; + } else if (value instanceof Character) { + return (int)(char)value; + } else { + return ((Number)value).intValue(); + } + } + + public static Long DefToLong(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return ((Boolean)value) ? 1L : 0; + } else if (value instanceof Character) { + return (long)(char)value; + } else { + return ((Number)value).longValue(); + } + } + + public static Float DefToFloat(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return ((Boolean)value) ? (float)1 : 0; + } else if (value instanceof Character) { + return (float)(char)value; + } else { + return ((Number)value).floatValue(); + } + } + + public static Double DefToDouble(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return ((Boolean)value) ? (double)1 : 0; + } else if (value instanceof Character) { + return (double)(char)value; + } else { + return ((Number)value).doubleValue(); + } + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Definition.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Definition.java new file mode 100644 index 00000000000..5c52a202919 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Definition.java @@ -0,0 +1,1809 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class Definition { + enum Sort { + VOID( void.class , 0 , true , false , false , false ), + BOOL( boolean.class , 1 , true , true , false , true ), + BYTE( byte.class , 1 , true , false , true , true ), + SHORT( short.class , 1 , true , false , true , true ), + CHAR( char.class , 1 , true , false , true , true ), + INT( int.class , 1 , true , false , true , true ), + LONG( long.class , 2 , true , false , true , true ), + FLOAT( float.class , 1 , true , false , true , true ), + DOUBLE( double.class , 2 , true , false , true , true ), + + VOID_OBJ( Void.class , 1 , true , false , false , false ), + BOOL_OBJ( Boolean.class , 1 , false , true , false , false ), + BYTE_OBJ( Byte.class , 1 , false , false , true , false ), + SHORT_OBJ( Short.class , 1 , false , false , true , false ), + CHAR_OBJ( Character.class , 1 , false , false , true , false ), + INT_OBJ( Integer.class , 1 , false , false , true , false ), + LONG_OBJ( Long.class , 1 , false , false , true , false ), + FLOAT_OBJ( Float.class , 1 , false , false , true , false ), + DOUBLE_OBJ( Double.class , 1 , false , false , true , false ), + + NUMBER( Number.class , 1 , false , false , true , false ), + STRING( String.class , 1 , false , false , false , true ), + + OBJECT( null , 1 , false , false , false , false ), + DEF( null , 1 , false , false , false , false ), + ARRAY( null , 1 , false , false , false , false ); + + final Class clazz; + final int size; + final boolean primitive; + final boolean bool; + final boolean numeric; + final boolean constant; + + Sort(final Class clazz, final int size, final boolean primitive, + final boolean bool, final boolean numeric, final boolean constant) { + this.clazz = clazz; + this.size = size; + this.bool = bool; + this.primitive = primitive; + this.numeric = numeric; + this.constant = constant; + } + } + + static class Type { + final String name; + final Struct struct; + final Class clazz; + final org.objectweb.asm.Type type; + final Sort sort; + + private Type(final String name, final Struct struct, final Class clazz, + final org.objectweb.asm.Type type, final Sort sort) { + this.name = name; + this.struct = struct; + this.clazz = clazz; + this.type = type; + this.sort = sort; + } + + @Override + public boolean equals(final Object object) { + if (this == object) { + return true; + } + + if (object == null || getClass() != object.getClass()) { + return false; + } + + final Type type = (Type)object; + + return this.type.equals(type.type) && struct.equals(type.struct); + } + + @Override + public int hashCode() { + int result = struct.hashCode(); + result = 31 * result + type.hashCode(); + + return result; + } + } + + static class Constructor { + final String name; + final Struct owner; + final List arguments; + final org.objectweb.asm.commons.Method method; + final java.lang.reflect.Constructor reflect; + + private Constructor(final String name, final Struct owner, final List arguments, + final org.objectweb.asm.commons.Method method, final java.lang.reflect.Constructor reflect) { + this.name = name; + this.owner = owner; + this.arguments = Collections.unmodifiableList(arguments); + this.method = method; + this.reflect = reflect; + } + } + + static class Method { + final String name; + final Struct owner; + final Type rtn; + final List arguments; + final org.objectweb.asm.commons.Method method; + final java.lang.reflect.Method reflect; + final MethodHandle handle; + + private Method(final String name, final Struct owner, final Type rtn, final List arguments, + final org.objectweb.asm.commons.Method method, final java.lang.reflect.Method reflect, + final MethodHandle handle) { + this.name = name; + this.owner = owner; + this.rtn = rtn; + this.arguments = Collections.unmodifiableList(arguments); + this.method = method; + this.reflect = reflect; + this.handle = handle; + } + } + + static class Field { + final String name; + final Struct owner; + final Type generic; + final Type type; + final java.lang.reflect.Field reflect; + final MethodHandle getter; + final MethodHandle setter; + + private Field(final String name, final Struct owner, final Type generic, final Type type, + final java.lang.reflect.Field reflect, final MethodHandle getter, final MethodHandle setter) { + this.name = name; + this.owner = owner; + this.generic = generic; + this.type = type; + this.reflect = reflect; + this.getter = getter; + this.setter = setter; + } + } + + static class Struct { + final String name; + final Class clazz; + final org.objectweb.asm.Type type; + + final Map constructors; + final Map functions; + final Map methods; + + final Map statics; + final Map members; + + private Struct(final String name, final Class clazz, final org.objectweb.asm.Type type) { + this.name = name; + this.clazz = clazz; + this.type = type; + + constructors = new HashMap<>(); + functions = new HashMap<>(); + methods = new HashMap<>(); + + statics = new HashMap<>(); + members = new HashMap<>(); + } + + private Struct(final Struct struct) { + name = struct.name; + clazz = struct.clazz; + type = struct.type; + + constructors = Collections.unmodifiableMap(struct.constructors); + functions = Collections.unmodifiableMap(struct.functions); + methods = Collections.unmodifiableMap(struct.methods); + + statics = Collections.unmodifiableMap(struct.statics); + members = Collections.unmodifiableMap(struct.members); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + + if (object == null || getClass() != object.getClass()) { + return false; + } + + Struct struct = (Struct)object; + + return name.equals(struct.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + } + + static class Pair { + final Type type0; + final Type type1; + + Pair(final Type type0, final Type type1) { + this.type0 = type0; + this.type1 = type1; + } + + @Override + public boolean equals(final Object object) { + if (this == object) { + return true; + } + + if (object == null || getClass() != object.getClass()) { + return false; + } + + final Pair pair = (Pair)object; + + return type0.equals(pair.type0) && type1.equals(pair.type1); + } + + @Override + public int hashCode() { + int result = type0.hashCode(); + result = 31 * result + type1.hashCode(); + + return result; + } + } + + static class Cast { + final Type from; + final Type to; + + Cast(final Type from, final Type to) { + this.from = from; + this.to = to; + } + + @Override + public boolean equals(final Object object) { + if (this == object) { + return true; + } + + if (object == null || getClass() != object.getClass()) { + return false; + } + + final Cast cast = (Cast)object; + + return from.equals(cast.from) && to.equals(cast.to); + } + + @Override + public int hashCode() { + int result = from.hashCode(); + result = 31 * result + to.hashCode(); + + return result; + } + } + + static class Transform extends Cast { + final Cast cast; + final Method method; + final Type upcast; + final Type downcast; + + private Transform(final Cast cast, Method method, final Type upcast, final Type downcast) { + super(cast.from, cast.to); + + this.cast = cast; + this.method = method; + this.upcast = upcast; + this.downcast = downcast; + } + } + + final Map structs; + final Map, Struct> classes; + final Map transforms; + final Map bounds; + + final Type voidType; + final Type booleanType; + final Type byteType; + final Type shortType; + final Type charType; + final Type intType; + final Type longType; + final Type floatType; + final Type doubleType; + + final Type voidobjType; + final Type booleanobjType; + final Type byteobjType; + final Type shortobjType; + final Type charobjType; + final Type intobjType; + final Type longobjType; + final Type floatobjType; + final Type doubleobjType; + + final Type objectType; + final Type defType; + final Type numberType; + final Type charseqType; + final Type stringType; + final Type mathType; + final Type utilityType; + final Type defobjType; + + final Type listType; + final Type arraylistType; + final Type mapType; + final Type hashmapType; + + final Type olistType; + final Type oarraylistType; + final Type omapType; + final Type ohashmapType; + + final Type smapType; + final Type shashmapType; + final Type somapType; + final Type sohashmapType; + + final Type execType; + + public Definition() { + structs = new HashMap<>(); + classes = new HashMap<>(); + transforms = new HashMap<>(); + bounds = new HashMap<>(); + + addDefaultStructs(); + addDefaultClasses(); + + voidType = getType("void"); + booleanType = getType("boolean"); + byteType = getType("byte"); + shortType = getType("short"); + charType = getType("char"); + intType = getType("int"); + longType = getType("long"); + floatType = getType("float"); + doubleType = getType("double"); + + voidobjType = getType("Void"); + booleanobjType = getType("Boolean"); + byteobjType = getType("Byte"); + shortobjType = getType("Short"); + charobjType = getType("Character"); + intobjType = getType("Integer"); + longobjType = getType("Long"); + floatobjType = getType("Float"); + doubleobjType = getType("Double"); + + objectType = getType("Object"); + defType = getType("def"); + numberType = getType("Number"); + charseqType = getType("CharSequence"); + stringType = getType("String"); + mathType = getType("Math"); + utilityType = getType("Utility"); + defobjType = getType("Def"); + + listType = getType("List"); + arraylistType = getType("ArrayList"); + mapType = getType("Map"); + hashmapType = getType("HashMap"); + + olistType = getType("List"); + oarraylistType = getType("ArrayList"); + omapType = getType("Map"); + ohashmapType = getType("HashMap"); + + smapType = getType("Map"); + shashmapType = getType("HashMap"); + somapType = getType("Map"); + sohashmapType = getType("HashMap"); + + execType = getType("Executable"); + + addDefaultElements(); + copyDefaultStructs(); + addDefaultTransforms(); + addDefaultBounds(); + } + + Definition(final Definition definition) { + final Map structs = new HashMap<>(); + + for (final Struct struct : definition.structs.values()) { + structs.put(struct.name, new Struct(struct)); + } + + this.structs = Collections.unmodifiableMap(structs); + + final Map, Struct> classes = new HashMap<>(); + + for (final Struct struct : definition.classes.values()) { + classes.put(struct.clazz, this.structs.get(struct.name)); + } + + this.classes = Collections.unmodifiableMap(classes); + + transforms = Collections.unmodifiableMap(definition.transforms); + bounds = Collections.unmodifiableMap(definition.bounds); + + voidType = definition.voidType; + booleanType = definition.booleanType; + byteType = definition.byteType; + shortType = definition.shortType; + charType = definition.charType; + intType = definition.intType; + longType = definition.longType; + floatType = definition.floatType; + doubleType = definition.doubleType; + + voidobjType = definition.voidobjType; + booleanobjType = definition.booleanobjType; + byteobjType = definition.byteobjType; + shortobjType = definition.shortobjType; + charobjType = definition.charobjType; + intobjType = definition.intobjType; + longobjType = definition.longobjType; + floatobjType = definition.floatobjType; + doubleobjType = definition.doubleobjType; + + objectType = definition.objectType; + defType = definition.defType; + numberType = definition.numberType; + charseqType = definition.charseqType; + stringType = definition.stringType; + mathType = definition.mathType; + utilityType = definition.utilityType; + defobjType = definition.defobjType; + + listType = definition.listType; + arraylistType = definition.arraylistType; + mapType = definition.mapType; + hashmapType = definition.hashmapType; + + olistType = definition.olistType; + oarraylistType = definition.oarraylistType; + omapType = definition.omapType; + ohashmapType = definition.ohashmapType; + + smapType = definition.smapType; + shashmapType = definition.shashmapType; + somapType = definition.somapType; + sohashmapType = definition.sohashmapType; + + execType = definition.execType; + } + + private void addDefaultStructs() { + addStruct( "void" , void.class ); + addStruct( "boolean" , boolean.class ); + addStruct( "byte" , byte.class ); + addStruct( "short" , short.class ); + addStruct( "char" , char.class ); + addStruct( "int" , int.class ); + addStruct( "long" , long.class ); + addStruct( "float" , float.class ); + addStruct( "double" , double.class ); + + addStruct( "Void" , Void.class ); + addStruct( "Boolean" , Boolean.class ); + addStruct( "Byte" , Byte.class ); + addStruct( "Short" , Short.class ); + addStruct( "Character" , Character.class ); + addStruct( "Integer" , Integer.class ); + addStruct( "Long" , Long.class ); + addStruct( "Float" , Float.class ); + addStruct( "Double" , Double.class ); + + addStruct( "Object" , Object.class ); + addStruct( "def" , Object.class ); + addStruct( "Number" , Number.class ); + addStruct( "CharSequence" , CharSequence.class ); + addStruct( "String" , String.class ); + addStruct( "Math" , Math.class ); + addStruct( "Utility" , Utility.class ); + addStruct( "Def" , Def.class ); + + addStruct( "List" , List.class ); + addStruct( "ArrayList" , ArrayList.class ); + addStruct( "Map" , Map.class ); + addStruct( "HashMap" , HashMap.class ); + + addStruct( "List" , List.class ); + addStruct( "ArrayList" , ArrayList.class ); + addStruct( "Map" , Map.class ); + addStruct( "HashMap" , HashMap.class ); + + addStruct( "Map" , Map.class ); + addStruct( "HashMap" , HashMap.class ); + addStruct( "Map" , Map.class ); + addStruct( "HashMap" , HashMap.class ); + + addStruct( "Executable" , Executable.class ); + } + + private void addDefaultClasses() { + addClass("boolean"); + addClass("byte"); + addClass("short"); + addClass("char"); + addClass("int"); + addClass("long"); + addClass("float"); + addClass("double"); + + addClass("Boolean"); + addClass("Byte"); + addClass("Short"); + addClass("Character"); + addClass("Integer"); + addClass("Long"); + addClass("Float"); + addClass("Double"); + + addClass("Object"); + addClass("Number"); + addClass("CharSequence"); + addClass("String"); + + addClass("List"); + addClass("ArrayList"); + addClass("Map"); + addClass("HashMap"); + } + + private void addDefaultElements() { + addMethod("Object", "toString", null, false, stringType, new Type[] {}, null, null); + addMethod("Object", "equals", null, false, booleanType, new Type[] {objectType}, null, null); + addMethod("Object", "hashCode", null, false, intType, new Type[] {}, null, null); + + addMethod("def", "toString", null, false, stringType, new Type[] {}, null, null); + addMethod("def", "equals", null, false, booleanType, new Type[] {objectType}, null, null); + addMethod("def", "hashCode", null, false, intType, new Type[] {}, null, null); + + addConstructor("Boolean", "new", new Type[] {booleanType}, null); + addMethod("Boolean", "valueOf", null, true, booleanobjType, new Type[] {booleanType}, null, null); + addMethod("Boolean", "booleanValue", null, false, booleanType, new Type[] {}, null, null); + + addConstructor("Byte", "new", new Type[]{byteType}, null); + addMethod("Byte", "valueOf", null, true, byteobjType, new Type[] {byteType}, null, null); + addMethod("Byte", "byteValue", null, false, byteType, new Type[] {}, null, null); + addField("Byte", "MIN_VALUE", null, true, byteType, null); + addField("Byte", "MAX_VALUE", null, true, byteType, null); + + addConstructor("Short", "new", new Type[]{shortType}, null); + addMethod("Short", "valueOf", null, true, shortobjType, new Type[] {shortType}, null, null); + addMethod("Short", "shortValue", null, false, shortType, new Type[] {}, null, null); + addField("Short", "MIN_VALUE", null, true, shortType, null); + addField("Short", "MAX_VALUE", null, true, shortType, null); + + addConstructor("Character", "new", new Type[]{charType}, null); + addMethod("Character", "valueOf", null, true, charobjType, new Type[] {charType}, null, null); + addMethod("Character", "charValue", null, false, charType, new Type[] {}, null, null); + addField("Character", "MIN_VALUE", null, true, charType, null); + addField("Character", "MAX_VALUE", null, true, charType, null); + + addConstructor("Integer", "new", new Type[]{intType}, null); + addMethod("Integer", "valueOf", null, true, intobjType, new Type[] {intType}, null, null); + addMethod("Integer", "intValue", null, false, intType, new Type[] {}, null, null); + addField("Integer", "MIN_VALUE", null, true, intType, null); + addField("Integer", "MAX_VALUE", null, true, intType, null); + + addConstructor("Long", "new", new Type[]{longType}, null); + addMethod("Long", "valueOf", null, true, longobjType, new Type[] {longType}, null, null); + addMethod("Long", "longValue", null, false, longType, new Type[] {}, null, null); + addField("Long", "MIN_VALUE", null, true, longType, null); + addField("Long", "MAX_VALUE", null, true, longType, null); + + addConstructor("Float", "new", new Type[]{floatType}, null); + addMethod("Float", "valueOf", null, true, floatobjType, new Type[] {floatType}, null, null); + addMethod("Float", "floatValue", null, false, floatType, new Type[] {}, null, null); + addField("Float", "MIN_VALUE", null, true, floatType, null); + addField("Float", "MAX_VALUE", null, true, floatType, null); + + addConstructor("Double", "new", new Type[]{doubleType}, null); + addMethod("Double", "valueOf", null, true, doubleobjType, new Type[] {doubleType}, null, null); + addMethod("Double", "doubleValue", null, false, doubleType, new Type[] {}, null, null); + addField("Double", "MIN_VALUE", null, true, doubleType, null); + addField("Double", "MAX_VALUE", null, true, doubleType, null); + + addMethod("Number", "byteValue", null, false, byteType, new Type[] {}, null, null); + addMethod("Number", "shortValue", null, false, shortType, new Type[] {}, null, null); + addMethod("Number", "intValue", null, false, intType, new Type[] {}, null, null); + addMethod("Number", "longValue", null, false, longType, new Type[] {}, null, null); + addMethod("Number", "floatValue", null, false, floatType, new Type[] {}, null, null); + addMethod("Number", "doubleValue", null, false, doubleType, new Type[] {}, null, null); + + addMethod("CharSequence", "charAt", null, false, charType, new Type[] {intType}, null, null); + addMethod("CharSequence", "length", null, false, intType, new Type[] {}, null, null); + + addConstructor("String", "new", new Type[] {}, null); + addMethod("String", "codePointAt", null, false, intType, new Type[] {intType}, null, null); + addMethod("String", "compareTo", null, false, intType, new Type[] {stringType}, null, null); + addMethod("String", "concat", null, false, stringType, new Type[] {stringType}, null, null); + addMethod("String", "endsWith", null, false, booleanType, new Type[] {stringType}, null, null); + addMethod("String", "indexOf", null, false, intType, new Type[] {stringType, intType}, null, null); + addMethod("String", "isEmpty", null, false, booleanType, new Type[] {}, null, null); + addMethod("String", "replace", null, false, stringType, new Type[] {charseqType, charseqType}, null, null); + addMethod("String", "startsWith", null, false, booleanType, new Type[] {stringType}, null, null); + addMethod("String", "substring", null, false, stringType, new Type[] {intType, intType}, null, null); + addMethod("String", "toCharArray", null, false, getType(charType.struct, 1), new Type[] {}, null, null); + addMethod("String", "trim", null, false, stringType, new Type[] {}, null, null); + + addMethod("Utility", "NumberToboolean", null, true, booleanType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberTochar", null, true, charType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToBoolean", null, true, booleanobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToByte", null, true, byteobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToShort", null, true, shortobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToCharacter", null, true, charobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToInteger", null, true, intobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToLong", null, true, longobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToFloat", null, true, floatobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "NumberToDouble", null, true, doubleobjType, new Type[] {numberType}, null, null); + addMethod("Utility", "booleanTobyte", null, true, byteType, new Type[] {booleanType}, null, null); + addMethod("Utility", "booleanToshort", null, true, shortType, new Type[] {booleanType}, null, null); + addMethod("Utility", "booleanTochar", null, true, charType, new Type[] {booleanType}, null, null); + addMethod("Utility", "booleanToint", null, true, intType, new Type[] {booleanType}, null, null); + addMethod("Utility", "booleanTolong", null, true, longType, new Type[] {booleanType}, null, null); + addMethod("Utility", "booleanTofloat", null, true, floatType, new Type[] {booleanType}, null, null); + addMethod("Utility", "booleanTodouble", null, true, doubleType, new Type[] {booleanType}, null, null); + addMethod("Utility", "booleanToInteger", null, true, intobjType, new Type[] {booleanType}, null, null); + addMethod("Utility", "BooleanTobyte", null, true, byteType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToshort", null, true, shortType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanTochar", null, true, charType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToint", null, true, intType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanTolong", null, true, longType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanTofloat", null, true, floatType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanTodouble", null, true, doubleType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToByte", null, true, byteobjType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToShort", null, true, shortobjType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToCharacter", null, true, charobjType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToInteger", null, true, intobjType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToLong", null, true, longobjType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToFloat", null, true, floatobjType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "BooleanToDouble", null, true, doubleobjType, new Type[] {booleanobjType}, null, null); + addMethod("Utility", "byteToboolean", null, true, booleanType, new Type[] {byteType}, null, null); + addMethod("Utility", "byteToShort", null, true, shortobjType, new Type[] {byteType}, null, null); + addMethod("Utility", "byteToCharacter", null, true, charobjType, new Type[] {byteType}, null, null); + addMethod("Utility", "byteToInteger", null, true, intobjType, new Type[] {byteType}, null, null); + addMethod("Utility", "byteToLong", null, true, longobjType, new Type[] {byteType}, null, null); + addMethod("Utility", "byteToFloat", null, true, floatobjType, new Type[] {byteType}, null, null); + addMethod("Utility", "byteToDouble", null, true, doubleobjType, new Type[] {byteType}, null, null); + addMethod("Utility", "ByteToboolean", null, true, booleanType, new Type[] {byteobjType}, null, null); + addMethod("Utility", "ByteTochar", null, true, charType, new Type[] {byteobjType}, null, null); + addMethod("Utility", "shortToboolean", null, true, booleanType, new Type[] {shortType}, null, null); + addMethod("Utility", "shortToByte", null, true, byteobjType, new Type[] {shortType}, null, null); + addMethod("Utility", "shortToCharacter", null, true, charobjType, new Type[] {shortType}, null, null); + addMethod("Utility", "shortToInteger", null, true, intobjType, new Type[] {shortType}, null, null); + addMethod("Utility", "shortToLong", null, true, longobjType, new Type[] {shortType}, null, null); + addMethod("Utility", "shortToFloat", null, true, floatobjType, new Type[] {shortType}, null, null); + addMethod("Utility", "shortToDouble", null, true, doubleobjType, new Type[] {shortType}, null, null); + addMethod("Utility", "ShortToboolean", null, true, booleanType, new Type[] {shortobjType}, null, null); + addMethod("Utility", "ShortTochar", null, true, charType, new Type[] {shortobjType}, null, null); + addMethod("Utility", "charToboolean", null, true, booleanType, new Type[] {charType}, null, null); + addMethod("Utility", "charToByte", null, true, byteobjType, new Type[] {charType}, null, null); + addMethod("Utility", "charToShort", null, true, shortobjType, new Type[] {charType}, null, null); + addMethod("Utility", "charToInteger", null, true, intobjType, new Type[] {charType}, null, null); + addMethod("Utility", "charToLong", null, true, longobjType, new Type[] {charType}, null, null); + addMethod("Utility", "charToFloat", null, true, floatobjType, new Type[] {charType}, null, null); + addMethod("Utility", "charToDouble", null, true, doubleobjType, new Type[] {charType}, null, null); + addMethod("Utility", "CharacterToboolean", null, true, booleanType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterTobyte", null, true, byteType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToshort", null, true, shortType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToint", null, true, intType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterTolong", null, true, longType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterTofloat", null, true, floatType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterTodouble", null, true, doubleType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToBoolean", null, true, booleanobjType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToByte", null, true, byteobjType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToShort", null, true, shortobjType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToInteger", null, true, intobjType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToLong", null, true, longobjType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToFloat", null, true, floatobjType, new Type[] {charobjType}, null, null); + addMethod("Utility", "CharacterToDouble", null, true, doubleobjType, new Type[] {charobjType}, null, null); + addMethod("Utility", "intToboolean", null, true, booleanType, new Type[] {intType}, null, null); + addMethod("Utility", "intToByte", null, true, byteobjType, new Type[] {intType}, null, null); + addMethod("Utility", "intToShort", null, true, shortobjType, new Type[] {intType}, null, null); + addMethod("Utility", "intToCharacter", null, true, charobjType, new Type[] {intType}, null, null); + addMethod("Utility", "intToLong", null, true, longobjType, new Type[] {intType}, null, null); + addMethod("Utility", "intToFloat", null, true, floatobjType, new Type[] {intType}, null, null); + addMethod("Utility", "intToDouble", null, true, doubleobjType, new Type[] {intType}, null, null); + addMethod("Utility", "IntegerToboolean", null, true, booleanType, new Type[] {intobjType}, null, null); + addMethod("Utility", "IntegerTochar", null, true, charType, new Type[] {intobjType}, null, null); + addMethod("Utility", "longToboolean", null, true, booleanType, new Type[] {longType}, null, null); + addMethod("Utility", "longToByte", null, true, byteobjType, new Type[] {longType}, null, null); + addMethod("Utility", "longToShort", null, true, shortobjType, new Type[] {longType}, null, null); + addMethod("Utility", "longToCharacter", null, true, charobjType, new Type[] {longType}, null, null); + addMethod("Utility", "longToInteger", null, true, intobjType, new Type[] {longType}, null, null); + addMethod("Utility", "longToFloat", null, true, floatobjType, new Type[] {longType}, null, null); + addMethod("Utility", "longToDouble", null, true, doubleobjType, new Type[] {longType}, null, null); + addMethod("Utility", "LongToboolean", null, true, booleanType, new Type[] {longobjType}, null, null); + addMethod("Utility", "LongTochar", null, true, charType, new Type[] {longobjType}, null, null); + addMethod("Utility", "floatToboolean", null, true, booleanType, new Type[] {floatType}, null, null); + addMethod("Utility", "floatToByte", null, true, byteobjType, new Type[] {floatType}, null, null); + addMethod("Utility", "floatToShort", null, true, shortobjType, new Type[] {floatType}, null, null); + addMethod("Utility", "floatToCharacter", null, true, charobjType, new Type[] {floatType}, null, null); + addMethod("Utility", "floatToInteger", null, true, intobjType, new Type[] {floatType}, null, null); + addMethod("Utility", "floatToLong", null, true, longobjType, new Type[] {floatType}, null, null); + addMethod("Utility", "floatToDouble", null, true, doubleobjType, new Type[] {floatType}, null, null); + addMethod("Utility", "FloatToboolean", null, true, booleanType, new Type[] {floatobjType}, null, null); + addMethod("Utility", "FloatTochar", null, true, charType, new Type[] {floatobjType}, null, null); + addMethod("Utility", "doubleToboolean", null, true, booleanType, new Type[] {doubleType}, null, null); + addMethod("Utility", "doubleToByte", null, true, byteobjType, new Type[] {doubleType}, null, null); + addMethod("Utility", "doubleToShort", null, true, shortobjType, new Type[] {doubleType}, null, null); + addMethod("Utility", "doubleToCharacter", null, true, charobjType, new Type[] {doubleType}, null, null); + addMethod("Utility", "doubleToInteger", null, true, intobjType, new Type[] {doubleType}, null, null); + addMethod("Utility", "doubleToLong", null, true, longobjType, new Type[] {doubleType}, null, null); + addMethod("Utility", "doubleToFloat", null, true, floatobjType, new Type[] {doubleType}, null, null); + addMethod("Utility", "DoubleToboolean", null, true, booleanType, new Type[] {doubleobjType}, null, null); + addMethod("Utility", "DoubleTochar", null, true, charType, new Type[] {doubleobjType}, null, null); + + addMethod("Math", "dmax", "max", true, doubleType, new Type[] {doubleType, doubleType}, null, null); + + addMethod("Def", "DefToboolean", null, true, booleanType, new Type[] {defType}, null, null); + addMethod("Def", "DefTobyte", null, true, byteType, new Type[] {defType}, null, null); + addMethod("Def", "DefToshort", null, true, shortType, new Type[] {defType}, null, null); + addMethod("Def", "DefTochar", null, true, charType, new Type[] {defType}, null, null); + addMethod("Def", "DefToint", null, true, intType, new Type[] {defType}, null, null); + addMethod("Def", "DefTolong", null, true, longType, new Type[] {defType}, null, null); + addMethod("Def", "DefTofloat", null, true, floatType, new Type[] {defType}, null, null); + addMethod("Def", "DefTodouble", null, true, doubleType, new Type[] {defType}, null, null); + addMethod("Def", "DefToBoolean", null, true, booleanobjType, new Type[] {defType}, null, null); + addMethod("Def", "DefToByte", null, true, byteobjType, new Type[] {defType}, null, null); + addMethod("Def", "DefToShort", null, true, shortobjType, new Type[] {defType}, null, null); + addMethod("Def", "DefToCharacter", null, true, charobjType, new Type[] {defType}, null, null); + addMethod("Def", "DefToInteger", null, true, intobjType, new Type[] {defType}, null, null); + addMethod("Def", "DefToLong", null, true, longobjType, new Type[] {defType}, null, null); + addMethod("Def", "DefToFloat", null, true, floatobjType, new Type[] {defType}, null, null); + addMethod("Def", "DefToDouble", null, true, doubleobjType, new Type[] {defType}, null, null); + + addMethod("List", "addLast", "add", false, booleanType, new Type[] {objectType}, null, new Type[] {defType}); + addMethod("List", "add", null, false, voidType, new Type[] {intType, objectType}, null, new Type[] {intType, defType}); + addMethod("List", "get", null, false, objectType, new Type[] {intType}, defType, null); + addMethod("List", "remove", null, false, objectType, new Type[] {intType}, defType, null); + addMethod("List", "size", null, false, intType, new Type[] {}, null, null); + addMethod("List", "isEmpty", null, false, booleanType, new Type[] {}, null, null); + + addConstructor("ArrayList", "new", new Type[] {}, null); + + addMethod("Map", "put", null, false, objectType, new Type[] {objectType, objectType}, defType, new Type[] {defType, defType}); + addMethod("Map", "get", null, false, objectType, new Type[] {objectType}, defType, new Type[] {defType}); + addMethod("Map", "remove", null, false, objectType, new Type[] {objectType}, null, null); + addMethod("Map", "size", null, false, intType, new Type[] {}, null, null); + addMethod("Map", "isEmpty", null, false, booleanType, new Type[] {}, null, null); + + addConstructor("HashMap", "new", new Type[] {}, null); + + addMethod("Map", "put", null, false, objectType, new Type[] {objectType, objectType}, defType, new Type[] {stringType, defType}); + addMethod("Map", "get", null, false, objectType, new Type[] {objectType}, defType, new Type[] {stringType}); + addMethod("Map", "remove", null, false, objectType, new Type[] {objectType}, defType, new Type[] {stringType}); + addMethod("Map", "size", null, false, intType, new Type[] {}, null, null); + addMethod("Map", "isEmpty", null, false, booleanType, new Type[] {}, null, null); + + addConstructor("HashMap", "new", new Type[] {}, null); + + addMethod("List", "addLast", "add", false, booleanType, new Type[] {objectType}, null, null); + addMethod("List", "add", null, false, voidType, new Type[] {intType, objectType}, null, null); + addMethod("List", "get", null, false, objectType, new Type[] {intType}, null, null); + addMethod("List", "remove", null, false, objectType, new Type[] {intType}, null, null); + addMethod("List", "size", null, false, intType, new Type[] {}, null, null); + addMethod("List", "isEmpty", null, false, booleanType, new Type[] {}, null, null); + + addConstructor("ArrayList", "new", new Type[] {}, null); + + addMethod("Map", "put", null, false, objectType, new Type[] {objectType, objectType}, null, null); + addMethod("Map", "get", null, false, objectType, new Type[] {objectType}, null, null); + addMethod("Map", "remove", null, false, objectType, new Type[] {objectType}, null, null); + addMethod("Map", "size", null, false, intType, new Type[] {}, null, null); + addMethod("Map", "isEmpty", null, false, booleanType, new Type[] {}, null, null); + + addConstructor("HashMap", "new", new Type[] {}, null); + + addMethod("Map", "put", null, false, objectType, new Type[] {objectType, objectType}, null, new Type[] {stringType, objectType}); + addMethod("Map", "get", null, false, objectType, new Type[] {objectType}, null, new Type[] {stringType}); + addMethod("Map", "remove", null, false, objectType, new Type[] {objectType}, null, new Type[] {stringType}); + addMethod("Map", "size", null, false, intType, new Type[] {}, null, null); + addMethod("Map", "isEmpty", null, false, booleanType, new Type[] {}, null, null); + + addConstructor("HashMap", "new", new Type[] {}, null); + } + + private void copyDefaultStructs() { + copyStruct("Void", "Object"); + copyStruct("Boolean", "Object"); + copyStruct("Byte", "Number", "Object"); + copyStruct("Short", "Number", "Object"); + copyStruct("Character", "Object"); + copyStruct("Integer", "Number", "Object"); + copyStruct("Long", "Number", "Object"); + copyStruct("Float", "Number", "Object"); + copyStruct("Double", "Number", "Object"); + + copyStruct("Number", "Object"); + copyStruct("CharSequence", "Object"); + copyStruct("String", "CharSequence", "Object"); + + copyStruct("List", "Object"); + copyStruct("ArrayList", "List", "Object"); + copyStruct("Map", "Object"); + copyStruct("HashMap", "Map", "Object"); + copyStruct("Map", "Object"); + copyStruct("HashMap", "Map", "Object"); + + copyStruct("List", "Object"); + copyStruct("ArrayList", "List", "Object"); + copyStruct("Map", "Object"); + copyStruct("HashMap", "Map", "Object"); + copyStruct("Map", "Object"); + copyStruct("HashMap", "Map", "Object"); + + copyStruct("Executable", "Object"); + } + + private void addDefaultTransforms() { + addTransform(booleanType, byteType, "Utility", "booleanTobyte", true); + addTransform(booleanType, shortType, "Utility", "booleanToshort", true); + addTransform(booleanType, charType, "Utility", "booleanTochar", true); + addTransform(booleanType, intType, "Utility", "booleanToint", true); + addTransform(booleanType, longType, "Utility", "booleanTolong", true); + addTransform(booleanType, floatType, "Utility", "booleanTofloat", true); + addTransform(booleanType, doubleType, "Utility", "booleanTodouble", true); + addTransform(booleanType, objectType, "Boolean", "valueOf", true); + addTransform(booleanType, defType, "Boolean", "valueOf", true); + addTransform(booleanType, numberType, "Utility", "booleanToInteger", true); + addTransform(booleanType, booleanobjType, "Boolean", "valueOf", true); + + addTransform(byteType, booleanType, "Utility", "byteToboolean", true); + addTransform(byteType, objectType, "Byte", "valueOf", true); + addTransform(byteType, defType, "Byte", "valueOf", true); + addTransform(byteType, numberType, "Byte", "valueOf", true); + addTransform(byteType, byteobjType, "Byte", "valueOf", true); + addTransform(byteType, shortobjType, "Utility", "byteToShort", true); + addTransform(byteType, charobjType, "Utility", "byteToCharacter", true); + addTransform(byteType, intobjType, "Utility", "byteToInteger", true); + addTransform(byteType, longobjType, "Utility", "byteToLong", true); + addTransform(byteType, floatobjType, "Utility", "byteToFloat", true); + addTransform(byteType, doubleobjType, "Utility", "byteToDouble", true); + + addTransform(shortType, booleanType, "Utility", "shortToboolean", true); + addTransform(shortType, objectType, "Short", "valueOf", true); + addTransform(shortType, defType, "Short", "valueOf", true); + addTransform(shortType, numberType, "Short", "valueOf", true); + addTransform(shortType, byteobjType, "Utility", "shortToByte", true); + addTransform(shortType, shortobjType, "Short", "valueOf", true); + addTransform(shortType, charobjType, "Utility", "shortToCharacter", true); + addTransform(shortType, intobjType, "Utility", "shortToInteger", true); + addTransform(shortType, longobjType, "Utility", "shortToLong", true); + addTransform(shortType, floatobjType, "Utility", "shortToFloat", true); + addTransform(shortType, doubleobjType, "Utility", "shortToDouble", true); + + addTransform(charType, booleanType, "Utility", "charToboolean", true); + addTransform(charType, objectType, "Character", "valueOf", true); + addTransform(charType, defType, "Character", "valueOf", true); + addTransform(charType, numberType, "Utility", "charToInteger", true); + addTransform(charType, byteobjType, "Utility", "charToByte", true); + addTransform(charType, shortobjType, "Utility", "charToShort", true); + addTransform(charType, charobjType, "Character", "valueOf", true); + addTransform(charType, intobjType, "Utility", "charToInteger", true); + addTransform(charType, longobjType, "Utility", "charToLong", true); + addTransform(charType, floatobjType, "Utility", "charToFloat", true); + addTransform(charType, doubleobjType, "Utility", "charToDouble", true); + + addTransform(intType, booleanType, "Utility", "intToboolean", true); + addTransform(intType, objectType, "Integer", "valueOf", true); + addTransform(intType, defType, "Integer", "valueOf", true); + addTransform(intType, numberType, "Integer", "valueOf", true); + addTransform(intType, byteobjType, "Utility", "intToByte", true); + addTransform(intType, shortobjType, "Utility", "intToShort", true); + addTransform(intType, charobjType, "Utility", "intToCharacter", true); + addTransform(intType, intobjType, "Integer", "valueOf", true); + addTransform(intType, longobjType, "Utility", "intToLong", true); + addTransform(intType, floatobjType, "Utility", "intToFloat", true); + addTransform(intType, doubleobjType, "Utility", "intToDouble", true); + + addTransform(longType, booleanType, "Utility", "longToboolean", true); + addTransform(longType, objectType, "Long", "valueOf", true); + addTransform(longType, defType, "Long", "valueOf", true); + addTransform(longType, numberType, "Long", "valueOf", true); + addTransform(longType, byteobjType, "Utility", "longToByte", true); + addTransform(longType, shortobjType, "Utility", "longToShort", true); + addTransform(longType, charobjType, "Utility", "longToCharacter", true); + addTransform(longType, intobjType, "Utility", "longToInteger", true); + addTransform(longType, longobjType, "Long", "valueOf", true); + addTransform(longType, floatobjType, "Utility", "longToFloat", true); + addTransform(longType, doubleobjType, "Utility", "longToDouble", true); + + addTransform(floatType, booleanType, "Utility", "floatToboolean", true); + addTransform(floatType, objectType, "Float", "valueOf", true); + addTransform(floatType, defType, "Float", "valueOf", true); + addTransform(floatType, numberType, "Float", "valueOf", true); + addTransform(floatType, byteobjType, "Utility", "floatToByte", true); + addTransform(floatType, shortobjType, "Utility", "floatToShort", true); + addTransform(floatType, charobjType, "Utility", "floatToCharacter", true); + addTransform(floatType, intobjType, "Utility", "floatToInteger", true); + addTransform(floatType, longobjType, "Utility", "floatToLong", true); + addTransform(floatType, floatobjType, "Float", "valueOf", true); + addTransform(floatType, doubleobjType, "Utility", "floatToDouble", true); + + addTransform(doubleType, booleanType, "Utility", "doubleToboolean", true); + addTransform(doubleType, objectType, "Double", "valueOf", true); + addTransform(doubleType, defType, "Double", "valueOf", true); + addTransform(doubleType, numberType, "Double", "valueOf", true); + addTransform(doubleType, byteobjType, "Utility", "doubleToByte", true); + addTransform(doubleType, shortobjType, "Utility", "doubleToShort", true); + addTransform(doubleType, charobjType, "Utility", "doubleToCharacter", true); + addTransform(doubleType, intobjType, "Utility", "doubleToInteger", true); + addTransform(doubleType, longobjType, "Utility", "doubleToLong", true); + addTransform(doubleType, floatobjType, "Utility", "doubleToFloat", true); + addTransform(doubleType, doubleobjType, "Double", "valueOf", true); + + addTransform(objectType, booleanType, "Boolean", "booleanValue", false); + addTransform(objectType, byteType, "Number", "byteValue", false); + addTransform(objectType, shortType, "Number", "shortValue", false); + addTransform(objectType, charType, "Character", "charValue", false); + addTransform(objectType, intType, "Number", "intValue", false); + addTransform(objectType, longType, "Number", "longValue", false); + addTransform(objectType, floatType, "Number", "floatValue", false); + addTransform(objectType, doubleType, "Number", "doubleValue", false); + + addTransform(defType, booleanType, "Def", "DefToboolean", true); + addTransform(defType, byteType, "Def", "DefTobyte", true); + addTransform(defType, shortType, "Def", "DefToshort", true); + addTransform(defType, charType, "Def", "DefTochar", true); + addTransform(defType, intType, "Def", "DefToint", true); + addTransform(defType, longType, "Def", "DefTolong", true); + addTransform(defType, floatType, "Def", "DefTofloat", true); + addTransform(defType, doubleType, "Def", "DefTodouble", true); + addTransform(defType, booleanobjType, "Def", "DefToBoolean", true); + addTransform(defType, byteobjType, "Def", "DefToByte", true); + addTransform(defType, shortobjType, "Def", "DefToShort", true); + addTransform(defType, charobjType, "Def", "DefToCharacter", true); + addTransform(defType, intobjType, "Def", "DefToInteger", true); + addTransform(defType, longobjType, "Def", "DefToLong", true); + addTransform(defType, floatobjType, "Def", "DefToFloat", true); + addTransform(defType, doubleobjType, "Def", "DefToDouble", true); + + addTransform(numberType, booleanType, "Utility", "NumberToboolean", true); + addTransform(numberType, byteType, "Number", "byteValue", false); + addTransform(numberType, shortType, "Number", "shortValue", false); + addTransform(numberType, charType, "Utility", "NumberTochar", true); + addTransform(numberType, intType, "Number", "intValue", false); + addTransform(numberType, longType, "Number", "longValue", false); + addTransform(numberType, floatType, "Number", "floatValue", false); + addTransform(numberType, doubleType, "Number", "doubleValue", false); + addTransform(numberType, booleanobjType, "Utility", "NumberToBoolean", true); + addTransform(numberType, byteobjType, "Utility", "NumberToByte", true); + addTransform(numberType, shortobjType, "Utility", "NumberToShort", true); + addTransform(numberType, charobjType, "Utility", "NumberToCharacter", true); + addTransform(numberType, intobjType, "Utility", "NumberToInteger", true); + addTransform(numberType, longobjType, "Utility", "NumberToLong", true); + addTransform(numberType, floatobjType, "Utility", "NumberToFloat", true); + addTransform(numberType, doubleobjType, "Utility", "NumberToDouble", true); + + addTransform(booleanobjType, booleanType, "Boolean", "booleanValue", false); + addTransform(booleanobjType, byteType, "Utility", "BooleanTobyte", true); + addTransform(booleanobjType, shortType, "Utility", "BooleanToshort", true); + addTransform(booleanobjType, charType, "Utility", "BooleanTochar", true); + addTransform(booleanobjType, intType, "Utility", "BooleanToint", true); + addTransform(booleanobjType, longType, "Utility", "BooleanTolong", true); + addTransform(booleanobjType, floatType, "Utility", "BooleanTofloat", true); + addTransform(booleanobjType, doubleType, "Utility", "BooleanTodouble", true); + addTransform(booleanobjType, numberType, "Utility", "BooleanToLong", true); + addTransform(booleanobjType, byteobjType, "Utility", "BooleanToByte", true); + addTransform(booleanobjType, shortobjType, "Utility", "BooleanToShort", true); + addTransform(booleanobjType, charobjType, "Utility", "BooleanToCharacter", true); + addTransform(booleanobjType, intobjType, "Utility", "BooleanToInteger", true); + addTransform(booleanobjType, longobjType, "Utility", "BooleanToLong", true); + addTransform(booleanobjType, floatobjType, "Utility", "BooleanToFloat", true); + addTransform(booleanobjType, doubleobjType, "Utility", "BooleanToDouble", true); + + addTransform(byteobjType, booleanType, "Utility", "ByteToboolean", true); + addTransform(byteobjType, byteType, "Byte", "byteValue", false); + addTransform(byteobjType, shortType, "Byte", "shortValue", false); + addTransform(byteobjType, charType, "Utility", "ByteTochar", true); + addTransform(byteobjType, intType, "Byte", "intValue", false); + addTransform(byteobjType, longType, "Byte", "longValue", false); + addTransform(byteobjType, floatType, "Byte", "floatValue", false); + addTransform(byteobjType, doubleType, "Byte", "doubleValue", false); + addTransform(byteobjType, booleanobjType, "Utility", "NumberToBoolean", true); + addTransform(byteobjType, shortobjType, "Utility", "NumberToShort", true); + addTransform(byteobjType, charobjType, "Utility", "NumberToCharacter", true); + addTransform(byteobjType, intobjType, "Utility", "NumberToInteger", true); + addTransform(byteobjType, longobjType, "Utility", "NumberToLong", true); + addTransform(byteobjType, floatobjType, "Utility", "NumberToFloat", true); + addTransform(byteobjType, doubleobjType, "Utility", "NumberToDouble", true); + + addTransform(shortobjType, booleanType, "Utility", "ShortToboolean", true); + addTransform(shortobjType, byteType, "Short", "byteValue", false); + addTransform(shortobjType, shortType, "Short", "shortValue", false); + addTransform(shortobjType, charType, "Utility", "ShortTochar", true); + addTransform(shortobjType, intType, "Short", "intValue", false); + addTransform(shortobjType, longType, "Short", "longValue", false); + addTransform(shortobjType, floatType, "Short", "floatValue", false); + addTransform(shortobjType, doubleType, "Short", "doubleValue", false); + addTransform(shortobjType, booleanobjType, "Utility", "NumberToBoolean", true); + addTransform(shortobjType, byteobjType, "Utility", "NumberToByte", true); + addTransform(shortobjType, charobjType, "Utility", "NumberToCharacter", true); + addTransform(shortobjType, intobjType, "Utility", "NumberToInteger", true); + addTransform(shortobjType, longobjType, "Utility", "NumberToLong", true); + addTransform(shortobjType, floatobjType, "Utility", "NumberToFloat", true); + addTransform(shortobjType, doubleobjType, "Utility", "NumberToDouble", true); + + addTransform(charobjType, booleanType, "Utility", "CharacterToboolean", true); + addTransform(charobjType, byteType, "Utility", "CharacterTobyte", true); + addTransform(charobjType, shortType, "Utility", "CharacterToshort", true); + addTransform(charobjType, charType, "Character", "charValue", false); + addTransform(charobjType, intType, "Utility", "CharacterToint", true); + addTransform(charobjType, longType, "Utility", "CharacterTolong", true); + addTransform(charobjType, floatType, "Utility", "CharacterTofloat", true); + addTransform(charobjType, doubleType, "Utility", "CharacterTodouble", true); + addTransform(charobjType, booleanobjType, "Utility", "CharacterToBoolean", true); + addTransform(charobjType, byteobjType, "Utility", "CharacterToByte", true); + addTransform(charobjType, shortobjType, "Utility", "CharacterToShort", true); + addTransform(charobjType, intobjType, "Utility", "CharacterToInteger", true); + addTransform(charobjType, longobjType, "Utility", "CharacterToLong", true); + addTransform(charobjType, floatobjType, "Utility", "CharacterToFloat", true); + addTransform(charobjType, doubleobjType, "Utility", "CharacterToDouble", true); + + addTransform(intobjType, booleanType, "Utility", "IntegerToboolean", true); + addTransform(intobjType, byteType, "Integer", "byteValue", false); + addTransform(intobjType, shortType, "Integer", "shortValue", false); + addTransform(intobjType, charType, "Utility", "IntegerTochar", true); + addTransform(intobjType, intType, "Integer", "intValue", false); + addTransform(intobjType, longType, "Integer", "longValue", false); + addTransform(intobjType, floatType, "Integer", "floatValue", false); + addTransform(intobjType, doubleType, "Integer", "doubleValue", false); + addTransform(intobjType, booleanobjType, "Utility", "NumberToBoolean", true); + addTransform(intobjType, byteobjType, "Utility", "NumberToByte", true); + addTransform(intobjType, shortobjType, "Utility", "NumberToShort", true); + addTransform(intobjType, charobjType, "Utility", "NumberToCharacter", true); + addTransform(intobjType, longobjType, "Utility", "NumberToLong", true); + addTransform(intobjType, floatobjType, "Utility", "NumberToFloat", true); + addTransform(intobjType, doubleobjType, "Utility", "NumberToDouble", true); + + addTransform(longobjType, booleanType, "Utility", "LongToboolean", true); + addTransform(longobjType, byteType, "Long", "byteValue", false); + addTransform(longobjType, shortType, "Long", "shortValue", false); + addTransform(longobjType, charType, "Utility", "LongTochar", true); + addTransform(longobjType, intType, "Long", "intValue", false); + addTransform(longobjType, longType, "Long", "longValue", false); + addTransform(longobjType, floatType, "Long", "floatValue", false); + addTransform(longobjType, doubleType, "Long", "doubleValue", false); + addTransform(longobjType, booleanobjType, "Utility", "NumberToBoolean", true); + addTransform(longobjType, byteobjType, "Utility", "NumberToByte", true); + addTransform(longobjType, shortobjType, "Utility", "NumberToShort", true); + addTransform(longobjType, charobjType, "Utility", "NumberToCharacter", true); + addTransform(longobjType, intobjType, "Utility", "NumberToInteger", true); + addTransform(longobjType, floatobjType, "Utility", "NumberToFloat", true); + addTransform(longobjType, doubleobjType, "Utility", "NumberToDouble", true); + + addTransform(floatobjType, booleanType, "Utility", "FloatToboolean", true); + addTransform(floatobjType, byteType, "Float", "byteValue", false); + addTransform(floatobjType, shortType, "Float", "shortValue", false); + addTransform(floatobjType, charType, "Utility", "FloatTochar", true); + addTransform(floatobjType, intType, "Float", "intValue", false); + addTransform(floatobjType, longType, "Float", "longValue", false); + addTransform(floatobjType, floatType, "Float", "floatValue", false); + addTransform(floatobjType, doubleType, "Float", "doubleValue", false); + addTransform(floatobjType, booleanobjType, "Utility", "NumberToBoolean", true); + addTransform(floatobjType, byteobjType, "Utility", "NumberToByte", true); + addTransform(floatobjType, shortobjType, "Utility", "NumberToShort", true); + addTransform(floatobjType, charobjType, "Utility", "NumberToCharacter", true); + addTransform(floatobjType, intobjType, "Utility", "NumberToInteger", true); + addTransform(floatobjType, longobjType, "Utility", "NumberToLong", true); + addTransform(floatobjType, doubleobjType, "Utility", "NumberToDouble", true); + + addTransform(doubleobjType, booleanType, "Utility", "DoubleToboolean", true); + addTransform(doubleobjType, byteType, "Double", "byteValue", false); + addTransform(doubleobjType, shortType, "Double", "shortValue", false); + addTransform(doubleobjType, charType, "Utility", "DoubleTochar", true); + addTransform(doubleobjType, intType, "Double", "intValue", false); + addTransform(doubleobjType, longType, "Double", "longValue", false); + addTransform(doubleobjType, floatType, "Double", "floatValue", false); + addTransform(doubleobjType, doubleType, "Double", "doubleValue", false); + addTransform(doubleobjType, booleanobjType, "Utility", "NumberToBoolean", true); + addTransform(doubleobjType, byteobjType, "Utility", "NumberToByte", true); + addTransform(doubleobjType, shortobjType, "Utility", "NumberToShort", true); + addTransform(doubleobjType, charobjType, "Utility", "NumberToCharacter", true); + addTransform(doubleobjType, intobjType, "Utility", "NumberToInteger", true); + addTransform(doubleobjType, longobjType, "Utility", "NumberToLong", true); + addTransform(doubleobjType, floatobjType, "Utility", "NumberToFloat", true); + } + + private void addDefaultBounds() { + addBound(byteobjType, numberType, numberType); + + addBound(shortobjType, numberType, numberType); + addBound(shortobjType, byteobjType, numberType); + + addBound(intobjType, numberType, numberType); + addBound(intobjType, byteobjType, numberType); + addBound(intobjType, shortobjType, numberType); + + addBound(longobjType, numberType, numberType); + addBound(longobjType, byteobjType, numberType); + addBound(longobjType, shortobjType, numberType); + addBound(longobjType, intobjType, numberType); + + addBound(floatobjType, numberType, numberType); + addBound(floatobjType, byteobjType, numberType); + addBound(floatobjType, shortobjType, numberType); + addBound(floatobjType, intobjType, numberType); + addBound(floatobjType, longobjType, numberType); + + addBound(doubleobjType, numberType, numberType); + addBound(doubleobjType, byteobjType, numberType); + addBound(doubleobjType, shortobjType, numberType); + addBound(doubleobjType, intobjType, numberType); + addBound(doubleobjType, longobjType, numberType); + addBound(doubleobjType, floatobjType, numberType); + + addBound(stringType, charseqType, charseqType); + + addBound(arraylistType, listType, listType); + addBound(olistType, listType, listType); + addBound(olistType, arraylistType, listType); + addBound(oarraylistType, listType, listType); + addBound(oarraylistType, olistType, olistType); + addBound(oarraylistType, arraylistType, arraylistType); + + addBound(hashmapType, mapType, mapType); + addBound(omapType, mapType, mapType); + addBound(omapType, hashmapType, mapType); + addBound(ohashmapType, mapType, mapType); + addBound(ohashmapType, hashmapType, hashmapType); + addBound(ohashmapType, omapType, omapType); + addBound(smapType, mapType, mapType); + addBound(smapType, hashmapType, mapType); + addBound(smapType, omapType, omapType); + addBound(smapType, ohashmapType, omapType); + addBound(shashmapType, mapType, mapType); + addBound(shashmapType, hashmapType, hashmapType); + addBound(shashmapType, omapType, omapType); + addBound(shashmapType, ohashmapType, ohashmapType); + addBound(shashmapType, smapType, smapType); + addBound(somapType, mapType, mapType); + addBound(somapType, hashmapType, mapType); + addBound(somapType, omapType, omapType); + addBound(somapType, ohashmapType, omapType); + addBound(somapType, smapType, smapType); + addBound(somapType, shashmapType, smapType); + addBound(sohashmapType, mapType, mapType); + addBound(sohashmapType, hashmapType, hashmapType); + addBound(sohashmapType, omapType, omapType); + addBound(sohashmapType, ohashmapType, ohashmapType); + addBound(sohashmapType, smapType, smapType); + addBound(sohashmapType, shashmapType, shashmapType); + addBound(sohashmapType, somapType, somapType); + } + + public final void addStruct(final String name, final Class clazz) { + if (!name.matches("^[_a-zA-Z][<>,_a-zA-Z0-9]*$")) { + throw new IllegalArgumentException("Invalid struct name [" + name + "]."); + } + + if (structs.containsKey(name)) { + throw new IllegalArgumentException("Duplicate struct name [" + name + "]."); + } + + final Struct struct = new Struct(name, clazz, org.objectweb.asm.Type.getType(clazz)); + + structs.put(name, struct); + } + + public final void addClass(final String name) { + final Struct struct = structs.get(name); + + if (struct == null) { + throw new IllegalArgumentException("Struct [" + name + "] is not defined."); + } + + if (classes.containsKey(struct.clazz)) { + throw new IllegalArgumentException("Duplicate struct class [" + struct.clazz + "] when defining dynamic."); + } + + classes.put(struct.clazz, struct); + } + + public final void addConstructor(final String struct, final String name, final Type[] args, final Type[] genargs) { + final Struct owner = structs.get(struct); + + if (owner == null) { + throw new IllegalArgumentException( + "Owner struct [" + struct + "] not defined for constructor [" + name + "]."); + } + + if (!name.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) { + throw new IllegalArgumentException( + "Invalid constructor name [" + name + "] with the struct [" + owner.name + "]."); + } + + if (owner.constructors.containsKey(name)) { + throw new IllegalArgumentException( + "Duplicate constructor name [" + name + "] found within the struct [" + owner.name + "]."); + } + + if (owner.statics.containsKey(name)) { + throw new IllegalArgumentException("Constructors and functions may not have the same name" + + " [" + name + "] within the same struct [" + owner.name + "]."); + } + + if (owner.methods.containsKey(name)) { + throw new IllegalArgumentException("Constructors and methods may not have the same name" + + " [" + name + "] within the same struct [" + owner.name + "]."); + } + + final Class[] classes = new Class[args.length]; + + for (int count = 0; count < classes.length; ++count) { + if (genargs != null) { + try { + genargs[count].clazz.asSubclass(args[count].clazz); + } catch (ClassCastException exception) { + throw new ClassCastException("Generic argument [" + genargs[count].name + "]" + + " is not a sub class of [" + args[count].name + "] in the constructor" + + " [" + name + " ] from the struct [" + owner.name + "]."); + } + } + + classes[count] = args[count].clazz; + } + + final java.lang.reflect.Constructor reflect; + + try { + reflect = owner.clazz.getConstructor(classes); + } catch (NoSuchMethodException exception) { + throw new IllegalArgumentException("Constructor [" + name + "] not found for class" + + " [" + owner.clazz.getName() + "] with arguments " + Arrays.toString(classes) + "."); + } + + final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect); + final Constructor constructor = + new Constructor(name, owner, Arrays.asList(genargs != null ? genargs : args), asm, reflect); + + owner.constructors.put(name, constructor); + } + + public final void addMethod(final String struct, final String name, final String alias, final boolean statik, + final Type rtn, final Type[] args, final Type genrtn, final Type[] genargs) { + final Struct owner = structs.get(struct); + + if (owner == null) { + throw new IllegalArgumentException("Owner struct [" + struct + "] not defined" + + " for " + (statik ? "function" : "method") + " [" + name + "]."); + } + + if (!name.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) { + throw new IllegalArgumentException("Invalid " + (statik ? "function" : "method") + + " name [" + name + "] with the struct [" + owner.name + "]."); + } + + if (owner.constructors.containsKey(name)) { + throw new IllegalArgumentException("Constructors and " + (statik ? "functions" : "methods") + + " may not have the same name [" + name + "] within the same struct" + + " [" + owner.name + "]."); + } + + if (owner.statics.containsKey(name)) { + if (statik) { + throw new IllegalArgumentException( + "Duplicate function name [" + name + "] found within the struct [" + owner.name + "]."); + } else { + throw new IllegalArgumentException("Functions and methods may not have the same name" + + " [" + name + "] within the same struct [" + owner.name + "]."); + } + } + + if (owner.methods.containsKey(name)) { + if (statik) { + throw new IllegalArgumentException("Functions and methods may not have the same name" + + " [" + name + "] within the same struct [" + owner.name + "]."); + } else { + throw new IllegalArgumentException("Duplicate method name [" + name + "]" + + " found within the struct [" + owner.name + "]."); + } + } + + if (genrtn != null) { + try { + genrtn.clazz.asSubclass(rtn.clazz); + } catch (ClassCastException exception) { + throw new ClassCastException("Generic return [" + genrtn.clazz.getCanonicalName() + "]" + + " is not a sub class of [" + rtn.clazz.getCanonicalName() + "] in the method" + + " [" + name + " ] from the struct [" + owner.name + "]."); + } + } + + if (genargs != null && genargs.length != args.length) { + throw new IllegalArgumentException("Generic arguments arity [" + genargs.length + "] is not the same as " + + (statik ? "function" : "method") + " [" + name + "] arguments arity" + + " [" + args.length + "] within the struct [" + owner.name + "]."); + } + + final Class[] classes = new Class[args.length]; + + for (int count = 0; count < classes.length; ++count) { + if (genargs != null) { + try { + genargs[count].clazz.asSubclass(args[count].clazz); + } catch (ClassCastException exception) { + throw new ClassCastException("Generic argument [" + genargs[count].name + "] is not a sub class" + + " of [" + args[count].name + "] in the " + (statik ? "function" : "method") + + " [" + name + " ] from the struct [" + owner.name + "]."); + } + } + + classes[count] = args[count].clazz; + } + + final java.lang.reflect.Method reflect; + + try { + reflect = owner.clazz.getMethod(alias == null ? name : alias, classes); + } catch (NoSuchMethodException exception) { + throw new IllegalArgumentException((statik ? "Function" : "Method") + + " [" + (alias == null ? name : alias) + "] not found for class [" + owner.clazz.getName() + "]" + + " with arguments " + Arrays.toString(classes) + "."); + } + + if (!reflect.getReturnType().equals(rtn.clazz)) { + throw new IllegalArgumentException("Specified return type class [" + rtn.clazz + "]" + + " does not match the found return type class [" + reflect.getReturnType() + "] for the " + + (statik ? "function" : "method") + " [" + name + "]" + + " within the struct [" + owner.name + "]."); + } + + final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect); + + MethodHandle handle; + + try { + if (statik) { + handle = MethodHandles.publicLookup().in(owner.clazz).findStatic( + owner.clazz, alias == null ? name : alias, MethodType.methodType(rtn.clazz, classes)); + } else { + handle = MethodHandles.publicLookup().in(owner.clazz).findVirtual( + owner.clazz, alias == null ? name : alias, MethodType.methodType(rtn.clazz, classes)); + } + } catch (NoSuchMethodException | IllegalAccessException exception) { + throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) + "]" + + " not found for class [" + owner.clazz.getName() + "]" + + " with arguments " + Arrays.toString(classes) + "."); + } + + final Method method = new Method(name, owner, genrtn != null ? genrtn : rtn, + Arrays.asList(genargs != null ? genargs : args), asm, reflect, handle); + final int modifiers = reflect.getModifiers(); + + if (statik) { + if (!java.lang.reflect.Modifier.isStatic(modifiers)) { + throw new IllegalArgumentException("Function [" + name + "]" + + " within the struct [" + owner.name + "] is not linked to a static Java method."); + } + + owner.functions.put(name, method); + } else { + if (java.lang.reflect.Modifier.isStatic(modifiers)) { + throw new IllegalArgumentException("Method [" + name + "]" + + " within the struct [" + owner.name + "] is not linked to a non-static Java method."); + } + + owner.methods.put(name, method); + } + } + + public final void addField(final String struct, final String name, final String alias, + final boolean statik, final Type type, final Type generic) { + final Struct owner = structs.get(struct); + + if (owner == null) { + throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for " + + (statik ? "static" : "member") + " [" + name + "]."); + } + + if (!name.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) { + throw new IllegalArgumentException("Invalid " + (statik ? "static" : "member") + + " name [" + name + "] with the struct [" + owner.name + "]."); + } + + if (owner.statics.containsKey(name)) { + if (statik) { + throw new IllegalArgumentException("Duplicate static name [" + name + "]" + + " found within the struct [" + owner.name + "]."); + } else { + throw new IllegalArgumentException("Statics and members may not have the same name " + + "[" + name + "] within the same struct [" + owner.name + "]."); + } + } + + if (owner.members.containsKey(name)) { + if (statik) { + throw new IllegalArgumentException("Statics and members may not have the same name " + + "[" + name + "] within the same struct [" + owner.name + "]."); + } else { + throw new IllegalArgumentException("Duplicate member name [" + name + "]" + + " found within the struct [" + owner.name + "]."); + } + } + + if (generic != null) { + try { + generic.clazz.asSubclass(type.clazz); + } catch (ClassCastException exception) { + throw new ClassCastException("Generic type [" + generic.clazz.getCanonicalName() + "]" + + " is not a sub class of [" + type.clazz.getCanonicalName() + "] for the field" + + " [" + name + " ] from the struct [" + owner.name + "]."); + } + } + + java.lang.reflect.Field reflect; + + try { + reflect = owner.clazz.getField(alias == null ? name : alias); + } catch (NoSuchFieldException exception) { + throw new IllegalArgumentException("Field [" + (alias == null ? name : alias) + "]" + + " not found for class [" + owner.clazz.getName() + "]."); + } + + MethodHandle getter = null; + MethodHandle setter = null; + + try { + if (!statik) { + getter = MethodHandles.publicLookup().in(owner.clazz).findGetter( + owner.clazz, alias == null ? name : alias, type.clazz); + setter = MethodHandles.publicLookup().in(owner.clazz).findSetter( + owner.clazz, alias == null ? name : alias, type.clazz); + } + } catch (NoSuchFieldException | IllegalAccessException exception) { + throw new IllegalArgumentException("Getter/Setter [" + (alias == null ? name : alias) + "]" + + " not found for class [" + owner.clazz.getName() + "]."); + } + + final Field field = new Field(name, owner, generic == null ? type : generic, type, reflect, getter, setter); + final int modifiers = reflect.getModifiers(); + + if (statik) { + if (!java.lang.reflect.Modifier.isStatic(modifiers)) { + throw new IllegalArgumentException(); + } + + if (!java.lang.reflect.Modifier.isFinal(modifiers)) { + throw new IllegalArgumentException("Static [" + name + "]" + + " within the struct [" + owner.name + "] is not linked to static Java field."); + } + + owner.statics.put(alias == null ? name : alias, field); + } else { + if (java.lang.reflect.Modifier.isStatic(modifiers)) { + throw new IllegalArgumentException("Member [" + name + "]" + + " within the struct [" + owner.name + "] is not linked to non-static Java field."); + } + + owner.members.put(alias == null ? name : alias, field); + } + } + + public final void copyStruct(final String struct, final String... children) { + final Struct owner = structs.get(struct); + + if (owner == null) { + throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy."); + } + + for (int count = 0; count < children.length; ++count) { + final Struct child = structs.get(children[count]); + + if (struct == null) { + throw new IllegalArgumentException("Child struct [" + children[count] + "]" + + " not defined for copy to owner struct [" + owner.name + "]."); + } + + try { + owner.clazz.asSubclass(child.clazz); + } catch (ClassCastException exception) { + throw new ClassCastException("Child struct [" + child.name + "]" + + " is not a super type of owner struct [" + owner.name + "] in copy."); + } + + final boolean object = child.clazz.equals(Object.class) && + java.lang.reflect.Modifier.isInterface(owner.clazz.getModifiers()); + + for (final Method method : child.methods.values()) { + if (owner.methods.get(method.name) == null) { + final Class clazz = object ? Object.class : owner.clazz; + + java.lang.reflect.Method reflect; + MethodHandle handle; + + try { + reflect = clazz.getMethod(method.method.getName(), method.reflect.getParameterTypes()); + } catch (NoSuchMethodException exception) { + throw new IllegalArgumentException("Method [" + method.method.getName() + "] not found for" + + " class [" + owner.clazz.getName() + "] with arguments " + + Arrays.toString(method.reflect.getParameterTypes()) + "."); + } + + try { + handle = MethodHandles.publicLookup().in(owner.clazz).findVirtual( + owner.clazz, method.method.getName(), + MethodType.methodType(method.reflect.getReturnType(), method.reflect.getParameterTypes())); + } catch (NoSuchMethodException | IllegalAccessException exception) { + throw new IllegalArgumentException("Method [" + method.method.getName() + "] not found for" + + " class [" + owner.clazz.getName() + "] with arguments " + + Arrays.toString(method.reflect.getParameterTypes()) + "."); + } + + owner.methods.put(method.name, + new Method(method.name, owner, method.rtn, method.arguments, method.method, reflect, handle)); + } + } + + for (final Field field : child.members.values()) { + if (owner.members.get(field.name) == null) { + java.lang.reflect.Field reflect; + MethodHandle getter; + MethodHandle setter; + + try { + reflect = owner.clazz.getField(field.reflect.getName()); + } catch (NoSuchFieldException exception) { + throw new IllegalArgumentException("Field [" + field.reflect.getName() + "]" + + " not found for class [" + owner.clazz.getName() + "]."); + } + + try { + getter = MethodHandles.publicLookup().in(owner.clazz).findGetter( + owner.clazz, field.name, field.type.clazz); + setter = MethodHandles.publicLookup().in(owner.clazz).findSetter( + owner.clazz, field.name, field.type.clazz); + } catch (NoSuchFieldException | IllegalAccessException exception) { + throw new IllegalArgumentException("Getter/Setter [" + field.name + "]" + + " not found for class [" + owner.clazz.getName() + "]."); + } + + owner.members.put(field.name, + new Field(field.name, owner, field.type, field.generic, reflect, getter, setter)); + } + } + } + } + + public final void addTransform(final Type from, final Type to, final String struct, + final String name, final boolean statik) { + final Struct owner = structs.get(struct); + + if (owner == null) { + throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for" + + " transform with cast type from [" + from.name + "] and cast type to [" + to.name + "]."); + } + + if (from.equals(to)) { + throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "] cannot" + + " have cast type from [" + from.name + "] be the same as cast type to [" + to.name + "]."); + } + + final Cast cast = new Cast(from, to); + + if (transforms.containsKey(cast)) { + throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + "] already defined."); + } + + Method method; + Type upcast = null; + Type downcast = null; + + if (statik) { + method = owner.functions.get(name); + + if (method == null) { + throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + + "] using a function [" + name + "] that is not defined."); + } + + if (method.arguments.size() != 1) { + throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + + "] using function [" + name + "] does not have a single type argument."); + } + + Type argument = method.arguments.get(0); + + try { + from.clazz.asSubclass(argument.clazz); + } catch (ClassCastException cce0) { + try { + argument.clazz.asSubclass(from.clazz); + upcast = argument; + } catch (ClassCastException cce1) { + throw new ClassCastException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" + + " function [" + name + "] cannot cast from type to the function input argument type."); + } + } + + final Type rtn = method.rtn; + + try { + rtn.clazz.asSubclass(to.clazz); + } catch (ClassCastException cce0) { + try { + to.clazz.asSubclass(rtn.clazz); + downcast = to; + } catch (ClassCastException cce1) { + throw new ClassCastException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" + + " function [" + name + "] cannot cast to type to the function return argument type."); + } + } + } else { + method = owner.methods.get(name); + + if (method == null) { + throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + + "] using a method [" + name + "] that is not defined."); + } + + if (!method.arguments.isEmpty()) { + throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + + "] using method [" + name + "] does not have a single type argument."); + } + + try { + from.clazz.asSubclass(owner.clazz); + } catch (ClassCastException cce0) { + try { + owner.clazz.asSubclass(from.clazz); + upcast = getType(owner.name); + } catch (ClassCastException cce1) { + throw new ClassCastException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" + + " method [" + name + "] cannot cast from type to the method input argument type."); + } + } + + final Type rtn = method.rtn; + + try { + rtn.clazz.asSubclass(to.clazz); + } catch (ClassCastException cce0) { + try { + to.clazz.asSubclass(rtn.clazz); + downcast = to; + } catch (ClassCastException cce1) { + throw new ClassCastException("Transform with owner struct [" + owner.name + "]" + + " and cast type from [" + from.name + "] to cast type to [" + to.name + "]" + + " using method [" + name + "] cannot cast to type to the method return argument type."); + } + } + } + + final Transform transform = new Transform(cast, method, upcast, downcast); + transforms.put(cast, transform); + } + + public final void addBound(final Type type0, final Type type1, final Type bound) { + final Pair pair0 = new Pair(type0, type1); + final Pair pair1 = new Pair(type1, type0); + + if (bounds.containsKey(pair0)) { + throw new IllegalArgumentException( + "Bound already defined for types [" + type0.name + "] and [" + type1.name + "]."); + } + + if (bounds.containsKey(pair1)) { + throw new IllegalArgumentException( + "Bound already defined for types [" + type1.name + "] and [" + type0.name + "]."); + } + + bounds.put(pair0, bound); + bounds.put(pair1, bound); + } + + Type getType(final String name) { + final int dimensions = getDimensions(name); + final String structstr = dimensions == 0 ? name : name.substring(0, name.indexOf('[')); + final Struct struct = structs.get(structstr); + + if (struct == null) { + throw new IllegalArgumentException("The struct with name [" + name + "] has not been defined."); + } + + return getType(struct, dimensions); + } + + Type getType(final Struct struct, final int dimensions) { + String name = struct.name; + org.objectweb.asm.Type type = struct.type; + Class clazz = struct.clazz; + Sort sort; + + if (dimensions > 0) { + final StringBuilder builder = new StringBuilder(name); + final char[] brackets = new char[dimensions]; + + for (int count = 0; count < dimensions; ++count) { + builder.append("[]"); + brackets[count] = '['; + } + + final String descriptor = new String(brackets) + struct.type.getDescriptor(); + + name = builder.toString(); + type = org.objectweb.asm.Type.getType(descriptor); + + try { + clazz = Class.forName(type.getInternalName().replace('/', '.')); + } catch (ClassNotFoundException exception) { + throw new IllegalArgumentException("The class [" + type.getInternalName() + "]" + + " could not be found to create type [" + name + "]."); + } + + sort = Sort.ARRAY; + } else if ("def".equals(struct.name)) { + sort = Sort.DEF; + } else { + sort = Sort.OBJECT; + + for (Sort value : Sort.values()) { + if (value.clazz == null) { + continue; + } + + if (value.clazz.equals(struct.clazz)) { + sort = value; + + break; + } + } + } + + return new Type(name, struct, clazz, type, sort); + } + + private int getDimensions(final String name) { + int dimensions = 0; + int index = name.indexOf('['); + + if (index != -1) { + final int length = name.length(); + + while (index < length) { + if (name.charAt(index) == '[' && ++index < length && name.charAt(index++) == ']') { + ++dimensions; + } else { + throw new IllegalArgumentException("Invalid array braces in canonical name [" + name + "]."); + } + } + } + + return dimensions; + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ErrorHandlingLexer.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ErrorHandlingLexer.java new file mode 100644 index 00000000000..95e3c93a354 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ErrorHandlingLexer.java @@ -0,0 +1,45 @@ +package org.elasticsearch.plan.a; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.text.ParseException; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.LexerNoViableAltException; +import org.antlr.v4.runtime.misc.Interval; + +class ErrorHandlingLexer extends PlanALexer { + public ErrorHandlingLexer(CharStream charStream) { + super(charStream); + } + + @Override + public void recover(LexerNoViableAltException lnvae) { + CharStream charStream = lnvae.getInputStream(); + int startIndex = lnvae.getStartIndex(); + String text = charStream.getText(Interval.of(startIndex, charStream.index())); + + ParseException parseException = new ParseException("Error [" + _tokenStartLine + ":" + + _tokenStartCharPositionInLine + "]: unexpected character [" + + getErrorDisplay(text) + "].", _tokenStartCharIndex); + parseException.initCause(lnvae); + throw new RuntimeException(parseException); + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Executable.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Executable.java new file mode 100644 index 00000000000..09e28cf8216 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Executable.java @@ -0,0 +1,50 @@ +package org.elasticsearch.plan.a; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.Map; + +public abstract class Executable { + protected final Definition definition; + + private final String name; + private final String source; + + public Executable(final Definition definition, final String name, final String source) { + this.definition = definition; + + this.name = name; + this.source = source; + } + + public String getName() { + return name; + } + + public String getSource() { + return source; + } + + public Definition getDefinition() { + return definition; + } + + public abstract Object execute(Map input); +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ParserErrorStrategy.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ParserErrorStrategy.java new file mode 100644 index 00000000000..3fe36034792 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ParserErrorStrategy.java @@ -0,0 +1,74 @@ +package org.elasticsearch.plan.a; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.text.ParseException; + +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; + +class ParserErrorStrategy extends DefaultErrorStrategy { + @Override + public void recover(Parser recognizer, RecognitionException re) { + Token token = re.getOffendingToken(); + String message; + + if (token == null) { + message = "Error: no parse token found."; + } else if (re instanceof InputMismatchException) { + message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + + " unexpected token [" + getTokenErrorDisplay(token) + "]" + + " was expecting one of [" + re.getExpectedTokens().toString(recognizer.getVocabulary()) + "]."; + } else if (re instanceof NoViableAltException) { + if (token.getType() == PlanAParser.EOF) { + message = "Error: unexpected end of script."; + } else { + message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + + "invalid sequence of tokens near [" + getTokenErrorDisplay(token) + "]."; + } + } else { + message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + + " unexpected token near [" + getTokenErrorDisplay(token) + "]."; + } + + ParseException parseException = new ParseException(message, token == null ? -1 : token.getStartIndex()); + parseException.initCause(re); + + throw new RuntimeException(parseException); + } + + @Override + public Token recoverInline(Parser recognizer) throws RecognitionException { + Token token = recognizer.getCurrentToken(); + String message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + + " unexpected token [" + getTokenErrorDisplay(token) + "]" + + " was expecting one of [" + recognizer.getExpectedTokens().toString(recognizer.getVocabulary()) + "]."; + ParseException parseException = new ParseException(message, token.getStartIndex()); + throw new RuntimeException(parseException); + } + + @Override + public void sync(Parser recognizer) { + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanALexer.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanALexer.java new file mode 100644 index 00000000000..a9e5ff623bf --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanALexer.java @@ -0,0 +1,390 @@ +// ANTLR GENERATED CODE: DO NOT EDIT +package org.elasticsearch.plan.a; + + import java.util.Set; + +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.*; + +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +class PlanALexer extends Lexer { + static { RuntimeMetaData.checkVersion("4.5.1", RuntimeMetaData.VERSION); } + + protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); + public static final int + WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, + COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, + BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, + MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, + LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, BWXOR=43, + BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, + AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, + ALSH=60, ARSH=61, AUSH=62, ACAT=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, + STRING=68, CHAR=69, TRUE=70, FALSE=71, NULL=72, TYPE=73, ID=74, EXTINTEGER=75, + EXTID=76; + public static final int EXT = 1; + public static String[] modeNames = { + "DEFAULT_MODE", "EXT" + }; + + public static final String[] ruleNames = { + "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", + "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", + "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", + "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", + "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", + "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", + "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "ACAT", + "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "CHAR", "TRUE", "FALSE", + "NULL", "TYPE", "GENERIC", "ID", "EXTINTEGER", "EXTID" + }; + + private static final String[] _LITERAL_NAMES = { + null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", + "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", + "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", + "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", + "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", + "'||'", "'?'", "':'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", + "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", "'..='", null, + null, null, null, null, null, "'true'", "'false'", "'null'" + }; + private static final String[] _SYMBOLIC_NAMES = { + null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", + "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", + "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", + "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", + "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", + "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", + "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "ACAT", + "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "CHAR", "TRUE", "FALSE", + "NULL", "TYPE", "ID", "EXTINTEGER", "EXTID" + }; + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = ""; + } + } + } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + + private Set types = null; + + void setTypes(Set types) { + this.types = types; + } + + + public PlanALexer(CharStream input) { + super(input); + _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); + } + + @Override + public String getGrammarFileName() { return "PlanALexer.g4"; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public String getSerializedATN() { return _serializedATN; } + + @Override + public String[] getModeNames() { return modeNames; } + + @Override + public ATN getATN() { return _ATN; } + + @Override + public void action(RuleContext _localctx, int ruleIndex, int actionIndex) { + switch (ruleIndex) { + case 67: + STRING_action((RuleContext)_localctx, actionIndex); + break; + case 68: + CHAR_action((RuleContext)_localctx, actionIndex); + break; + case 72: + TYPE_action((RuleContext)_localctx, actionIndex); + break; + } + } + private void STRING_action(RuleContext _localctx, int actionIndex) { + switch (actionIndex) { + case 0: + setText(getText().substring(1, getText().length() - 1)); + break; + } + } + private void CHAR_action(RuleContext _localctx, int actionIndex) { + switch (actionIndex) { + case 1: + setText(getText().substring(1, getText().length() - 1)); + break; + } + } + private void TYPE_action(RuleContext _localctx, int actionIndex) { + switch (actionIndex) { + case 2: + setText(getText().replace(" ", "")); + break; + } + } + @Override + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { + switch (ruleIndex) { + case 72: + return TYPE_sempred((RuleContext)_localctx, predIndex); + } + return true; + } + private boolean TYPE_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 0: + return types.contains(getText().replace(" ", "")); + } + return true; + } + + public static final String _serializedATN = + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2N\u0236\b\1\b\1\4"+ + "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+ + "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+ + "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+ + "\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t"+ + " \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t"+ + "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+ + "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+ + "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+ + "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\3\2\6\2\u00a0\n\2\r\2\16\2\u00a1\3"+ + "\2\3\2\3\3\3\3\3\3\3\3\7\3\u00aa\n\3\f\3\16\3\u00ad\13\3\3\3\3\3\3\3\3"+ + "\3\3\3\7\3\u00b4\n\3\f\3\16\3\u00b7\13\3\3\3\3\3\5\3\u00bb\n\3\3\3\3\3"+ + "\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13"+ + "\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17"+ + "\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22"+ + "\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24"+ + "\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27"+ + "\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33"+ + "\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3"+ + "\"\3\"\3\"\3#\3#\3$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3"+ + ")\3)\3*\3*\3*\3*\3+\3+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3"+ + "\61\3\62\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3"+ + "\66\3\67\3\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3="+ + "\3=\3=\3>\3>\3>\3>\3?\3?\3?\3?\3?\3@\3@\3@\3@\3A\3A\6A\u0185\nA\rA\16"+ + "A\u0186\3A\5A\u018a\nA\3B\3B\3B\6B\u018f\nB\rB\16B\u0190\3B\5B\u0194\n"+ + "B\3C\3C\3C\7C\u0199\nC\fC\16C\u019c\13C\5C\u019e\nC\3C\5C\u01a1\nC\3D"+ + "\3D\3D\7D\u01a6\nD\fD\16D\u01a9\13D\5D\u01ab\nD\3D\3D\7D\u01af\nD\fD\16"+ + "D\u01b2\13D\3D\3D\5D\u01b6\nD\3D\6D\u01b9\nD\rD\16D\u01ba\5D\u01bd\nD"+ + "\3D\5D\u01c0\nD\3E\3E\3E\3E\3E\3E\7E\u01c8\nE\fE\16E\u01cb\13E\3E\3E\3"+ + "E\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3H\3H\3H\3H\3H\3H\3I\3I\3I\3I\3I\3J\3"+ + "J\5J\u01e7\nJ\3J\3J\3J\3K\7K\u01ed\nK\fK\16K\u01f0\13K\3K\3K\7K\u01f4"+ + "\nK\fK\16K\u01f7\13K\3K\3K\5K\u01fb\nK\3K\7K\u01fe\nK\fK\16K\u0201\13"+ + "K\3K\3K\7K\u0205\nK\fK\16K\u0208\13K\3K\3K\5K\u020c\nK\3K\7K\u020f\nK"+ + "\fK\16K\u0212\13K\7K\u0214\nK\fK\16K\u0217\13K\3K\3K\3L\3L\7L\u021d\n"+ + "L\fL\16L\u0220\13L\3M\3M\3M\7M\u0225\nM\fM\16M\u0228\13M\5M\u022a\nM\3"+ + "M\3M\3N\3N\7N\u0230\nN\fN\16N\u0233\13N\3N\3N\5\u00ab\u00b5\u01c9\2O\4"+ + "\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21"+ + "\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!"+ + "B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:"+ + "t;v|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090"+ + "I\u0092J\u0094K\u0096\2\u0098L\u009aM\u009cN\4\2\3\21\5\2\13\f\17\17\""+ + "\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b"+ + "\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aac|\6\2\62;C\\a"+ + "ac|\u0255\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2"+ + "\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30"+ + "\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2"+ + "\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60"+ + "\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2"+ + "\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H"+ + "\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2"+ + "\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2"+ + "\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2"+ + "n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3"+ + "\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3"+ + "\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2"+ + "\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0098"+ + "\3\2\2\2\3\u009a\3\2\2\2\3\u009c\3\2\2\2\4\u009f\3\2\2\2\6\u00ba\3\2\2"+ + "\2\b\u00be\3\2\2\2\n\u00c0\3\2\2\2\f\u00c2\3\2\2\2\16\u00c4\3\2\2\2\20"+ + "\u00c6\3\2\2\2\22\u00c8\3\2\2\2\24\u00ca\3\2\2\2\26\u00ce\3\2\2\2\30\u00d0"+ + "\3\2\2\2\32\u00d2\3\2\2\2\34\u00d5\3\2\2\2\36\u00da\3\2\2\2 \u00e0\3\2"+ + "\2\2\"\u00e3\3\2\2\2$\u00e7\3\2\2\2&\u00f0\3\2\2\2(\u00f6\3\2\2\2*\u00fd"+ + "\3\2\2\2,\u0101\3\2\2\2.\u0105\3\2\2\2\60\u010b\3\2\2\2\62\u0111\3\2\2"+ + "\2\64\u0113\3\2\2\2\66\u0115\3\2\2\28\u0117\3\2\2\2:\u0119\3\2\2\2<\u011b"+ + "\3\2\2\2>\u011d\3\2\2\2@\u011f\3\2\2\2B\u0122\3\2\2\2D\u0125\3\2\2\2F"+ + "\u0129\3\2\2\2H\u012b\3\2\2\2J\u012e\3\2\2\2L\u0130\3\2\2\2N\u0133\3\2"+ + "\2\2P\u0136\3\2\2\2R\u013a\3\2\2\2T\u013d\3\2\2\2V\u0141\3\2\2\2X\u0143"+ + "\3\2\2\2Z\u0145\3\2\2\2\\\u0147\3\2\2\2^\u014a\3\2\2\2`\u014d\3\2\2\2"+ + "b\u014f\3\2\2\2d\u0151\3\2\2\2f\u0154\3\2\2\2h\u0157\3\2\2\2j\u0159\3"+ + "\2\2\2l\u015c\3\2\2\2n\u015f\3\2\2\2p\u0162\3\2\2\2r\u0165\3\2\2\2t\u0168"+ + "\3\2\2\2v\u016b\3\2\2\2x\u016e\3\2\2\2z\u0171\3\2\2\2|\u0175\3\2\2\2~"+ + "\u0179\3\2\2\2\u0080\u017e\3\2\2\2\u0082\u0182\3\2\2\2\u0084\u018b\3\2"+ + "\2\2\u0086\u019d\3\2\2\2\u0088\u01aa\3\2\2\2\u008a\u01c1\3\2\2\2\u008c"+ + "\u01cf\3\2\2\2\u008e\u01d4\3\2\2\2\u0090\u01d9\3\2\2\2\u0092\u01df\3\2"+ + "\2\2\u0094\u01e4\3\2\2\2\u0096\u01ee\3\2\2\2\u0098\u021a\3\2\2\2\u009a"+ + "\u0229\3\2\2\2\u009c\u022d\3\2\2\2\u009e\u00a0\t\2\2\2\u009f\u009e\3\2"+ + "\2\2\u00a0\u00a1\3\2\2\2\u00a1\u009f\3\2\2\2\u00a1\u00a2\3\2\2\2\u00a2"+ + "\u00a3\3\2\2\2\u00a3\u00a4\b\2\2\2\u00a4\5\3\2\2\2\u00a5\u00a6\7\61\2"+ + "\2\u00a6\u00a7\7\61\2\2\u00a7\u00ab\3\2\2\2\u00a8\u00aa\13\2\2\2\u00a9"+ + "\u00a8\3\2\2\2\u00aa\u00ad\3\2\2\2\u00ab\u00ac\3\2\2\2\u00ab\u00a9\3\2"+ + "\2\2\u00ac\u00ae\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ae\u00bb\t\3\2\2\u00af"+ + "\u00b0\7\61\2\2\u00b0\u00b1\7,\2\2\u00b1\u00b5\3\2\2\2\u00b2\u00b4\13"+ + "\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\u00b7\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b5"+ + "\u00b3\3\2\2\2\u00b6\u00b8\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8\u00b9\7,"+ + "\2\2\u00b9\u00bb\7\61\2\2\u00ba\u00a5\3\2\2\2\u00ba\u00af\3\2\2\2\u00bb"+ + "\u00bc\3\2\2\2\u00bc\u00bd\b\3\2\2\u00bd\7\3\2\2\2\u00be\u00bf\7}\2\2"+ + "\u00bf\t\3\2\2\2\u00c0\u00c1\7\177\2\2\u00c1\13\3\2\2\2\u00c2\u00c3\7"+ + "]\2\2\u00c3\r\3\2\2\2\u00c4\u00c5\7_\2\2\u00c5\17\3\2\2\2\u00c6\u00c7"+ + "\7*\2\2\u00c7\21\3\2\2\2\u00c8\u00c9\7+\2\2\u00c9\23\3\2\2\2\u00ca\u00cb"+ + "\7\60\2\2\u00cb\u00cc\3\2\2\2\u00cc\u00cd\b\n\3\2\u00cd\25\3\2\2\2\u00ce"+ + "\u00cf\7.\2\2\u00cf\27\3\2\2\2\u00d0\u00d1\7=\2\2\u00d1\31\3\2\2\2\u00d2"+ + "\u00d3\7k\2\2\u00d3\u00d4\7h\2\2\u00d4\33\3\2\2\2\u00d5\u00d6\7g\2\2\u00d6"+ + "\u00d7\7n\2\2\u00d7\u00d8\7u\2\2\u00d8\u00d9\7g\2\2\u00d9\35\3\2\2\2\u00da"+ + "\u00db\7y\2\2\u00db\u00dc\7j\2\2\u00dc\u00dd\7k\2\2\u00dd\u00de\7n\2\2"+ + "\u00de\u00df\7g\2\2\u00df\37\3\2\2\2\u00e0\u00e1\7f\2\2\u00e1\u00e2\7"+ + "q\2\2\u00e2!\3\2\2\2\u00e3\u00e4\7h\2\2\u00e4\u00e5\7q\2\2\u00e5\u00e6"+ + "\7t\2\2\u00e6#\3\2\2\2\u00e7\u00e8\7e\2\2\u00e8\u00e9\7q\2\2\u00e9\u00ea"+ + "\7p\2\2\u00ea\u00eb\7v\2\2\u00eb\u00ec\7k\2\2\u00ec\u00ed\7p\2\2\u00ed"+ + "\u00ee\7w\2\2\u00ee\u00ef\7g\2\2\u00ef%\3\2\2\2\u00f0\u00f1\7d\2\2\u00f1"+ + "\u00f2\7t\2\2\u00f2\u00f3\7g\2\2\u00f3\u00f4\7c\2\2\u00f4\u00f5\7m\2\2"+ + "\u00f5\'\3\2\2\2\u00f6\u00f7\7t\2\2\u00f7\u00f8\7g\2\2\u00f8\u00f9\7v"+ + "\2\2\u00f9\u00fa\7w\2\2\u00fa\u00fb\7t\2\2\u00fb\u00fc\7p\2\2\u00fc)\3"+ + "\2\2\2\u00fd\u00fe\7p\2\2\u00fe\u00ff\7g\2\2\u00ff\u0100\7y\2\2\u0100"+ + "+\3\2\2\2\u0101\u0102\7v\2\2\u0102\u0103\7t\2\2\u0103\u0104\7{\2\2\u0104"+ + "-\3\2\2\2\u0105\u0106\7e\2\2\u0106\u0107\7c\2\2\u0107\u0108\7v\2\2\u0108"+ + "\u0109\7e\2\2\u0109\u010a\7j\2\2\u010a/\3\2\2\2\u010b\u010c\7v\2\2\u010c"+ + "\u010d\7j\2\2\u010d\u010e\7t\2\2\u010e\u010f\7q\2\2\u010f\u0110\7y\2\2"+ + "\u0110\61\3\2\2\2\u0111\u0112\7#\2\2\u0112\63\3\2\2\2\u0113\u0114\7\u0080"+ + "\2\2\u0114\65\3\2\2\2\u0115\u0116\7,\2\2\u0116\67\3\2\2\2\u0117\u0118"+ + "\7\61\2\2\u01189\3\2\2\2\u0119\u011a\7\'\2\2\u011a;\3\2\2\2\u011b\u011c"+ + "\7-\2\2\u011c=\3\2\2\2\u011d\u011e\7/\2\2\u011e?\3\2\2\2\u011f\u0120\7"+ + ">\2\2\u0120\u0121\7>\2\2\u0121A\3\2\2\2\u0122\u0123\7@\2\2\u0123\u0124"+ + "\7@\2\2\u0124C\3\2\2\2\u0125\u0126\7@\2\2\u0126\u0127\7@\2\2\u0127\u0128"+ + "\7@\2\2\u0128E\3\2\2\2\u0129\u012a\7>\2\2\u012aG\3\2\2\2\u012b\u012c\7"+ + ">\2\2\u012c\u012d\7?\2\2\u012dI\3\2\2\2\u012e\u012f\7@\2\2\u012fK\3\2"+ + "\2\2\u0130\u0131\7@\2\2\u0131\u0132\7?\2\2\u0132M\3\2\2\2\u0133\u0134"+ + "\7?\2\2\u0134\u0135\7?\2\2\u0135O\3\2\2\2\u0136\u0137\7?\2\2\u0137\u0138"+ + "\7?\2\2\u0138\u0139\7?\2\2\u0139Q\3\2\2\2\u013a\u013b\7#\2\2\u013b\u013c"+ + "\7?\2\2\u013cS\3\2\2\2\u013d\u013e\7#\2\2\u013e\u013f\7?\2\2\u013f\u0140"+ + "\7?\2\2\u0140U\3\2\2\2\u0141\u0142\7(\2\2\u0142W\3\2\2\2\u0143\u0144\7"+ + "`\2\2\u0144Y\3\2\2\2\u0145\u0146\7~\2\2\u0146[\3\2\2\2\u0147\u0148\7("+ + "\2\2\u0148\u0149\7(\2\2\u0149]\3\2\2\2\u014a\u014b\7~\2\2\u014b\u014c"+ + "\7~\2\2\u014c_\3\2\2\2\u014d\u014e\7A\2\2\u014ea\3\2\2\2\u014f\u0150\7"+ + "<\2\2\u0150c\3\2\2\2\u0151\u0152\7-\2\2\u0152\u0153\7-\2\2\u0153e\3\2"+ + "\2\2\u0154\u0155\7/\2\2\u0155\u0156\7/\2\2\u0156g\3\2\2\2\u0157\u0158"+ + "\7?\2\2\u0158i\3\2\2\2\u0159\u015a\7-\2\2\u015a\u015b\7?\2\2\u015bk\3"+ + "\2\2\2\u015c\u015d\7/\2\2\u015d\u015e\7?\2\2\u015em\3\2\2\2\u015f\u0160"+ + "\7,\2\2\u0160\u0161\7?\2\2\u0161o\3\2\2\2\u0162\u0163\7\61\2\2\u0163\u0164"+ + "\7?\2\2\u0164q\3\2\2\2\u0165\u0166\7\'\2\2\u0166\u0167\7?\2\2\u0167s\3"+ + "\2\2\2\u0168\u0169\7(\2\2\u0169\u016a\7?\2\2\u016au\3\2\2\2\u016b\u016c"+ + "\7`\2\2\u016c\u016d\7?\2\2\u016dw\3\2\2\2\u016e\u016f\7~\2\2\u016f\u0170"+ + "\7?\2\2\u0170y\3\2\2\2\u0171\u0172\7>\2\2\u0172\u0173\7>\2\2\u0173\u0174"+ + "\7?\2\2\u0174{\3\2\2\2\u0175\u0176\7@\2\2\u0176\u0177\7@\2\2\u0177\u0178"+ + "\7?\2\2\u0178}\3\2\2\2\u0179\u017a\7@\2\2\u017a\u017b\7@\2\2\u017b\u017c"+ + "\7@\2\2\u017c\u017d\7?\2\2\u017d\177\3\2\2\2\u017e\u017f\7\60\2\2\u017f"+ + "\u0180\7\60\2\2\u0180\u0181\7?\2\2\u0181\u0081\3\2\2\2\u0182\u0184\7\62"+ + "\2\2\u0183\u0185\t\4\2\2\u0184\u0183\3\2\2\2\u0185\u0186\3\2\2\2\u0186"+ + "\u0184\3\2\2\2\u0186\u0187\3\2\2\2\u0187\u0189\3\2\2\2\u0188\u018a\t\5"+ + "\2\2\u0189\u0188\3\2\2\2\u0189\u018a\3\2\2\2\u018a\u0083\3\2\2\2\u018b"+ + "\u018c\7\62\2\2\u018c\u018e\t\6\2\2\u018d\u018f\t\7\2\2\u018e\u018d\3"+ + "\2\2\2\u018f\u0190\3\2\2\2\u0190\u018e\3\2\2\2\u0190\u0191\3\2\2\2\u0191"+ + "\u0193\3\2\2\2\u0192\u0194\t\5\2\2\u0193\u0192\3\2\2\2\u0193\u0194\3\2"+ + "\2\2\u0194\u0085\3\2\2\2\u0195\u019e\7\62\2\2\u0196\u019a\t\b\2\2\u0197"+ + "\u0199\t\t\2\2\u0198\u0197\3\2\2\2\u0199\u019c\3\2\2\2\u019a\u0198\3\2"+ + "\2\2\u019a\u019b\3\2\2\2\u019b\u019e\3\2\2\2\u019c\u019a\3\2\2\2\u019d"+ + "\u0195\3\2\2\2\u019d\u0196\3\2\2\2\u019e\u01a0\3\2\2\2\u019f\u01a1\t\n"+ + "\2\2\u01a0\u019f\3\2\2\2\u01a0\u01a1\3\2\2\2\u01a1\u0087\3\2\2\2\u01a2"+ + "\u01ab\7\62\2\2\u01a3\u01a7\t\b\2\2\u01a4\u01a6\t\t\2\2\u01a5\u01a4\3"+ + "\2\2\2\u01a6\u01a9\3\2\2\2\u01a7\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8"+ + "\u01ab\3\2\2\2\u01a9\u01a7\3\2\2\2\u01aa\u01a2\3\2\2\2\u01aa\u01a3\3\2"+ + "\2\2\u01ab\u01ac\3\2\2\2\u01ac\u01b0\5\24\n\2\u01ad\u01af\t\t\2\2\u01ae"+ + "\u01ad\3\2\2\2\u01af\u01b2\3\2\2\2\u01b0\u01ae\3\2\2\2\u01b0\u01b1\3\2"+ + "\2\2\u01b1\u01bc\3\2\2\2\u01b2\u01b0\3\2\2\2\u01b3\u01b5\t\13\2\2\u01b4"+ + "\u01b6\t\f\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b8\3\2"+ + "\2\2\u01b7\u01b9\t\t\2\2\u01b8\u01b7\3\2\2\2\u01b9\u01ba\3\2\2\2\u01ba"+ + "\u01b8\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb\u01bd\3\2\2\2\u01bc\u01b3\3\2"+ + "\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01bf\3\2\2\2\u01be\u01c0\t\r\2\2\u01bf"+ + "\u01be\3\2\2\2\u01bf\u01c0\3\2\2\2\u01c0\u0089\3\2\2\2\u01c1\u01c9\7$"+ + "\2\2\u01c2\u01c3\7^\2\2\u01c3\u01c8\7$\2\2\u01c4\u01c5\7^\2\2\u01c5\u01c8"+ + "\7^\2\2\u01c6\u01c8\n\16\2\2\u01c7\u01c2\3\2\2\2\u01c7\u01c4\3\2\2\2\u01c7"+ + "\u01c6\3\2\2\2\u01c8\u01cb\3\2\2\2\u01c9\u01ca\3\2\2\2\u01c9\u01c7\3\2"+ + "\2\2\u01ca\u01cc\3\2\2\2\u01cb\u01c9\3\2\2\2\u01cc\u01cd\7$\2\2\u01cd"+ + "\u01ce\bE\4\2\u01ce\u008b\3\2\2\2\u01cf\u01d0\7)\2\2\u01d0\u01d1\13\2"+ + "\2\2\u01d1\u01d2\7)\2\2\u01d2\u01d3\bF\5\2\u01d3\u008d\3\2\2\2\u01d4\u01d5"+ + "\7v\2\2\u01d5\u01d6\7t\2\2\u01d6\u01d7\7w\2\2\u01d7\u01d8\7g\2\2\u01d8"+ + "\u008f\3\2\2\2\u01d9\u01da\7h\2\2\u01da\u01db\7c\2\2\u01db\u01dc\7n\2"+ + "\2\u01dc\u01dd\7u\2\2\u01dd\u01de\7g\2\2\u01de\u0091\3\2\2\2\u01df\u01e0"+ + "\7p\2\2\u01e0\u01e1\7w\2\2\u01e1\u01e2\7n\2\2\u01e2\u01e3\7n\2\2\u01e3"+ + "\u0093\3\2\2\2\u01e4\u01e6\5\u0098L\2\u01e5\u01e7\5\u0096K\2\u01e6\u01e5"+ + "\3\2\2\2\u01e6\u01e7\3\2\2\2\u01e7\u01e8\3\2\2\2\u01e8\u01e9\6J\2\2\u01e9"+ + "\u01ea\bJ\6\2\u01ea\u0095\3\2\2\2\u01eb\u01ed\7\"\2\2\u01ec\u01eb\3\2"+ + "\2\2\u01ed\u01f0\3\2\2\2\u01ee\u01ec\3\2\2\2\u01ee\u01ef\3\2\2\2\u01ef"+ + "\u01f1\3\2\2\2\u01f0\u01ee\3\2\2\2\u01f1\u01f5\7>\2\2\u01f2\u01f4\7\""+ + "\2\2\u01f3\u01f2\3\2\2\2\u01f4\u01f7\3\2\2\2\u01f5\u01f3\3\2\2\2\u01f5"+ + "\u01f6\3\2\2\2\u01f6\u01f8\3\2\2\2\u01f7\u01f5\3\2\2\2\u01f8\u01fa\5\u0098"+ + "L\2\u01f9\u01fb\5\u0096K\2\u01fa\u01f9\3\2\2\2\u01fa\u01fb\3\2\2\2\u01fb"+ + "\u01ff\3\2\2\2\u01fc\u01fe\7\"\2\2\u01fd\u01fc\3\2\2\2\u01fe\u0201\3\2"+ + "\2\2\u01ff\u01fd\3\2\2\2\u01ff\u0200\3\2\2\2\u0200\u0215\3\2\2\2\u0201"+ + "\u01ff\3\2\2\2\u0202\u0206\5\26\13\2\u0203\u0205\7\"\2\2\u0204\u0203\3"+ + "\2\2\2\u0205\u0208\3\2\2\2\u0206\u0204\3\2\2\2\u0206\u0207\3\2\2\2\u0207"+ + "\u0209\3\2\2\2\u0208\u0206\3\2\2\2\u0209\u020b\5\u0098L\2\u020a\u020c"+ + "\5\u0096K\2\u020b\u020a\3\2\2\2\u020b\u020c\3\2\2\2\u020c\u0210\3\2\2"+ + "\2\u020d\u020f\7\"\2\2\u020e\u020d\3\2\2\2\u020f\u0212\3\2\2\2\u0210\u020e"+ + "\3\2\2\2\u0210\u0211\3\2\2\2\u0211\u0214\3\2\2\2\u0212\u0210\3\2\2\2\u0213"+ + "\u0202\3\2\2\2\u0214\u0217\3\2\2\2\u0215\u0213\3\2\2\2\u0215\u0216\3\2"+ + "\2\2\u0216\u0218\3\2\2\2\u0217\u0215\3\2\2\2\u0218\u0219\7@\2\2\u0219"+ + "\u0097\3\2\2\2\u021a\u021e\t\17\2\2\u021b\u021d\t\20\2\2\u021c\u021b\3"+ + "\2\2\2\u021d\u0220\3\2\2\2\u021e\u021c\3\2\2\2\u021e\u021f\3\2\2\2\u021f"+ + "\u0099\3\2\2\2\u0220\u021e\3\2\2\2\u0221\u022a\7\62\2\2\u0222\u0226\t"+ + "\b\2\2\u0223\u0225\t\t\2\2\u0224\u0223\3\2\2\2\u0225\u0228\3\2\2\2\u0226"+ + "\u0224\3\2\2\2\u0226\u0227\3\2\2\2\u0227\u022a\3\2\2\2\u0228\u0226\3\2"+ + "\2\2\u0229\u0221\3\2\2\2\u0229\u0222\3\2\2\2\u022a\u022b\3\2\2\2\u022b"+ + "\u022c\bM\7\2\u022c\u009b\3\2\2\2\u022d\u0231\t\17\2\2\u022e\u0230\t\20"+ + "\2\2\u022f\u022e\3\2\2\2\u0230\u0233\3\2\2\2\u0231\u022f\3\2\2\2\u0231"+ + "\u0232\3\2\2\2\u0232\u0234\3\2\2\2\u0233\u0231\3\2\2\2\u0234\u0235\bN"+ + "\7\2\u0235\u009d\3\2\2\2%\2\3\u00a1\u00ab\u00b5\u00ba\u0186\u0189\u0190"+ + "\u0193\u019a\u019d\u01a0\u01a7\u01aa\u01b0\u01b5\u01ba\u01bc\u01bf\u01c7"+ + "\u01c9\u01e6\u01ee\u01f5\u01fa\u01ff\u0206\u020b\u0210\u0215\u021e\u0226"+ + "\u0229\u0231\b\b\2\2\4\3\2\3E\2\3F\3\3J\4\4\2\2"; + public static final ATN _ATN = + new ATNDeserializer().deserialize(_serializedATN.toCharArray()); + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) { + _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i); + } + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParser.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParser.java new file mode 100644 index 00000000000..13f61acb495 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParser.java @@ -0,0 +1,2884 @@ +// ANTLR GENERATED CODE: DO NOT EDIT +package org.elasticsearch.plan.a; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.misc.*; +import org.antlr.v4.runtime.tree.*; +import java.util.List; +import java.util.Iterator; +import java.util.ArrayList; + +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +class PlanAParser extends Parser { + static { RuntimeMetaData.checkVersion("4.5.1", RuntimeMetaData.VERSION); } + + protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); + public static final int + WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, + COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, + BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, + MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, + LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, BWXOR=43, + BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, + AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, + ALSH=60, ARSH=61, AUSH=62, ACAT=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, + STRING=68, CHAR=69, TRUE=70, FALSE=71, NULL=72, TYPE=73, ID=74, EXTINTEGER=75, + EXTID=76; + public static final int + RULE_source = 0, RULE_statement = 1, RULE_block = 2, RULE_empty = 3, RULE_initializer = 4, + RULE_afterthought = 5, RULE_declaration = 6, RULE_decltype = 7, RULE_declvar = 8, + RULE_expression = 9, RULE_extstart = 10, RULE_extprec = 11, RULE_extcast = 12, + RULE_extbrace = 13, RULE_extdot = 14, RULE_exttype = 15, RULE_extcall = 16, + RULE_extvar = 17, RULE_extfield = 18, RULE_extnew = 19, RULE_extstring = 20, + RULE_arguments = 21, RULE_increment = 22; + public static final String[] ruleNames = { + "source", "statement", "block", "empty", "initializer", "afterthought", + "declaration", "decltype", "declvar", "expression", "extstart", "extprec", + "extcast", "extbrace", "extdot", "exttype", "extcall", "extvar", "extfield", + "extnew", "extstring", "arguments", "increment" + }; + + private static final String[] _LITERAL_NAMES = { + null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", + "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", + "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", + "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", + "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", + "'||'", "'?'", "':'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", + "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", "'..='", null, + null, null, null, null, null, "'true'", "'false'", "'null'" + }; + private static final String[] _SYMBOLIC_NAMES = { + null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", + "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", + "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", + "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", + "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", + "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", + "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "ACAT", + "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "CHAR", "TRUE", "FALSE", + "NULL", "TYPE", "ID", "EXTINTEGER", "EXTID" + }; + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = ""; + } + } + } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + @Override + public String getGrammarFileName() { return "PlanAParser.g4"; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public String getSerializedATN() { return _serializedATN; } + + @Override + public ATN getATN() { return _ATN; } + + public PlanAParser(TokenStream input) { + super(input); + _interp = new ParserATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); + } + public static class SourceContext extends ParserRuleContext { + public TerminalNode EOF() { return getToken(PlanAParser.EOF, 0); } + public List statement() { + return getRuleContexts(StatementContext.class); + } + public StatementContext statement(int i) { + return getRuleContext(StatementContext.class,i); + } + public SourceContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_source; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitSource(this); + else return visitor.visitChildren(this); + } + } + + public final SourceContext source() throws RecognitionException { + SourceContext _localctx = new SourceContext(_ctx, getState()); + enterRule(_localctx, 0, RULE_source); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(47); + _errHandler.sync(this); + _la = _input.LA(1); + do { + { + { + setState(46); + statement(); + } + } + setState(49); + _errHandler.sync(this); + _la = _input.LA(1); + } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (TYPE - 64)) | (1L << (ID - 64)))) != 0) ); + setState(51); + match(EOF); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class StatementContext extends ParserRuleContext { + public StatementContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_statement; } + + public StatementContext() { } + public void copyFrom(StatementContext ctx) { + super.copyFrom(ctx); + } + } + public static class DeclContext extends StatementContext { + public DeclarationContext declaration() { + return getRuleContext(DeclarationContext.class,0); + } + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public DeclContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitDecl(this); + else return visitor.visitChildren(this); + } + } + public static class BreakContext extends StatementContext { + public TerminalNode BREAK() { return getToken(PlanAParser.BREAK, 0); } + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public BreakContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitBreak(this); + else return visitor.visitChildren(this); + } + } + public static class ThrowContext extends StatementContext { + public TerminalNode THROW() { return getToken(PlanAParser.THROW, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public ThrowContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitThrow(this); + else return visitor.visitChildren(this); + } + } + public static class ContinueContext extends StatementContext { + public TerminalNode CONTINUE() { return getToken(PlanAParser.CONTINUE, 0); } + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public ContinueContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitContinue(this); + else return visitor.visitChildren(this); + } + } + public static class ForContext extends StatementContext { + public TerminalNode FOR() { return getToken(PlanAParser.FOR, 0); } + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public List SEMICOLON() { return getTokens(PlanAParser.SEMICOLON); } + public TerminalNode SEMICOLON(int i) { + return getToken(PlanAParser.SEMICOLON, i); + } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public BlockContext block() { + return getRuleContext(BlockContext.class,0); + } + public EmptyContext empty() { + return getRuleContext(EmptyContext.class,0); + } + public InitializerContext initializer() { + return getRuleContext(InitializerContext.class,0); + } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public AfterthoughtContext afterthought() { + return getRuleContext(AfterthoughtContext.class,0); + } + public ForContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitFor(this); + else return visitor.visitChildren(this); + } + } + public static class TryContext extends StatementContext { + public TerminalNode TRY() { return getToken(PlanAParser.TRY, 0); } + public List block() { + return getRuleContexts(BlockContext.class); + } + public BlockContext block(int i) { + return getRuleContext(BlockContext.class,i); + } + public List CATCH() { return getTokens(PlanAParser.CATCH); } + public TerminalNode CATCH(int i) { + return getToken(PlanAParser.CATCH, i); + } + public List LP() { return getTokens(PlanAParser.LP); } + public TerminalNode LP(int i) { + return getToken(PlanAParser.LP, i); + } + public List RP() { return getTokens(PlanAParser.RP); } + public TerminalNode RP(int i) { + return getToken(PlanAParser.RP, i); + } + public List TYPE() { return getTokens(PlanAParser.TYPE); } + public TerminalNode TYPE(int i) { + return getToken(PlanAParser.TYPE, i); + } + public List ID() { return getTokens(PlanAParser.ID); } + public TerminalNode ID(int i) { + return getToken(PlanAParser.ID, i); + } + public TryContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitTry(this); + else return visitor.visitChildren(this); + } + } + public static class ExprContext extends StatementContext { + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public ExprContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExpr(this); + else return visitor.visitChildren(this); + } + } + public static class DoContext extends StatementContext { + public TerminalNode DO() { return getToken(PlanAParser.DO, 0); } + public BlockContext block() { + return getRuleContext(BlockContext.class,0); + } + public TerminalNode WHILE() { return getToken(PlanAParser.WHILE, 0); } + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public DoContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitDo(this); + else return visitor.visitChildren(this); + } + } + public static class WhileContext extends StatementContext { + public TerminalNode WHILE() { return getToken(PlanAParser.WHILE, 0); } + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public BlockContext block() { + return getRuleContext(BlockContext.class,0); + } + public EmptyContext empty() { + return getRuleContext(EmptyContext.class,0); + } + public WhileContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitWhile(this); + else return visitor.visitChildren(this); + } + } + public static class IfContext extends StatementContext { + public TerminalNode IF() { return getToken(PlanAParser.IF, 0); } + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public List block() { + return getRuleContexts(BlockContext.class); + } + public BlockContext block(int i) { + return getRuleContext(BlockContext.class,i); + } + public TerminalNode ELSE() { return getToken(PlanAParser.ELSE, 0); } + public IfContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitIf(this); + else return visitor.visitChildren(this); + } + } + public static class ReturnContext extends StatementContext { + public TerminalNode RETURN() { return getToken(PlanAParser.RETURN, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public ReturnContext(StatementContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitReturn(this); + else return visitor.visitChildren(this); + } + } + + public final StatementContext statement() throws RecognitionException { + StatementContext _localctx = new StatementContext(_ctx, getState()); + enterRule(_localctx, 2, RULE_statement); + int _la; + try { + int _alt; + setState(136); + switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { + case 1: + _localctx = new IfContext(_localctx); + enterOuterAlt(_localctx, 1); + { + setState(53); + match(IF); + setState(54); + match(LP); + setState(55); + expression(0); + setState(56); + match(RP); + setState(57); + block(); + setState(60); + switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { + case 1: + { + setState(58); + match(ELSE); + setState(59); + block(); + } + break; + } + } + break; + case 2: + _localctx = new WhileContext(_localctx); + enterOuterAlt(_localctx, 2); + { + setState(62); + match(WHILE); + setState(63); + match(LP); + setState(64); + expression(0); + setState(65); + match(RP); + setState(68); + switch (_input.LA(1)) { + case LBRACK: + case LP: + case IF: + case WHILE: + case DO: + case FOR: + case CONTINUE: + case BREAK: + case RETURN: + case NEW: + case TRY: + case THROW: + case BOOLNOT: + case BWNOT: + case ADD: + case SUB: + case INCR: + case DECR: + case OCTAL: + case HEX: + case INTEGER: + case DECIMAL: + case STRING: + case CHAR: + case TRUE: + case FALSE: + case NULL: + case TYPE: + case ID: + { + setState(66); + block(); + } + break; + case SEMICOLON: + { + setState(67); + empty(); + } + break; + default: + throw new NoViableAltException(this); + } + } + break; + case 3: + _localctx = new DoContext(_localctx); + enterOuterAlt(_localctx, 3); + { + setState(70); + match(DO); + setState(71); + block(); + setState(72); + match(WHILE); + setState(73); + match(LP); + setState(74); + expression(0); + setState(75); + match(RP); + setState(77); + _la = _input.LA(1); + if (_la==SEMICOLON) { + { + setState(76); + match(SEMICOLON); + } + } + + } + break; + case 4: + _localctx = new ForContext(_localctx); + enterOuterAlt(_localctx, 4); + { + setState(79); + match(FOR); + setState(80); + match(LP); + setState(82); + _la = _input.LA(1); + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (TYPE - 64)) | (1L << (ID - 64)))) != 0)) { + { + setState(81); + initializer(); + } + } + + setState(84); + match(SEMICOLON); + setState(86); + _la = _input.LA(1); + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (TYPE - 64)) | (1L << (ID - 64)))) != 0)) { + { + setState(85); + expression(0); + } + } + + setState(88); + match(SEMICOLON); + setState(90); + _la = _input.LA(1); + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (TYPE - 64)) | (1L << (ID - 64)))) != 0)) { + { + setState(89); + afterthought(); + } + } + + setState(92); + match(RP); + setState(95); + switch (_input.LA(1)) { + case LBRACK: + case LP: + case IF: + case WHILE: + case DO: + case FOR: + case CONTINUE: + case BREAK: + case RETURN: + case NEW: + case TRY: + case THROW: + case BOOLNOT: + case BWNOT: + case ADD: + case SUB: + case INCR: + case DECR: + case OCTAL: + case HEX: + case INTEGER: + case DECIMAL: + case STRING: + case CHAR: + case TRUE: + case FALSE: + case NULL: + case TYPE: + case ID: + { + setState(93); + block(); + } + break; + case SEMICOLON: + { + setState(94); + empty(); + } + break; + default: + throw new NoViableAltException(this); + } + } + break; + case 5: + _localctx = new DeclContext(_localctx); + enterOuterAlt(_localctx, 5); + { + setState(97); + declaration(); + setState(99); + _la = _input.LA(1); + if (_la==SEMICOLON) { + { + setState(98); + match(SEMICOLON); + } + } + + } + break; + case 6: + _localctx = new ContinueContext(_localctx); + enterOuterAlt(_localctx, 6); + { + setState(101); + match(CONTINUE); + setState(103); + _la = _input.LA(1); + if (_la==SEMICOLON) { + { + setState(102); + match(SEMICOLON); + } + } + + } + break; + case 7: + _localctx = new BreakContext(_localctx); + enterOuterAlt(_localctx, 7); + { + setState(105); + match(BREAK); + setState(107); + _la = _input.LA(1); + if (_la==SEMICOLON) { + { + setState(106); + match(SEMICOLON); + } + } + + } + break; + case 8: + _localctx = new ReturnContext(_localctx); + enterOuterAlt(_localctx, 8); + { + setState(109); + match(RETURN); + setState(110); + expression(0); + setState(112); + _la = _input.LA(1); + if (_la==SEMICOLON) { + { + setState(111); + match(SEMICOLON); + } + } + + } + break; + case 9: + _localctx = new TryContext(_localctx); + enterOuterAlt(_localctx, 9); + { + setState(114); + match(TRY); + setState(115); + block(); + setState(123); + _errHandler.sync(this); + _alt = 1; + do { + switch (_alt) { + case 1: + { + { + setState(116); + match(CATCH); + setState(117); + match(LP); + { + setState(118); + match(TYPE); + setState(119); + match(ID); + } + setState(121); + match(RP); + setState(122); + block(); + } + } + break; + default: + throw new NoViableAltException(this); + } + setState(125); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,12,_ctx); + } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); + } + break; + case 10: + _localctx = new ThrowContext(_localctx); + enterOuterAlt(_localctx, 10); + { + setState(127); + match(THROW); + setState(128); + expression(0); + setState(130); + _la = _input.LA(1); + if (_la==SEMICOLON) { + { + setState(129); + match(SEMICOLON); + } + } + + } + break; + case 11: + _localctx = new ExprContext(_localctx); + enterOuterAlt(_localctx, 11); + { + setState(132); + expression(0); + setState(134); + _la = _input.LA(1); + if (_la==SEMICOLON) { + { + setState(133); + match(SEMICOLON); + } + } + + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class BlockContext extends ParserRuleContext { + public BlockContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_block; } + + public BlockContext() { } + public void copyFrom(BlockContext ctx) { + super.copyFrom(ctx); + } + } + public static class SingleContext extends BlockContext { + public StatementContext statement() { + return getRuleContext(StatementContext.class,0); + } + public SingleContext(BlockContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitSingle(this); + else return visitor.visitChildren(this); + } + } + public static class MultipleContext extends BlockContext { + public TerminalNode LBRACK() { return getToken(PlanAParser.LBRACK, 0); } + public TerminalNode RBRACK() { return getToken(PlanAParser.RBRACK, 0); } + public List statement() { + return getRuleContexts(StatementContext.class); + } + public StatementContext statement(int i) { + return getRuleContext(StatementContext.class,i); + } + public MultipleContext(BlockContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitMultiple(this); + else return visitor.visitChildren(this); + } + } + + public final BlockContext block() throws RecognitionException { + BlockContext _localctx = new BlockContext(_ctx, getState()); + enterRule(_localctx, 4, RULE_block); + int _la; + try { + setState(147); + switch (_input.LA(1)) { + case LBRACK: + _localctx = new MultipleContext(_localctx); + enterOuterAlt(_localctx, 1); + { + setState(138); + match(LBRACK); + setState(142); + _errHandler.sync(this); + _la = _input.LA(1); + while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (TYPE - 64)) | (1L << (ID - 64)))) != 0)) { + { + { + setState(139); + statement(); + } + } + setState(144); + _errHandler.sync(this); + _la = _input.LA(1); + } + setState(145); + match(RBRACK); + } + break; + case LP: + case IF: + case WHILE: + case DO: + case FOR: + case CONTINUE: + case BREAK: + case RETURN: + case NEW: + case TRY: + case THROW: + case BOOLNOT: + case BWNOT: + case ADD: + case SUB: + case INCR: + case DECR: + case OCTAL: + case HEX: + case INTEGER: + case DECIMAL: + case STRING: + case CHAR: + case TRUE: + case FALSE: + case NULL: + case TYPE: + case ID: + _localctx = new SingleContext(_localctx); + enterOuterAlt(_localctx, 2); + { + setState(146); + statement(); + } + break; + default: + throw new NoViableAltException(this); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class EmptyContext extends ParserRuleContext { + public TerminalNode SEMICOLON() { return getToken(PlanAParser.SEMICOLON, 0); } + public EmptyContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_empty; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitEmpty(this); + else return visitor.visitChildren(this); + } + } + + public final EmptyContext empty() throws RecognitionException { + EmptyContext _localctx = new EmptyContext(_ctx, getState()); + enterRule(_localctx, 6, RULE_empty); + try { + enterOuterAlt(_localctx, 1); + { + setState(149); + match(SEMICOLON); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class InitializerContext extends ParserRuleContext { + public DeclarationContext declaration() { + return getRuleContext(DeclarationContext.class,0); + } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public InitializerContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_initializer; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitInitializer(this); + else return visitor.visitChildren(this); + } + } + + public final InitializerContext initializer() throws RecognitionException { + InitializerContext _localctx = new InitializerContext(_ctx, getState()); + enterRule(_localctx, 8, RULE_initializer); + try { + setState(153); + switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(151); + declaration(); + } + break; + case 2: + enterOuterAlt(_localctx, 2); + { + setState(152); + expression(0); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class AfterthoughtContext extends ParserRuleContext { + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public AfterthoughtContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_afterthought; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitAfterthought(this); + else return visitor.visitChildren(this); + } + } + + public final AfterthoughtContext afterthought() throws RecognitionException { + AfterthoughtContext _localctx = new AfterthoughtContext(_ctx, getState()); + enterRule(_localctx, 10, RULE_afterthought); + try { + enterOuterAlt(_localctx, 1); + { + setState(155); + expression(0); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class DeclarationContext extends ParserRuleContext { + public DecltypeContext decltype() { + return getRuleContext(DecltypeContext.class,0); + } + public List declvar() { + return getRuleContexts(DeclvarContext.class); + } + public DeclvarContext declvar(int i) { + return getRuleContext(DeclvarContext.class,i); + } + public List COMMA() { return getTokens(PlanAParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(PlanAParser.COMMA, i); + } + public DeclarationContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_declaration; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitDeclaration(this); + else return visitor.visitChildren(this); + } + } + + public final DeclarationContext declaration() throws RecognitionException { + DeclarationContext _localctx = new DeclarationContext(_ctx, getState()); + enterRule(_localctx, 12, RULE_declaration); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(157); + decltype(); + setState(158); + declvar(); + setState(163); + _errHandler.sync(this); + _la = _input.LA(1); + while (_la==COMMA) { + { + { + setState(159); + match(COMMA); + setState(160); + declvar(); + } + } + setState(165); + _errHandler.sync(this); + _la = _input.LA(1); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class DecltypeContext extends ParserRuleContext { + public TerminalNode TYPE() { return getToken(PlanAParser.TYPE, 0); } + public List LBRACE() { return getTokens(PlanAParser.LBRACE); } + public TerminalNode LBRACE(int i) { + return getToken(PlanAParser.LBRACE, i); + } + public List RBRACE() { return getTokens(PlanAParser.RBRACE); } + public TerminalNode RBRACE(int i) { + return getToken(PlanAParser.RBRACE, i); + } + public DecltypeContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_decltype; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitDecltype(this); + else return visitor.visitChildren(this); + } + } + + public final DecltypeContext decltype() throws RecognitionException { + DecltypeContext _localctx = new DecltypeContext(_ctx, getState()); + enterRule(_localctx, 14, RULE_decltype); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(166); + match(TYPE); + setState(171); + _errHandler.sync(this); + _la = _input.LA(1); + while (_la==LBRACE) { + { + { + setState(167); + match(LBRACE); + setState(168); + match(RBRACE); + } + } + setState(173); + _errHandler.sync(this); + _la = _input.LA(1); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class DeclvarContext extends ParserRuleContext { + public TerminalNode ID() { return getToken(PlanAParser.ID, 0); } + public TerminalNode ASSIGN() { return getToken(PlanAParser.ASSIGN, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public DeclvarContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_declvar; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitDeclvar(this); + else return visitor.visitChildren(this); + } + } + + public final DeclvarContext declvar() throws RecognitionException { + DeclvarContext _localctx = new DeclvarContext(_ctx, getState()); + enterRule(_localctx, 16, RULE_declvar); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(174); + match(ID); + setState(177); + _la = _input.LA(1); + if (_la==ASSIGN) { + { + setState(175); + match(ASSIGN); + setState(176); + expression(0); + } + } + + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExpressionContext extends ParserRuleContext { + public ExpressionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_expression; } + + public ExpressionContext() { } + public void copyFrom(ExpressionContext ctx) { + super.copyFrom(ctx); + } + } + public static class CompContext extends ExpressionContext { + public List expression() { + return getRuleContexts(ExpressionContext.class); + } + public ExpressionContext expression(int i) { + return getRuleContext(ExpressionContext.class,i); + } + public TerminalNode LT() { return getToken(PlanAParser.LT, 0); } + public TerminalNode LTE() { return getToken(PlanAParser.LTE, 0); } + public TerminalNode GT() { return getToken(PlanAParser.GT, 0); } + public TerminalNode GTE() { return getToken(PlanAParser.GTE, 0); } + public TerminalNode EQ() { return getToken(PlanAParser.EQ, 0); } + public TerminalNode EQR() { return getToken(PlanAParser.EQR, 0); } + public TerminalNode NE() { return getToken(PlanAParser.NE, 0); } + public TerminalNode NER() { return getToken(PlanAParser.NER, 0); } + public CompContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitComp(this); + else return visitor.visitChildren(this); + } + } + public static class BoolContext extends ExpressionContext { + public List expression() { + return getRuleContexts(ExpressionContext.class); + } + public ExpressionContext expression(int i) { + return getRuleContext(ExpressionContext.class,i); + } + public TerminalNode BOOLAND() { return getToken(PlanAParser.BOOLAND, 0); } + public TerminalNode BOOLOR() { return getToken(PlanAParser.BOOLOR, 0); } + public BoolContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitBool(this); + else return visitor.visitChildren(this); + } + } + public static class ConditionalContext extends ExpressionContext { + public List expression() { + return getRuleContexts(ExpressionContext.class); + } + public ExpressionContext expression(int i) { + return getRuleContext(ExpressionContext.class,i); + } + public TerminalNode COND() { return getToken(PlanAParser.COND, 0); } + public TerminalNode COLON() { return getToken(PlanAParser.COLON, 0); } + public ConditionalContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitConditional(this); + else return visitor.visitChildren(this); + } + } + public static class AssignmentContext extends ExpressionContext { + public ExtstartContext extstart() { + return getRuleContext(ExtstartContext.class,0); + } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode ASSIGN() { return getToken(PlanAParser.ASSIGN, 0); } + public TerminalNode AADD() { return getToken(PlanAParser.AADD, 0); } + public TerminalNode ASUB() { return getToken(PlanAParser.ASUB, 0); } + public TerminalNode AMUL() { return getToken(PlanAParser.AMUL, 0); } + public TerminalNode ADIV() { return getToken(PlanAParser.ADIV, 0); } + public TerminalNode AREM() { return getToken(PlanAParser.AREM, 0); } + public TerminalNode AAND() { return getToken(PlanAParser.AAND, 0); } + public TerminalNode AXOR() { return getToken(PlanAParser.AXOR, 0); } + public TerminalNode AOR() { return getToken(PlanAParser.AOR, 0); } + public TerminalNode ALSH() { return getToken(PlanAParser.ALSH, 0); } + public TerminalNode ARSH() { return getToken(PlanAParser.ARSH, 0); } + public TerminalNode AUSH() { return getToken(PlanAParser.AUSH, 0); } + public AssignmentContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitAssignment(this); + else return visitor.visitChildren(this); + } + } + public static class FalseContext extends ExpressionContext { + public TerminalNode FALSE() { return getToken(PlanAParser.FALSE, 0); } + public FalseContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitFalse(this); + else return visitor.visitChildren(this); + } + } + public static class NumericContext extends ExpressionContext { + public TerminalNode OCTAL() { return getToken(PlanAParser.OCTAL, 0); } + public TerminalNode HEX() { return getToken(PlanAParser.HEX, 0); } + public TerminalNode INTEGER() { return getToken(PlanAParser.INTEGER, 0); } + public TerminalNode DECIMAL() { return getToken(PlanAParser.DECIMAL, 0); } + public NumericContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitNumeric(this); + else return visitor.visitChildren(this); + } + } + public static class UnaryContext extends ExpressionContext { + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode BOOLNOT() { return getToken(PlanAParser.BOOLNOT, 0); } + public TerminalNode BWNOT() { return getToken(PlanAParser.BWNOT, 0); } + public TerminalNode ADD() { return getToken(PlanAParser.ADD, 0); } + public TerminalNode SUB() { return getToken(PlanAParser.SUB, 0); } + public UnaryContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitUnary(this); + else return visitor.visitChildren(this); + } + } + public static class PrecedenceContext extends ExpressionContext { + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public PrecedenceContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitPrecedence(this); + else return visitor.visitChildren(this); + } + } + public static class PreincContext extends ExpressionContext { + public IncrementContext increment() { + return getRuleContext(IncrementContext.class,0); + } + public ExtstartContext extstart() { + return getRuleContext(ExtstartContext.class,0); + } + public PreincContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitPreinc(this); + else return visitor.visitChildren(this); + } + } + public static class PostincContext extends ExpressionContext { + public ExtstartContext extstart() { + return getRuleContext(ExtstartContext.class,0); + } + public IncrementContext increment() { + return getRuleContext(IncrementContext.class,0); + } + public PostincContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitPostinc(this); + else return visitor.visitChildren(this); + } + } + public static class CastContext extends ExpressionContext { + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public DecltypeContext decltype() { + return getRuleContext(DecltypeContext.class,0); + } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public CastContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitCast(this); + else return visitor.visitChildren(this); + } + } + public static class ExternalContext extends ExpressionContext { + public ExtstartContext extstart() { + return getRuleContext(ExtstartContext.class,0); + } + public ExternalContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExternal(this); + else return visitor.visitChildren(this); + } + } + public static class NullContext extends ExpressionContext { + public TerminalNode NULL() { return getToken(PlanAParser.NULL, 0); } + public NullContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitNull(this); + else return visitor.visitChildren(this); + } + } + public static class BinaryContext extends ExpressionContext { + public List expression() { + return getRuleContexts(ExpressionContext.class); + } + public ExpressionContext expression(int i) { + return getRuleContext(ExpressionContext.class,i); + } + public TerminalNode MUL() { return getToken(PlanAParser.MUL, 0); } + public TerminalNode DIV() { return getToken(PlanAParser.DIV, 0); } + public TerminalNode REM() { return getToken(PlanAParser.REM, 0); } + public TerminalNode ADD() { return getToken(PlanAParser.ADD, 0); } + public TerminalNode SUB() { return getToken(PlanAParser.SUB, 0); } + public TerminalNode LSH() { return getToken(PlanAParser.LSH, 0); } + public TerminalNode RSH() { return getToken(PlanAParser.RSH, 0); } + public TerminalNode USH() { return getToken(PlanAParser.USH, 0); } + public TerminalNode BWAND() { return getToken(PlanAParser.BWAND, 0); } + public TerminalNode BWXOR() { return getToken(PlanAParser.BWXOR, 0); } + public TerminalNode BWOR() { return getToken(PlanAParser.BWOR, 0); } + public BinaryContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitBinary(this); + else return visitor.visitChildren(this); + } + } + public static class CharContext extends ExpressionContext { + public TerminalNode CHAR() { return getToken(PlanAParser.CHAR, 0); } + public CharContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitChar(this); + else return visitor.visitChildren(this); + } + } + public static class TrueContext extends ExpressionContext { + public TerminalNode TRUE() { return getToken(PlanAParser.TRUE, 0); } + public TrueContext(ExpressionContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitTrue(this); + else return visitor.visitChildren(this); + } + } + + public final ExpressionContext expression() throws RecognitionException { + return expression(0); + } + + private ExpressionContext expression(int _p) throws RecognitionException { + ParserRuleContext _parentctx = _ctx; + int _parentState = getState(); + ExpressionContext _localctx = new ExpressionContext(_ctx, _parentState); + ExpressionContext _prevctx = _localctx; + int _startState = 18; + enterRecursionRule(_localctx, 18, RULE_expression, _p); + int _la; + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + setState(207); + switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { + case 1: + { + _localctx = new UnaryContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + + setState(180); + _la = _input.LA(1); + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(181); + expression(14); + } + break; + case 2: + { + _localctx = new CastContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(182); + match(LP); + setState(183); + decltype(); + setState(184); + match(RP); + setState(185); + expression(13); + } + break; + case 3: + { + _localctx = new AssignmentContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(187); + extstart(); + setState(188); + _la = _input.LA(1); + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(189); + expression(1); + } + break; + case 4: + { + _localctx = new PrecedenceContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(191); + match(LP); + setState(192); + expression(0); + setState(193); + match(RP); + } + break; + case 5: + { + _localctx = new NumericContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(195); + _la = _input.LA(1); + if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)))) != 0)) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + } + break; + case 6: + { + _localctx = new CharContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(196); + match(CHAR); + } + break; + case 7: + { + _localctx = new TrueContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(197); + match(TRUE); + } + break; + case 8: + { + _localctx = new FalseContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(198); + match(FALSE); + } + break; + case 9: + { + _localctx = new NullContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(199); + match(NULL); + } + break; + case 10: + { + _localctx = new PostincContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(200); + extstart(); + setState(201); + increment(); + } + break; + case 11: + { + _localctx = new PreincContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(203); + increment(); + setState(204); + extstart(); + } + break; + case 12: + { + _localctx = new ExternalContext(_localctx); + _ctx = _localctx; + _prevctx = _localctx; + setState(206); + extstart(); + } + break; + } + _ctx.stop = _input.LT(-1); + setState(247); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,24,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + if ( _parseListeners!=null ) triggerExitRuleEvent(); + _prevctx = _localctx; + { + setState(245); + switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) { + case 1: + { + _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(209); + if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)"); + setState(210); + _la = _input.LA(1); + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(211); + expression(13); + } + break; + case 2: + { + _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(212); + if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)"); + setState(213); + _la = _input.LA(1); + if ( !(_la==ADD || _la==SUB) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(214); + expression(12); + } + break; + case 3: + { + _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(215); + if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)"); + setState(216); + _la = _input.LA(1); + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(217); + expression(11); + } + break; + case 4: + { + _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(218); + if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)"); + setState(219); + _la = _input.LA(1); + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(220); + expression(10); + } + break; + case 5: + { + _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(221); + if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)"); + setState(222); + _la = _input.LA(1); + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(223); + expression(9); + } + break; + case 6: + { + _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(224); + if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)"); + setState(225); + match(BWAND); + setState(226); + expression(8); + } + break; + case 7: + { + _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(227); + if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)"); + setState(228); + match(BWXOR); + setState(229); + expression(7); + } + break; + case 8: + { + _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(230); + if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); + setState(231); + match(BWOR); + setState(232); + expression(6); + } + break; + case 9: + { + _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(233); + if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); + setState(234); + match(BOOLAND); + setState(235); + expression(5); + } + break; + case 10: + { + _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(236); + if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)"); + setState(237); + match(BOOLOR); + setState(238); + expression(4); + } + break; + case 11: + { + _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState)); + pushNewRecursionContext(_localctx, _startState, RULE_expression); + setState(239); + if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); + setState(240); + match(COND); + setState(241); + expression(0); + setState(242); + match(COLON); + setState(243); + expression(2); + } + break; + } + } + } + setState(249); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,24,_ctx); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + unrollRecursionContexts(_parentctx); + } + return _localctx; + } + + public static class ExtstartContext extends ParserRuleContext { + public ExtprecContext extprec() { + return getRuleContext(ExtprecContext.class,0); + } + public ExtcastContext extcast() { + return getRuleContext(ExtcastContext.class,0); + } + public ExttypeContext exttype() { + return getRuleContext(ExttypeContext.class,0); + } + public ExtvarContext extvar() { + return getRuleContext(ExtvarContext.class,0); + } + public ExtnewContext extnew() { + return getRuleContext(ExtnewContext.class,0); + } + public ExtstringContext extstring() { + return getRuleContext(ExtstringContext.class,0); + } + public ExtstartContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extstart; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtstart(this); + else return visitor.visitChildren(this); + } + } + + public final ExtstartContext extstart() throws RecognitionException { + ExtstartContext _localctx = new ExtstartContext(_ctx, getState()); + enterRule(_localctx, 20, RULE_extstart); + try { + setState(256); + switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(250); + extprec(); + } + break; + case 2: + enterOuterAlt(_localctx, 2); + { + setState(251); + extcast(); + } + break; + case 3: + enterOuterAlt(_localctx, 3); + { + setState(252); + exttype(); + } + break; + case 4: + enterOuterAlt(_localctx, 4); + { + setState(253); + extvar(); + } + break; + case 5: + enterOuterAlt(_localctx, 5); + { + setState(254); + extnew(); + } + break; + case 6: + enterOuterAlt(_localctx, 6); + { + setState(255); + extstring(); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtprecContext extends ParserRuleContext { + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public ExtprecContext extprec() { + return getRuleContext(ExtprecContext.class,0); + } + public ExtcastContext extcast() { + return getRuleContext(ExtcastContext.class,0); + } + public ExttypeContext exttype() { + return getRuleContext(ExttypeContext.class,0); + } + public ExtvarContext extvar() { + return getRuleContext(ExtvarContext.class,0); + } + public ExtnewContext extnew() { + return getRuleContext(ExtnewContext.class,0); + } + public ExtstringContext extstring() { + return getRuleContext(ExtstringContext.class,0); + } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExtbraceContext extbrace() { + return getRuleContext(ExtbraceContext.class,0); + } + public ExtprecContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extprec; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtprec(this); + else return visitor.visitChildren(this); + } + } + + public final ExtprecContext extprec() throws RecognitionException { + ExtprecContext _localctx = new ExtprecContext(_ctx, getState()); + enterRule(_localctx, 22, RULE_extprec); + try { + enterOuterAlt(_localctx, 1); + { + setState(258); + match(LP); + setState(265); + switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { + case 1: + { + setState(259); + extprec(); + } + break; + case 2: + { + setState(260); + extcast(); + } + break; + case 3: + { + setState(261); + exttype(); + } + break; + case 4: + { + setState(262); + extvar(); + } + break; + case 5: + { + setState(263); + extnew(); + } + break; + case 6: + { + setState(264); + extstring(); + } + break; + } + setState(267); + match(RP); + setState(270); + switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) { + case 1: + { + setState(268); + extdot(); + } + break; + case 2: + { + setState(269); + extbrace(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtcastContext extends ParserRuleContext { + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public DecltypeContext decltype() { + return getRuleContext(DecltypeContext.class,0); + } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public ExtprecContext extprec() { + return getRuleContext(ExtprecContext.class,0); + } + public ExtcastContext extcast() { + return getRuleContext(ExtcastContext.class,0); + } + public ExttypeContext exttype() { + return getRuleContext(ExttypeContext.class,0); + } + public ExtvarContext extvar() { + return getRuleContext(ExtvarContext.class,0); + } + public ExtnewContext extnew() { + return getRuleContext(ExtnewContext.class,0); + } + public ExtstringContext extstring() { + return getRuleContext(ExtstringContext.class,0); + } + public ExtcastContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extcast; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtcast(this); + else return visitor.visitChildren(this); + } + } + + public final ExtcastContext extcast() throws RecognitionException { + ExtcastContext _localctx = new ExtcastContext(_ctx, getState()); + enterRule(_localctx, 24, RULE_extcast); + try { + enterOuterAlt(_localctx, 1); + { + setState(272); + match(LP); + setState(273); + decltype(); + setState(274); + match(RP); + setState(281); + switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { + case 1: + { + setState(275); + extprec(); + } + break; + case 2: + { + setState(276); + extcast(); + } + break; + case 3: + { + setState(277); + exttype(); + } + break; + case 4: + { + setState(278); + extvar(); + } + break; + case 5: + { + setState(279); + extnew(); + } + break; + case 6: + { + setState(280); + extstring(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtbraceContext extends ParserRuleContext { + public TerminalNode LBRACE() { return getToken(PlanAParser.LBRACE, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode RBRACE() { return getToken(PlanAParser.RBRACE, 0); } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExtbraceContext extbrace() { + return getRuleContext(ExtbraceContext.class,0); + } + public ExtbraceContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extbrace; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtbrace(this); + else return visitor.visitChildren(this); + } + } + + public final ExtbraceContext extbrace() throws RecognitionException { + ExtbraceContext _localctx = new ExtbraceContext(_ctx, getState()); + enterRule(_localctx, 26, RULE_extbrace); + try { + enterOuterAlt(_localctx, 1); + { + setState(283); + match(LBRACE); + setState(284); + expression(0); + setState(285); + match(RBRACE); + setState(288); + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + case 1: + { + setState(286); + extdot(); + } + break; + case 2: + { + setState(287); + extbrace(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtdotContext extends ParserRuleContext { + public TerminalNode DOT() { return getToken(PlanAParser.DOT, 0); } + public ExtcallContext extcall() { + return getRuleContext(ExtcallContext.class,0); + } + public ExtfieldContext extfield() { + return getRuleContext(ExtfieldContext.class,0); + } + public ExtdotContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extdot; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtdot(this); + else return visitor.visitChildren(this); + } + } + + public final ExtdotContext extdot() throws RecognitionException { + ExtdotContext _localctx = new ExtdotContext(_ctx, getState()); + enterRule(_localctx, 28, RULE_extdot); + try { + enterOuterAlt(_localctx, 1); + { + setState(290); + match(DOT); + setState(293); + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + case 1: + { + setState(291); + extcall(); + } + break; + case 2: + { + setState(292); + extfield(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExttypeContext extends ParserRuleContext { + public TerminalNode TYPE() { return getToken(PlanAParser.TYPE, 0); } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExttypeContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_exttype; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExttype(this); + else return visitor.visitChildren(this); + } + } + + public final ExttypeContext exttype() throws RecognitionException { + ExttypeContext _localctx = new ExttypeContext(_ctx, getState()); + enterRule(_localctx, 30, RULE_exttype); + try { + enterOuterAlt(_localctx, 1); + { + setState(295); + match(TYPE); + setState(296); + extdot(); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtcallContext extends ParserRuleContext { + public TerminalNode EXTID() { return getToken(PlanAParser.EXTID, 0); } + public ArgumentsContext arguments() { + return getRuleContext(ArgumentsContext.class,0); + } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExtbraceContext extbrace() { + return getRuleContext(ExtbraceContext.class,0); + } + public ExtcallContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extcall; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtcall(this); + else return visitor.visitChildren(this); + } + } + + public final ExtcallContext extcall() throws RecognitionException { + ExtcallContext _localctx = new ExtcallContext(_ctx, getState()); + enterRule(_localctx, 32, RULE_extcall); + try { + enterOuterAlt(_localctx, 1); + { + setState(298); + match(EXTID); + setState(299); + arguments(); + setState(302); + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { + case 1: + { + setState(300); + extdot(); + } + break; + case 2: + { + setState(301); + extbrace(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtvarContext extends ParserRuleContext { + public TerminalNode ID() { return getToken(PlanAParser.ID, 0); } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExtbraceContext extbrace() { + return getRuleContext(ExtbraceContext.class,0); + } + public ExtvarContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extvar; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtvar(this); + else return visitor.visitChildren(this); + } + } + + public final ExtvarContext extvar() throws RecognitionException { + ExtvarContext _localctx = new ExtvarContext(_ctx, getState()); + enterRule(_localctx, 34, RULE_extvar); + try { + enterOuterAlt(_localctx, 1); + { + setState(304); + match(ID); + setState(307); + switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { + case 1: + { + setState(305); + extdot(); + } + break; + case 2: + { + setState(306); + extbrace(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtfieldContext extends ParserRuleContext { + public TerminalNode EXTID() { return getToken(PlanAParser.EXTID, 0); } + public TerminalNode EXTINTEGER() { return getToken(PlanAParser.EXTINTEGER, 0); } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExtbraceContext extbrace() { + return getRuleContext(ExtbraceContext.class,0); + } + public ExtfieldContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extfield; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtfield(this); + else return visitor.visitChildren(this); + } + } + + public final ExtfieldContext extfield() throws RecognitionException { + ExtfieldContext _localctx = new ExtfieldContext(_ctx, getState()); + enterRule(_localctx, 36, RULE_extfield); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(309); + _la = _input.LA(1); + if ( !(_la==EXTINTEGER || _la==EXTID) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + setState(312); + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { + case 1: + { + setState(310); + extdot(); + } + break; + case 2: + { + setState(311); + extbrace(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtnewContext extends ParserRuleContext { + public TerminalNode NEW() { return getToken(PlanAParser.NEW, 0); } + public TerminalNode TYPE() { return getToken(PlanAParser.TYPE, 0); } + public ArgumentsContext arguments() { + return getRuleContext(ArgumentsContext.class,0); + } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExtbraceContext extbrace() { + return getRuleContext(ExtbraceContext.class,0); + } + public List LBRACE() { return getTokens(PlanAParser.LBRACE); } + public TerminalNode LBRACE(int i) { + return getToken(PlanAParser.LBRACE, i); + } + public List expression() { + return getRuleContexts(ExpressionContext.class); + } + public ExpressionContext expression(int i) { + return getRuleContext(ExpressionContext.class,i); + } + public List RBRACE() { return getTokens(PlanAParser.RBRACE); } + public TerminalNode RBRACE(int i) { + return getToken(PlanAParser.RBRACE, i); + } + public ExtnewContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extnew; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtnew(this); + else return visitor.visitChildren(this); + } + } + + public final ExtnewContext extnew() throws RecognitionException { + ExtnewContext _localctx = new ExtnewContext(_ctx, getState()); + enterRule(_localctx, 38, RULE_extnew); + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + setState(314); + match(NEW); + setState(315); + match(TYPE); + setState(332); + switch (_input.LA(1)) { + case LP: + { + { + setState(316); + arguments(); + setState(319); + switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + case 1: + { + setState(317); + extdot(); + } + break; + case 2: + { + setState(318); + extbrace(); + } + break; + } + } + } + break; + case LBRACE: + { + { + setState(325); + _errHandler.sync(this); + _alt = 1; + do { + switch (_alt) { + case 1: + { + { + setState(321); + match(LBRACE); + setState(322); + expression(0); + setState(323); + match(RBRACE); + } + } + break; + default: + throw new NoViableAltException(this); + } + setState(327); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); + setState(330); + switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) { + case 1: + { + setState(329); + extdot(); + } + break; + } + } + } + break; + default: + throw new NoViableAltException(this); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtstringContext extends ParserRuleContext { + public TerminalNode STRING() { return getToken(PlanAParser.STRING, 0); } + public ExtdotContext extdot() { + return getRuleContext(ExtdotContext.class,0); + } + public ExtbraceContext extbrace() { + return getRuleContext(ExtbraceContext.class,0); + } + public ExtstringContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extstring; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitExtstring(this); + else return visitor.visitChildren(this); + } + } + + public final ExtstringContext extstring() throws RecognitionException { + ExtstringContext _localctx = new ExtstringContext(_ctx, getState()); + enterRule(_localctx, 40, RULE_extstring); + try { + enterOuterAlt(_localctx, 1); + { + setState(334); + match(STRING); + setState(337); + switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { + case 1: + { + setState(335); + extdot(); + } + break; + case 2: + { + setState(336); + extbrace(); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ArgumentsContext extends ParserRuleContext { + public TerminalNode LP() { return getToken(PlanAParser.LP, 0); } + public TerminalNode RP() { return getToken(PlanAParser.RP, 0); } + public List expression() { + return getRuleContexts(ExpressionContext.class); + } + public ExpressionContext expression(int i) { + return getRuleContext(ExpressionContext.class,i); + } + public List COMMA() { return getTokens(PlanAParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(PlanAParser.COMMA, i); + } + public ArgumentsContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_arguments; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitArguments(this); + else return visitor.visitChildren(this); + } + } + + public final ArgumentsContext arguments() throws RecognitionException { + ArgumentsContext _localctx = new ArgumentsContext(_ctx, getState()); + enterRule(_localctx, 42, RULE_arguments); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + { + setState(339); + match(LP); + setState(348); + _la = _input.LA(1); + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (TYPE - 64)) | (1L << (ID - 64)))) != 0)) { + { + setState(340); + expression(0); + setState(345); + _errHandler.sync(this); + _la = _input.LA(1); + while (_la==COMMA) { + { + { + setState(341); + match(COMMA); + setState(342); + expression(0); + } + } + setState(347); + _errHandler.sync(this); + _la = _input.LA(1); + } + } + } + + setState(350); + match(RP); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class IncrementContext extends ParserRuleContext { + public TerminalNode INCR() { return getToken(PlanAParser.INCR, 0); } + public TerminalNode DECR() { return getToken(PlanAParser.DECR, 0); } + public IncrementContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_increment; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PlanAParserVisitor ) return ((PlanAParserVisitor)visitor).visitIncrement(this); + else return visitor.visitChildren(this); + } + } + + public final IncrementContext increment() throws RecognitionException { + IncrementContext _localctx = new IncrementContext(_ctx, getState()); + enterRule(_localctx, 44, RULE_increment); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(352); + _la = _input.LA(1); + if ( !(_la==INCR || _la==DECR) ) { + _errHandler.recoverInline(this); + } else { + consume(); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { + switch (ruleIndex) { + case 9: + return expression_sempred((ExpressionContext)_localctx, predIndex); + } + return true; + } + private boolean expression_sempred(ExpressionContext _localctx, int predIndex) { + switch (predIndex) { + case 0: + return precpred(_ctx, 12); + case 1: + return precpred(_ctx, 11); + case 2: + return precpred(_ctx, 10); + case 3: + return precpred(_ctx, 9); + case 4: + return precpred(_ctx, 8); + case 5: + return precpred(_ctx, 7); + case 6: + return precpred(_ctx, 6); + case 7: + return precpred(_ctx, 5); + case 8: + return precpred(_ctx, 4); + case 9: + return precpred(_ctx, 3); + case 10: + return precpred(_ctx, 2); + } + return true; + } + + public static final String _serializedATN = + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3N\u0165\4\2\t\2\4"+ + "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+ + "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+ + "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\3\2\6\2\62"+ + "\n\2\r\2\16\2\63\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3?\n\3\3\3\3\3"+ + "\3\3\3\3\3\3\3\3\5\3G\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3P\n\3\3\3\3\3"+ + "\3\3\5\3U\n\3\3\3\3\3\5\3Y\n\3\3\3\3\3\5\3]\n\3\3\3\3\3\3\3\5\3b\n\3\3"+ + "\3\3\3\5\3f\n\3\3\3\3\3\5\3j\n\3\3\3\3\3\5\3n\n\3\3\3\3\3\3\3\5\3s\n\3"+ + "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\6\3~\n\3\r\3\16\3\177\3\3\3\3\3\3"+ + "\5\3\u0085\n\3\3\3\3\3\5\3\u0089\n\3\5\3\u008b\n\3\3\4\3\4\7\4\u008f\n"+ + "\4\f\4\16\4\u0092\13\4\3\4\3\4\5\4\u0096\n\4\3\5\3\5\3\6\3\6\5\6\u009c"+ + "\n\6\3\7\3\7\3\b\3\b\3\b\3\b\7\b\u00a4\n\b\f\b\16\b\u00a7\13\b\3\t\3\t"+ + "\3\t\7\t\u00ac\n\t\f\t\16\t\u00af\13\t\3\n\3\n\3\n\5\n\u00b4\n\n\3\13"+ + "\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13"+ + "\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\5\13"+ + "\u00d2\n\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13"+ + "\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13"+ + "\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\7\13\u00f8\n\13\f\13"+ + "\16\13\u00fb\13\13\3\f\3\f\3\f\3\f\3\f\3\f\5\f\u0103\n\f\3\r\3\r\3\r\3"+ + "\r\3\r\3\r\3\r\5\r\u010c\n\r\3\r\3\r\3\r\5\r\u0111\n\r\3\16\3\16\3\16"+ + "\3\16\3\16\3\16\3\16\3\16\3\16\5\16\u011c\n\16\3\17\3\17\3\17\3\17\3\17"+ + "\5\17\u0123\n\17\3\20\3\20\3\20\5\20\u0128\n\20\3\21\3\21\3\21\3\22\3"+ + "\22\3\22\3\22\5\22\u0131\n\22\3\23\3\23\3\23\5\23\u0136\n\23\3\24\3\24"+ + "\3\24\5\24\u013b\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0142\n\25\3\25\3"+ + "\25\3\25\3\25\6\25\u0148\n\25\r\25\16\25\u0149\3\25\5\25\u014d\n\25\5"+ + "\25\u014f\n\25\3\26\3\26\3\26\5\26\u0154\n\26\3\27\3\27\3\27\3\27\7\27"+ + "\u015a\n\27\f\27\16\27\u015d\13\27\5\27\u015f\n\27\3\27\3\27\3\30\3\30"+ + "\3\30\2\3\24\31\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\2\f\4"+ + "\2\32\33\37 \3\2\65@\3\2BE\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2MN"+ + "\3\2\63\64\u01a5\2\61\3\2\2\2\4\u008a\3\2\2\2\6\u0095\3\2\2\2\b\u0097"+ + "\3\2\2\2\n\u009b\3\2\2\2\f\u009d\3\2\2\2\16\u009f\3\2\2\2\20\u00a8\3\2"+ + "\2\2\22\u00b0\3\2\2\2\24\u00d1\3\2\2\2\26\u0102\3\2\2\2\30\u0104\3\2\2"+ + "\2\32\u0112\3\2\2\2\34\u011d\3\2\2\2\36\u0124\3\2\2\2 \u0129\3\2\2\2\""+ + "\u012c\3\2\2\2$\u0132\3\2\2\2&\u0137\3\2\2\2(\u013c\3\2\2\2*\u0150\3\2"+ + "\2\2,\u0155\3\2\2\2.\u0162\3\2\2\2\60\62\5\4\3\2\61\60\3\2\2\2\62\63\3"+ + "\2\2\2\63\61\3\2\2\2\63\64\3\2\2\2\64\65\3\2\2\2\65\66\7\2\2\3\66\3\3"+ + "\2\2\2\678\7\16\2\289\7\t\2\29:\5\24\13\2:;\7\n\2\2;>\5\6\4\2<=\7\17\2"+ + "\2=?\5\6\4\2><\3\2\2\2>?\3\2\2\2?\u008b\3\2\2\2@A\7\20\2\2AB\7\t\2\2B"+ + "C\5\24\13\2CF\7\n\2\2DG\5\6\4\2EG\5\b\5\2FD\3\2\2\2FE\3\2\2\2G\u008b\3"+ + "\2\2\2HI\7\21\2\2IJ\5\6\4\2JK\7\20\2\2KL\7\t\2\2LM\5\24\13\2MO\7\n\2\2"+ + "NP\7\r\2\2ON\3\2\2\2OP\3\2\2\2P\u008b\3\2\2\2QR\7\22\2\2RT\7\t\2\2SU\5"+ + "\n\6\2TS\3\2\2\2TU\3\2\2\2UV\3\2\2\2VX\7\r\2\2WY\5\24\13\2XW\3\2\2\2X"+ + "Y\3\2\2\2YZ\3\2\2\2Z\\\7\r\2\2[]\5\f\7\2\\[\3\2\2\2\\]\3\2\2\2]^\3\2\2"+ + "\2^a\7\n\2\2_b\5\6\4\2`b\5\b\5\2a_\3\2\2\2a`\3\2\2\2b\u008b\3\2\2\2ce"+ + "\5\16\b\2df\7\r\2\2ed\3\2\2\2ef\3\2\2\2f\u008b\3\2\2\2gi\7\23\2\2hj\7"+ + "\r\2\2ih\3\2\2\2ij\3\2\2\2j\u008b\3\2\2\2km\7\24\2\2ln\7\r\2\2ml\3\2\2"+ + "\2mn\3\2\2\2n\u008b\3\2\2\2op\7\25\2\2pr\5\24\13\2qs\7\r\2\2rq\3\2\2\2"+ + "rs\3\2\2\2s\u008b\3\2\2\2tu\7\27\2\2u}\5\6\4\2vw\7\30\2\2wx\7\t\2\2xy"+ + "\7K\2\2yz\7L\2\2z{\3\2\2\2{|\7\n\2\2|~\5\6\4\2}v\3\2\2\2~\177\3\2\2\2"+ + "\177}\3\2\2\2\177\u0080\3\2\2\2\u0080\u008b\3\2\2\2\u0081\u0082\7\31\2"+ + "\2\u0082\u0084\5\24\13\2\u0083\u0085\7\r\2\2\u0084\u0083\3\2\2\2\u0084"+ + "\u0085\3\2\2\2\u0085\u008b\3\2\2\2\u0086\u0088\5\24\13\2\u0087\u0089\7"+ + "\r\2\2\u0088\u0087\3\2\2\2\u0088\u0089\3\2\2\2\u0089\u008b\3\2\2\2\u008a"+ + "\67\3\2\2\2\u008a@\3\2\2\2\u008aH\3\2\2\2\u008aQ\3\2\2\2\u008ac\3\2\2"+ + "\2\u008ag\3\2\2\2\u008ak\3\2\2\2\u008ao\3\2\2\2\u008at\3\2\2\2\u008a\u0081"+ + "\3\2\2\2\u008a\u0086\3\2\2\2\u008b\5\3\2\2\2\u008c\u0090\7\5\2\2\u008d"+ + "\u008f\5\4\3\2\u008e\u008d\3\2\2\2\u008f\u0092\3\2\2\2\u0090\u008e\3\2"+ + "\2\2\u0090\u0091\3\2\2\2\u0091\u0093\3\2\2\2\u0092\u0090\3\2\2\2\u0093"+ + "\u0096\7\6\2\2\u0094\u0096\5\4\3\2\u0095\u008c\3\2\2\2\u0095\u0094\3\2"+ + "\2\2\u0096\7\3\2\2\2\u0097\u0098\7\r\2\2\u0098\t\3\2\2\2\u0099\u009c\5"+ + "\16\b\2\u009a\u009c\5\24\13\2\u009b\u0099\3\2\2\2\u009b\u009a\3\2\2\2"+ + "\u009c\13\3\2\2\2\u009d\u009e\5\24\13\2\u009e\r\3\2\2\2\u009f\u00a0\5"+ + "\20\t\2\u00a0\u00a5\5\22\n\2\u00a1\u00a2\7\f\2\2\u00a2\u00a4\5\22\n\2"+ + "\u00a3\u00a1\3\2\2\2\u00a4\u00a7\3\2\2\2\u00a5\u00a3\3\2\2\2\u00a5\u00a6"+ + "\3\2\2\2\u00a6\17\3\2\2\2\u00a7\u00a5\3\2\2\2\u00a8\u00ad\7K\2\2\u00a9"+ + "\u00aa\7\7\2\2\u00aa\u00ac\7\b\2\2\u00ab\u00a9\3\2\2\2\u00ac\u00af\3\2"+ + "\2\2\u00ad\u00ab\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\21\3\2\2\2\u00af\u00ad"+ + "\3\2\2\2\u00b0\u00b3\7L\2\2\u00b1\u00b2\7\65\2\2\u00b2\u00b4\5\24\13\2"+ + "\u00b3\u00b1\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4\23\3\2\2\2\u00b5\u00b6"+ + "\b\13\1\2\u00b6\u00b7\t\2\2\2\u00b7\u00d2\5\24\13\20\u00b8\u00b9\7\t\2"+ + "\2\u00b9\u00ba\5\20\t\2\u00ba\u00bb\7\n\2\2\u00bb\u00bc\5\24\13\17\u00bc"+ + "\u00d2\3\2\2\2\u00bd\u00be\5\26\f\2\u00be\u00bf\t\3\2\2\u00bf\u00c0\5"+ + "\24\13\3\u00c0\u00d2\3\2\2\2\u00c1\u00c2\7\t\2\2\u00c2\u00c3\5\24\13\2"+ + "\u00c3\u00c4\7\n\2\2\u00c4\u00d2\3\2\2\2\u00c5\u00d2\t\4\2\2\u00c6\u00d2"+ + "\7G\2\2\u00c7\u00d2\7H\2\2\u00c8\u00d2\7I\2\2\u00c9\u00d2\7J\2\2\u00ca"+ + "\u00cb\5\26\f\2\u00cb\u00cc\5.\30\2\u00cc\u00d2\3\2\2\2\u00cd\u00ce\5"+ + ".\30\2\u00ce\u00cf\5\26\f\2\u00cf\u00d2\3\2\2\2\u00d0\u00d2\5\26\f\2\u00d1"+ + "\u00b5\3\2\2\2\u00d1\u00b8\3\2\2\2\u00d1\u00bd\3\2\2\2\u00d1\u00c1\3\2"+ + "\2\2\u00d1\u00c5\3\2\2\2\u00d1\u00c6\3\2\2\2\u00d1\u00c7\3\2\2\2\u00d1"+ + "\u00c8\3\2\2\2\u00d1\u00c9\3\2\2\2\u00d1\u00ca\3\2\2\2\u00d1\u00cd\3\2"+ + "\2\2\u00d1\u00d0\3\2\2\2\u00d2\u00f9\3\2\2\2\u00d3\u00d4\f\16\2\2\u00d4"+ + "\u00d5\t\5\2\2\u00d5\u00f8\5\24\13\17\u00d6\u00d7\f\r\2\2\u00d7\u00d8"+ + "\t\6\2\2\u00d8\u00f8\5\24\13\16\u00d9\u00da\f\f\2\2\u00da\u00db\t\7\2"+ + "\2\u00db\u00f8\5\24\13\r\u00dc\u00dd\f\13\2\2\u00dd\u00de\t\b\2\2\u00de"+ + "\u00f8\5\24\13\f\u00df\u00e0\f\n\2\2\u00e0\u00e1\t\t\2\2\u00e1\u00f8\5"+ + "\24\13\13\u00e2\u00e3\f\t\2\2\u00e3\u00e4\7,\2\2\u00e4\u00f8\5\24\13\n"+ + "\u00e5\u00e6\f\b\2\2\u00e6\u00e7\7-\2\2\u00e7\u00f8\5\24\13\t\u00e8\u00e9"+ + "\f\7\2\2\u00e9\u00ea\7.\2\2\u00ea\u00f8\5\24\13\b\u00eb\u00ec\f\6\2\2"+ + "\u00ec\u00ed\7/\2\2\u00ed\u00f8\5\24\13\7\u00ee\u00ef\f\5\2\2\u00ef\u00f0"+ + "\7\60\2\2\u00f0\u00f8\5\24\13\6\u00f1\u00f2\f\4\2\2\u00f2\u00f3\7\61\2"+ + "\2\u00f3\u00f4\5\24\13\2\u00f4\u00f5\7\62\2\2\u00f5\u00f6\5\24\13\4\u00f6"+ + "\u00f8\3\2\2\2\u00f7\u00d3\3\2\2\2\u00f7\u00d6\3\2\2\2\u00f7\u00d9\3\2"+ + "\2\2\u00f7\u00dc\3\2\2\2\u00f7\u00df\3\2\2\2\u00f7\u00e2\3\2\2\2\u00f7"+ + "\u00e5\3\2\2\2\u00f7\u00e8\3\2\2\2\u00f7\u00eb\3\2\2\2\u00f7\u00ee\3\2"+ + "\2\2\u00f7\u00f1\3\2\2\2\u00f8\u00fb\3\2\2\2\u00f9\u00f7\3\2\2\2\u00f9"+ + "\u00fa\3\2\2\2\u00fa\25\3\2\2\2\u00fb\u00f9\3\2\2\2\u00fc\u0103\5\30\r"+ + "\2\u00fd\u0103\5\32\16\2\u00fe\u0103\5 \21\2\u00ff\u0103\5$\23\2\u0100"+ + "\u0103\5(\25\2\u0101\u0103\5*\26\2\u0102\u00fc\3\2\2\2\u0102\u00fd\3\2"+ + "\2\2\u0102\u00fe\3\2\2\2\u0102\u00ff\3\2\2\2\u0102\u0100\3\2\2\2\u0102"+ + "\u0101\3\2\2\2\u0103\27\3\2\2\2\u0104\u010b\7\t\2\2\u0105\u010c\5\30\r"+ + "\2\u0106\u010c\5\32\16\2\u0107\u010c\5 \21\2\u0108\u010c\5$\23\2\u0109"+ + "\u010c\5(\25\2\u010a\u010c\5*\26\2\u010b\u0105\3\2\2\2\u010b\u0106\3\2"+ + "\2\2\u010b\u0107\3\2\2\2\u010b\u0108\3\2\2\2\u010b\u0109\3\2\2\2\u010b"+ + "\u010a\3\2\2\2\u010c\u010d\3\2\2\2\u010d\u0110\7\n\2\2\u010e\u0111\5\36"+ + "\20\2\u010f\u0111\5\34\17\2\u0110\u010e\3\2\2\2\u0110\u010f\3\2\2\2\u0110"+ + "\u0111\3\2\2\2\u0111\31\3\2\2\2\u0112\u0113\7\t\2\2\u0113\u0114\5\20\t"+ + "\2\u0114\u011b\7\n\2\2\u0115\u011c\5\30\r\2\u0116\u011c\5\32\16\2\u0117"+ + "\u011c\5 \21\2\u0118\u011c\5$\23\2\u0119\u011c\5(\25\2\u011a\u011c\5*"+ + "\26\2\u011b\u0115\3\2\2\2\u011b\u0116\3\2\2\2\u011b\u0117\3\2\2\2\u011b"+ + "\u0118\3\2\2\2\u011b\u0119\3\2\2\2\u011b\u011a\3\2\2\2\u011c\33\3\2\2"+ + "\2\u011d\u011e\7\7\2\2\u011e\u011f\5\24\13\2\u011f\u0122\7\b\2\2\u0120"+ + "\u0123\5\36\20\2\u0121\u0123\5\34\17\2\u0122\u0120\3\2\2\2\u0122\u0121"+ + "\3\2\2\2\u0122\u0123\3\2\2\2\u0123\35\3\2\2\2\u0124\u0127\7\13\2\2\u0125"+ + "\u0128\5\"\22\2\u0126\u0128\5&\24\2\u0127\u0125\3\2\2\2\u0127\u0126\3"+ + "\2\2\2\u0128\37\3\2\2\2\u0129\u012a\7K\2\2\u012a\u012b\5\36\20\2\u012b"+ + "!\3\2\2\2\u012c\u012d\7N\2\2\u012d\u0130\5,\27\2\u012e\u0131\5\36\20\2"+ + "\u012f\u0131\5\34\17\2\u0130\u012e\3\2\2\2\u0130\u012f\3\2\2\2\u0130\u0131"+ + "\3\2\2\2\u0131#\3\2\2\2\u0132\u0135\7L\2\2\u0133\u0136\5\36\20\2\u0134"+ + "\u0136\5\34\17\2\u0135\u0133\3\2\2\2\u0135\u0134\3\2\2\2\u0135\u0136\3"+ + "\2\2\2\u0136%\3\2\2\2\u0137\u013a\t\n\2\2\u0138\u013b\5\36\20\2\u0139"+ + "\u013b\5\34\17\2\u013a\u0138\3\2\2\2\u013a\u0139\3\2\2\2\u013a\u013b\3"+ + "\2\2\2\u013b\'\3\2\2\2\u013c\u013d\7\26\2\2\u013d\u014e\7K\2\2\u013e\u0141"+ + "\5,\27\2\u013f\u0142\5\36\20\2\u0140\u0142\5\34\17\2\u0141\u013f\3\2\2"+ + "\2\u0141\u0140\3\2\2\2\u0141\u0142\3\2\2\2\u0142\u014f\3\2\2\2\u0143\u0144"+ + "\7\7\2\2\u0144\u0145\5\24\13\2\u0145\u0146\7\b\2\2\u0146\u0148\3\2\2\2"+ + "\u0147\u0143\3\2\2\2\u0148\u0149\3\2\2\2\u0149\u0147\3\2\2\2\u0149\u014a"+ + "\3\2\2\2\u014a\u014c\3\2\2\2\u014b\u014d\5\36\20\2\u014c\u014b\3\2\2\2"+ + "\u014c\u014d\3\2\2\2\u014d\u014f\3\2\2\2\u014e\u013e\3\2\2\2\u014e\u0147"+ + "\3\2\2\2\u014f)\3\2\2\2\u0150\u0153\7F\2\2\u0151\u0154\5\36\20\2\u0152"+ + "\u0154\5\34\17\2\u0153\u0151\3\2\2\2\u0153\u0152\3\2\2\2\u0153\u0154\3"+ + "\2\2\2\u0154+\3\2\2\2\u0155\u015e\7\t\2\2\u0156\u015b\5\24\13\2\u0157"+ + "\u0158\7\f\2\2\u0158\u015a\5\24\13\2\u0159\u0157\3\2\2\2\u015a\u015d\3"+ + "\2\2\2\u015b\u0159\3\2\2\2\u015b\u015c\3\2\2\2\u015c\u015f\3\2\2\2\u015d"+ + "\u015b\3\2\2\2\u015e\u0156\3\2\2\2\u015e\u015f\3\2\2\2\u015f\u0160\3\2"+ + "\2\2\u0160\u0161\7\n\2\2\u0161-\3\2\2\2\u0162\u0163\t\13\2\2\u0163/\3"+ + "\2\2\2+\63>FOTX\\aeimr\177\u0084\u0088\u008a\u0090\u0095\u009b\u00a5\u00ad"+ + "\u00b3\u00d1\u00f7\u00f9\u0102\u010b\u0110\u011b\u0122\u0127\u0130\u0135"+ + "\u013a\u0141\u0149\u014c\u014e\u0153\u015b\u015e"; + public static final ATN _ATN = + new ATNDeserializer().deserialize(_serializedATN.toCharArray()); + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) { + _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i); + } + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParserBaseVisitor.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParserBaseVisitor.java new file mode 100644 index 00000000000..d731b57676b --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParserBaseVisitor.java @@ -0,0 +1,357 @@ +// ANTLR GENERATED CODE: DO NOT EDIT +package org.elasticsearch.plan.a; +import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor; + +/** + * This class provides an empty implementation of {@link PlanAParserVisitor}, + * which can be extended to create a visitor which only needs to handle a subset + * of the available methods. + * + * @param The return type of the visit operation. Use {@link Void} for + * operations with no return type. + */ +class PlanAParserBaseVisitor extends AbstractParseTreeVisitor implements PlanAParserVisitor { + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitSource(PlanAParser.SourceContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitIf(PlanAParser.IfContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitWhile(PlanAParser.WhileContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitDo(PlanAParser.DoContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitFor(PlanAParser.ForContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitDecl(PlanAParser.DeclContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitContinue(PlanAParser.ContinueContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitBreak(PlanAParser.BreakContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitReturn(PlanAParser.ReturnContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitTry(PlanAParser.TryContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitThrow(PlanAParser.ThrowContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExpr(PlanAParser.ExprContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitMultiple(PlanAParser.MultipleContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitSingle(PlanAParser.SingleContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitEmpty(PlanAParser.EmptyContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitInitializer(PlanAParser.InitializerContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitAfterthought(PlanAParser.AfterthoughtContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitDeclaration(PlanAParser.DeclarationContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitDecltype(PlanAParser.DecltypeContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitDeclvar(PlanAParser.DeclvarContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitComp(PlanAParser.CompContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitBool(PlanAParser.BoolContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitConditional(PlanAParser.ConditionalContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitAssignment(PlanAParser.AssignmentContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitFalse(PlanAParser.FalseContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitNumeric(PlanAParser.NumericContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitUnary(PlanAParser.UnaryContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitPrecedence(PlanAParser.PrecedenceContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitPreinc(PlanAParser.PreincContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitPostinc(PlanAParser.PostincContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitCast(PlanAParser.CastContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExternal(PlanAParser.ExternalContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitNull(PlanAParser.NullContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitBinary(PlanAParser.BinaryContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitChar(PlanAParser.CharContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitTrue(PlanAParser.TrueContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtstart(PlanAParser.ExtstartContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtprec(PlanAParser.ExtprecContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtcast(PlanAParser.ExtcastContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtbrace(PlanAParser.ExtbraceContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtdot(PlanAParser.ExtdotContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExttype(PlanAParser.ExttypeContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtcall(PlanAParser.ExtcallContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtvar(PlanAParser.ExtvarContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtfield(PlanAParser.ExtfieldContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtnew(PlanAParser.ExtnewContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtstring(PlanAParser.ExtstringContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitArguments(PlanAParser.ArgumentsContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitIncrement(PlanAParser.IncrementContext ctx) { return visitChildren(ctx); } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParserVisitor.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParserVisitor.java new file mode 100644 index 00000000000..7470f3b6ad5 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAParserVisitor.java @@ -0,0 +1,336 @@ +// ANTLR GENERATED CODE: DO NOT EDIT +package org.elasticsearch.plan.a; +import org.antlr.v4.runtime.tree.ParseTreeVisitor; + +/** + * This interface defines a complete generic visitor for a parse tree produced + * by {@link PlanAParser}. + * + * @param The return type of the visit operation. Use {@link Void} for + * operations with no return type. + */ +interface PlanAParserVisitor extends ParseTreeVisitor { + /** + * Visit a parse tree produced by {@link PlanAParser#source}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitSource(PlanAParser.SourceContext ctx); + /** + * Visit a parse tree produced by the {@code if} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitIf(PlanAParser.IfContext ctx); + /** + * Visit a parse tree produced by the {@code while} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitWhile(PlanAParser.WhileContext ctx); + /** + * Visit a parse tree produced by the {@code do} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDo(PlanAParser.DoContext ctx); + /** + * Visit a parse tree produced by the {@code for} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFor(PlanAParser.ForContext ctx); + /** + * Visit a parse tree produced by the {@code decl} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDecl(PlanAParser.DeclContext ctx); + /** + * Visit a parse tree produced by the {@code continue} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitContinue(PlanAParser.ContinueContext ctx); + /** + * Visit a parse tree produced by the {@code break} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitBreak(PlanAParser.BreakContext ctx); + /** + * Visit a parse tree produced by the {@code return} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitReturn(PlanAParser.ReturnContext ctx); + /** + * Visit a parse tree produced by the {@code try} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitTry(PlanAParser.TryContext ctx); + /** + * Visit a parse tree produced by the {@code throw} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitThrow(PlanAParser.ThrowContext ctx); + /** + * Visit a parse tree produced by the {@code expr} + * labeled alternative in {@link PlanAParser#statement}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExpr(PlanAParser.ExprContext ctx); + /** + * Visit a parse tree produced by the {@code multiple} + * labeled alternative in {@link PlanAParser#block}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitMultiple(PlanAParser.MultipleContext ctx); + /** + * Visit a parse tree produced by the {@code single} + * labeled alternative in {@link PlanAParser#block}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitSingle(PlanAParser.SingleContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#empty}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitEmpty(PlanAParser.EmptyContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#initializer}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitInitializer(PlanAParser.InitializerContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#afterthought}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitAfterthought(PlanAParser.AfterthoughtContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#declaration}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDeclaration(PlanAParser.DeclarationContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#decltype}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDecltype(PlanAParser.DecltypeContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#declvar}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDeclvar(PlanAParser.DeclvarContext ctx); + /** + * Visit a parse tree produced by the {@code comp} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitComp(PlanAParser.CompContext ctx); + /** + * Visit a parse tree produced by the {@code bool} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitBool(PlanAParser.BoolContext ctx); + /** + * Visit a parse tree produced by the {@code conditional} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitConditional(PlanAParser.ConditionalContext ctx); + /** + * Visit a parse tree produced by the {@code assignment} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitAssignment(PlanAParser.AssignmentContext ctx); + /** + * Visit a parse tree produced by the {@code false} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFalse(PlanAParser.FalseContext ctx); + /** + * Visit a parse tree produced by the {@code numeric} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitNumeric(PlanAParser.NumericContext ctx); + /** + * Visit a parse tree produced by the {@code unary} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitUnary(PlanAParser.UnaryContext ctx); + /** + * Visit a parse tree produced by the {@code precedence} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitPrecedence(PlanAParser.PrecedenceContext ctx); + /** + * Visit a parse tree produced by the {@code preinc} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitPreinc(PlanAParser.PreincContext ctx); + /** + * Visit a parse tree produced by the {@code postinc} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitPostinc(PlanAParser.PostincContext ctx); + /** + * Visit a parse tree produced by the {@code cast} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitCast(PlanAParser.CastContext ctx); + /** + * Visit a parse tree produced by the {@code external} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExternal(PlanAParser.ExternalContext ctx); + /** + * Visit a parse tree produced by the {@code null} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitNull(PlanAParser.NullContext ctx); + /** + * Visit a parse tree produced by the {@code binary} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitBinary(PlanAParser.BinaryContext ctx); + /** + * Visit a parse tree produced by the {@code char} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitChar(PlanAParser.CharContext ctx); + /** + * Visit a parse tree produced by the {@code true} + * labeled alternative in {@link PlanAParser#expression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitTrue(PlanAParser.TrueContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extstart}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtstart(PlanAParser.ExtstartContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extprec}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtprec(PlanAParser.ExtprecContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extcast}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtcast(PlanAParser.ExtcastContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extbrace}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtbrace(PlanAParser.ExtbraceContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extdot}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtdot(PlanAParser.ExtdotContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#exttype}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExttype(PlanAParser.ExttypeContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extcall}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtcall(PlanAParser.ExtcallContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extvar}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtvar(PlanAParser.ExtvarContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extfield}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtfield(PlanAParser.ExtfieldContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extnew}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtnew(PlanAParser.ExtnewContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#extstring}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtstring(PlanAParser.ExtstringContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#arguments}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitArguments(PlanAParser.ArgumentsContext ctx); + /** + * Visit a parse tree produced by {@link PlanAParser#increment}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitIncrement(PlanAParser.IncrementContext ctx); +} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFTestCompressor.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAPlugin.java similarity index 63% rename from core/src/test/java/org/elasticsearch/common/compress/lzf/LZFTestCompressor.java rename to plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAPlugin.java index 8f21b0cbf0b..c893cd38324 100644 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFTestCompressor.java +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAPlugin.java @@ -17,18 +17,24 @@ * under the License. */ -package org.elasticsearch.common.compress.lzf; +package org.elasticsearch.plan.a; -import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptModule; -import java.io.IOException; - -// LZF compressor with write support, for testing only -public class LZFTestCompressor extends LZFCompressor { +public final class PlanAPlugin extends Plugin { @Override - public StreamOutput streamOutput(StreamOutput out) throws IOException { - return new LZFCompressedStreamOutput(out); + public String name() { + return "lang-plan-a"; } + @Override + public String description() { + return "Plan A scripting language for Elasticsearch"; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(PlanAScriptEngineService.class); + } } diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAScriptEngineService.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAScriptEngineService.java new file mode 100644 index 00000000000..6b3cd834715 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/PlanAScriptEngineService.java @@ -0,0 +1,140 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.Permissions; +import java.security.PrivilegedAction; +import java.security.ProtectionDomain; +import java.util.Map; + +public class PlanAScriptEngineService extends AbstractComponent implements ScriptEngineService { + + public static final String NAME = "plan-a"; + // TODO: this should really be per-script since scripts do so many different things? + private static final CompilerSettings compilerSettings = new CompilerSettings(); + + public static final String NUMERIC_OVERFLOW = "plan-a.numeric_overflow"; + + // TODO: how should custom definitions be specified? + private Definition definition = null; + + @Inject + public PlanAScriptEngineService(Settings settings) { + super(settings); + compilerSettings.setNumericOverflow(settings.getAsBoolean(NUMERIC_OVERFLOW, compilerSettings.getNumericOverflow())); + } + + public void setDefinition(final Definition definition) { + this.definition = new Definition(definition); + } + + @Override + public String[] types() { + return new String[] { NAME }; + } + + @Override + public String[] extensions() { + return new String[] { NAME }; + } + + @Override + public boolean sandboxed() { + return true; + } + + // context used during compilation + private static final AccessControlContext COMPILATION_CONTEXT; + static { + Permissions none = new Permissions(); + none.setReadOnly(); + COMPILATION_CONTEXT = new AccessControlContext(new ProtectionDomain[] { + new ProtectionDomain(null, none) + }); + } + + @Override + public Object compile(String script) { + // check we ourselves are not being called by unprivileged code + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + // create our loader (which loads compiled code with no permissions) + Compiler.Loader loader = AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Compiler.Loader run() { + return new Compiler.Loader(getClass().getClassLoader()); + } + }); + // drop all permissions to actually compile the code itself + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Executable run() { + return Compiler.compile(loader, "something", script, definition, compilerSettings); + } + }, COMPILATION_CONTEXT); + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map vars) { + return new ScriptImpl((Executable) compiledScript.compiled(), vars, null); + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + return new SearchScript() { + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + return new ScriptImpl((Executable) compiledScript.compiled(), vars, lookup.getLeafSearchLookup(context)); + } + + @Override + public boolean needsScores() { + return true; // TODO: maybe even do these different and more like expressions. + } + }; + } + + @Override + public void scriptRemoved(CompiledScript script) { + // nothing to do + } + + @Override + public void close() throws IOException { + // nothing to do + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ScriptImpl.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ScriptImpl.java new file mode 100644 index 00000000000..3910cdc96f7 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/ScriptImpl.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.apache.lucene.search.Scorer; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.ScoreAccessor; +import org.elasticsearch.search.lookup.LeafSearchLookup; + +import java.util.HashMap; +import java.util.Map; + +final class ScriptImpl implements ExecutableScript, LeafSearchScript { + final Executable executable; + final Map variables; + final LeafSearchLookup lookup; + + ScriptImpl(Executable executable, Map vars, LeafSearchLookup lookup) { + this.executable = executable; + this.lookup = lookup; + this.variables = new HashMap<>(); + if (vars != null) { + variables.putAll(vars); + } + if (lookup != null) { + variables.putAll(lookup.asMap()); + } + } + + @Override + public void setNextVar(String name, Object value) { + variables.put(name, value); + } + + @Override + public Object run() { + return executable.execute(variables); + } + + @Override + public float runAsFloat() { + return ((Number) run()).floatValue(); + } + + @Override + public long runAsLong() { + return ((Number) run()).longValue(); + } + + @Override + public double runAsDouble() { + return ((Number) run()).doubleValue(); + } + + @Override + public Object unwrap(Object value) { + return value; + } + + @Override + public void setScorer(Scorer scorer) { + variables.put("_score", new ScoreAccessor(scorer)); + } + + @Override + public void setDocument(int doc) { + if (lookup != null) { + lookup.setDocument(doc); + } + } + + @Override + public void setSource(Map source) { + if (lookup != null) { + lookup.source().setSource(source); + } + } +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Utility.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Utility.java new file mode 100644 index 00000000000..3bb5ae463e7 --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Utility.java @@ -0,0 +1,801 @@ +package org.elasticsearch.plan.a; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +public class Utility { + public static boolean NumberToboolean(final Number value) { + return value.longValue() != 0; + } + + public static char NumberTochar(final Number value) { + return (char)value.intValue(); + } + + public static Boolean NumberToBoolean(final Number value) { + return value.longValue() != 0; + } + + public static Byte NumberToByte(final Number value) { + return value == null ? null : value.byteValue(); + } + + public static Short NumberToShort(final Number value) { + return value == null ? null : value.shortValue(); + } + + public static Character NumberToCharacter(final Number value) { + return value == null ? null : (char)value.intValue(); + } + + public static Integer NumberToInteger(final Number value) { + return value == null ? null : value.intValue(); + } + + public static Long NumberToLong(final Number value) { + return value == null ? null : value.longValue(); + } + + public static Float NumberToFloat(final Number value) { + return value == null ? null : value.floatValue(); + } + + public static Double NumberToDouble(final Number value) { + return value == null ? null : value.doubleValue(); + } + + public static byte booleanTobyte(final boolean value) { + return (byte)(value ? 1 : 0); + } + + public static short booleanToshort(final boolean value) { + return (short)(value ? 1 : 0); + } + + public static char booleanTochar(final boolean value) { + return (char)(value ? 1 : 0); + } + + public static int booleanToint(final boolean value) { + return value ? 1 : 0; + } + + public static long booleanTolong(final boolean value) { + return value ? 1 : 0; + } + + public static float booleanTofloat(final boolean value) { + return value ? 1 : 0; + } + + public static double booleanTodouble(final boolean value) { + return value ? 1 : 0; + } + + public static Integer booleanToInteger(final boolean value) { + return value ? 1 : 0; + } + + public static byte BooleanTobyte(final Boolean value) { + return (byte)(value ? 1 : 0); + } + + public static short BooleanToshort(final Boolean value) { + return (short)(value ? 1 : 0); + } + + public static char BooleanTochar(final Boolean value) { + return (char)(value ? 1 : 0); + } + + public static int BooleanToint(final Boolean value) { + return value ? 1 : 0; + } + + public static long BooleanTolong(final Boolean value) { + return value ? 1 : 0; + } + + public static float BooleanTofloat(final Boolean value) { + return value ? 1 : 0; + } + + public static double BooleanTodouble(final Boolean value) { + return value ? 1 : 0; + } + + public static Byte BooleanToByte(final Boolean value) { + return value == null ? null : (byte)(value ? 1 : 0); + } + + public static Short BooleanToShort(final Boolean value) { + return value == null ? null : (short)(value ? 1 : 0); + } + + public static Character BooleanToCharacter(final Boolean value) { + return value == null ? null : (char)(value ? 1 : 0); + } + + public static Integer BooleanToInteger(final Boolean value) { + return value == null ? null : value ? 1 : 0; + } + + public static Long BooleanToLong(final Boolean value) { + return value == null ? null : value ? 1L : 0L; + } + + public static Float BooleanToFloat(final Boolean value) { + return value == null ? null : value ? 1F : 0F; + } + + public static Double BooleanToDouble(final Boolean value) { + return value == null ? null : value ? 1D : 0D; + } + + public static boolean byteToboolean(final byte value) { + return value != 0; + } + + public static Short byteToShort(final byte value) { + return (short)value; + } + + public static Character byteToCharacter(final byte value) { + return (char)(byte)value; + } + + public static Integer byteToInteger(final byte value) { + return (int)value; + } + + public static Long byteToLong(final byte value) { + return (long)value; + } + + public static Float byteToFloat(final byte value) { + return (float)value; + } + + public static Double byteToDouble(final byte value) { + return (double)value; + } + + public static boolean ByteToboolean(final Byte value) { + return value != 0; + } + + public static char ByteTochar(final Byte value) { + return (char)value.byteValue(); + } + + public static boolean shortToboolean(final short value) { + return value != 0; + } + + public static Byte shortToByte(final short value) { + return (byte)value; + } + + public static Character shortToCharacter(final short value) { + return (char)(short)value; + } + + public static Integer shortToInteger(final short value) { + return (int)value; + } + + public static Long shortToLong(final short value) { + return (long)value; + } + + public static Float shortToFloat(final short value) { + return (float)value; + } + + public static Double shortToDouble(final short value) { + return (double)value; + } + + public static boolean ShortToboolean(final Short value) { + return value != 0; + } + + public static char ShortTochar(final Short value) { + return (char)value.shortValue(); + } + + public static boolean charToboolean(final char value) { + return value != 0; + } + + public static Byte charToByte(final char value) { + return (byte)value; + } + + public static Short charToShort(final char value) { + return (short)value; + } + + public static Integer charToInteger(final char value) { + return (int)value; + } + + public static Long charToLong(final char value) { + return (long)value; + } + + public static Float charToFloat(final char value) { + return (float)value; + } + + public static Double charToDouble(final char value) { + return (double)value; + } + + public static boolean CharacterToboolean(final Character value) { + return value != 0; + } + + public static byte CharacterTobyte(final Character value) { + return (byte)value.charValue(); + } + + public static short CharacterToshort(final Character value) { + return (short)value.charValue(); + } + + public static int CharacterToint(final Character value) { + return (int)value; + } + + public static long CharacterTolong(final Character value) { + return (long)value; + } + + public static float CharacterTofloat(final Character value) { + return (float)value; + } + + public static double CharacterTodouble(final Character value) { + return (double)value; + } + + public static Boolean CharacterToBoolean(final Character value) { + return value == null ? null : value != 0; + } + + public static Byte CharacterToByte(final Character value) { + return value == null ? null : (byte)value.charValue(); + } + + public static Short CharacterToShort(final Character value) { + return value == null ? null : (short)value.charValue(); + } + + public static Integer CharacterToInteger(final Character value) { + return value == null ? null : (int)value; + } + + public static Long CharacterToLong(final Character value) { + return value == null ? null : (long)value; + } + + public static Float CharacterToFloat(final Character value) { + return value == null ? null : (float)value; + } + + public static Double CharacterToDouble(final Character value) { + return value == null ? null : (double)value; + } + + public static boolean intToboolean(final int value) { + return value != 0; + } + + public static Byte intToByte(final int value) { + return (byte)value; + } + + public static Short intToShort(final int value) { + return (short)value; + } + + public static Character intToCharacter(final int value) { + return (char)(int)value; + } + + public static Long intToLong(final int value) { + return (long)value; + } + + public static Float intToFloat(final int value) { + return (float)value; + } + + public static Double intToDouble(final int value) { + return (double)value; + } + + public static boolean IntegerToboolean(final Integer value) { + return value != 0; + } + + public static char IntegerTochar(final Integer value) { + return (char)value.intValue(); + } + + public static boolean longToboolean(final long value) { + return value != 0; + } + + public static Byte longToByte(final long value) { + return (byte)value; + } + + public static Short longToShort(final long value) { + return (short)value; + } + + public static Character longToCharacter(final long value) { + return (char)(long)value; + } + + public static Integer longToInteger(final long value) { + return (int)value; + } + + public static Float longToFloat(final long value) { + return (float)value; + } + + public static Double longToDouble(final long value) { + return (double)value; + } + + public static boolean LongToboolean(final Long value) { + return value != 0; + } + + public static char LongTochar(final Long value) { + return (char)value.longValue(); + } + + public static boolean floatToboolean(final float value) { + return value != 0; + } + + public static Byte floatToByte(final float value) { + return (byte)value; + } + + public static Short floatToShort(final float value) { + return (short)value; + } + + public static Character floatToCharacter(final float value) { + return (char)(float)value; + } + + public static Integer floatToInteger(final float value) { + return (int)value; + } + + public static Long floatToLong(final float value) { + return (long)value; + } + + public static Double floatToDouble(final float value) { + return (double)value; + } + + public static boolean FloatToboolean(final Float value) { + return value != 0; + } + + public static char FloatTochar(final Float value) { + return (char)value.floatValue(); + } + + public static boolean doubleToboolean(final double value) { + return value != 0; + } + + public static Byte doubleToByte(final double value) { + return (byte)value; + } + + public static Short doubleToShort(final double value) { + return (short)value; + } + + public static Character doubleToCharacter(final double value) { + return (char)(double)value; + } + + public static Integer doubleToInteger(final double value) { + return (int)value; + } + + public static Long doubleToLong(final double value) { + return (long)value; + } + + public static Float doubleToFloat(final double value) { + return (float)value; + } + + public static boolean DoubleToboolean(final Double value) { + return value != 0; + } + + public static char DoubleTochar(final Double value) { + return (char)value.doubleValue(); + } + + // although divide by zero is guaranteed, the special overflow case is not caught. + // its not needed for remainder because it is not possible there. + // see https://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.17.2 + + /** + * Integer divide without overflow + * @throws ArithmeticException on overflow or divide-by-zero + */ + public static int divideWithoutOverflow(int x, int y) { + if (x == Integer.MIN_VALUE && y == -1) { + throw new ArithmeticException("integer overflow"); + } + return x / y; + } + + /** + * Long divide without overflow + * @throws ArithmeticException on overflow or divide-by-zero + */ + public static long divideWithoutOverflow(long x, long y) { + if (x == Long.MIN_VALUE && y == -1L) { + throw new ArithmeticException("long overflow"); + } + return x / y; + } + + // byte, short, and char are promoted to int for normal operations, + // so the JDK exact methods are typically used, and the result has a wider range. + // but compound assignments and increment/decrement operators (e.g. byte b = Byte.MAX_VALUE; b++;) + // implicitly cast back to the original type: so these need to be checked against the original range. + + /** + * Like {@link Math#toIntExact(long)} but for byte range. + */ + public static byte toByteExact(int value) { + byte s = (byte) value; + if (s != value) { + throw new ArithmeticException("byte overflow"); + } + return s; + } + + /** + * Like {@link Math#toIntExact(long)} but for byte range. + */ + public static byte toByteExact(long value) { + byte s = (byte) value; + if (s != value) { + throw new ArithmeticException("byte overflow"); + } + return s; + } + + /** + * Like {@link Math#toIntExact(long)} but for byte range. + */ + public static byte toByteWithoutOverflow(float value) { + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw new ArithmeticException("byte overflow"); + } + return (byte)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for byte range. + */ + public static byte toByteWithoutOverflow(double value) { + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw new ArithmeticException("byte overflow"); + } + return (byte)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for short range. + */ + public static short toShortExact(int value) { + short s = (short) value; + if (s != value) { + throw new ArithmeticException("short overflow"); + } + return s; + } + + /** + * Like {@link Math#toIntExact(long)} but for short range. + */ + public static short toShortExact(long value) { + short s = (short) value; + if (s != value) { + throw new ArithmeticException("short overflow"); + } + return s; + } + + /** + * Like {@link Math#toIntExact(long)} but for short range. + */ + public static short toShortWithoutOverflow(float value) { + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw new ArithmeticException("short overflow"); + } + return (short)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for short range. + */ + public static short toShortExact(double value) { + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw new ArithmeticException("short overflow"); + } + return (short)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for char range. + */ + public static char toCharExact(int value) { + char s = (char) value; + if (s != value) { + throw new ArithmeticException("char overflow"); + } + return s; + } + + /** + * Like {@link Math#toIntExact(long)} but for char range. + */ + public static char toCharExact(long value) { + char s = (char) value; + if (s != value) { + throw new ArithmeticException("char overflow"); + } + return s; + } + + /** + * Like {@link Math#toIntExact(long)} but for char range. + */ + public static char toCharWithoutOverflow(float value) { + if (value < Character.MIN_VALUE || value > Character.MAX_VALUE) { + throw new ArithmeticException("char overflow"); + } + return (char)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for char range. + */ + public static char toCharWithoutOverflow(double value) { + if (value < Character.MIN_VALUE || value > Character.MAX_VALUE) { + throw new ArithmeticException("char overflow"); + } + return (char)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for int range. + */ + public static int toIntWithoutOverflow(float value) { + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new ArithmeticException("int overflow"); + } + return (int)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for int range. + */ + public static int toIntWithoutOverflow(double value) { + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new ArithmeticException("int overflow"); + } + return (int)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for long range. + */ + public static long toLongExactWithoutOverflow(float value) { + if (value < Long.MIN_VALUE || value > Long.MAX_VALUE) { + throw new ArithmeticException("long overflow"); + } + return (long)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for long range. + */ + public static float toLongExactWithoutOverflow(double value) { + if (value < Long.MIN_VALUE || value > Long.MAX_VALUE) { + throw new ArithmeticException("long overflow"); + } + return (long)value; + } + + /** + * Like {@link Math#toIntExact(long)} but for float range. + */ + public static float toFloatWithoutOverflow(double value) { + if (value < Float.MIN_VALUE || value > Float.MAX_VALUE) { + throw new ArithmeticException("float overflow"); + } + return (float)value; + } + + /** + * Checks for overflow, result is infinite but operands are finite + * @throws ArithmeticException if overflow occurred + */ + private static float checkInfFloat(float x, float y, float z) { + if (Float.isInfinite(z)) { + if (Float.isFinite(x) && Float.isFinite(y)) { + throw new ArithmeticException("float overflow"); + } + } + return z; + } + + /** + * Checks for NaN, result is NaN but operands are finite + * @throws ArithmeticException if overflow occurred + */ + private static float checkNaNFloat(float x, float y, float z) { + if (Float.isNaN(z)) { + if (Float.isFinite(x) && Float.isFinite(y)) { + throw new ArithmeticException("NaN"); + } + } + return z; + } + + /** + * Checks for NaN, result is infinite but operands are finite + * @throws ArithmeticException if overflow occurred + */ + private static double checkInfDouble(double x, double y, double z) { + if (Double.isInfinite(z)) { + if (Double.isFinite(x) && Double.isFinite(y)) { + throw new ArithmeticException("double overflow"); + } + } + return z; + } + + /** + * Checks for NaN, result is NaN but operands are finite + * @throws ArithmeticException if overflow occurred + */ + private static double checkNaNDouble(double x, double y, double z) { + if (Double.isNaN(z)) { + if (Double.isFinite(x) && Double.isFinite(y)) { + throw new ArithmeticException("NaN"); + } + } + return z; + } + + /** + * Adds two floats but throws {@code ArithmeticException} + * if the result overflows. + */ + public static float addWithoutOverflow(float x, float y) { + return checkInfFloat(x, y, x + y); + } + + /** + * Adds two doubles but throws {@code ArithmeticException} + * if the result overflows. + */ + public static double addWithoutOverflow(double x, double y) { + return checkInfDouble(x, y, x + y); + } + + /** + * Subtracts two floats but throws {@code ArithmeticException} + * if the result overflows. + */ + public static float subtractWithoutOverflow(float x, float y) { + return checkInfFloat(x, y, x - y); + } + + /** + * Subtracts two doubles but throws {@code ArithmeticException} + * if the result overflows. + */ + public static double subtractWithoutOverflow(double x, double y) { + return checkInfDouble(x, y , x - y); + } + + /** + * Multiplies two floats but throws {@code ArithmeticException} + * if the result overflows. + */ + public static float multiplyWithoutOverflow(float x, float y) { + return checkInfFloat(x, y, x * y); + } + + /** + * Multiplies two doubles but throws {@code ArithmeticException} + * if the result overflows. + */ + public static double multiplyWithoutOverflow(double x, double y) { + return checkInfDouble(x, y, x * y); + } + + /** + * Divides two floats but throws {@code ArithmeticException} + * if the result overflows, or would create NaN from finite + * inputs ({@code x == 0, y == 0}) + */ + public static float divideWithoutOverflow(float x, float y) { + return checkNaNFloat(x, y, checkInfFloat(x, y, x / y)); + } + + /** + * Divides two doubles but throws {@code ArithmeticException} + * if the result overflows, or would create NaN from finite + * inputs ({@code x == 0, y == 0}) + */ + public static double divideWithoutOverflow(double x, double y) { + return checkNaNDouble(x, y, checkInfDouble(x, y, x / y)); + } + + /** + * Takes remainder two floats but throws {@code ArithmeticException} + * if the result would create NaN from finite inputs ({@code y == 0}) + */ + public static float remainderWithoutOverflow(float x, float y) { + return checkNaNFloat(x, y, x % y); + } + + /** + * Divides two doubles but throws {@code ArithmeticException} + * if the result would create NaN from finite inputs ({@code y == 0}) + */ + public static double remainderWithoutOverflow(double x, double y) { + return checkNaNDouble(x, y, x % y); + } + + public static boolean checkEquals(final Object left, final Object right) { + if (left != null && right != null) { + return left.equals(right); + } + + return left == null && right == null; + } + + private Utility() {} +} diff --git a/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Writer.java b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Writer.java new file mode 100644 index 00000000000..3756e02f8dc --- /dev/null +++ b/plugins/lang-plan-a/src/main/java/org/elasticsearch/plan/a/Writer.java @@ -0,0 +1,2224 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ParseTree; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Label; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.GeneratorAdapter; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.plan.a.Adapter.*; +import static org.elasticsearch.plan.a.Definition.*; +import static org.elasticsearch.plan.a.PlanAParser.*; + +class Writer extends PlanAParserBaseVisitor { + private static class Branch { + final ParserRuleContext source; + + Label begin; + Label end; + Label tru; + Label fals; + + private Branch(final ParserRuleContext source) { + this.source = source; + + begin = null; + end = null; + tru = null; + fals = null; + } + } + + final static String BASE_CLASS_NAME = Executable.class.getName(); + final static String CLASS_NAME = BASE_CLASS_NAME + "$CompiledPlanAExecutable"; + private final static org.objectweb.asm.Type BASE_CLASS_TYPE = org.objectweb.asm.Type.getType(Executable.class); + private final static org.objectweb.asm.Type CLASS_TYPE = + org.objectweb.asm.Type.getType("L" + CLASS_NAME.replace(".", "/") + ";"); + + private final static org.objectweb.asm.commons.Method CONSTRUCTOR = org.objectweb.asm.commons.Method.getMethod( + "void (org.elasticsearch.plan.a.Definition, java.lang.String, java.lang.String)"); + private final static org.objectweb.asm.commons.Method EXECUTE = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object execute(java.util.Map)"); + private final static String SIGNATURE = "(Ljava/util/Map;)Ljava/lang/Object;"; + + private final static org.objectweb.asm.Type DEFINITION_TYPE = org.objectweb.asm.Type.getType(Definition.class); + + private final static org.objectweb.asm.commons.Method DEF_METHOD_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object methodCall(java.lang.Object, java.lang.String, " + + "org.elasticsearch.plan.a.Definition, java.lang.Object[], boolean[])"); + private final static org.objectweb.asm.commons.Method DEF_ARRAY_STORE = org.objectweb.asm.commons.Method.getMethod( + "void arrayStore(java.lang.Object, java.lang.Object, java.lang.Object, " + + "org.elasticsearch.plan.a.Definition, boolean, boolean)"); + private final static org.objectweb.asm.commons.Method DEF_ARRAY_LOAD = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object arrayLoad(java.lang.Object, java.lang.Object, " + + "org.elasticsearch.plan.a.Definition, boolean)"); + private final static org.objectweb.asm.commons.Method DEF_FIELD_STORE = org.objectweb.asm.commons.Method.getMethod( + "void fieldStore(java.lang.Object, java.lang.Object, java.lang.String, " + + "org.elasticsearch.plan.a.Definition, boolean)"); + private final static org.objectweb.asm.commons.Method DEF_FIELD_LOAD = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object fieldLoad(java.lang.Object, java.lang.String, org.elasticsearch.plan.a.Definition)"); + + private final static org.objectweb.asm.commons.Method DEF_NOT_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object not(java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_NEG_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object neg(java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_MUL_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object mul(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_DIV_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object div(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_REM_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object rem(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_ADD_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object add(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_SUB_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object sub(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_LSH_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object lsh(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_RSH_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object rsh(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_USH_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object ush(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_AND_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object and(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_XOR_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object xor(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_OR_CALL = org.objectweb.asm.commons.Method.getMethod( + "java.lang.Object or(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_EQ_CALL = org.objectweb.asm.commons.Method.getMethod( + "boolean eq(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_LT_CALL = org.objectweb.asm.commons.Method.getMethod( + "boolean lt(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_LTE_CALL = org.objectweb.asm.commons.Method.getMethod( + "boolean lte(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_GT_CALL = org.objectweb.asm.commons.Method.getMethod( + "boolean gt(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method DEF_GTE_CALL = org.objectweb.asm.commons.Method.getMethod( + "boolean gte(java.lang.Object, java.lang.Object)"); + + private final static org.objectweb.asm.Type STRINGBUILDER_TYPE = org.objectweb.asm.Type.getType(StringBuilder.class); + + private final static org.objectweb.asm.commons.Method STRINGBUILDER_CONSTRUCTOR = + org.objectweb.asm.commons.Method.getMethod("void ()"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_BOOLEAN = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(boolean)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_CHAR = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(char)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_INT = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(int)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_LONG = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(long)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_FLOAT = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(float)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(double)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_STRING = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(java.lang.String)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_OBJECT = + org.objectweb.asm.commons.Method.getMethod("java.lang.StringBuilder append(java.lang.Object)"); + private final static org.objectweb.asm.commons.Method STRINGBUILDER_TOSTRING = + org.objectweb.asm.commons.Method.getMethod("java.lang.String toString()"); + + private final static org.objectweb.asm.commons.Method TOINTEXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("int toIntExact(long)"); + private final static org.objectweb.asm.commons.Method NEGATEEXACT_INT = + org.objectweb.asm.commons.Method.getMethod("int negateExact(int)"); + private final static org.objectweb.asm.commons.Method NEGATEEXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("long negateExact(long)"); + private final static org.objectweb.asm.commons.Method MULEXACT_INT = + org.objectweb.asm.commons.Method.getMethod("int multiplyExact(int, int)"); + private final static org.objectweb.asm.commons.Method MULEXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("long multiplyExact(long, long)"); + private final static org.objectweb.asm.commons.Method ADDEXACT_INT = + org.objectweb.asm.commons.Method.getMethod("int addExact(int, int)"); + private final static org.objectweb.asm.commons.Method ADDEXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("long addExact(long, long)"); + private final static org.objectweb.asm.commons.Method SUBEXACT_INT = + org.objectweb.asm.commons.Method.getMethod("int subtractExact(int, int)"); + private final static org.objectweb.asm.commons.Method SUBEXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("long subtractExact(long, long)"); + + private final static org.objectweb.asm.commons.Method CHECKEQUALS = + org.objectweb.asm.commons.Method.getMethod("boolean checkEquals(java.lang.Object, java.lang.Object)"); + private final static org.objectweb.asm.commons.Method TOBYTEEXACT_INT = + org.objectweb.asm.commons.Method.getMethod("byte toByteExact(int)"); + private final static org.objectweb.asm.commons.Method TOBYTEEXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("byte toByteExact(long)"); + private final static org.objectweb.asm.commons.Method TOBYTEWOOVERFLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("byte toByteWithoutOverflow(float)"); + private final static org.objectweb.asm.commons.Method TOBYTEWOOVERFLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("byte toByteWithoutOverflow(double)"); + private final static org.objectweb.asm.commons.Method TOSHORTEXACT_INT = + org.objectweb.asm.commons.Method.getMethod("short toShortExact(int)"); + private final static org.objectweb.asm.commons.Method TOSHORTEXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("short toShortExact(long)"); + private final static org.objectweb.asm.commons.Method TOSHORTWOOVERFLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("short toShortWithoutOverflow(float)"); + private final static org.objectweb.asm.commons.Method TOSHORTWOOVERFLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("short toShortWithoutOverflow(double)"); + private final static org.objectweb.asm.commons.Method TOCHAREXACT_INT = + org.objectweb.asm.commons.Method.getMethod("char toCharExact(int)"); + private final static org.objectweb.asm.commons.Method TOCHAREXACT_LONG = + org.objectweb.asm.commons.Method.getMethod("char toCharExact(long)"); + private final static org.objectweb.asm.commons.Method TOCHARWOOVERFLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("char toCharWithoutOverflow(float)"); + private final static org.objectweb.asm.commons.Method TOCHARWOOVERFLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("char toCharWithoutOverflow(double)"); + private final static org.objectweb.asm.commons.Method TOINTWOOVERFLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("int toIntWithoutOverflow(float)"); + private final static org.objectweb.asm.commons.Method TOINTWOOVERFLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("int toIntWithoutOverflow(double)"); + private final static org.objectweb.asm.commons.Method TOLONGWOOVERFLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("long toLongExactWithoutOverflow(float)"); + private final static org.objectweb.asm.commons.Method TOLONGWOOVERFLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("long toLongExactWithoutOverflow(double)"); + private final static org.objectweb.asm.commons.Method TOFLOATWOOVERFLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("float toFloatWithoutOverflow(double)"); + private final static org.objectweb.asm.commons.Method MULWOOVERLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("float multiplyWithoutOverflow(float, float)"); + private final static org.objectweb.asm.commons.Method MULWOOVERLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("double multiplyWithoutOverflow(double, double)"); + private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_INT = + org.objectweb.asm.commons.Method.getMethod("int divideWithoutOverflow(int, int)"); + private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_LONG = + org.objectweb.asm.commons.Method.getMethod("long divideWithoutOverflow(long, long)"); + private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("float divideWithoutOverflow(float, float)"); + private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("double divideWithoutOverflow(double, double)"); + private final static org.objectweb.asm.commons.Method REMWOOVERLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("float remainderWithoutOverflow(float, float)"); + private final static org.objectweb.asm.commons.Method REMWOOVERLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("double remainderWithoutOverflow(double, double)"); + private final static org.objectweb.asm.commons.Method ADDWOOVERLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("float addWithoutOverflow(float, float)"); + private final static org.objectweb.asm.commons.Method ADDWOOVERLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("double addWithoutOverflow(double, double)"); + private final static org.objectweb.asm.commons.Method SUBWOOVERLOW_FLOAT = + org.objectweb.asm.commons.Method.getMethod("float subtractWithoutOverflow(float, float)"); + private final static org.objectweb.asm.commons.Method SUBWOOVERLOW_DOUBLE = + org.objectweb.asm.commons.Method.getMethod("double subtractWithoutOverflow(double, double)"); + + static byte[] write(Adapter adapter) { + Writer writer = new Writer(adapter); + + return writer.getBytes(); + } + + private final Adapter adapter; + private final Definition definition; + private final ParseTree root; + private final String source; + private final CompilerSettings settings; + + private final Map branches; + private final Deque jumps; + private final Set strings; + + private ClassWriter writer; + private GeneratorAdapter execute; + + private Writer(final Adapter adapter) { + this.adapter = adapter; + definition = adapter.definition; + root = adapter.root; + source = adapter.source; + settings = adapter.settings; + + branches = new HashMap<>(); + jumps = new ArrayDeque<>(); + strings = new HashSet<>(); + + writeBegin(); + writeConstructor(); + writeExecute(); + writeEnd(); + } + + private Branch markBranch(final ParserRuleContext source, final ParserRuleContext... nodes) { + final Branch branch = new Branch(source); + + for (final ParserRuleContext node : nodes) { + branches.put(node, branch); + } + + return branch; + } + + private void copyBranch(final Branch branch, final ParserRuleContext... nodes) { + for (final ParserRuleContext node : nodes) { + branches.put(node, branch); + } + } + + private Branch getBranch(final ParserRuleContext source) { + return branches.get(source); + } + + private void writeBegin() { + final int compute = ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS; + final int version = Opcodes.V1_7; + final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL | Opcodes.ACC_SYNTHETIC; + final String base = BASE_CLASS_TYPE.getInternalName(); + final String name = CLASS_TYPE.getInternalName(); + + writer = new ClassWriter(compute); + writer.visit(version, access, name, null, base, null); + writer.visitSource(source, null); + } + + private void writeConstructor() { + final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SYNTHETIC; + final GeneratorAdapter constructor = new GeneratorAdapter(access, CONSTRUCTOR, null, null, writer); + constructor.loadThis(); + constructor.loadArgs(); + constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR); + constructor.returnValue(); + constructor.endMethod(); + } + + private void writeExecute() { + final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SYNTHETIC; + execute = new GeneratorAdapter(access, EXECUTE, SIGNATURE, null, writer); + visit(root); + execute.endMethod(); + } + + @Override + public Void visitSource(final SourceContext ctx) { + final StatementMetadata sourcesmd = adapter.getStatementMetadata(ctx); + + for (final StatementContext sctx : ctx.statement()) { + visit(sctx); + } + + if (!sourcesmd.allReturn) { + execute.visitInsn(Opcodes.ACONST_NULL); + execute.returnValue(); + } + + return null; + } + + @Override + public Void visitIf(final IfContext ctx) { + final ExpressionContext exprctx = ctx.expression(); + final boolean els = ctx.ELSE() != null; + final Branch branch = markBranch(ctx, exprctx); + branch.end = new Label(); + branch.fals = els ? new Label() : branch.end; + + visit(exprctx); + + final BlockContext blockctx0 = ctx.block(0); + final StatementMetadata blockmd0 = adapter.getStatementMetadata(blockctx0); + visit(blockctx0); + + if (els) { + if (!blockmd0.allExit) { + execute.goTo(branch.end); + } + + execute.mark(branch.fals); + visit(ctx.block(1)); + } + + execute.mark(branch.end); + + return null; + } + + @Override + public Void visitWhile(final WhileContext ctx) { + final ExpressionContext exprctx = ctx.expression(); + final Branch branch = markBranch(ctx, exprctx); + branch.begin = new Label(); + branch.end = new Label(); + branch.fals = branch.end; + + jumps.push(branch); + execute.mark(branch.begin); + visit(exprctx); + + final BlockContext blockctx = ctx.block(); + boolean allexit = false; + + if (blockctx != null) { + StatementMetadata blocksmd = adapter.getStatementMetadata(blockctx); + allexit = blocksmd.allExit; + visit(blockctx); + } + + if (!allexit) { + execute.goTo(branch.begin); + } + + execute.mark(branch.end); + jumps.pop(); + + return null; + } + + @Override + public Void visitDo(final DoContext ctx) { + final ExpressionContext exprctx = ctx.expression(); + final Branch branch = markBranch(ctx, exprctx); + branch.begin = new Label(); + branch.end = new Label(); + branch.fals = branch.end; + + jumps.push(branch); + execute.mark(branch.begin); + + final BlockContext bctx = ctx.block(); + final StatementMetadata blocksmd = adapter.getStatementMetadata(bctx); + visit(bctx); + + visit(exprctx); + + if (!blocksmd.allExit) { + execute.goTo(branch.begin); + } + + execute.mark(branch.end); + jumps.pop(); + + return null; + } + + @Override + public Void visitFor(final ForContext ctx) { + final ExpressionContext exprctx = ctx.expression(); + final AfterthoughtContext atctx = ctx.afterthought(); + final Branch branch = markBranch(ctx, exprctx); + final Label start = new Label(); + branch.begin = atctx == null ? start : new Label(); + branch.end = new Label(); + branch.fals = branch.end; + + jumps.push(branch); + + if (ctx.initializer() != null) { + visit(ctx.initializer()); + } + + execute.mark(start); + + if (exprctx != null) { + visit(exprctx); + } + + final BlockContext blockctx = ctx.block(); + boolean allexit = false; + + if (blockctx != null) { + StatementMetadata blocksmd = adapter.getStatementMetadata(blockctx); + allexit = blocksmd.allExit; + visit(blockctx); + } + + if (atctx != null) { + execute.mark(branch.begin); + visit(atctx); + } + + if (atctx != null || !allexit) { + execute.goTo(start); + } + + execute.mark(branch.end); + jumps.pop(); + + return null; + } + + @Override + public Void visitDecl(final DeclContext ctx) { + visit(ctx.declaration()); + + return null; + } + + @Override + public Void visitContinue(final ContinueContext ctx) { + final Branch jump = jumps.peek(); + execute.goTo(jump.begin); + + return null; + } + + @Override + public Void visitBreak(final BreakContext ctx) { + final Branch jump = jumps.peek(); + execute.goTo(jump.end); + + return null; + } + + @Override + public Void visitReturn(final ReturnContext ctx) { + visit(ctx.expression()); + execute.returnValue(); + + return null; + } + + @Override + public Void visitExpr(final ExprContext ctx) { + final StatementMetadata exprsmd = adapter.getStatementMetadata(ctx); + final ExpressionContext exprctx = ctx.expression(); + final ExpressionMetadata expremd = adapter.getExpressionMetadata(exprctx); + visit(exprctx); + + if (exprsmd.allReturn) { + execute.returnValue(); + } else { + writePop(expremd.to.type.getSize()); + } + + return null; + } + + @Override + public Void visitMultiple(final MultipleContext ctx) { + for (final StatementContext sctx : ctx.statement()) { + visit(sctx); + } + + return null; + } + + @Override + public Void visitSingle(final SingleContext ctx) { + visit(ctx.statement()); + + return null; + } + + @Override + public Void visitEmpty(final EmptyContext ctx) { + throw new UnsupportedOperationException(error(ctx) + "Unexpected writer state."); + } + + @Override + public Void visitInitializer(InitializerContext ctx) { + final DeclarationContext declctx = ctx.declaration(); + final ExpressionContext exprctx = ctx.expression(); + + if (declctx != null) { + visit(declctx); + } else if (exprctx != null) { + visit(exprctx); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + return null; + } + + @Override + public Void visitAfterthought(AfterthoughtContext ctx) { + visit(ctx.expression()); + + return null; + } + + @Override + public Void visitDeclaration(DeclarationContext ctx) { + for (final DeclvarContext declctx : ctx.declvar()) { + visit(declctx); + } + + return null; + } + + @Override + public Void visitDecltype(final DecltypeContext ctx) { + throw new UnsupportedOperationException(error(ctx) + "Unexpected writer state."); + } + + @Override + public Void visitDeclvar(final DeclvarContext ctx) { + final ExpressionMetadata declvaremd = adapter.getExpressionMetadata(ctx); + final org.objectweb.asm.Type type = declvaremd.to.type; + final Sort sort = declvaremd.to.sort; + final int slot = (int)declvaremd.postConst; + + final ExpressionContext exprctx = ctx.expression(); + final boolean initialize = exprctx == null; + + if (!initialize) { + visit(exprctx); + } + + switch (sort) { + case VOID: throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + case BOOL: + case BYTE: + case SHORT: + case CHAR: + case INT: if (initialize) execute.push(0); break; + case LONG: if (initialize) execute.push(0L); break; + case FLOAT: if (initialize) execute.push(0.0F); break; + case DOUBLE: if (initialize) execute.push(0.0); break; + default: if (initialize) execute.visitInsn(Opcodes.ACONST_NULL); + } + + execute.visitVarInsn(type.getOpcode(Opcodes.ISTORE), slot); + + return null; + } + + @Override + public Void visitPrecedence(final PrecedenceContext ctx) { + throw new UnsupportedOperationException(error(ctx) + "Unexpected writer state."); + } + + @Override + public Void visitNumeric(final NumericContext ctx) { + final ExpressionMetadata numericemd = adapter.getExpressionMetadata(ctx); + final Object postConst = numericemd.postConst; + + if (postConst == null) { + writeNumeric(ctx, numericemd.preConst); + checkWriteCast(numericemd); + } else { + writeConstant(ctx, postConst); + } + + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitChar(final CharContext ctx) { + final ExpressionMetadata charemd = adapter.getExpressionMetadata(ctx); + final Object postConst = charemd.postConst; + + if (postConst == null) { + writeNumeric(ctx, (int)(char)charemd.preConst); + checkWriteCast(charemd); + } else { + writeConstant(ctx, postConst); + } + + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitTrue(final TrueContext ctx) { + final ExpressionMetadata trueemd = adapter.getExpressionMetadata(ctx); + final Object postConst = trueemd.postConst; + final Branch branch = getBranch(ctx); + + if (branch == null) { + if (postConst == null) { + writeBoolean(ctx, true); + checkWriteCast(trueemd); + } else { + writeConstant(ctx, postConst); + } + } else if (branch.tru != null) { + execute.goTo(branch.tru); + } + + return null; + } + + @Override + public Void visitFalse(final FalseContext ctx) { + final ExpressionMetadata falseemd = adapter.getExpressionMetadata(ctx); + final Object postConst = falseemd.postConst; + final Branch branch = getBranch(ctx); + + if (branch == null) { + if (postConst == null) { + writeBoolean(ctx, false); + checkWriteCast(falseemd); + } else { + writeConstant(ctx, postConst); + } + } else if (branch.fals != null) { + execute.goTo(branch.fals); + } + + return null; + } + + @Override + public Void visitNull(final NullContext ctx) { + final ExpressionMetadata nullemd = adapter.getExpressionMetadata(ctx); + + execute.visitInsn(Opcodes.ACONST_NULL); + checkWriteCast(nullemd); + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitExternal(final ExternalContext ctx) { + final ExpressionMetadata expremd = adapter.getExpressionMetadata(ctx); + visit(ctx.extstart()); + checkWriteCast(expremd); + checkWriteBranch(ctx); + + return null; + } + + + @Override + public Void visitPostinc(final PostincContext ctx) { + final ExpressionMetadata expremd = adapter.getExpressionMetadata(ctx); + visit(ctx.extstart()); + checkWriteCast(expremd); + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitPreinc(final PreincContext ctx) { + final ExpressionMetadata expremd = adapter.getExpressionMetadata(ctx); + visit(ctx.extstart()); + checkWriteCast(expremd); + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitUnary(final UnaryContext ctx) { + final ExpressionMetadata unaryemd = adapter.getExpressionMetadata(ctx); + final Object postConst = unaryemd.postConst; + final Object preConst = unaryemd.preConst; + final Branch branch = getBranch(ctx); + + if (postConst != null) { + if (ctx.BOOLNOT() != null) { + if (branch == null) { + writeConstant(ctx, postConst); + } else { + if ((boolean)postConst && branch.tru != null) { + execute.goTo(branch.tru); + } else if (!(boolean)postConst && branch.fals != null) { + execute.goTo(branch.fals); + } + } + } else { + writeConstant(ctx, postConst); + checkWriteBranch(ctx); + } + } else if (preConst != null) { + if (branch == null) { + writeConstant(ctx, preConst); + checkWriteCast(unaryemd); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + } else { + final ExpressionContext exprctx = ctx.expression(); + + if (ctx.BOOLNOT() != null) { + final Branch local = markBranch(ctx, exprctx); + + if (branch == null) { + local.fals = new Label(); + final Label aend = new Label(); + + visit(exprctx); + + execute.push(false); + execute.goTo(aend); + execute.mark(local.fals); + execute.push(true); + execute.mark(aend); + + checkWriteCast(unaryemd); + } else { + local.tru = branch.fals; + local.fals = branch.tru; + + visit(exprctx); + } + } else { + final org.objectweb.asm.Type type = unaryemd.from.type; + final Sort sort = unaryemd.from.sort; + + visit(exprctx); + + if (ctx.BWNOT() != null) { + if (sort == Sort.DEF) { + execute.invokeStatic(definition.defobjType.type, DEF_NOT_CALL); + } else { + if (sort == Sort.INT) { + writeConstant(ctx, -1); + } else if (sort == Sort.LONG) { + writeConstant(ctx, -1L); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + execute.math(GeneratorAdapter.XOR, type); + } + } else if (ctx.SUB() != null) { + if (sort == Sort.DEF) { + execute.invokeStatic(definition.defobjType.type, DEF_NEG_CALL); + } else { + if (settings.getNumericOverflow()) { + execute.math(GeneratorAdapter.NEG, type); + } else { + if (sort == Sort.INT) { + execute.invokeStatic(definition.mathType.type, NEGATEEXACT_INT); + } else if (sort == Sort.LONG) { + execute.invokeStatic(definition.mathType.type, NEGATEEXACT_LONG); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + } + } + } else if (ctx.ADD() == null) { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + checkWriteCast(unaryemd); + checkWriteBranch(ctx); + } + } + + return null; + } + + @Override + public Void visitCast(final CastContext ctx) { + final ExpressionMetadata castemd = adapter.getExpressionMetadata(ctx); + final Object postConst = castemd.postConst; + + if (postConst == null) { + visit(ctx.expression()); + checkWriteCast(castemd); + } else { + writeConstant(ctx, postConst); + } + + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitBinary(final BinaryContext ctx) { + final ExpressionMetadata binaryemd = adapter.getExpressionMetadata(ctx); + final Object postConst = binaryemd.postConst; + final Object preConst = binaryemd.preConst; + final Branch branch = getBranch(ctx); + + if (postConst != null) { + writeConstant(ctx, postConst); + } else if (preConst != null) { + if (branch == null) { + writeConstant(ctx, preConst); + checkWriteCast(binaryemd); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + } else if (binaryemd.from.sort == Sort.STRING) { + final boolean marked = strings.contains(ctx); + + if (!marked) { + writeNewStrings(); + } + + final ExpressionContext exprctx0 = ctx.expression(0); + final ExpressionMetadata expremd0 = adapter.getExpressionMetadata(exprctx0); + strings.add(exprctx0); + visit(exprctx0); + + if (strings.contains(exprctx0)) { + writeAppendStrings(expremd0.from.sort); + strings.remove(exprctx0); + } + + final ExpressionContext exprctx1 = ctx.expression(1); + final ExpressionMetadata expremd1 = adapter.getExpressionMetadata(exprctx1); + strings.add(exprctx1); + visit(exprctx1); + + if (strings.contains(exprctx1)) { + writeAppendStrings(expremd1.from.sort); + strings.remove(exprctx1); + } + + if (marked) { + strings.remove(ctx); + } else { + writeToStrings(); + } + + checkWriteCast(binaryemd); + } else { + final ExpressionContext exprctx0 = ctx.expression(0); + final ExpressionContext exprctx1 = ctx.expression(1); + + visit(exprctx0); + visit(exprctx1); + + final Type type = binaryemd.from; + + if (ctx.MUL() != null) writeBinaryInstruction(ctx, type, MUL); + else if (ctx.DIV() != null) writeBinaryInstruction(ctx, type, DIV); + else if (ctx.REM() != null) writeBinaryInstruction(ctx, type, REM); + else if (ctx.ADD() != null) writeBinaryInstruction(ctx, type, ADD); + else if (ctx.SUB() != null) writeBinaryInstruction(ctx, type, SUB); + else if (ctx.LSH() != null) writeBinaryInstruction(ctx, type, LSH); + else if (ctx.USH() != null) writeBinaryInstruction(ctx, type, USH); + else if (ctx.RSH() != null) writeBinaryInstruction(ctx, type, RSH); + else if (ctx.BWAND() != null) writeBinaryInstruction(ctx, type, BWAND); + else if (ctx.BWXOR() != null) writeBinaryInstruction(ctx, type, BWXOR); + else if (ctx.BWOR() != null) writeBinaryInstruction(ctx, type, BWOR); + else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + checkWriteCast(binaryemd); + } + + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitComp(final CompContext ctx) { + final ExpressionMetadata compemd = adapter.getExpressionMetadata(ctx); + final Object postConst = compemd.postConst; + final Object preConst = compemd.preConst; + final Branch branch = getBranch(ctx); + + if (postConst != null) { + if (branch == null) { + writeConstant(ctx, postConst); + } else { + if ((boolean)postConst && branch.tru != null) { + execute.mark(branch.tru); + } else if (!(boolean)postConst && branch.fals != null) { + execute.mark(branch.fals); + } + } + } else if (preConst != null) { + if (branch == null) { + writeConstant(ctx, preConst); + checkWriteCast(compemd); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + } else { + final ExpressionContext exprctx0 = ctx.expression(0); + final ExpressionMetadata expremd0 = adapter.getExpressionMetadata(exprctx0); + + final ExpressionContext exprctx1 = ctx.expression(1); + final ExpressionMetadata expremd1 = adapter.getExpressionMetadata(exprctx1); + final org.objectweb.asm.Type type = expremd1.to.type; + final Sort sort1 = expremd1.to.sort; + + visit(exprctx0); + + if (!expremd1.isNull) { + visit(exprctx1); + } + + final boolean tru = branch != null && branch.tru != null; + final boolean fals = branch != null && branch.fals != null; + final Label jump = tru ? branch.tru : fals ? branch.fals : new Label(); + final Label end = new Label(); + + final boolean eq = (ctx.EQ() != null || ctx.EQR() != null) && (tru || !fals) || + (ctx.NE() != null || ctx.NER() != null) && fals; + final boolean ne = (ctx.NE() != null || ctx.NER() != null) && (tru || !fals) || + (ctx.EQ() != null || ctx.EQR() != null) && fals; + final boolean lt = ctx.LT() != null && (tru || !fals) || ctx.GTE() != null && fals; + final boolean lte = ctx.LTE() != null && (tru || !fals) || ctx.GT() != null && fals; + final boolean gt = ctx.GT() != null && (tru || !fals) || ctx.LTE() != null && fals; + final boolean gte = ctx.GTE() != null && (tru || !fals) || ctx.LT() != null && fals; + + boolean writejump = true; + + switch (sort1) { + case VOID: + case BYTE: + case SHORT: + case CHAR: + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + case BOOL: + if (eq) execute.ifZCmp(GeneratorAdapter.EQ, jump); + else if (ne) execute.ifZCmp(GeneratorAdapter.NE, jump); + else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + break; + case INT: + case LONG: + case FLOAT: + case DOUBLE: + if (eq) execute.ifCmp(type, GeneratorAdapter.EQ, jump); + else if (ne) execute.ifCmp(type, GeneratorAdapter.NE, jump); + else if (lt) execute.ifCmp(type, GeneratorAdapter.LT, jump); + else if (lte) execute.ifCmp(type, GeneratorAdapter.LE, jump); + else if (gt) execute.ifCmp(type, GeneratorAdapter.GT, jump); + else if (gte) execute.ifCmp(type, GeneratorAdapter.GE, jump); + else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + break; + case DEF: + if (eq) { + if (expremd1.isNull) { + execute.ifNull(jump); + } else if (!expremd0.isNull && ctx.EQ() != null) { + execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL); + } else { + execute.ifCmp(type, GeneratorAdapter.EQ, jump); + } + } else if (ne) { + if (expremd1.isNull) { + execute.ifNonNull(jump); + } else if (!expremd0.isNull && ctx.NE() != null) { + execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL); + execute.ifZCmp(GeneratorAdapter.EQ, jump); + } else { + execute.ifCmp(type, GeneratorAdapter.NE, jump); + } + } else if (lt) { + execute.invokeStatic(definition.defobjType.type, DEF_LT_CALL); + } else if (lte) { + execute.invokeStatic(definition.defobjType.type, DEF_LTE_CALL); + } else if (gt) { + execute.invokeStatic(definition.defobjType.type, DEF_GT_CALL); + } else if (gte) { + execute.invokeStatic(definition.defobjType.type, DEF_GTE_CALL); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + writejump = expremd1.isNull || ne || ctx.EQR() != null; + + if (branch != null && !writejump) { + execute.ifZCmp(GeneratorAdapter.NE, jump); + } + + break; + default: + if (eq) { + if (expremd1.isNull) { + execute.ifNull(jump); + } else if (ctx.EQ() != null) { + execute.invokeStatic(definition.utilityType.type, CHECKEQUALS); + + if (branch != null) { + execute.ifZCmp(GeneratorAdapter.NE, jump); + } + + writejump = false; + } else { + execute.ifCmp(type, GeneratorAdapter.EQ, jump); + } + } else if (ne) { + if (expremd1.isNull) { + execute.ifNonNull(jump); + } else if (ctx.NE() != null) { + execute.invokeStatic(definition.utilityType.type, CHECKEQUALS); + execute.ifZCmp(GeneratorAdapter.EQ, jump); + } else { + execute.ifCmp(type, GeneratorAdapter.NE, jump); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + } + + if (branch == null) { + if (writejump) { + execute.push(false); + execute.goTo(end); + execute.mark(jump); + execute.push(true); + execute.mark(end); + } + + checkWriteCast(compemd); + } + } + + return null; + } + + @Override + public Void visitBool(final BoolContext ctx) { + final ExpressionMetadata boolemd = adapter.getExpressionMetadata(ctx); + final Object postConst = boolemd.postConst; + final Object preConst = boolemd.preConst; + final Branch branch = getBranch(ctx); + + if (postConst != null) { + if (branch == null) { + writeConstant(ctx, postConst); + } else { + if ((boolean)postConst && branch.tru != null) { + execute.mark(branch.tru); + } else if (!(boolean)postConst && branch.fals != null) { + execute.mark(branch.fals); + } + } + } else if (preConst != null) { + if (branch == null) { + writeConstant(ctx, preConst); + checkWriteCast(boolemd); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + } else { + final ExpressionContext exprctx0 = ctx.expression(0); + final ExpressionContext exprctx1 = ctx.expression(1); + + if (branch == null) { + if (ctx.BOOLAND() != null) { + final Branch local = markBranch(ctx, exprctx0, exprctx1); + local.fals = new Label(); + final Label end = new Label(); + + visit(exprctx0); + visit(exprctx1); + + execute.push(true); + execute.goTo(end); + execute.mark(local.fals); + execute.push(false); + execute.mark(end); + } else if (ctx.BOOLOR() != null) { + final Branch branch0 = markBranch(ctx, exprctx0); + branch0.tru = new Label(); + final Branch branch1 = markBranch(ctx, exprctx1); + branch1.fals = new Label(); + final Label aend = new Label(); + + visit(exprctx0); + visit(exprctx1); + + execute.mark(branch0.tru); + execute.push(true); + execute.goTo(aend); + execute.mark(branch1.fals); + execute.push(false); + execute.mark(aend); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + checkWriteCast(boolemd); + } else { + if (ctx.BOOLAND() != null) { + final Branch branch0 = markBranch(ctx, exprctx0); + branch0.fals = branch.fals == null ? new Label() : branch.fals; + final Branch branch1 = markBranch(ctx, exprctx1); + branch1.tru = branch.tru; + branch1.fals = branch.fals; + + visit(exprctx0); + visit(exprctx1); + + if (branch.fals == null) { + execute.mark(branch0.fals); + } + } else if (ctx.BOOLOR() != null) { + final Branch branch0 = markBranch(ctx, exprctx0); + branch0.tru = branch.tru == null ? new Label() : branch.tru; + final Branch branch1 = markBranch(ctx, exprctx1); + branch1.tru = branch.tru; + branch1.fals = branch.fals; + + visit(exprctx0); + visit(exprctx1); + + if (branch.tru == null) { + execute.mark(branch0.tru); + } + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + } + } + + return null; + } + + @Override + public Void visitConditional(final ConditionalContext ctx) { + final ExpressionMetadata condemd = adapter.getExpressionMetadata(ctx); + final Branch branch = getBranch(ctx); + + final ExpressionContext expr0 = ctx.expression(0); + final ExpressionContext expr1 = ctx.expression(1); + final ExpressionContext expr2 = ctx.expression(2); + + final Branch local = markBranch(ctx, expr0); + local.fals = new Label(); + local.end = new Label(); + + if (branch != null) { + copyBranch(branch, expr1, expr2); + } + + visit(expr0); + visit(expr1); + execute.goTo(local.end); + execute.mark(local.fals); + visit(expr2); + execute.mark(local.end); + + if (branch == null) { + checkWriteCast(condemd); + } + + return null; + } + + @Override + public Void visitAssignment(final AssignmentContext ctx) { + final ExpressionMetadata expremd = adapter.getExpressionMetadata(ctx); + visit(ctx.extstart()); + checkWriteCast(expremd); + checkWriteBranch(ctx); + + return null; + } + + @Override + public Void visitExtstart(ExtstartContext ctx) { + final ExternalMetadata startemd = adapter.getExternalMetadata(ctx); + + if (startemd.token == ADD) { + final ExpressionMetadata storeemd = adapter.getExpressionMetadata(startemd.storeExpr); + + if (startemd.current.sort == Sort.STRING || storeemd.from.sort == Sort.STRING) { + writeNewStrings(); + strings.add(startemd.storeExpr); + } + } + + final ExtprecContext precctx = ctx.extprec(); + final ExtcastContext castctx = ctx.extcast(); + final ExttypeContext typectx = ctx.exttype(); + final ExtvarContext varctx = ctx.extvar(); + final ExtnewContext newctx = ctx.extnew(); + final ExtstringContext stringctx = ctx.extstring(); + + if (precctx != null) { + visit(precctx); + } else if (castctx != null) { + visit(castctx); + } else if (typectx != null) { + visit(typectx); + } else if (varctx != null) { + visit(varctx); + } else if (newctx != null) { + visit(newctx); + } else if (stringctx != null) { + visit(stringctx); + } else { + throw new IllegalStateException(); + } + + return null; + } + + @Override + public Void visitExtprec(final ExtprecContext ctx) { + final ExtprecContext precctx = ctx.extprec(); + final ExtcastContext castctx = ctx.extcast(); + final ExttypeContext typectx = ctx.exttype(); + final ExtvarContext varctx = ctx.extvar(); + final ExtnewContext newctx = ctx.extnew(); + final ExtstringContext stringctx = ctx.extstring(); + + if (precctx != null) { + visit(precctx); + } else if (castctx != null) { + visit(castctx); + } else if (typectx != null) { + visit(typectx); + } else if (varctx != null) { + visit(varctx); + } else if (newctx != null) { + visit(newctx); + } else if (stringctx != null) { + visit(stringctx); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null) { + visit(dotctx); + } else if (bracectx != null) { + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtcast(final ExtcastContext ctx) { + ExtNodeMetadata castenmd = adapter.getExtNodeMetadata(ctx); + + final ExtprecContext precctx = ctx.extprec(); + final ExtcastContext castctx = ctx.extcast(); + final ExttypeContext typectx = ctx.exttype(); + final ExtvarContext varctx = ctx.extvar(); + final ExtnewContext newctx = ctx.extnew(); + final ExtstringContext stringctx = ctx.extstring(); + + if (precctx != null) { + visit(precctx); + } else if (castctx != null) { + visit(castctx); + } else if (typectx != null) { + visit(typectx); + } else if (varctx != null) { + visit(varctx); + } else if (newctx != null) { + visit(newctx); + } else if (stringctx != null) { + visit(stringctx); + } else { + throw new IllegalStateException(error(ctx) + "Unexpected writer state."); + } + + checkWriteCast(ctx, castenmd.castTo); + + return null; + } + + @Override + public Void visitExtbrace(final ExtbraceContext ctx) { + final ExpressionContext exprctx = adapter.updateExpressionTree(ctx.expression()); + + visit(exprctx); + writeLoadStoreExternal(ctx); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null) { + visit(dotctx); + } else if (bracectx != null) { + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtdot(final ExtdotContext ctx) { + final ExtcallContext callctx = ctx.extcall(); + final ExtfieldContext fieldctx = ctx.extfield(); + + if (callctx != null) { + visit(callctx); + } else if (fieldctx != null) { + visit(fieldctx); + } + + return null; + } + + @Override + public Void visitExttype(final ExttypeContext ctx) { + visit(ctx.extdot()); + + return null; + } + + @Override + public Void visitExtcall(final ExtcallContext ctx) { + writeCallExternal(ctx); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null) { + visit(dotctx); + } else if (bracectx != null) { + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtvar(final ExtvarContext ctx) { + writeLoadStoreExternal(ctx); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null) { + visit(dotctx); + } else if (bracectx != null) { + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtfield(final ExtfieldContext ctx) { + writeLoadStoreExternal(ctx); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null) { + visit(dotctx); + } else if (bracectx != null) { + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtnew(ExtnewContext ctx) { + writeNewExternal(ctx); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null) { + visit(dotctx); + } else if (bracectx != null) { + visit(bracectx); + } + + return null; + } + + @Override + public Void visitExtstring(ExtstringContext ctx) { + final ExtNodeMetadata stringenmd = adapter.getExtNodeMetadata(ctx); + + writeConstant(ctx, stringenmd.target); + + final ExtdotContext dotctx = ctx.extdot(); + final ExtbraceContext bracectx = ctx.extbrace(); + + if (dotctx != null) { + visit(dotctx); + } else if (bracectx != null) { + visit(bracectx); + } + + return null; + } + + @Override + public Void visitArguments(final ArgumentsContext ctx) { + throw new UnsupportedOperationException(error(ctx) + "Unexpected writer state."); + } + + @Override + public Void visitIncrement(IncrementContext ctx) { + final ExpressionMetadata incremd = adapter.getExpressionMetadata(ctx); + final Object postConst = incremd.postConst; + + if (postConst == null) { + writeNumeric(ctx, incremd.preConst); + checkWriteCast(incremd); + } else { + writeConstant(ctx, postConst); + } + + checkWriteBranch(ctx); + + return null; + } + + private void writeConstant(final ParserRuleContext source, final Object constant) { + if (constant instanceof Number) { + writeNumeric(source, constant); + } else if (constant instanceof Character) { + writeNumeric(source, (int)(char)constant); + } else if (constant instanceof String) { + writeString(source, constant); + } else if (constant instanceof Boolean) { + writeBoolean(source, constant); + } else if (constant != null) { + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + } + + private void writeNumeric(final ParserRuleContext source, final Object numeric) { + if (numeric instanceof Double) { + execute.push((double)numeric); + } else if (numeric instanceof Float) { + execute.push((float)numeric); + } else if (numeric instanceof Long) { + execute.push((long)numeric); + } else if (numeric instanceof Number) { + execute.push(((Number)numeric).intValue()); + } else { + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + } + + private void writeString(final ParserRuleContext source, final Object string) { + if (string instanceof String) { + execute.push((String)string); + } else { + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + } + + private void writeBoolean(final ParserRuleContext source, final Object bool) { + if (bool instanceof Boolean) { + execute.push((boolean)bool); + } else { + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + } + + private void writeNewStrings() { + execute.newInstance(STRINGBUILDER_TYPE); + execute.dup(); + execute.invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR); + } + + private void writeAppendStrings(final Sort sort) { + switch (sort) { + case BOOL: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break; + case CHAR: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR); break; + case BYTE: + case SHORT: + case INT: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT); break; + case LONG: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG); break; + case FLOAT: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT); break; + case DOUBLE: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE); break; + case STRING: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING); break; + default: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT); + } + } + + private void writeToStrings() { + execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING); + } + + private void writeBinaryInstruction(final ParserRuleContext source, final Type type, final int token) { + final Sort sort = type.sort; + final boolean exact = !settings.getNumericOverflow() && + ((sort == Sort.INT || sort == Sort.LONG) && + (token == MUL || token == DIV || token == ADD || token == SUB) || + (sort == Sort.FLOAT || sort == Sort.DOUBLE) && + (token == MUL || token == DIV || token == REM || token == ADD || token == SUB)); + + // if its a 64-bit shift, fixup the last argument to truncate to 32-bits + // note unlike java, this means we still do binary promotion of shifts, + // but it keeps things simple -- this check works because we promote shifts. + if (sort == Sort.LONG && (token == LSH || token == USH || token == RSH)) { + execute.cast(org.objectweb.asm.Type.LONG_TYPE, org.objectweb.asm.Type.INT_TYPE); + } + + if (exact) { + switch (sort) { + case INT: + switch (token) { + case MUL: execute.invokeStatic(definition.mathType.type, MULEXACT_INT); break; + case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break; + case ADD: execute.invokeStatic(definition.mathType.type, ADDEXACT_INT); break; + case SUB: execute.invokeStatic(definition.mathType.type, SUBEXACT_INT); break; + default: + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + + break; + case LONG: + switch (token) { + case MUL: execute.invokeStatic(definition.mathType.type, MULEXACT_LONG); break; + case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break; + case ADD: execute.invokeStatic(definition.mathType.type, ADDEXACT_LONG); break; + case SUB: execute.invokeStatic(definition.mathType.type, SUBEXACT_LONG); break; + default: + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + + break; + case FLOAT: + switch (token) { + case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break; + case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break; + case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break; + case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break; + case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break; + default: + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + + break; + case DOUBLE: + switch (token) { + case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break; + case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break; + case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break; + case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break; + case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break; + default: + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + + break; + default: + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + } else { + if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) && + (token == LSH || token == USH || token == RSH || token == BWAND || token == BWXOR || token == BWOR)) { + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + + if (sort == Sort.DEF) { + switch (token) { + case MUL: execute.invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break; + case DIV: execute.invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break; + case REM: execute.invokeStatic(definition.defobjType.type, DEF_REM_CALL); break; + case ADD: execute.invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break; + case SUB: execute.invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break; + case LSH: execute.invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break; + case USH: execute.invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break; + case RSH: execute.invokeStatic(definition.defobjType.type, DEF_USH_CALL); break; + case BWAND: execute.invokeStatic(definition.defobjType.type, DEF_AND_CALL); break; + case BWXOR: execute.invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break; + case BWOR: execute.invokeStatic(definition.defobjType.type, DEF_OR_CALL); break; + default: + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + } else { + switch (token) { + case MUL: execute.math(GeneratorAdapter.MUL, type.type); break; + case DIV: execute.math(GeneratorAdapter.DIV, type.type); break; + case REM: execute.math(GeneratorAdapter.REM, type.type); break; + case ADD: execute.math(GeneratorAdapter.ADD, type.type); break; + case SUB: execute.math(GeneratorAdapter.SUB, type.type); break; + case LSH: execute.math(GeneratorAdapter.SHL, type.type); break; + case USH: execute.math(GeneratorAdapter.USHR, type.type); break; + case RSH: execute.math(GeneratorAdapter.SHR, type.type); break; + case BWAND: execute.math(GeneratorAdapter.AND, type.type); break; + case BWXOR: execute.math(GeneratorAdapter.XOR, type.type); break; + case BWOR: execute.math(GeneratorAdapter.OR, type.type); break; + default: + throw new IllegalStateException(error(source) + "Unexpected writer state."); + } + } + } + } + + /** + * Called for any compound assignment (including increment/decrement instructions). + * We have to be stricter than writeBinary, and do overflow checks against the original type's size + * instead of the promoted type's size, since the result will be implicitly cast back. + * + * @return true if an instruction is written, false otherwise + */ + private boolean writeExactInstruction(final Sort osort, final Sort psort) { + if (psort == Sort.DOUBLE) { + if (osort == Sort.FLOAT) { + execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE); + } else if (osort == Sort.FLOAT_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE); + execute.checkCast(definition.floatobjType.type); + } else if (osort == Sort.LONG) { + execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE); + } else if (osort == Sort.LONG_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE); + execute.checkCast(definition.longobjType.type); + } else if (osort == Sort.INT) { + execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE); + } else if (osort == Sort.INT_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE); + execute.checkCast(definition.intobjType.type); + } else if (osort == Sort.CHAR) { + execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE); + } else if (osort == Sort.CHAR_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE); + execute.checkCast(definition.charobjType.type); + } else if (osort == Sort.SHORT) { + execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE); + } else if (osort == Sort.SHORT_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE); + execute.checkCast(definition.shortobjType.type); + } else if (osort == Sort.BYTE) { + execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE); + } else if (osort == Sort.BYTE_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE); + execute.checkCast(definition.byteobjType.type); + } else { + return false; + } + } else if (psort == Sort.FLOAT) { + if (osort == Sort.LONG) { + execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT); + } else if (osort == Sort.LONG_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT); + execute.checkCast(definition.longobjType.type); + } else if (osort == Sort.INT) { + execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT); + } else if (osort == Sort.INT_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT); + execute.checkCast(definition.intobjType.type); + } else if (osort == Sort.CHAR) { + execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT); + } else if (osort == Sort.CHAR_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT); + execute.checkCast(definition.charobjType.type); + } else if (osort == Sort.SHORT) { + execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT); + } else if (osort == Sort.SHORT_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT); + execute.checkCast(definition.shortobjType.type); + } else if (osort == Sort.BYTE) { + execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT); + } else if (osort == Sort.BYTE_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT); + execute.checkCast(definition.byteobjType.type); + } else { + return false; + } + } else if (psort == Sort.LONG) { + if (osort == Sort.INT) { + execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG); + } else if (osort == Sort.INT_OBJ) { + execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG); + execute.checkCast(definition.intobjType.type); + } else if (osort == Sort.CHAR) { + execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG); + } else if (osort == Sort.CHAR_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG); + execute.checkCast(definition.charobjType.type); + } else if (osort == Sort.SHORT) { + execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG); + } else if (osort == Sort.SHORT_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG); + execute.checkCast(definition.shortobjType.type); + } else if (osort == Sort.BYTE) { + execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG); + } else if (osort == Sort.BYTE_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG); + execute.checkCast(definition.byteobjType.type); + } else { + return false; + } + } else if (psort == Sort.INT) { + if (osort == Sort.CHAR) { + execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT); + } else if (osort == Sort.CHAR_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT); + execute.checkCast(definition.charobjType.type); + } else if (osort == Sort.SHORT) { + execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT); + } else if (osort == Sort.SHORT_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT); + execute.checkCast(definition.shortobjType.type); + } else if (osort == Sort.BYTE) { + execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT); + } else if (osort == Sort.BYTE_OBJ) { + execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT); + execute.checkCast(definition.byteobjType.type); + } else { + return false; + } + } else { + return false; + } + + return true; + } + + private void writeLoadStoreExternal(final ParserRuleContext source) { + final ExtNodeMetadata sourceenmd = adapter.getExtNodeMetadata(source); + final ExternalMetadata parentemd = adapter.getExternalMetadata(sourceenmd.parent); + + final boolean length = "#length".equals(sourceenmd.target); + final boolean array = "#brace".equals(sourceenmd.target); + final boolean name = sourceenmd.target instanceof String && !length && !array; + final boolean variable = sourceenmd.target instanceof Integer; + final boolean field = sourceenmd.target instanceof Field; + final boolean shortcut = sourceenmd.target instanceof Object[]; + + if (!length && !variable && !field && !array && !name && !shortcut) { + throw new IllegalStateException(error(source) + "Target not found for load/store."); + } + + final boolean maplist = shortcut && (boolean)((Object[])sourceenmd.target)[2]; + final Object constant = shortcut ? ((Object[])sourceenmd.target)[3] : null; + + final boolean x1 = field || name || (shortcut && !maplist); + final boolean x2 = array || (shortcut && maplist); + + if (length) { + execute.arrayLength(); + } else if (sourceenmd.last && parentemd.storeExpr != null) { + final ExpressionMetadata expremd = adapter.getExpressionMetadata(parentemd.storeExpr); + final boolean cat = strings.contains(parentemd.storeExpr); + + if (cat) { + if (field || name || shortcut) { + execute.dupX1(); + } else if (array) { + execute.dup2X1(); + } + + if (maplist) { + if (constant != null) { + writeConstant(source, constant); + } + + execute.dupX2(); + } + + writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut); + writeAppendStrings(sourceenmd.type.sort); + visit(parentemd.storeExpr); + + if (strings.contains(parentemd.storeExpr)) { + writeAppendStrings(expremd.to.sort); + strings.remove(parentemd.storeExpr); + } + + writeToStrings(); + checkWriteCast(source, sourceenmd.castTo); + + if (parentemd.read) { + writeDup(sourceenmd.type.sort.size, x1, x2); + } + + writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut); + } else if (parentemd.token > 0) { + final int token = parentemd.token; + + if (field || name || shortcut) { + execute.dup(); + } else if (array) { + execute.dup2(); + } + + if (maplist) { + if (constant != null) { + writeConstant(source, constant); + } + + execute.dupX1(); + } + + writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut); + + if (parentemd.read && parentemd.post) { + writeDup(sourceenmd.type.sort.size, x1, x2); + } + + checkWriteCast(source, sourceenmd.castFrom); + visit(parentemd.storeExpr); + + writeBinaryInstruction(source, sourceenmd.promote, token); + + boolean exact = false; + + if (!settings.getNumericOverflow() && expremd.typesafe && sourceenmd.type.sort != Sort.DEF && + (token == MUL || token == DIV || token == REM || token == ADD || token == SUB)) { + exact = writeExactInstruction(sourceenmd.type.sort, sourceenmd.promote.sort); + } + + if (!exact) { + checkWriteCast(source, sourceenmd.castTo); + } + + if (parentemd.read && !parentemd.post) { + writeDup(sourceenmd.type.sort.size, x1, x2); + } + + writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut); + } else { + if (constant != null) { + writeConstant(source, constant); + } + + visit(parentemd.storeExpr); + + if (parentemd.read) { + writeDup(sourceenmd.type.sort.size, x1, x2); + } + + writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut); + } + } else { + if (constant != null) { + writeConstant(source, constant); + } + + writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut); + } + } + + private void writeLoadStoreInstruction(final ParserRuleContext source, + final boolean store, final boolean variable, + final boolean field, final boolean name, + final boolean array, final boolean shortcut) { + final ExtNodeMetadata sourceemd = adapter.getExtNodeMetadata(source); + + if (variable) { + writeLoadStoreVariable(source, store, sourceemd.type, (int)sourceemd.target); + } else if (field) { + writeLoadStoreField(store, (Field)sourceemd.target); + } else if (name) { + writeLoadStoreField(source, store, (String)sourceemd.target); + } else if (array) { + writeLoadStoreArray(source, store, sourceemd.type); + } else if (shortcut) { + Object[] targets = (Object[])sourceemd.target; + writeLoadStoreShortcut(store, (Method)targets[0], (Method)targets[1]); + } else { + throw new IllegalStateException(error(source) + "Load/Store requires a variable, field, or array."); + } + } + + private void writeLoadStoreVariable(final ParserRuleContext source, final boolean store, + final Type type, final int slot) { + if (type.sort == Sort.VOID) { + throw new IllegalStateException(error(source) + "Cannot load/store void type."); + } + + if (store) { + execute.visitVarInsn(type.type.getOpcode(Opcodes.ISTORE), slot); + } else { + execute.visitVarInsn(type.type.getOpcode(Opcodes.ILOAD), slot); + } + } + + private void writeLoadStoreField(final boolean store, final Field field) { + if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) { + if (store) { + execute.putStatic(field.owner.type, field.reflect.getName(), field.type.type); + } else { + execute.getStatic(field.owner.type, field.reflect.getName(), field.type.type); + + if (!field.generic.clazz.equals(field.type.clazz)) { + execute.checkCast(field.generic.type); + } + } + } else { + if (store) { + execute.putField(field.owner.type, field.reflect.getName(), field.type.type); + } else { + execute.getField(field.owner.type, field.reflect.getName(), field.type.type); + + if (!field.generic.clazz.equals(field.type.clazz)) { + execute.checkCast(field.generic.type); + } + } + } + } + + private void writeLoadStoreField(final ParserRuleContext source, final boolean store, final String name) { + if (store) { + final ExtNodeMetadata sourceemd = adapter.getExtNodeMetadata(source); + final ExternalMetadata parentemd = adapter.getExternalMetadata(sourceemd.parent); + final ExpressionMetadata expremd = adapter.getExpressionMetadata(parentemd.storeExpr); + + execute.push(name); + execute.loadThis(); + execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE); + execute.push(parentemd.token == 0 && expremd.typesafe); + execute.invokeStatic(definition.defobjType.type, DEF_FIELD_STORE); + } else { + execute.push(name); + execute.loadThis(); + execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE); + execute.invokeStatic(definition.defobjType.type, DEF_FIELD_LOAD); + } + } + + private void writeLoadStoreArray(final ParserRuleContext source, final boolean store, final Type type) { + if (type.sort == Sort.VOID) { + throw new IllegalStateException(error(source) + "Cannot load/store void type."); + } + + if (type.sort == Sort.DEF) { + final ExtbraceContext bracectx = (ExtbraceContext)source; + final ExpressionMetadata expremd0 = adapter.getExpressionMetadata(bracectx.expression()); + + if (store) { + final ExtNodeMetadata braceenmd = adapter.getExtNodeMetadata(bracectx); + final ExternalMetadata parentemd = adapter.getExternalMetadata(braceenmd.parent); + final ExpressionMetadata expremd1 = adapter.getExpressionMetadata(parentemd.storeExpr); + + execute.loadThis(); + execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE); + execute.push(expremd0.typesafe); + execute.push(parentemd.token == 0 && expremd1.typesafe); + execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_STORE); + } else { + execute.loadThis(); + execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE); + execute.push(expremd0.typesafe); + execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_LOAD); + } + } else { + if (store) { + execute.arrayStore(type.type); + } else { + execute.arrayLoad(type.type); + } + } + } + + private void writeLoadStoreShortcut(final boolean store, final Method getter, final Method setter) { + final Method method = store ? setter : getter; + + if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) { + execute.invokeInterface(method.owner.type, method.method); + } else { + execute.invokeVirtual(method.owner.type, method.method); + } + + if (store) { + writePop(method.rtn.type.getSize()); + } else if (!method.rtn.clazz.equals(method.handle.type().returnType())) { + execute.checkCast(method.rtn.type); + } + } + + private void writeDup(final int size, final boolean x1, final boolean x2) { + if (size == 1) { + if (x2) { + execute.dupX2(); + } else if (x1) { + execute.dupX1(); + } else { + execute.dup(); + } + } else if (size == 2) { + if (x2) { + execute.dup2X2(); + } else if (x1) { + execute.dup2X1(); + } else { + execute.dup2(); + } + } + } + + private void writeNewExternal(final ExtnewContext source) { + final ExtNodeMetadata sourceenmd = adapter.getExtNodeMetadata(source); + final ExternalMetadata parentemd = adapter.getExternalMetadata(sourceenmd.parent); + + final boolean makearray = "#makearray".equals(sourceenmd.target); + final boolean constructor = sourceenmd.target instanceof Constructor; + + if (!makearray && !constructor) { + throw new IllegalStateException(error(source) + "Target not found for new call."); + } + + if (makearray) { + for (final ExpressionContext exprctx : source.expression()) { + visit(exprctx); + } + + if (sourceenmd.type.sort == Sort.ARRAY) { + execute.visitMultiANewArrayInsn(sourceenmd.type.type.getDescriptor(), sourceenmd.type.type.getDimensions()); + } else { + execute.newArray(sourceenmd.type.type); + } + } else { + execute.newInstance(sourceenmd.type.type); + + if (parentemd.read) { + execute.dup(); + } + + for (final ExpressionContext exprctx : source.arguments().expression()) { + visit(exprctx); + } + + final Constructor target = (Constructor)sourceenmd.target; + execute.invokeConstructor(target.owner.type, target.method); + } + } + + private void writeCallExternal(final ExtcallContext source) { + final ExtNodeMetadata sourceenmd = adapter.getExtNodeMetadata(source); + + final boolean method = sourceenmd.target instanceof Method; + final boolean def = sourceenmd.target instanceof String; + + if (!method && !def) { + throw new IllegalStateException(error(source) + "Target not found for call."); + } + + final List arguments = source.arguments().expression(); + + if (method) { + for (final ExpressionContext exprctx : arguments) { + visit(exprctx); + } + + final Method target = (Method)sourceenmd.target; + + if (java.lang.reflect.Modifier.isStatic(target.reflect.getModifiers())) { + execute.invokeStatic(target.owner.type, target.method); + } else if (java.lang.reflect.Modifier.isInterface(target.owner.clazz.getModifiers())) { + execute.invokeInterface(target.owner.type, target.method); + } else { + execute.invokeVirtual(target.owner.type, target.method); + } + + if (!target.rtn.clazz.equals(target.handle.type().returnType())) { + execute.checkCast(target.rtn.type); + } + } else { + execute.push((String)sourceenmd.target); + execute.loadThis(); + execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE); + + execute.push(arguments.size()); + execute.newArray(definition.defType.type); + + for (int argument = 0; argument < arguments.size(); ++argument) { + execute.dup(); + execute.push(argument); + visit(arguments.get(argument)); + execute.arrayStore(definition.defType.type); + } + + execute.push(arguments.size()); + execute.newArray(definition.booleanType.type); + + for (int argument = 0; argument < arguments.size(); ++argument) { + execute.dup(); + execute.push(argument); + execute.push(adapter.getExpressionMetadata(arguments.get(argument)).typesafe); + execute.arrayStore(definition.booleanType.type); + } + + execute.invokeStatic(definition.defobjType.type, DEF_METHOD_CALL); + } + } + + private void writePop(final int size) { + if (size == 1) { + execute.pop(); + } else if (size == 2) { + execute.pop2(); + } + } + + private void checkWriteCast(final ExpressionMetadata sort) { + checkWriteCast(sort.source, sort.cast); + } + + private void checkWriteCast(final ParserRuleContext source, final Cast cast) { + if (cast instanceof Transform) { + writeTransform((Transform)cast); + } else if (cast != null) { + writeCast(cast); + } else { + throw new IllegalStateException(error(source) + "Unexpected cast object."); + } + } + + private void writeCast(final Cast cast) { + final Type from = cast.from; + final Type to = cast.to; + + if (from.equals(to)) { + return; + } + + if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) { + execute.cast(from.type, to.type); + } else { + try { + from.clazz.asSubclass(to.clazz); + } catch (ClassCastException exception) { + execute.checkCast(to.type); + } + } + } + + private void writeTransform(final Transform transform) { + if (transform.upcast != null) { + execute.checkCast(transform.upcast.type); + } + + if (java.lang.reflect.Modifier.isStatic(transform.method.reflect.getModifiers())) { + execute.invokeStatic(transform.method.owner.type, transform.method.method); + } else if (java.lang.reflect.Modifier.isInterface(transform.method.owner.clazz.getModifiers())) { + execute.invokeInterface(transform.method.owner.type, transform.method.method); + } else { + execute.invokeVirtual(transform.method.owner.type, transform.method.method); + } + + if (transform.downcast != null) { + execute.checkCast(transform.downcast.type); + } + } + + void checkWriteBranch(final ParserRuleContext source) { + final Branch branch = getBranch(source); + + if (branch != null) { + if (branch.tru != null) { + execute.visitJumpInsn(Opcodes.IFNE, branch.tru); + } else if (branch.fals != null) { + execute.visitJumpInsn(Opcodes.IFEQ, branch.fals); + } + } + } + + private void writeEnd() { + writer.visitEnd(); + } + + private byte[] getBytes() { + return writer.toByteArray(); + } +} diff --git a/plugins/lang-expression/src/main/plugin-metadata/plugin-security.policy b/plugins/lang-plan-a/src/main/plugin-metadata/plugin-security.policy similarity index 100% rename from plugins/lang-expression/src/main/plugin-metadata/plugin-security.policy rename to plugins/lang-plan-a/src/main/plugin-metadata/plugin-security.policy diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/AdditionTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/AdditionTests.java new file mode 100644 index 00000000000..af7eb25a6c0 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/AdditionTests.java @@ -0,0 +1,199 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.HashMap; +import java.util.Map; + +/** Tests for addition operator across all types */ +//TODO: NaN/Inf/overflow/... +public class AdditionTests extends ScriptTestCase { + + public void testInt() throws Exception { + assertEquals(1+1, exec("int x = 1; int y = 1; return x+y;")); + assertEquals(1+2, exec("int x = 1; int y = 2; return x+y;")); + assertEquals(5+10, exec("int x = 5; int y = 10; return x+y;")); + assertEquals(1+1+2, exec("int x = 1; int y = 1; int z = 2; return x+y+z;")); + assertEquals((1+1)+2, exec("int x = 1; int y = 1; int z = 2; return (x+y)+z;")); + assertEquals(1+(1+2), exec("int x = 1; int y = 1; int z = 2; return x+(y+z);")); + assertEquals(0+1, exec("int x = 0; int y = 1; return x+y;")); + assertEquals(1+0, exec("int x = 1; int y = 0; return x+y;")); + assertEquals(0+0, exec("int x = 0; int y = 0; return x+y;")); + assertEquals(0+0, exec("int x = 0; int y = 0; return x+y;")); + } + + public void testIntConst() throws Exception { + assertEquals(1+1, exec("return 1+1;")); + assertEquals(1+2, exec("return 1+2;")); + assertEquals(5+10, exec("return 5+10;")); + assertEquals(1+1+2, exec("return 1+1+2;")); + assertEquals((1+1)+2, exec("return (1+1)+2;")); + assertEquals(1+(1+2), exec("return 1+(1+2);")); + assertEquals(0+1, exec("return 0+1;")); + assertEquals(1+0, exec("return 1+0;")); + assertEquals(0+0, exec("return 0+0;")); + } + + public void testByte() throws Exception { + assertEquals((byte)1+(byte)1, exec("byte x = 1; byte y = 1; return x+y;")); + assertEquals((byte)1+(byte)2, exec("byte x = 1; byte y = 2; return x+y;")); + assertEquals((byte)5+(byte)10, exec("byte x = 5; byte y = 10; return x+y;")); + assertEquals((byte)1+(byte)1+(byte)2, exec("byte x = 1; byte y = 1; byte z = 2; return x+y+z;")); + assertEquals(((byte)1+(byte)1)+(byte)2, exec("byte x = 1; byte y = 1; byte z = 2; return (x+y)+z;")); + assertEquals((byte)1+((byte)1+(byte)2), exec("byte x = 1; byte y = 1; byte z = 2; return x+(y+z);")); + assertEquals((byte)0+(byte)1, exec("byte x = 0; byte y = 1; return x+y;")); + assertEquals((byte)1+(byte)0, exec("byte x = 1; byte y = 0; return x+y;")); + assertEquals((byte)0+(byte)0, exec("byte x = 0; byte y = 0; return x+y;")); + } + + public void testByteConst() throws Exception { + assertEquals((byte)1+(byte)1, exec("return (byte)1+(byte)1;")); + assertEquals((byte)1+(byte)2, exec("return (byte)1+(byte)2;")); + assertEquals((byte)5+(byte)10, exec("return (byte)5+(byte)10;")); + assertEquals((byte)1+(byte)1+(byte)2, exec("return (byte)1+(byte)1+(byte)2;")); + assertEquals(((byte)1+(byte)1)+(byte)2, exec("return ((byte)1+(byte)1)+(byte)2;")); + assertEquals((byte)1+((byte)1+(byte)2), exec("return (byte)1+((byte)1+(byte)2);")); + assertEquals((byte)0+(byte)1, exec("return (byte)0+(byte)1;")); + assertEquals((byte)1+(byte)0, exec("return (byte)1+(byte)0;")); + assertEquals((byte)0+(byte)0, exec("return (byte)0+(byte)0;")); + } + + public void testChar() throws Exception { + assertEquals((char)1+(char)1, exec("char x = 1; char y = 1; return x+y;")); + assertEquals((char)1+(char)2, exec("char x = 1; char y = 2; return x+y;")); + assertEquals((char)5+(char)10, exec("char x = 5; char y = 10; return x+y;")); + assertEquals((char)1+(char)1+(char)2, exec("char x = 1; char y = 1; char z = 2; return x+y+z;")); + assertEquals(((char)1+(char)1)+(char)2, exec("char x = 1; char y = 1; char z = 2; return (x+y)+z;")); + assertEquals((char)1+((char)1+(char)2), exec("char x = 1; char y = 1; char z = 2; return x+(y+z);")); + assertEquals((char)0+(char)1, exec("char x = 0; char y = 1; return x+y;")); + assertEquals((char)1+(char)0, exec("char x = 1; char y = 0; return x+y;")); + assertEquals((char)0+(char)0, exec("char x = 0; char y = 0; return x+y;")); + } + + public void testCharConst() throws Exception { + assertEquals((char)1+(char)1, exec("return (char)1+(char)1;")); + assertEquals((char)1+(char)2, exec("return (char)1+(char)2;")); + assertEquals((char)5+(char)10, exec("return (char)5+(char)10;")); + assertEquals((char)1+(char)1+(char)2, exec("return (char)1+(char)1+(char)2;")); + assertEquals(((char)1+(char)1)+(char)2, exec("return ((char)1+(char)1)+(char)2;")); + assertEquals((char)1+((char)1+(char)2), exec("return (char)1+((char)1+(char)2);")); + assertEquals((char)0+(char)1, exec("return (char)0+(char)1;")); + assertEquals((char)1+(char)0, exec("return (char)1+(char)0;")); + assertEquals((char)0+(char)0, exec("return (char)0+(char)0;")); + } + + public void testShort() throws Exception { + assertEquals((short)1+(short)1, exec("short x = 1; short y = 1; return x+y;")); + assertEquals((short)1+(short)2, exec("short x = 1; short y = 2; return x+y;")); + assertEquals((short)5+(short)10, exec("short x = 5; short y = 10; return x+y;")); + assertEquals((short)1+(short)1+(short)2, exec("short x = 1; short y = 1; short z = 2; return x+y+z;")); + assertEquals(((short)1+(short)1)+(short)2, exec("short x = 1; short y = 1; short z = 2; return (x+y)+z;")); + assertEquals((short)1+((short)1+(short)2), exec("short x = 1; short y = 1; short z = 2; return x+(y+z);")); + assertEquals((short)0+(short)1, exec("short x = 0; short y = 1; return x+y;")); + assertEquals((short)1+(short)0, exec("short x = 1; short y = 0; return x+y;")); + assertEquals((short)0+(short)0, exec("short x = 0; short y = 0; return x+y;")); + } + + public void testShortConst() throws Exception { + assertEquals((short)1+(short)1, exec("return (short)1+(short)1;")); + assertEquals((short)1+(short)2, exec("return (short)1+(short)2;")); + assertEquals((short)5+(short)10, exec("return (short)5+(short)10;")); + assertEquals((short)1+(short)1+(short)2, exec("return (short)1+(short)1+(short)2;")); + assertEquals(((short)1+(short)1)+(short)2, exec("return ((short)1+(short)1)+(short)2;")); + assertEquals((short)1+((short)1+(short)2), exec("return (short)1+((short)1+(short)2);")); + assertEquals((short)0+(short)1, exec("return (short)0+(short)1;")); + assertEquals((short)1+(short)0, exec("return (short)1+(short)0;")); + assertEquals((short)0+(short)0, exec("return (short)0+(short)0;")); + } + + public void testLong() throws Exception { + assertEquals(1L+1L, exec("long x = 1; long y = 1; return x+y;")); + assertEquals(1L+2L, exec("long x = 1; long y = 2; return x+y;")); + assertEquals(5L+10L, exec("long x = 5; long y = 10; return x+y;")); + assertEquals(1L+1L+2L, exec("long x = 1; long y = 1; long z = 2; return x+y+z;")); + assertEquals((1L+1L)+2L, exec("long x = 1; long y = 1; long z = 2; return (x+y)+z;")); + assertEquals(1L+(1L+2L), exec("long x = 1; long y = 1; long z = 2; return x+(y+z);")); + assertEquals(0L+1L, exec("long x = 0; long y = 1; return x+y;")); + assertEquals(1L+0L, exec("long x = 1; long y = 0; return x+y;")); + assertEquals(0L+0L, exec("long x = 0; long y = 0; return x+y;")); + } + + public void testLongConst() throws Exception { + assertEquals(1L+1L, exec("return 1L+1L;")); + assertEquals(1L+2L, exec("return 1L+2L;")); + assertEquals(5L+10L, exec("return 5L+10L;")); + assertEquals(1L+1L+2L, exec("return 1L+1L+2L;")); + assertEquals((1L+1L)+2L, exec("return (1L+1L)+2L;")); + assertEquals(1L+(1L+2L), exec("return 1L+(1L+2L);")); + assertEquals(0L+1L, exec("return 0L+1L;")); + assertEquals(1L+0L, exec("return 1L+0L;")); + assertEquals(0L+0L, exec("return 0L+0L;")); + } + + public void testFloat() throws Exception { + assertEquals(1F+1F, exec("float x = 1F; float y = 1F; return x+y;")); + assertEquals(1F+2F, exec("float x = 1F; float y = 2F; return x+y;")); + assertEquals(5F+10F, exec("float x = 5F; float y = 10F; return x+y;")); + assertEquals(1F+1F+2F, exec("float x = 1F; float y = 1F; float z = 2F; return x+y+z;")); + assertEquals((1F+1F)+2F, exec("float x = 1F; float y = 1F; float z = 2F; return (x+y)+z;")); + assertEquals((1F+1F)+2F, exec("float x = 1F; float y = 1F; float z = 2F; return x+(y+z);")); + assertEquals(0F+1F, exec("float x = 0F; float y = 1F; return x+y;")); + assertEquals(1F+0F, exec("float x = 1F; float y = 0F; return x+y;")); + assertEquals(0F+0F, exec("float x = 0F; float y = 0F; return x+y;")); + } + + public void testFloatConst() throws Exception { + assertEquals(1F+1F, exec("return 1F+1F;")); + assertEquals(1F+2F, exec("return 1F+2F;")); + assertEquals(5F+10F, exec("return 5F+10F;")); + assertEquals(1F+1F+2F, exec("return 1F+1F+2F;")); + assertEquals((1F+1F)+2F, exec("return (1F+1F)+2F;")); + assertEquals(1F+(1F+2F), exec("return 1F+(1F+2F);")); + assertEquals(0F+1F, exec("return 0F+1F;")); + assertEquals(1F+0F, exec("return 1F+0F;")); + assertEquals(0F+0F, exec("return 0F+0F;")); + } + + public void testDouble() throws Exception { + assertEquals(1.0+1.0, exec("double x = 1.0; double y = 1.0; return x+y;")); + assertEquals(1.0+2.0, exec("double x = 1.0; double y = 2.0; return x+y;")); + assertEquals(5.0+10.0, exec("double x = 5.0; double y = 10.0; return x+y;")); + assertEquals(1.0+1.0+2.0, exec("double x = 1.0; double y = 1.0; double z = 2.0; return x+y+z;")); + assertEquals((1.0+1.0)+2.0, exec("double x = 1.0; double y = 1.0; double z = 2.0; return (x+y)+z;")); + assertEquals(1.0+(1.0+2.0), exec("double x = 1.0; double y = 1.0; double z = 2.0; return x+(y+z);")); + assertEquals(0.0+1.0, exec("double x = 0.0; double y = 1.0; return x+y;")); + assertEquals(1.0+0.0, exec("double x = 1.0; double y = 0.0; return x+y;")); + assertEquals(0.0+0.0, exec("double x = 0.0; double y = 0.0; return x+y;")); + } + + public void testDoubleConst() throws Exception { + assertEquals(1.0+1.0, exec("return 1.0+1.0;")); + assertEquals(1.0+2.0, exec("return 1.0+2.0;")); + assertEquals(5.0+10.0, exec("return 5.0+10.0;")); + assertEquals(1.0+1.0+2.0, exec("return 1.0+1.0+2.0;")); + assertEquals((1.0+1.0)+2.0, exec("return (1.0+1.0)+2.0;")); + assertEquals(1.0+(1.0+2.0), exec("return 1.0+(1.0+2.0);")); + assertEquals(0.0+1.0, exec("return 0.0+1.0;")); + assertEquals(1.0+0.0, exec("return 1.0+0.0;")); + assertEquals(0.0+0.0, exec("return 0.0+0.0;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/AndTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/AndTests.java new file mode 100644 index 00000000000..6a4168415dc --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/AndTests.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for and operator across all types */ +public class AndTests extends ScriptTestCase { + + public void testInt() throws Exception { + assertEquals(5 & 12, exec("int x = 5; int y = 12; return x & y;")); + assertEquals(5 & -12, exec("int x = 5; int y = -12; return x & y;")); + assertEquals(7 & 15 & 3, exec("int x = 7; int y = 15; int z = 3; return x & y & z;")); + } + + public void testIntConst() throws Exception { + assertEquals(5 & 12, exec("return 5 & 12;")); + assertEquals(5 & -12, exec("return 5 & -12;")); + assertEquals(7 & 15 & 3, exec("return 7 & 15 & 3;")); + } + + public void testLong() throws Exception { + assertEquals(5L & 12L, exec("long x = 5; long y = 12; return x & y;")); + assertEquals(5L & -12L, exec("long x = 5; long y = -12; return x & y;")); + assertEquals(7L & 15L & 3L, exec("long x = 7; long y = 15; long z = 3; return x & y & z;")); + } + + public void testLongConst() throws Exception { + assertEquals(5L & 12L, exec("return 5L & 12L;")); + assertEquals(5L & -12L, exec("return 5L & -12L;")); + assertEquals(7L & 15L & 3L, exec("return 7L & 15L & 3L;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BasicExpressionTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BasicExpressionTests.java new file mode 100644 index 00000000000..6af8adab564 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BasicExpressionTests.java @@ -0,0 +1,126 @@ +package org.elasticsearch.plan.a; + +import java.util.Collections; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +public class BasicExpressionTests extends ScriptTestCase { + + /** simple tests returning a constant value */ + public void testReturnConstant() { + assertEquals(5, exec("return 5;")); + assertEquals(7L, exec("return 7L;")); + assertEquals(7.0, exec("return 7.0;")); + assertEquals(32.0F, exec("return 32.0F;")); + assertEquals((byte)255, exec("return (byte)255;")); + assertEquals((short)5, exec("return (short)5;")); + assertEquals("string", exec("return \"string\";")); + assertEquals(true, exec("return true;")); + assertEquals(false, exec("return false;")); + assertNull(exec("return null;")); + } + + public void testReturnConstantChar() { + assertEquals('x', exec("return 'x';")); + } + + public void testConstantCharTruncation() { + assertEquals('蚠', exec("return (char)100000;")); + } + + /** declaring variables for primitive types */ + public void testDeclareVariable() { + assertEquals(5, exec("int i = 5; return i;")); + assertEquals(7L, exec("long l = 7; return l;")); + assertEquals(7.0, exec("double d = 7; return d;")); + assertEquals(32.0F, exec("float f = 32F; return f;")); + assertEquals((byte)255, exec("byte b = (byte)255; return b;")); + assertEquals((short)5, exec("short s = (short)5; return s;")); + assertEquals("string", exec("String s = \"string\"; return s;")); + assertEquals(true, exec("boolean v = true; return v;")); + assertEquals(false, exec("boolean v = false; return v;")); + } + + public void testCast() { + assertEquals(1, exec("return (int)1.0;")); + assertEquals((byte)100, exec("double x = 100; return (byte)x;")); + + assertEquals(3, exec( + "Map x = new HashMap();\n" + + "Object y = x;\n" + + "((Map)y).put(2, 3);\n" + + "return x.get(2);\n")); + } + + public void testCat() { + assertEquals("aaabbb", exec("return \"aaa\" + \"bbb\";")); + assertEquals("aaabbb", exec("String aaa = \"aaa\", bbb = \"bbb\"; return aaa + bbb;")); + + assertEquals("aaabbbbbbbbb", exec( + "String aaa = \"aaa\", bbb = \"bbb\"; int x;\n" + + "for (; x < 3; ++x) \n" + + " aaa += bbb;\n" + + "return aaa;")); + } + + public void testComp() { + assertEquals(true, exec("return 2 < 3;")); + assertEquals(false, exec("int x = 4; char y = 2; return x < y;")); + assertEquals(true, exec("return 3 <= 3;")); + assertEquals(true, exec("int x = 3; char y = 3; return x <= y;")); + assertEquals(false, exec("return 2 > 3;")); + assertEquals(true, exec("int x = 4; long y = 2; return x > y;")); + assertEquals(false, exec("return 3 >= 4;")); + assertEquals(true, exec("double x = 3; float y = 3; return x >= y;")); + assertEquals(false, exec("return 3 == 4;")); + assertEquals(true, exec("double x = 3; float y = 3; return x == y;")); + assertEquals(true, exec("return 3 != 4;")); + assertEquals(false, exec("double x = 3; float y = 3; return x != y;")); + } + + /** + * Test boxed objects in various places + */ + public void testBoxing() { + // return + assertEquals(4, exec("return input.get(\"x\");", Collections.singletonMap("x", 4))); + // assignment + assertEquals(4, exec("int y = (Integer)input.get(\"x\"); return y;", Collections.singletonMap("x", 4))); + // comparison + assertEquals(true, exec("return 5 > (Integer)input.get(\"x\");", Collections.singletonMap("x", 4))); + } + + public void testBool() { + assertEquals(true, exec("return true && true;")); + assertEquals(false, exec("boolean a = true, b = false; return a && b;")); + assertEquals(true, exec("return true || true;")); + assertEquals(true, exec("boolean a = true, b = false; return a || b;")); + } + + public void testConditional() { + assertEquals(1, exec("int x = 5; return x > 3 ? 1 : 0;")); + assertEquals(0, exec("String a = null; return a != null ? 1 : 0;")); + } + + public void testPrecedence() { + assertEquals(2, exec("int x = 5; return (x+x)/x;")); + assertEquals(true, exec("boolean t = true, f = false; return t && (f || t);")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BasicStatementTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BasicStatementTests.java new file mode 100644 index 00000000000..07ad32d74af --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BasicStatementTests.java @@ -0,0 +1,178 @@ +package org.elasticsearch.plan.a; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.HashMap; +import java.util.Map; + +public class BasicStatementTests extends ScriptTestCase { + + public void testIfStatement() { + assertEquals(1, exec("int x = 5; if (x == 5) return 1; return 0;")); + assertEquals(0, exec("int x = 4; if (x == 5) return 1; else return 0;")); + assertEquals(2, exec("int x = 4; if (x == 5) return 1; else if (x == 4) return 2; else return 0;")); + assertEquals(1, exec("int x = 4; if (x == 5) return 1; else if (x == 4) return 1; else return 0;")); + + assertEquals(3, exec( + "int x = 5;\n" + + "if (x == 5) {\n" + + " int y = 2;\n" + + " \n" + + " if (y == 2) {\n" + + " x = 3;\n" + + " }\n" + + " \n" + + "}\n" + + "\n" + + "return x;\n")); + } + + public void testWhileStatement() { + + assertEquals("aaaaaa", exec("String c = \"a\"; int x; while (x < 5) { c += \"a\"; ++x; } return c;")); + + Object value = exec( + " byte[][] b = new byte[5][5]; \n" + + " byte x = 0, y; \n" + + " \n" + + " while (x < 5) { \n" + + " y = 0; \n" + + " \n" + + " while (y < 5) { \n" + + " b[x][y] = (byte)(x*y); \n" + + " ++y; \n" + + " } \n" + + " \n" + + " ++x; \n" + + " } \n" + + " \n" + + " return b; \n"); + + byte[][] b = (byte[][])value; + + for (byte x = 0; x < 5; ++x) { + for (byte y = 0; y < 5; ++y) { + assertEquals(x*y, b[x][y]); + } + } + } + + public void testDoWhileStatement() { + assertEquals("aaaaaa", exec("String c = \"a\"; int x; do { c += \"a\"; ++x; } while (x < 5); return c;")); + + Object value = exec( + " int[][] b = new int[5][5]; \n" + + " int x = 0, y; \n" + + " \n" + + " do { \n" + + " y = 0; \n" + + " \n" + + " do { \n" + + " b[x][y] = x*y; \n" + + " ++y; \n" + + " } while (y < 5); \n" + + " \n" + + " ++x; \n" + + " } while (x < 5); \n" + + " \n" + + " return b; \n"); + + int[][] b = (int[][])value; + + for (byte x = 0; x < 5; ++x) { + for (byte y = 0; y < 5; ++y) { + assertEquals(x*y, b[x][y]); + } + } + } + + public void testForStatement() { + assertEquals("aaaaaa", exec("String c = \"a\"; for (int x = 0; x < 5; ++x) c += \"a\"; return c;")); + + Object value = exec( + " int[][] b = new int[5][5]; \n" + + " for (int x = 0; x < 5; ++x) { \n" + + " for (int y = 0; y < 5; ++y) { \n" + + " b[x][y] = x*y; \n" + + " } \n" + + " } \n" + + " \n" + + " return b; \n"); + + int[][] b = (int[][])value; + + for (byte x = 0; x < 5; ++x) { + for (byte y = 0; y < 5; ++y) { + assertEquals(x*y, b[x][y]); + } + } + } + + public void testDeclarationStatement() { + assertEquals((byte)2, exec("byte a = 2; return a;")); + assertEquals((short)2, exec("short a = 2; return a;")); + assertEquals((char)2, exec("char a = 2; return a;")); + assertEquals(2, exec("int a = 2; return a;")); + assertEquals(2L, exec("long a = 2; return a;")); + assertEquals(2F, exec("float a = 2; return a;")); + assertEquals(2.0, exec("double a = 2; return a;")); + assertEquals(false, exec("boolean a = false; return a;")); + assertEquals("string", exec("String a = \"string\"; return a;")); + assertEquals(HashMap.class, exec("Map a = new HashMap(); return a;").getClass()); + + assertEquals(byte[].class, exec("byte[] a = new byte[1]; return a;").getClass()); + assertEquals(short[].class, exec("short[] a = new short[1]; return a;").getClass()); + assertEquals(char[].class, exec("char[] a = new char[1]; return a;").getClass()); + assertEquals(int[].class, exec("int[] a = new int[1]; return a;").getClass()); + assertEquals(long[].class, exec("long[] a = new long[1]; return a;").getClass()); + assertEquals(float[].class, exec("float[] a = new float[1]; return a;").getClass()); + assertEquals(double[].class, exec("double[] a = new double[1]; return a;").getClass()); + assertEquals(boolean[].class, exec("boolean[] a = new boolean[1]; return a;").getClass()); + assertEquals(String[].class, exec("String[] a = new String[1]; return a;").getClass()); + assertEquals(Map[].class, exec("Map[] a = new Map[1]; return a;").getClass()); + + assertEquals(byte[][].class, exec("byte[][] a = new byte[1][2]; return a;").getClass()); + assertEquals(short[][][].class, exec("short[][][] a = new short[1][2][3]; return a;").getClass()); + assertEquals(char[][][][].class, exec("char[][][][] a = new char[1][2][3][4]; return a;").getClass()); + assertEquals(int[][][][][].class, exec("int[][][][][] a = new int[1][2][3][4][5]; return a;").getClass()); + assertEquals(long[][].class, exec("long[][] a = new long[1][2]; return a;").getClass()); + assertEquals(float[][][].class, exec("float[][][] a = new float[1][2][3]; return a;").getClass()); + assertEquals(double[][][][].class, exec("double[][][][] a = new double[1][2][3][4]; return a;").getClass()); + assertEquals(boolean[][][][][].class, exec("boolean[][][][][] a = new boolean[1][2][3][4][5]; return a;").getClass()); + assertEquals(String[][].class, exec("String[][] a = new String[1][2]; return a;").getClass()); + assertEquals(Map[][][].class, exec("Map[][][] a = new Map[1][2][3]; return a;").getClass()); + } + + public void testContinueStatement() { + assertEquals(9, exec("int x = 0, y = 0; while (x < 10) { ++x; if (x == 1) continue; ++y; } return y;")); + } + + public void testBreakStatement() { + assertEquals(4, exec("int x = 0, y = 0; while (x < 10) { ++x; if (x == 5) break; ++y; } return y;")); + } + + public void testReturnStatement() { + assertEquals(10, exec("return 10;")); + assertEquals(5, exec("int x = 5; return x;")); + assertEquals(4, exec("int[] x = new int[2]; x[1] = 4; return x[1];")); + assertEquals(5, ((short[])exec("short[] s = new short[3]; s[1] = 5; return s;"))[1]); + assertEquals(10, ((Map)exec("Map s = new HashMap< String , Object >(); s.put(\"x\", 10); return s;")).get("x")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BinaryOperatorTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BinaryOperatorTests.java new file mode 100644 index 00000000000..032cdcde5e0 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/BinaryOperatorTests.java @@ -0,0 +1,294 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** + * Tests binary operators across different types + */ +// TODO: NaN/Inf/overflow/... +public class BinaryOperatorTests extends ScriptTestCase { + + // TODO: move to per-type tests and test for each type + public void testBasics() { + assertEquals(2.25F / 1.5F, exec("return 2.25F / 1.5F;")); + assertEquals(2.25F % 1.5F, exec("return 2.25F % 1.5F;")); + assertEquals(2 - 1, exec("return 2 - 1;")); + assertEquals(1 << 2, exec("return 1 << 2;")); + assertEquals(4 >> 2, exec("return 4 >> 2;")); + assertEquals(-1 >>> 29, exec("return -1 >>> 29;")); + assertEquals(5 & 3, exec("return 5 & 3;")); + assertEquals(5 & 3L, exec("return 5 & 3L;")); + assertEquals(5L & 3, exec("return 5L & 3;")); + assertEquals(5 | 3, exec("return 5 | 3;")); + assertEquals(5L | 3, exec("return 5L | 3;")); + assertEquals(5 | 3L, exec("return 5 | 3L;")); + assertEquals(9 ^ 3, exec("return 9 ^ 3;")); + assertEquals(9L ^ 3, exec("return 9L ^ 3;")); + assertEquals(9 ^ 3L, exec("return 9 ^ 3L;")); + } + + public void testLongShifts() { + // note: we always promote the results of shifts too (unlike java) + assertEquals(1L << 2, exec("long x = 1L; int y = 2; return x << y;")); + assertEquals(1L << 2L, exec("long x = 1L; long y = 2L; return x << y;")); + assertEquals(4L >> 2L, exec("long x = 4L; long y = 2L; return x >> y;")); + assertEquals(4L >> 2, exec("long x = 4L; int y = 2; return x >> y;")); + assertEquals(-1L >>> 29, exec("long x = -1L; int y = 29; return x >>> y;")); + assertEquals(-1L >>> 29L, exec("long x = -1L; long y = 29L; return x >>> y;")); + } + + public void testLongShiftsConst() { + // note: we always promote the results of shifts too (unlike java) + assertEquals(1L << 2, exec("return 1L << 2;")); + assertEquals(1L << 2L, exec("return 1 << 2L;")); + assertEquals(4L >> 2L, exec("return 4 >> 2L;")); + assertEquals(4L >> 2, exec("return 4L >> 2;")); + assertEquals(-1L >>> 29, exec("return -1L >>> 29;")); + assertEquals(-1L >>> 29L, exec("return -1 >>> 29L;")); + } + + public void testMixedTypes() { + assertEquals(8, exec("int x = 4; char y = 2; return x*y;")); + assertEquals(0.5, exec("double x = 1; float y = 2; return x / y;")); + assertEquals(1, exec("int x = 3; int y = 2; return x % y;")); + assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;")); + assertEquals(-1, exec("int x = 1; char y = 2; return x - y;")); + assertEquals(4, exec("int x = 1; char y = 2; return x << y;")); + assertEquals(-1, exec("int x = -1; char y = 29; return x >> y;")); + assertEquals(3, exec("int x = -1; char y = 30; return x >>> y;")); + assertEquals(1L, exec("int x = 5; long y = 3; return x & y;")); + assertEquals(7, exec("short x = 5; byte y = 3; return x | y;")); + assertEquals(10, exec("short x = 9; char y = 3; return x ^ y;")); + } + + public void testBinaryPromotion() throws Exception { + // byte/byte + assertEquals((byte)1 + (byte)1, exec("byte x = 1; byte y = 1; return x+y;")); + // byte/char + assertEquals((byte)1 + (char)1, exec("byte x = 1; char y = 1; return x+y;")); + // byte/short + assertEquals((byte)1 + (short)1, exec("byte x = 1; short y = 1; return x+y;")); + // byte/int + assertEquals((byte)1 + 1, exec("byte x = 1; int y = 1; return x+y;")); + // byte/long + assertEquals((byte)1 + 1L, exec("byte x = 1; long y = 1; return x+y;")); + // byte/float + assertEquals((byte)1 + 1F, exec("byte x = 1; float y = 1; return x+y;")); + // byte/double + assertEquals((byte)1 + 1.0, exec("byte x = 1; double y = 1; return x+y;")); + + // char/byte + assertEquals((char)1 + (byte)1, exec("char x = 1; byte y = 1; return x+y;")); + // char/char + assertEquals((char)1 + (char)1, exec("char x = 1; char y = 1; return x+y;")); + // char/short + assertEquals((char)1 + (short)1, exec("char x = 1; short y = 1; return x+y;")); + // char/int + assertEquals((char)1 + 1, exec("char x = 1; int y = 1; return x+y;")); + // char/long + assertEquals((char)1 + 1L, exec("char x = 1; long y = 1; return x+y;")); + // char/float + assertEquals((char)1 + 1F, exec("char x = 1; float y = 1; return x+y;")); + // char/double + assertEquals((char)1 + 1.0, exec("char x = 1; double y = 1; return x+y;")); + + // short/byte + assertEquals((short)1 + (byte)1, exec("short x = 1; byte y = 1; return x+y;")); + // short/char + assertEquals((short)1 + (char)1, exec("short x = 1; char y = 1; return x+y;")); + // short/short + assertEquals((short)1 + (short)1, exec("short x = 1; short y = 1; return x+y;")); + // short/int + assertEquals((short)1 + 1, exec("short x = 1; int y = 1; return x+y;")); + // short/long + assertEquals((short)1 + 1L, exec("short x = 1; long y = 1; return x+y;")); + // short/float + assertEquals((short)1 + 1F, exec("short x = 1; float y = 1; return x+y;")); + // short/double + assertEquals((short)1 + 1.0, exec("short x = 1; double y = 1; return x+y;")); + + // int/byte + assertEquals(1 + (byte)1, exec("int x = 1; byte y = 1; return x+y;")); + // int/char + assertEquals(1 + (char)1, exec("int x = 1; char y = 1; return x+y;")); + // int/short + assertEquals(1 + (short)1, exec("int x = 1; short y = 1; return x+y;")); + // int/int + assertEquals(1 + 1, exec("int x = 1; int y = 1; return x+y;")); + // int/long + assertEquals(1 + 1L, exec("int x = 1; long y = 1; return x+y;")); + // int/float + assertEquals(1 + 1F, exec("int x = 1; float y = 1; return x+y;")); + // int/double + assertEquals(1 + 1.0, exec("int x = 1; double y = 1; return x+y;")); + + // long/byte + assertEquals(1L + (byte)1, exec("long x = 1; byte y = 1; return x+y;")); + // long/char + assertEquals(1L + (char)1, exec("long x = 1; char y = 1; return x+y;")); + // long/short + assertEquals(1L + (short)1, exec("long x = 1; short y = 1; return x+y;")); + // long/int + assertEquals(1L + 1, exec("long x = 1; int y = 1; return x+y;")); + // long/long + assertEquals(1L + 1L, exec("long x = 1; long y = 1; return x+y;")); + // long/float + assertEquals(1L + 1F, exec("long x = 1; float y = 1; return x+y;")); + // long/double + assertEquals(1L + 1.0, exec("long x = 1; double y = 1; return x+y;")); + + // float/byte + assertEquals(1F + (byte)1, exec("float x = 1; byte y = 1; return x+y;")); + // float/char + assertEquals(1F + (char)1, exec("float x = 1; char y = 1; return x+y;")); + // float/short + assertEquals(1F + (short)1, exec("float x = 1; short y = 1; return x+y;")); + // float/int + assertEquals(1F + 1, exec("float x = 1; int y = 1; return x+y;")); + // float/long + assertEquals(1F + 1L, exec("float x = 1; long y = 1; return x+y;")); + // float/float + assertEquals(1F + 1F, exec("float x = 1; float y = 1; return x+y;")); + // float/double + assertEquals(1F + 1.0, exec("float x = 1; double y = 1; return x+y;")); + + // double/byte + assertEquals(1.0 + (byte)1, exec("double x = 1; byte y = 1; return x+y;")); + // double/char + assertEquals(1.0 + (char)1, exec("double x = 1; char y = 1; return x+y;")); + // double/short + assertEquals(1.0 + (short)1, exec("double x = 1; short y = 1; return x+y;")); + // double/int + assertEquals(1.0 + 1, exec("double x = 1; int y = 1; return x+y;")); + // double/long + assertEquals(1.0 + 1L, exec("double x = 1; long y = 1; return x+y;")); + // double/float + assertEquals(1.0 + 1F, exec("double x = 1; float y = 1; return x+y;")); + // double/double + assertEquals(1.0 + 1.0, exec("double x = 1; double y = 1; return x+y;")); + } + + public void testBinaryPromotionConst() throws Exception { + // byte/byte + assertEquals((byte)1 + (byte)1, exec("return (byte)1 + (byte)1;")); + // byte/char + assertEquals((byte)1 + (char)1, exec("return (byte)1 + (char)1;")); + // byte/short + assertEquals((byte)1 + (short)1, exec("return (byte)1 + (short)1;")); + // byte/int + assertEquals((byte)1 + 1, exec("return (byte)1 + 1;")); + // byte/long + assertEquals((byte)1 + 1L, exec("return (byte)1 + 1L;")); + // byte/float + assertEquals((byte)1 + 1F, exec("return (byte)1 + 1F;")); + // byte/double + assertEquals((byte)1 + 1.0, exec("return (byte)1 + 1.0;")); + + // char/byte + assertEquals((char)1 + (byte)1, exec("return (char)1 + (byte)1;")); + // char/char + assertEquals((char)1 + (char)1, exec("return (char)1 + (char)1;")); + // char/short + assertEquals((char)1 + (short)1, exec("return (char)1 + (short)1;")); + // char/int + assertEquals((char)1 + 1, exec("return (char)1 + 1;")); + // char/long + assertEquals((char)1 + 1L, exec("return (char)1 + 1L;")); + // char/float + assertEquals((char)1 + 1F, exec("return (char)1 + 1F;")); + // char/double + assertEquals((char)1 + 1.0, exec("return (char)1 + 1.0;")); + + // short/byte + assertEquals((short)1 + (byte)1, exec("return (short)1 + (byte)1;")); + // short/char + assertEquals((short)1 + (char)1, exec("return (short)1 + (char)1;")); + // short/short + assertEquals((short)1 + (short)1, exec("return (short)1 + (short)1;")); + // short/int + assertEquals((short)1 + 1, exec("return (short)1 + 1;")); + // short/long + assertEquals((short)1 + 1L, exec("return (short)1 + 1L;")); + // short/float + assertEquals((short)1 + 1F, exec("return (short)1 + 1F;")); + // short/double + assertEquals((short)1 + 1.0, exec("return (short)1 + 1.0;")); + + // int/byte + assertEquals(1 + (byte)1, exec("return 1 + (byte)1;")); + // int/char + assertEquals(1 + (char)1, exec("return 1 + (char)1;")); + // int/short + assertEquals(1 + (short)1, exec("return 1 + (short)1;")); + // int/int + assertEquals(1 + 1, exec("return 1 + 1;")); + // int/long + assertEquals(1 + 1L, exec("return 1 + 1L;")); + // int/float + assertEquals(1 + 1F, exec("return 1 + 1F;")); + // int/double + assertEquals(1 + 1.0, exec("return 1 + 1.0;")); + + // long/byte + assertEquals(1L + (byte)1, exec("return 1L + (byte)1;")); + // long/char + assertEquals(1L + (char)1, exec("return 1L + (char)1;")); + // long/short + assertEquals(1L + (short)1, exec("return 1L + (short)1;")); + // long/int + assertEquals(1L + 1, exec("return 1L + 1;")); + // long/long + assertEquals(1L + 1L, exec("return 1L + 1L;")); + // long/float + assertEquals(1L + 1F, exec("return 1L + 1F;")); + // long/double + assertEquals(1L + 1.0, exec("return 1L + 1.0;")); + + // float/byte + assertEquals(1F + (byte)1, exec("return 1F + (byte)1;")); + // float/char + assertEquals(1F + (char)1, exec("return 1F + (char)1;")); + // float/short + assertEquals(1F + (short)1, exec("return 1F + (short)1;")); + // float/int + assertEquals(1F + 1, exec("return 1F + 1;")); + // float/long + assertEquals(1F + 1L, exec("return 1F + 1L;")); + // float/float + assertEquals(1F + 1F, exec("return 1F + 1F;")); + // float/double + assertEquals(1F + 1.0, exec("return 1F + 1.0;")); + + // double/byte + assertEquals(1.0 + (byte)1, exec("return 1.0 + (byte)1;")); + // double/char + assertEquals(1.0 + (char)1, exec("return 1.0 + (char)1;")); + // double/short + assertEquals(1.0 + (short)1, exec("return 1.0 + (short)1;")); + // double/int + assertEquals(1.0 + 1, exec("return 1.0 + 1;")); + // double/long + assertEquals(1.0 + 1L, exec("return 1.0 + 1L;")); + // double/float + assertEquals(1.0 + 1F, exec("return 1.0 + 1F;")); + // double/double + assertEquals(1.0 + 1.0, exec("return 1.0 + 1.0;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/CompoundAssignmentTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/CompoundAssignmentTests.java new file mode 100644 index 00000000000..3af440ad02c --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/CompoundAssignmentTests.java @@ -0,0 +1,319 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** + * Tests compound assignments (+=, etc) across all data types + */ +public class CompoundAssignmentTests extends ScriptTestCase { + public void testAddition() { + // byte + assertEquals((byte) 15, exec("byte x = 5; x += 10; return x;")); + assertEquals((byte) -5, exec("byte x = 5; x += -10; return x;")); + + // short + assertEquals((short) 15, exec("short x = 5; x += 10; return x;")); + assertEquals((short) -5, exec("short x = 5; x += -10; return x;")); + // char + assertEquals((char) 15, exec("char x = 5; x += 10; return x;")); + assertEquals((char) 5, exec("char x = 10; x += -5; return x;")); + // int + assertEquals(15, exec("int x = 5; x += 10; return x;")); + assertEquals(-5, exec("int x = 5; x += -10; return x;")); + // long + assertEquals(15L, exec("long x = 5; x += 10; return x;")); + assertEquals(-5L, exec("long x = 5; x += -10; return x;")); + // float + assertEquals(15F, exec("float x = 5f; x += 10; return x;")); + assertEquals(-5F, exec("float x = 5f; x += -10; return x;")); + // double + assertEquals(15D, exec("double x = 5.0; x += 10; return x;")); + assertEquals(-5D, exec("double x = 5.0; x += -10; return x;")); + } + + public void testSubtraction() { + // byte + assertEquals((byte) 15, exec("byte x = 5; x -= -10; return x;")); + assertEquals((byte) -5, exec("byte x = 5; x -= 10; return x;")); + // short + assertEquals((short) 15, exec("short x = 5; x -= -10; return x;")); + assertEquals((short) -5, exec("short x = 5; x -= 10; return x;")); + // char + assertEquals((char) 15, exec("char x = 5; x -= -10; return x;")); + assertEquals((char) 5, exec("char x = 10; x -= 5; return x;")); + // int + assertEquals(15, exec("int x = 5; x -= -10; return x;")); + assertEquals(-5, exec("int x = 5; x -= 10; return x;")); + // long + assertEquals(15L, exec("long x = 5; x -= -10; return x;")); + assertEquals(-5L, exec("long x = 5; x -= 10; return x;")); + // float + assertEquals(15F, exec("float x = 5f; x -= -10; return x;")); + assertEquals(-5F, exec("float x = 5f; x -= 10; return x;")); + // double + assertEquals(15D, exec("double x = 5.0; x -= -10; return x;")); + assertEquals(-5D, exec("double x = 5.0; x -= 10; return x;")); + } + + public void testMultiplication() { + // byte + assertEquals((byte) 15, exec("byte x = 5; x *= 3; return x;")); + assertEquals((byte) -5, exec("byte x = 5; x *= -1; return x;")); + // short + assertEquals((short) 15, exec("short x = 5; x *= 3; return x;")); + assertEquals((short) -5, exec("short x = 5; x *= -1; return x;")); + // char + assertEquals((char) 15, exec("char x = 5; x *= 3; return x;")); + // int + assertEquals(15, exec("int x = 5; x *= 3; return x;")); + assertEquals(-5, exec("int x = 5; x *= -1; return x;")); + // long + assertEquals(15L, exec("long x = 5; x *= 3; return x;")); + assertEquals(-5L, exec("long x = 5; x *= -1; return x;")); + // float + assertEquals(15F, exec("float x = 5f; x *= 3; return x;")); + assertEquals(-5F, exec("float x = 5f; x *= -1; return x;")); + // double + assertEquals(15D, exec("double x = 5.0; x *= 3; return x;")); + assertEquals(-5D, exec("double x = 5.0; x *= -1; return x;")); + } + + public void testDivision() { + // byte + assertEquals((byte) 15, exec("byte x = 45; x /= 3; return x;")); + assertEquals((byte) -5, exec("byte x = 5; x /= -1; return x;")); + // short + assertEquals((short) 15, exec("short x = 45; x /= 3; return x;")); + assertEquals((short) -5, exec("short x = 5; x /= -1; return x;")); + // char + assertEquals((char) 15, exec("char x = 45; x /= 3; return x;")); + // int + assertEquals(15, exec("int x = 45; x /= 3; return x;")); + assertEquals(-5, exec("int x = 5; x /= -1; return x;")); + // long + assertEquals(15L, exec("long x = 45; x /= 3; return x;")); + assertEquals(-5L, exec("long x = 5; x /= -1; return x;")); + // float + assertEquals(15F, exec("float x = 45f; x /= 3; return x;")); + assertEquals(-5F, exec("float x = 5f; x /= -1; return x;")); + // double + assertEquals(15D, exec("double x = 45.0; x /= 3; return x;")); + assertEquals(-5D, exec("double x = 5.0; x /= -1; return x;")); + } + + public void testDivisionByZero() { + // byte + try { + exec("byte x = 1; x /= 0; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + // short + try { + exec("short x = 1; x /= 0; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + // char + try { + exec("char x = 1; x /= 0; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + // int + try { + exec("int x = 1; x /= 0; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + // long + try { + exec("long x = 1; x /= 0; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testRemainder() { + // byte + assertEquals((byte) 3, exec("byte x = 15; x %= 4; return x;")); + assertEquals((byte) -3, exec("byte x = (byte) -15; x %= 4; return x;")); + // short + assertEquals((short) 3, exec("short x = 15; x %= 4; return x;")); + assertEquals((short) -3, exec("short x = (short) -15; x %= 4; return x;")); + // char + assertEquals((char) 3, exec("char x = (char) 15; x %= 4; return x;")); + // int + assertEquals(3, exec("int x = 15; x %= 4; return x;")); + assertEquals(-3, exec("int x = -15; x %= 4; return x;")); + // long + assertEquals(3L, exec("long x = 15L; x %= 4; return x;")); + assertEquals(-3L, exec("long x = -15L; x %= 4; return x;")); + // float + assertEquals(3F, exec("float x = 15F; x %= 4; return x;")); + assertEquals(-3F, exec("float x = -15F; x %= 4; return x;")); + // double + assertEquals(3D, exec("double x = 15.0; x %= 4; return x;")); + assertEquals(-3D, exec("double x = -15.0; x %= 4; return x;")); + } + + public void testLeftShift() { + // byte + assertEquals((byte) 60, exec("byte x = 15; x <<= 2; return x;")); + assertEquals((byte) -60, exec("byte x = (byte) -15; x <<= 2; return x;")); + // short + assertEquals((short) 60, exec("short x = 15; x <<= 2; return x;")); + assertEquals((short) -60, exec("short x = (short) -15; x <<= 2; return x;")); + // char + assertEquals((char) 60, exec("char x = (char) 15; x <<= 2; return x;")); + // int + assertEquals(60, exec("int x = 15; x <<= 2; return x;")); + assertEquals(-60, exec("int x = -15; x <<= 2; return x;")); + // long + assertEquals(60L, exec("long x = 15L; x <<= 2; return x;")); + assertEquals(-60L, exec("long x = -15L; x <<= 2; return x;")); + } + + public void testRightShift() { + // byte + assertEquals((byte) 15, exec("byte x = 60; x >>= 2; return x;")); + assertEquals((byte) -15, exec("byte x = (byte) -60; x >>= 2; return x;")); + // short + assertEquals((short) 15, exec("short x = 60; x >>= 2; return x;")); + assertEquals((short) -15, exec("short x = (short) -60; x >>= 2; return x;")); + // char + assertEquals((char) 15, exec("char x = (char) 60; x >>= 2; return x;")); + // int + assertEquals(15, exec("int x = 60; x >>= 2; return x;")); + assertEquals(-15, exec("int x = -60; x >>= 2; return x;")); + // long + assertEquals(15L, exec("long x = 60L; x >>= 2; return x;")); + assertEquals(-15L, exec("long x = -60L; x >>= 2; return x;")); + } + + public void testUnsignedRightShift() { + // byte + assertEquals((byte) 15, exec("byte x = 60; x >>>= 2; return x;")); + assertEquals((byte) -15, exec("byte x = (byte) -60; x >>>= 2; return x;")); + // short + assertEquals((short) 15, exec("short x = 60; x >>>= 2; return x;")); + assertEquals((short) -15, exec("short x = (short) -60; x >>>= 2; return x;")); + // char + assertEquals((char) 15, exec("char x = (char) 60; x >>>= 2; return x;")); + // int + assertEquals(15, exec("int x = 60; x >>>= 2; return x;")); + assertEquals(-60 >>> 2, exec("int x = -60; x >>>= 2; return x;")); + // long + assertEquals(15L, exec("long x = 60L; x >>>= 2; return x;")); + assertEquals(-60L >>> 2, exec("long x = -60L; x >>>= 2; return x;")); + } + + public void testAnd() { + // boolean + assertEquals(true, exec("boolean x = true; x &= true; return x;")); + assertEquals(false, exec("boolean x = true; x &= false; return x;")); + assertEquals(false, exec("boolean x = false; x &= true; return x;")); + assertEquals(false, exec("boolean x = false; x &= false; return x;")); + assertEquals(true, exec("Boolean x = true; x &= true; return x;")); + assertEquals(false, exec("Boolean x = true; x &= false; return x;")); + assertEquals(false, exec("Boolean x = false; x &= true; return x;")); + assertEquals(false, exec("Boolean x = false; x &= false; return x;")); + assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= true; return x[0];")); + assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= false; return x[0];")); + assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= true; return x[0];")); + assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= false; return x[0];")); + assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] &= true; return x[0];")); + assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] &= false; return x[0];")); + assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] &= true; return x[0];")); + assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] &= false; return x[0];")); + + // byte + assertEquals((byte) (13 & 14), exec("byte x = 13; x &= 14; return x;")); + // short + assertEquals((short) (13 & 14), exec("short x = 13; x &= 14; return x;")); + // char + assertEquals((char) (13 & 14), exec("char x = 13; x &= 14; return x;")); + // int + assertEquals(13 & 14, exec("int x = 13; x &= 14; return x;")); + // long + assertEquals((long) (13 & 14), exec("long x = 13L; x &= 14; return x;")); + } + + public void testOr() { + // boolean + assertEquals(true, exec("boolean x = true; x |= true; return x;")); + assertEquals(true, exec("boolean x = true; x |= false; return x;")); + assertEquals(true, exec("boolean x = false; x |= true; return x;")); + assertEquals(false, exec("boolean x = false; x |= false; return x;")); + assertEquals(true, exec("Boolean x = true; x |= true; return x;")); + assertEquals(true, exec("Boolean x = true; x |= false; return x;")); + assertEquals(true, exec("Boolean x = false; x |= true; return x;")); + assertEquals(false, exec("Boolean x = false; x |= false; return x;")); + assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= true; return x[0];")); + assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= false; return x[0];")); + assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= true; return x[0];")); + assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= false; return x[0];")); + assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] |= true; return x[0];")); + assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] |= false; return x[0];")); + assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] |= true; return x[0];")); + assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] |= false; return x[0];")); + + // byte + assertEquals((byte) (13 | 14), exec("byte x = 13; x |= 14; return x;")); + // short + assertEquals((short) (13 | 14), exec("short x = 13; x |= 14; return x;")); + // char + assertEquals((char) (13 | 14), exec("char x = 13; x |= 14; return x;")); + // int + assertEquals(13 | 14, exec("int x = 13; x |= 14; return x;")); + // long + assertEquals((long) (13 | 14), exec("long x = 13L; x |= 14; return x;")); + } + + public void testXor() { + // boolean + assertEquals(false, exec("boolean x = true; x ^= true; return x;")); + assertEquals(true, exec("boolean x = true; x ^= false; return x;")); + assertEquals(true, exec("boolean x = false; x ^= true; return x;")); + assertEquals(false, exec("boolean x = false; x ^= false; return x;")); + assertEquals(false, exec("Boolean x = true; x ^= true; return x;")); + assertEquals(true, exec("Boolean x = true; x ^= false; return x;")); + assertEquals(true, exec("Boolean x = false; x ^= true; return x;")); + assertEquals(false, exec("Boolean x = false; x ^= false; return x;")); + assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= true; return x[0];")); + assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= false; return x[0];")); + assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= true; return x[0];")); + assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= false; return x[0];")); + assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] ^= true; return x[0];")); + assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] ^= false; return x[0];")); + assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] ^= true; return x[0];")); + assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] ^= false; return x[0];")); + + // byte + assertEquals((byte) (13 ^ 14), exec("byte x = 13; x ^= 14; return x;")); + // short + assertEquals((short) (13 ^ 14), exec("short x = 13; x ^= 14; return x;")); + // char + assertEquals((char) (13 ^ 14), exec("char x = 13; x ^= 14; return x;")); + // int + assertEquals(13 ^ 14, exec("int x = 13; x ^= 14; return x;")); + // long + assertEquals((long) (13 ^ 14), exec("long x = 13L; x ^= 14; return x;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ConditionalTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ConditionalTests.java new file mode 100644 index 00000000000..bc466427da7 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ConditionalTests.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.util.ArrayList; +import java.util.HashMap; + +public class ConditionalTests extends ScriptTestCase { + public void testBasic() { + assertEquals(2, exec("boolean x = true; return x ? 2 : 3;")); + assertEquals(3, exec("boolean x = false; return x ? 2 : 3;")); + assertEquals(3, exec("boolean x = false, y = true; return x && y ? 2 : 3;")); + assertEquals(2, exec("boolean x = true, y = true; return x && y ? 2 : 3;")); + assertEquals(2, exec("boolean x = true, y = false; return x || y ? 2 : 3;")); + assertEquals(3, exec("boolean x = false, y = false; return x || y ? 2 : 3;")); + } + + public void testPrecedence() { + assertEquals(4, exec("boolean x = false, y = true; return x ? (y ? 2 : 3) : 4;")); + assertEquals(2, exec("boolean x = true, y = true; return x ? (y ? 2 : 3) : 4;")); + assertEquals(3, exec("boolean x = true, y = false; return x ? (y ? 2 : 3) : 4;")); + assertEquals(2, exec("boolean x = true, y = true; return x ? y ? 2 : 3 : 4;")); + assertEquals(4, exec("boolean x = false, y = true; return x ? y ? 2 : 3 : 4;")); + assertEquals(3, exec("boolean x = true, y = false; return x ? y ? 2 : 3 : 4;")); + assertEquals(3, exec("boolean x = false, y = true; return x ? 2 : y ? 3 : 4;")); + assertEquals(2, exec("boolean x = true, y = false; return x ? 2 : y ? 3 : 4;")); + assertEquals(4, exec("boolean x = false, y = false; return x ? 2 : y ? 3 : 4;")); + assertEquals(4, exec("boolean x = false, y = false; return (x ? true : y) ? 3 : 4;")); + assertEquals(4, exec("boolean x = true, y = false; return (x ? false : y) ? 3 : 4;")); + assertEquals(3, exec("boolean x = false, y = true; return (x ? false : y) ? 3 : 4;")); + assertEquals(2, exec("boolean x = true, y = false; return (x ? false : y) ? (x ? 3 : 4) : x ? 2 : 1;")); + assertEquals(2, exec("boolean x = true, y = false; return (x ? false : y) ? x ? 3 : 4 : x ? 2 : 1;")); + assertEquals(4, exec("boolean x = false, y = true; return x ? false : y ? x ? 3 : 4 : x ? 2 : 1;")); + } + + public void testAssignment() { + assertEquals(4D, exec("boolean x = false; double z = x ? 2 : 4.0F; return z;")); + assertEquals((byte)7, exec("boolean x = false; int y = 2; byte z = x ? (byte)y : 7; return z;")); + assertEquals((byte)7, exec("boolean x = false; int y = 2; byte z = (byte)(x ? y : 7); return z;")); + assertEquals(ArrayList.class, exec("boolean x = false; Object z = x ? new HashMap() : new ArrayList(); return z;").getClass()); + } + + public void testNullArguments() { + assertEquals(null, exec("boolean b = false, c = true; Object x; Map y; return b && c ? x : y;")); + assertEquals(HashMap.class, exec("boolean b = false, c = true; Object x; Map y = new HashMap(); return b && c ? x : y;").getClass()); + } + + public void testPromotion() { + assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? 2 : 4.0F);")); + assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? new Long(2) : new Float(4.0F));")); + assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? new HashMap() : new ArrayList()) == (y ? new Long(2) : new Float(4.0F));")); + assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? new HashMap() : new ArrayList());")); + } + + public void testIncompatibleAssignment() { + try { + exec("boolean x = false; byte z = x ? 2 : 4.0F; return z;"); + fail("expected class cast exception"); + } catch (ClassCastException expected) {} + + try { + exec("boolean x = false; Map z = x ? 4 : (byte)7; return z;"); + fail("expected class cast exception"); + } catch (ClassCastException expected) {} + + try { + exec("boolean x = false; Map z = x ? new HashMap() : new ArrayList(); return z;"); + fail("expected class cast exception"); + } catch (ClassCastException expected) {} + + try { + exec("boolean x = false; int y = 2; byte z = x ? y : 7; return z;"); + fail("expected class cast exception"); + } catch (ClassCastException expected) {} + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/DefTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/DefTests.java new file mode 100644 index 00000000000..6ff51131fe5 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/DefTests.java @@ -0,0 +1,914 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +public class DefTests extends ScriptTestCase { + public void testNot() { + assertEquals(~1, exec("def x = (byte)1 return ~x")); + assertEquals(~1, exec("def x = (short)1 return ~x")); + assertEquals(~1, exec("def x = (char)1 return ~x")); + assertEquals(~1, exec("def x = 1 return ~x")); + assertEquals(~1L, exec("def x = 1L return ~x")); + } + + public void testNeg() { + assertEquals(-1, exec("def x = (byte)1 return -x")); + assertEquals(-1, exec("def x = (short)1 return -x")); + assertEquals(-1, exec("def x = (char)1 return -x")); + assertEquals(-1, exec("def x = 1 return -x")); + assertEquals(-1L, exec("def x = 1L return -x")); + assertEquals(-1.0F, exec("def x = 1F return -x")); + assertEquals(-1.0, exec("def x = 1.0 return -x")); + } + + public void testMul() { + assertEquals(4, exec("def x = (byte)2 def y = (byte)2 return x * y")); + assertEquals(4, exec("def x = (short)2 def y = (byte)2 return x * y")); + assertEquals(4, exec("def x = (char)2 def y = (byte)2 return x * y")); + assertEquals(4, exec("def x = (int)2 def y = (byte)2 return x * y")); + assertEquals(4L, exec("def x = (long)2 def y = (byte)2 return x * y")); + assertEquals(4F, exec("def x = (float)2 def y = (byte)2 return x * y")); + assertEquals(4D, exec("def x = (double)2 def y = (byte)2 return x * y")); + + assertEquals(4, exec("def x = (byte)2 def y = (short)2 return x * y")); + assertEquals(4, exec("def x = (short)2 def y = (short)2 return x * y")); + assertEquals(4, exec("def x = (char)2 def y = (short)2 return x * y")); + assertEquals(4, exec("def x = (int)2 def y = (short)2 return x * y")); + assertEquals(4L, exec("def x = (long)2 def y = (short)2 return x * y")); + assertEquals(4F, exec("def x = (float)2 def y = (short)2 return x * y")); + assertEquals(4D, exec("def x = (double)2 def y = (short)2 return x * y")); + + assertEquals(4, exec("def x = (byte)2 def y = (char)2 return x * y")); + assertEquals(4, exec("def x = (short)2 def y = (char)2 return x * y")); + assertEquals(4, exec("def x = (char)2 def y = (char)2 return x * y")); + assertEquals(4, exec("def x = (int)2 def y = (char)2 return x * y")); + assertEquals(4L, exec("def x = (long)2 def y = (char)2 return x * y")); + assertEquals(4F, exec("def x = (float)2 def y = (char)2 return x * y")); + assertEquals(4D, exec("def x = (double)2 def y = (char)2 return x * y")); + + assertEquals(4, exec("def x = (byte)2 def y = (int)2 return x * y")); + assertEquals(4, exec("def x = (short)2 def y = (int)2 return x * y")); + assertEquals(4, exec("def x = (char)2 def y = (int)2 return x * y")); + assertEquals(4, exec("def x = (int)2 def y = (int)2 return x * y")); + assertEquals(4L, exec("def x = (long)2 def y = (int)2 return x * y")); + assertEquals(4F, exec("def x = (float)2 def y = (int)2 return x * y")); + assertEquals(4D, exec("def x = (double)2 def y = (int)2 return x * y")); + + assertEquals(4L, exec("def x = (byte)2 def y = (long)2 return x * y")); + assertEquals(4L, exec("def x = (short)2 def y = (long)2 return x * y")); + assertEquals(4L, exec("def x = (char)2 def y = (long)2 return x * y")); + assertEquals(4L, exec("def x = (int)2 def y = (long)2 return x * y")); + assertEquals(4L, exec("def x = (long)2 def y = (long)2 return x * y")); + assertEquals(4F, exec("def x = (float)2 def y = (long)2 return x * y")); + assertEquals(4D, exec("def x = (double)2 def y = (long)2 return x * y")); + + assertEquals(4F, exec("def x = (byte)2 def y = (float)2 return x * y")); + assertEquals(4F, exec("def x = (short)2 def y = (float)2 return x * y")); + assertEquals(4F, exec("def x = (char)2 def y = (float)2 return x * y")); + assertEquals(4F, exec("def x = (int)2 def y = (float)2 return x * y")); + assertEquals(4F, exec("def x = (long)2 def y = (float)2 return x * y")); + assertEquals(4F, exec("def x = (float)2 def y = (float)2 return x * y")); + assertEquals(4D, exec("def x = (double)2 def y = (float)2 return x * y")); + + assertEquals(4D, exec("def x = (byte)2 def y = (double)2 return x * y")); + assertEquals(4D, exec("def x = (short)2 def y = (double)2 return x * y")); + assertEquals(4D, exec("def x = (char)2 def y = (double)2 return x * y")); + assertEquals(4D, exec("def x = (int)2 def y = (double)2 return x * y")); + assertEquals(4D, exec("def x = (long)2 def y = (double)2 return x * y")); + assertEquals(4D, exec("def x = (float)2 def y = (double)2 return x * y")); + assertEquals(4D, exec("def x = (double)2 def y = (double)2 return x * y")); + + assertEquals(4, exec("def x = (Byte)2 def y = (byte)2 return x * y")); + assertEquals(4, exec("def x = (Short)2 def y = (short)2 return x * y")); + assertEquals(4, exec("def x = (Character)2 def y = (char)2 return x * y")); + assertEquals(4, exec("def x = (Integer)2 def y = (int)2 return x * y")); + assertEquals(4L, exec("def x = (Long)2 def y = (long)2 return x * y")); + assertEquals(4F, exec("def x = (Float)2 def y = (float)2 return x * y")); + assertEquals(4D, exec("def x = (Double)2 def y = (double)2 return x * y")); + } + + public void testDiv() { + assertEquals(1, exec("def x = (byte)2 def y = (byte)2 return x / y")); + assertEquals(1, exec("def x = (short)2 def y = (byte)2 return x / y")); + assertEquals(1, exec("def x = (char)2 def y = (byte)2 return x / y")); + assertEquals(1, exec("def x = (int)2 def y = (byte)2 return x / y")); + assertEquals(1L, exec("def x = (long)2 def y = (byte)2 return x / y")); + assertEquals(1F, exec("def x = (float)2 def y = (byte)2 return x / y")); + assertEquals(1D, exec("def x = (double)2 def y = (byte)2 return x / y")); + + assertEquals(1, exec("def x = (byte)2 def y = (short)2 return x / y")); + assertEquals(1, exec("def x = (short)2 def y = (short)2 return x / y")); + assertEquals(1, exec("def x = (char)2 def y = (short)2 return x / y")); + assertEquals(1, exec("def x = (int)2 def y = (short)2 return x / y")); + assertEquals(1L, exec("def x = (long)2 def y = (short)2 return x / y")); + assertEquals(1F, exec("def x = (float)2 def y = (short)2 return x / y")); + assertEquals(1D, exec("def x = (double)2 def y = (short)2 return x / y")); + + assertEquals(1, exec("def x = (byte)2 def y = (char)2 return x / y")); + assertEquals(1, exec("def x = (short)2 def y = (char)2 return x / y")); + assertEquals(1, exec("def x = (char)2 def y = (char)2 return x / y")); + assertEquals(1, exec("def x = (int)2 def y = (char)2 return x / y")); + assertEquals(1L, exec("def x = (long)2 def y = (char)2 return x / y")); + assertEquals(1F, exec("def x = (float)2 def y = (char)2 return x / y")); + assertEquals(1D, exec("def x = (double)2 def y = (char)2 return x / y")); + + assertEquals(1, exec("def x = (byte)2 def y = (int)2 return x / y")); + assertEquals(1, exec("def x = (short)2 def y = (int)2 return x / y")); + assertEquals(1, exec("def x = (char)2 def y = (int)2 return x / y")); + assertEquals(1, exec("def x = (int)2 def y = (int)2 return x / y")); + assertEquals(1L, exec("def x = (long)2 def y = (int)2 return x / y")); + assertEquals(1F, exec("def x = (float)2 def y = (int)2 return x / y")); + assertEquals(1D, exec("def x = (double)2 def y = (int)2 return x / y")); + + assertEquals(1L, exec("def x = (byte)2 def y = (long)2 return x / y")); + assertEquals(1L, exec("def x = (short)2 def y = (long)2 return x / y")); + assertEquals(1L, exec("def x = (char)2 def y = (long)2 return x / y")); + assertEquals(1L, exec("def x = (int)2 def y = (long)2 return x / y")); + assertEquals(1L, exec("def x = (long)2 def y = (long)2 return x / y")); + assertEquals(1F, exec("def x = (float)2 def y = (long)2 return x / y")); + assertEquals(1D, exec("def x = (double)2 def y = (long)2 return x / y")); + + assertEquals(1F, exec("def x = (byte)2 def y = (float)2 return x / y")); + assertEquals(1F, exec("def x = (short)2 def y = (float)2 return x / y")); + assertEquals(1F, exec("def x = (char)2 def y = (float)2 return x / y")); + assertEquals(1F, exec("def x = (int)2 def y = (float)2 return x / y")); + assertEquals(1F, exec("def x = (long)2 def y = (float)2 return x / y")); + assertEquals(1F, exec("def x = (float)2 def y = (float)2 return x / y")); + assertEquals(1D, exec("def x = (double)2 def y = (float)2 return x / y")); + + assertEquals(1D, exec("def x = (byte)2 def y = (double)2 return x / y")); + assertEquals(1D, exec("def x = (short)2 def y = (double)2 return x / y")); + assertEquals(1D, exec("def x = (char)2 def y = (double)2 return x / y")); + assertEquals(1D, exec("def x = (int)2 def y = (double)2 return x / y")); + assertEquals(1D, exec("def x = (long)2 def y = (double)2 return x / y")); + assertEquals(1D, exec("def x = (float)2 def y = (double)2 return x / y")); + assertEquals(1D, exec("def x = (double)2 def y = (double)2 return x / y")); + + assertEquals(1, exec("def x = (Byte)2 def y = (byte)2 return x / y")); + assertEquals(1, exec("def x = (Short)2 def y = (short)2 return x / y")); + assertEquals(1, exec("def x = (Character)2 def y = (char)2 return x / y")); + assertEquals(1, exec("def x = (Integer)2 def y = (int)2 return x / y")); + assertEquals(1L, exec("def x = (Long)2 def y = (long)2 return x / y")); + assertEquals(1F, exec("def x = (Float)2 def y = (float)2 return x / y")); + assertEquals(1D, exec("def x = (Double)2 def y = (double)2 return x / y")); + } + + public void testRem() { + assertEquals(0, exec("def x = (byte)2 def y = (byte)2 return x % y")); + assertEquals(0, exec("def x = (short)2 def y = (byte)2 return x % y")); + assertEquals(0, exec("def x = (char)2 def y = (byte)2 return x % y")); + assertEquals(0, exec("def x = (int)2 def y = (byte)2 return x % y")); + assertEquals(0L, exec("def x = (long)2 def y = (byte)2 return x % y")); + assertEquals(0F, exec("def x = (float)2 def y = (byte)2 return x % y")); + assertEquals(0D, exec("def x = (double)2 def y = (byte)2 return x % y")); + + assertEquals(0, exec("def x = (byte)2 def y = (short)2 return x % y")); + assertEquals(0, exec("def x = (short)2 def y = (short)2 return x % y")); + assertEquals(0, exec("def x = (char)2 def y = (short)2 return x % y")); + assertEquals(0, exec("def x = (int)2 def y = (short)2 return x % y")); + assertEquals(0L, exec("def x = (long)2 def y = (short)2 return x % y")); + assertEquals(0F, exec("def x = (float)2 def y = (short)2 return x % y")); + assertEquals(0D, exec("def x = (double)2 def y = (short)2 return x % y")); + + assertEquals(0, exec("def x = (byte)2 def y = (char)2 return x % y")); + assertEquals(0, exec("def x = (short)2 def y = (char)2 return x % y")); + assertEquals(0, exec("def x = (char)2 def y = (char)2 return x % y")); + assertEquals(0, exec("def x = (int)2 def y = (char)2 return x % y")); + assertEquals(0L, exec("def x = (long)2 def y = (char)2 return x % y")); + assertEquals(0F, exec("def x = (float)2 def y = (char)2 return x % y")); + assertEquals(0D, exec("def x = (double)2 def y = (char)2 return x % y")); + + assertEquals(0, exec("def x = (byte)2 def y = (int)2 return x % y")); + assertEquals(0, exec("def x = (short)2 def y = (int)2 return x % y")); + assertEquals(0, exec("def x = (char)2 def y = (int)2 return x % y")); + assertEquals(0, exec("def x = (int)2 def y = (int)2 return x % y")); + assertEquals(0L, exec("def x = (long)2 def y = (int)2 return x % y")); + assertEquals(0F, exec("def x = (float)2 def y = (int)2 return x % y")); + assertEquals(0D, exec("def x = (double)2 def y = (int)2 return x % y")); + + assertEquals(0L, exec("def x = (byte)2 def y = (long)2 return x % y")); + assertEquals(0L, exec("def x = (short)2 def y = (long)2 return x % y")); + assertEquals(0L, exec("def x = (char)2 def y = (long)2 return x % y")); + assertEquals(0L, exec("def x = (int)2 def y = (long)2 return x % y")); + assertEquals(0L, exec("def x = (long)2 def y = (long)2 return x % y")); + assertEquals(0F, exec("def x = (float)2 def y = (long)2 return x % y")); + assertEquals(0D, exec("def x = (double)2 def y = (long)2 return x % y")); + + assertEquals(0F, exec("def x = (byte)2 def y = (float)2 return x % y")); + assertEquals(0F, exec("def x = (short)2 def y = (float)2 return x % y")); + assertEquals(0F, exec("def x = (char)2 def y = (float)2 return x % y")); + assertEquals(0F, exec("def x = (int)2 def y = (float)2 return x % y")); + assertEquals(0F, exec("def x = (long)2 def y = (float)2 return x % y")); + assertEquals(0F, exec("def x = (float)2 def y = (float)2 return x % y")); + assertEquals(0D, exec("def x = (double)2 def y = (float)2 return x % y")); + + assertEquals(0D, exec("def x = (byte)2 def y = (double)2 return x % y")); + assertEquals(0D, exec("def x = (short)2 def y = (double)2 return x % y")); + assertEquals(0D, exec("def x = (char)2 def y = (double)2 return x % y")); + assertEquals(0D, exec("def x = (int)2 def y = (double)2 return x % y")); + assertEquals(0D, exec("def x = (long)2 def y = (double)2 return x % y")); + assertEquals(0D, exec("def x = (float)2 def y = (double)2 return x % y")); + assertEquals(0D, exec("def x = (double)2 def y = (double)2 return x % y")); + + assertEquals(0, exec("def x = (Byte)2 def y = (byte)2 return x % y")); + assertEquals(0, exec("def x = (Short)2 def y = (short)2 return x % y")); + assertEquals(0, exec("def x = (Character)2 def y = (char)2 return x % y")); + assertEquals(0, exec("def x = (Integer)2 def y = (int)2 return x % y")); + assertEquals(0L, exec("def x = (Long)2 def y = (long)2 return x % y")); + assertEquals(0F, exec("def x = (Float)2 def y = (float)2 return x % y")); + assertEquals(0D, exec("def x = (Double)2 def y = (double)2 return x % y")); + } + + public void testAdd() { + assertEquals(2, exec("def x = (byte)1 def y = (byte)1 return x + y")); + assertEquals(2, exec("def x = (short)1 def y = (byte)1 return x + y")); + assertEquals(2, exec("def x = (char)1 def y = (byte)1 return x + y")); + assertEquals(2, exec("def x = (int)1 def y = (byte)1 return x + y")); + assertEquals(2L, exec("def x = (long)1 def y = (byte)1 return x + y")); + assertEquals(2F, exec("def x = (float)1 def y = (byte)1 return x + y")); + assertEquals(2D, exec("def x = (double)1 def y = (byte)1 return x + y")); + + assertEquals(2, exec("def x = (byte)1 def y = (short)1 return x + y")); + assertEquals(2, exec("def x = (short)1 def y = (short)1 return x + y")); + assertEquals(2, exec("def x = (char)1 def y = (short)1 return x + y")); + assertEquals(2, exec("def x = (int)1 def y = (short)1 return x + y")); + assertEquals(2L, exec("def x = (long)1 def y = (short)1 return x + y")); + assertEquals(2F, exec("def x = (float)1 def y = (short)1 return x + y")); + assertEquals(2D, exec("def x = (double)1 def y = (short)1 return x + y")); + + assertEquals(2, exec("def x = (byte)1 def y = (char)1 return x + y")); + assertEquals(2, exec("def x = (short)1 def y = (char)1 return x + y")); + assertEquals(2, exec("def x = (char)1 def y = (char)1 return x + y")); + assertEquals(2, exec("def x = (int)1 def y = (char)1 return x + y")); + assertEquals(2L, exec("def x = (long)1 def y = (char)1 return x + y")); + assertEquals(2F, exec("def x = (float)1 def y = (char)1 return x + y")); + assertEquals(2D, exec("def x = (double)1 def y = (char)1 return x + y")); + + assertEquals(2, exec("def x = (byte)1 def y = (int)1 return x + y")); + assertEquals(2, exec("def x = (short)1 def y = (int)1 return x + y")); + assertEquals(2, exec("def x = (char)1 def y = (int)1 return x + y")); + assertEquals(2, exec("def x = (int)1 def y = (int)1 return x + y")); + assertEquals(2L, exec("def x = (long)1 def y = (int)1 return x + y")); + assertEquals(2F, exec("def x = (float)1 def y = (int)1 return x + y")); + assertEquals(2D, exec("def x = (double)1 def y = (int)1 return x + y")); + + assertEquals(2L, exec("def x = (byte)1 def y = (long)1 return x + y")); + assertEquals(2L, exec("def x = (short)1 def y = (long)1 return x + y")); + assertEquals(2L, exec("def x = (char)1 def y = (long)1 return x + y")); + assertEquals(2L, exec("def x = (int)1 def y = (long)1 return x + y")); + assertEquals(2L, exec("def x = (long)1 def y = (long)1 return x + y")); + assertEquals(2F, exec("def x = (float)1 def y = (long)1 return x + y")); + assertEquals(2D, exec("def x = (double)1 def y = (long)1 return x + y")); + + assertEquals(2F, exec("def x = (byte)1 def y = (float)1 return x + y")); + assertEquals(2F, exec("def x = (short)1 def y = (float)1 return x + y")); + assertEquals(2F, exec("def x = (char)1 def y = (float)1 return x + y")); + assertEquals(2F, exec("def x = (int)1 def y = (float)1 return x + y")); + assertEquals(2F, exec("def x = (long)1 def y = (float)1 return x + y")); + assertEquals(2F, exec("def x = (float)1 def y = (float)1 return x + y")); + assertEquals(2D, exec("def x = (double)1 def y = (float)1 return x + y")); + + assertEquals(2D, exec("def x = (byte)1 def y = (double)1 return x + y")); + assertEquals(2D, exec("def x = (short)1 def y = (double)1 return x + y")); + assertEquals(2D, exec("def x = (char)1 def y = (double)1 return x + y")); + assertEquals(2D, exec("def x = (int)1 def y = (double)1 return x + y")); + assertEquals(2D, exec("def x = (long)1 def y = (double)1 return x + y")); + assertEquals(2D, exec("def x = (float)1 def y = (double)1 return x + y")); + assertEquals(2D, exec("def x = (double)1 def y = (double)1 return x + y")); + + assertEquals(2, exec("def x = (Byte)1 def y = (byte)1 return x + y")); + assertEquals(2, exec("def x = (Short)1 def y = (short)1 return x + y")); + assertEquals(2, exec("def x = (Character)1 def y = (char)1 return x + y")); + assertEquals(2, exec("def x = (Integer)1 def y = (int)1 return x + y")); + assertEquals(2L, exec("def x = (Long)1 def y = (long)1 return x + y")); + assertEquals(2F, exec("def x = (Float)1 def y = (float)1 return x + y")); + assertEquals(2D, exec("def x = (Double)1 def y = (double)1 return x + y")); + } + + public void testSub() { + assertEquals(0, exec("def x = (byte)1 def y = (byte)1 return x - y")); + assertEquals(0, exec("def x = (short)1 def y = (byte)1 return x - y")); + assertEquals(0, exec("def x = (char)1 def y = (byte)1 return x - y")); + assertEquals(0, exec("def x = (int)1 def y = (byte)1 return x - y")); + assertEquals(0L, exec("def x = (long)1 def y = (byte)1 return x - y")); + assertEquals(0F, exec("def x = (float)1 def y = (byte)1 return x - y")); + assertEquals(0D, exec("def x = (double)1 def y = (byte)1 return x - y")); + + assertEquals(0, exec("def x = (byte)1 def y = (short)1 return x - y")); + assertEquals(0, exec("def x = (short)1 def y = (short)1 return x - y")); + assertEquals(0, exec("def x = (char)1 def y = (short)1 return x - y")); + assertEquals(0, exec("def x = (int)1 def y = (short)1 return x - y")); + assertEquals(0L, exec("def x = (long)1 def y = (short)1 return x - y")); + assertEquals(0F, exec("def x = (float)1 def y = (short)1 return x - y")); + assertEquals(0D, exec("def x = (double)1 def y = (short)1 return x - y")); + + assertEquals(0, exec("def x = (byte)1 def y = (char)1 return x - y")); + assertEquals(0, exec("def x = (short)1 def y = (char)1 return x - y")); + assertEquals(0, exec("def x = (char)1 def y = (char)1 return x - y")); + assertEquals(0, exec("def x = (int)1 def y = (char)1 return x - y")); + assertEquals(0L, exec("def x = (long)1 def y = (char)1 return x - y")); + assertEquals(0F, exec("def x = (float)1 def y = (char)1 return x - y")); + assertEquals(0D, exec("def x = (double)1 def y = (char)1 return x - y")); + + assertEquals(0, exec("def x = (byte)1 def y = (int)1 return x - y")); + assertEquals(0, exec("def x = (short)1 def y = (int)1 return x - y")); + assertEquals(0, exec("def x = (char)1 def y = (int)1 return x - y")); + assertEquals(0, exec("def x = (int)1 def y = (int)1 return x - y")); + assertEquals(0L, exec("def x = (long)1 def y = (int)1 return x - y")); + assertEquals(0F, exec("def x = (float)1 def y = (int)1 return x - y")); + assertEquals(0D, exec("def x = (double)1 def y = (int)1 return x - y")); + + assertEquals(0L, exec("def x = (byte)1 def y = (long)1 return x - y")); + assertEquals(0L, exec("def x = (short)1 def y = (long)1 return x - y")); + assertEquals(0L, exec("def x = (char)1 def y = (long)1 return x - y")); + assertEquals(0L, exec("def x = (int)1 def y = (long)1 return x - y")); + assertEquals(0L, exec("def x = (long)1 def y = (long)1 return x - y")); + assertEquals(0F, exec("def x = (float)1 def y = (long)1 return x - y")); + assertEquals(0D, exec("def x = (double)1 def y = (long)1 return x - y")); + + assertEquals(0F, exec("def x = (byte)1 def y = (float)1 return x - y")); + assertEquals(0F, exec("def x = (short)1 def y = (float)1 return x - y")); + assertEquals(0F, exec("def x = (char)1 def y = (float)1 return x - y")); + assertEquals(0F, exec("def x = (int)1 def y = (float)1 return x - y")); + assertEquals(0F, exec("def x = (long)1 def y = (float)1 return x - y")); + assertEquals(0F, exec("def x = (float)1 def y = (float)1 return x - y")); + assertEquals(0D, exec("def x = (double)1 def y = (float)1 return x - y")); + + assertEquals(0D, exec("def x = (byte)1 def y = (double)1 return x - y")); + assertEquals(0D, exec("def x = (short)1 def y = (double)1 return x - y")); + assertEquals(0D, exec("def x = (char)1 def y = (double)1 return x - y")); + assertEquals(0D, exec("def x = (int)1 def y = (double)1 return x - y")); + assertEquals(0D, exec("def x = (long)1 def y = (double)1 return x - y")); + assertEquals(0D, exec("def x = (float)1 def y = (double)1 return x - y")); + assertEquals(0D, exec("def x = (double)1 def y = (double)1 return x - y")); + + assertEquals(0, exec("def x = (Byte)1 def y = (byte)1 return x - y")); + assertEquals(0, exec("def x = (Short)1 def y = (short)1 return x - y")); + assertEquals(0, exec("def x = (Character)1 def y = (char)1 return x - y")); + assertEquals(0, exec("def x = (Integer)1 def y = (int)1 return x - y")); + assertEquals(0L, exec("def x = (Long)1 def y = (long)1 return x - y")); + assertEquals(0F, exec("def x = (Float)1 def y = (float)1 return x - y")); + assertEquals(0D, exec("def x = (Double)1 def y = (double)1 return x - y")); + } + + public void testLsh() { + assertEquals(2, exec("def x = (byte)1 def y = (byte)1 return x << y")); + assertEquals(2, exec("def x = (short)1 def y = (byte)1 return x << y")); + assertEquals(2, exec("def x = (char)1 def y = (byte)1 return x << y")); + assertEquals(2, exec("def x = (int)1 def y = (byte)1 return x << y")); + assertEquals(2L, exec("def x = (long)1 def y = (byte)1 return x << y")); + assertEquals(2L, exec("def x = (float)1 def y = (byte)1 return x << y")); + assertEquals(2L, exec("def x = (double)1 def y = (byte)1 return x << y")); + + assertEquals(2, exec("def x = (byte)1 def y = (short)1 return x << y")); + assertEquals(2, exec("def x = (short)1 def y = (short)1 return x << y")); + assertEquals(2, exec("def x = (char)1 def y = (short)1 return x << y")); + assertEquals(2, exec("def x = (int)1 def y = (short)1 return x << y")); + assertEquals(2L, exec("def x = (long)1 def y = (short)1 return x << y")); + assertEquals(2L, exec("def x = (float)1 def y = (short)1 return x << y")); + assertEquals(2L, exec("def x = (double)1 def y = (short)1 return x << y")); + + assertEquals(2, exec("def x = (byte)1 def y = (char)1 return x << y")); + assertEquals(2, exec("def x = (short)1 def y = (char)1 return x << y")); + assertEquals(2, exec("def x = (char)1 def y = (char)1 return x << y")); + assertEquals(2, exec("def x = (int)1 def y = (char)1 return x << y")); + assertEquals(2L, exec("def x = (long)1 def y = (char)1 return x << y")); + assertEquals(2L, exec("def x = (float)1 def y = (char)1 return x << y")); + assertEquals(2L, exec("def x = (double)1 def y = (char)1 return x << y")); + + assertEquals(2, exec("def x = (byte)1 def y = (int)1 return x << y")); + assertEquals(2, exec("def x = (short)1 def y = (int)1 return x << y")); + assertEquals(2, exec("def x = (char)1 def y = (int)1 return x << y")); + assertEquals(2, exec("def x = (int)1 def y = (int)1 return x << y")); + assertEquals(2L, exec("def x = (long)1 def y = (int)1 return x << y")); + assertEquals(2L, exec("def x = (float)1 def y = (int)1 return x << y")); + assertEquals(2L, exec("def x = (double)1 def y = (int)1 return x << y")); + + assertEquals(2L, exec("def x = (byte)1 def y = (long)1 return x << y")); + assertEquals(2L, exec("def x = (short)1 def y = (long)1 return x << y")); + assertEquals(2L, exec("def x = (char)1 def y = (long)1 return x << y")); + assertEquals(2L, exec("def x = (int)1 def y = (long)1 return x << y")); + assertEquals(2L, exec("def x = (long)1 def y = (long)1 return x << y")); + assertEquals(2L, exec("def x = (float)1 def y = (long)1 return x << y")); + assertEquals(2L, exec("def x = (double)1 def y = (long)1 return x << y")); + + assertEquals(2L, exec("def x = (byte)1 def y = (float)1 return x << y")); + assertEquals(2L, exec("def x = (short)1 def y = (float)1 return x << y")); + assertEquals(2L, exec("def x = (char)1 def y = (float)1 return x << y")); + assertEquals(2L, exec("def x = (int)1 def y = (float)1 return x << y")); + assertEquals(2L, exec("def x = (long)1 def y = (float)1 return x << y")); + assertEquals(2L, exec("def x = (float)1 def y = (float)1 return x << y")); + assertEquals(2L, exec("def x = (double)1 def y = (float)1 return x << y")); + + assertEquals(2L, exec("def x = (byte)1 def y = (double)1 return x << y")); + assertEquals(2L, exec("def x = (short)1 def y = (double)1 return x << y")); + assertEquals(2L, exec("def x = (char)1 def y = (double)1 return x << y")); + assertEquals(2L, exec("def x = (int)1 def y = (double)1 return x << y")); + assertEquals(2L, exec("def x = (long)1 def y = (double)1 return x << y")); + assertEquals(2L, exec("def x = (float)1 def y = (double)1 return x << y")); + assertEquals(2L, exec("def x = (double)1 def y = (double)1 return x << y")); + + assertEquals(2, exec("def x = (Byte)1 def y = (byte)1 return x << y")); + assertEquals(2, exec("def x = (Short)1 def y = (short)1 return x << y")); + assertEquals(2, exec("def x = (Character)1 def y = (char)1 return x << y")); + assertEquals(2, exec("def x = (Integer)1 def y = (int)1 return x << y")); + assertEquals(2L, exec("def x = (Long)1 def y = (long)1 return x << y")); + assertEquals(2L, exec("def x = (Float)1 def y = (float)1 return x << y")); + assertEquals(2L, exec("def x = (Double)1 def y = (double)1 return x << y")); + } + + public void testRsh() { + assertEquals(2, exec("def x = (byte)4 def y = (byte)1 return x >> y")); + assertEquals(2, exec("def x = (short)4 def y = (byte)1 return x >> y")); + assertEquals(2, exec("def x = (char)4 def y = (byte)1 return x >> y")); + assertEquals(2, exec("def x = (int)4 def y = (byte)1 return x >> y")); + assertEquals(2L, exec("def x = (long)4 def y = (byte)1 return x >> y")); + assertEquals(2L, exec("def x = (float)4 def y = (byte)1 return x >> y")); + assertEquals(2L, exec("def x = (double)4 def y = (byte)1 return x >> y")); + + assertEquals(2, exec("def x = (byte)4 def y = (short)1 return x >> y")); + assertEquals(2, exec("def x = (short)4 def y = (short)1 return x >> y")); + assertEquals(2, exec("def x = (char)4 def y = (short)1 return x >> y")); + assertEquals(2, exec("def x = (int)4 def y = (short)1 return x >> y")); + assertEquals(2L, exec("def x = (long)4 def y = (short)1 return x >> y")); + assertEquals(2L, exec("def x = (float)4 def y = (short)1 return x >> y")); + assertEquals(2L, exec("def x = (double)4 def y = (short)1 return x >> y")); + + assertEquals(2, exec("def x = (byte)4 def y = (char)1 return x >> y")); + assertEquals(2, exec("def x = (short)4 def y = (char)1 return x >> y")); + assertEquals(2, exec("def x = (char)4 def y = (char)1 return x >> y")); + assertEquals(2, exec("def x = (int)4 def y = (char)1 return x >> y")); + assertEquals(2L, exec("def x = (long)4 def y = (char)1 return x >> y")); + assertEquals(2L, exec("def x = (float)4 def y = (char)1 return x >> y")); + assertEquals(2L, exec("def x = (double)4 def y = (char)1 return x >> y")); + + assertEquals(2, exec("def x = (byte)4 def y = (int)1 return x >> y")); + assertEquals(2, exec("def x = (short)4 def y = (int)1 return x >> y")); + assertEquals(2, exec("def x = (char)4 def y = (int)1 return x >> y")); + assertEquals(2, exec("def x = (int)4 def y = (int)1 return x >> y")); + assertEquals(2L, exec("def x = (long)4 def y = (int)1 return x >> y")); + assertEquals(2L, exec("def x = (float)4 def y = (int)1 return x >> y")); + assertEquals(2L, exec("def x = (double)4 def y = (int)1 return x >> y")); + + assertEquals(2L, exec("def x = (byte)4 def y = (long)1 return x >> y")); + assertEquals(2L, exec("def x = (short)4 def y = (long)1 return x >> y")); + assertEquals(2L, exec("def x = (char)4 def y = (long)1 return x >> y")); + assertEquals(2L, exec("def x = (int)4 def y = (long)1 return x >> y")); + assertEquals(2L, exec("def x = (long)4 def y = (long)1 return x >> y")); + assertEquals(2L, exec("def x = (float)4 def y = (long)1 return x >> y")); + assertEquals(2L, exec("def x = (double)4 def y = (long)1 return x >> y")); + + assertEquals(2L, exec("def x = (byte)4 def y = (float)1 return x >> y")); + assertEquals(2L, exec("def x = (short)4 def y = (float)1 return x >> y")); + assertEquals(2L, exec("def x = (char)4 def y = (float)1 return x >> y")); + assertEquals(2L, exec("def x = (int)4 def y = (float)1 return x >> y")); + assertEquals(2L, exec("def x = (long)4 def y = (float)1 return x >> y")); + assertEquals(2L, exec("def x = (float)4 def y = (float)1 return x >> y")); + assertEquals(2L, exec("def x = (double)4 def y = (float)1 return x >> y")); + + assertEquals(2L, exec("def x = (byte)4 def y = (double)1 return x >> y")); + assertEquals(2L, exec("def x = (short)4 def y = (double)1 return x >> y")); + assertEquals(2L, exec("def x = (char)4 def y = (double)1 return x >> y")); + assertEquals(2L, exec("def x = (int)4 def y = (double)1 return x >> y")); + assertEquals(2L, exec("def x = (long)4 def y = (double)1 return x >> y")); + assertEquals(2L, exec("def x = (float)4 def y = (double)1 return x >> y")); + assertEquals(2L, exec("def x = (double)4 def y = (double)1 return x >> y")); + + assertEquals(2, exec("def x = (Byte)4 def y = (byte)1 return x >> y")); + assertEquals(2, exec("def x = (Short)4 def y = (short)1 return x >> y")); + assertEquals(2, exec("def x = (Character)4 def y = (char)1 return x >> y")); + assertEquals(2, exec("def x = (Integer)4 def y = (int)1 return x >> y")); + assertEquals(2L, exec("def x = (Long)4 def y = (long)1 return x >> y")); + assertEquals(2L, exec("def x = (Float)4 def y = (float)1 return x >> y")); + assertEquals(2L, exec("def x = (Double)4 def y = (double)1 return x >> y")); + } + + public void testUsh() { + assertEquals(2, exec("def x = (byte)4 def y = (byte)1 return x >>> y")); + assertEquals(2, exec("def x = (short)4 def y = (byte)1 return x >>> y")); + assertEquals(2, exec("def x = (char)4 def y = (byte)1 return x >>> y")); + assertEquals(2, exec("def x = (int)4 def y = (byte)1 return x >>> y")); + assertEquals(2L, exec("def x = (long)4 def y = (byte)1 return x >>> y")); + assertEquals(2L, exec("def x = (float)4 def y = (byte)1 return x >>> y")); + assertEquals(2L, exec("def x = (double)4 def y = (byte)1 return x >>> y")); + + assertEquals(2, exec("def x = (byte)4 def y = (short)1 return x >>> y")); + assertEquals(2, exec("def x = (short)4 def y = (short)1 return x >>> y")); + assertEquals(2, exec("def x = (char)4 def y = (short)1 return x >>> y")); + assertEquals(2, exec("def x = (int)4 def y = (short)1 return x >>> y")); + assertEquals(2L, exec("def x = (long)4 def y = (short)1 return x >>> y")); + assertEquals(2L, exec("def x = (float)4 def y = (short)1 return x >>> y")); + assertEquals(2L, exec("def x = (double)4 def y = (short)1 return x >>> y")); + + assertEquals(2, exec("def x = (byte)4 def y = (char)1 return x >>> y")); + assertEquals(2, exec("def x = (short)4 def y = (char)1 return x >>> y")); + assertEquals(2, exec("def x = (char)4 def y = (char)1 return x >>> y")); + assertEquals(2, exec("def x = (int)4 def y = (char)1 return x >>> y")); + assertEquals(2L, exec("def x = (long)4 def y = (char)1 return x >>> y")); + assertEquals(2L, exec("def x = (float)4 def y = (char)1 return x >>> y")); + assertEquals(2L, exec("def x = (double)4 def y = (char)1 return x >>> y")); + + assertEquals(2, exec("def x = (byte)4 def y = (int)1 return x >>> y")); + assertEquals(2, exec("def x = (short)4 def y = (int)1 return x >>> y")); + assertEquals(2, exec("def x = (char)4 def y = (int)1 return x >>> y")); + assertEquals(2, exec("def x = (int)4 def y = (int)1 return x >>> y")); + assertEquals(2L, exec("def x = (long)4 def y = (int)1 return x >>> y")); + assertEquals(2L, exec("def x = (float)4 def y = (int)1 return x >>> y")); + assertEquals(2L, exec("def x = (double)4 def y = (int)1 return x >>> y")); + + assertEquals(2L, exec("def x = (byte)4 def y = (long)1 return x >>> y")); + assertEquals(2L, exec("def x = (short)4 def y = (long)1 return x >>> y")); + assertEquals(2L, exec("def x = (char)4 def y = (long)1 return x >>> y")); + assertEquals(2L, exec("def x = (int)4 def y = (long)1 return x >>> y")); + assertEquals(2L, exec("def x = (long)4 def y = (long)1 return x >>> y")); + assertEquals(2L, exec("def x = (float)4 def y = (long)1 return x >>> y")); + assertEquals(2L, exec("def x = (double)4 def y = (long)1 return x >>> y")); + + assertEquals(2L, exec("def x = (byte)4 def y = (float)1 return x >>> y")); + assertEquals(2L, exec("def x = (short)4 def y = (float)1 return x >>> y")); + assertEquals(2L, exec("def x = (char)4 def y = (float)1 return x >>> y")); + assertEquals(2L, exec("def x = (int)4 def y = (float)1 return x >>> y")); + assertEquals(2L, exec("def x = (long)4 def y = (float)1 return x >>> y")); + assertEquals(2L, exec("def x = (float)4 def y = (float)1 return x >>> y")); + assertEquals(2L, exec("def x = (double)4 def y = (float)1 return x >>> y")); + + assertEquals(2L, exec("def x = (byte)4 def y = (double)1 return x >>> y")); + assertEquals(2L, exec("def x = (short)4 def y = (double)1 return x >>> y")); + assertEquals(2L, exec("def x = (char)4 def y = (double)1 return x >>> y")); + assertEquals(2L, exec("def x = (int)4 def y = (double)1 return x >>> y")); + assertEquals(2L, exec("def x = (long)4 def y = (double)1 return x >>> y")); + assertEquals(2L, exec("def x = (float)4 def y = (double)1 return x >>> y")); + assertEquals(2L, exec("def x = (double)4 def y = (double)1 return x >>> y")); + + assertEquals(2, exec("def x = (Byte)4 def y = (byte)1 return x >>> y")); + assertEquals(2, exec("def x = (Short)4 def y = (short)1 return x >>> y")); + assertEquals(2, exec("def x = (Character)4 def y = (char)1 return x >>> y")); + assertEquals(2, exec("def x = (Integer)4 def y = (int)1 return x >>> y")); + assertEquals(2L, exec("def x = (Long)4 def y = (long)1 return x >>> y")); + assertEquals(2L, exec("def x = (Float)4 def y = (float)1 return x >>> y")); + assertEquals(2L, exec("def x = (Double)4 def y = (double)1 return x >>> y")); + } + + public void testAnd() { + assertEquals(0, exec("def x = (byte)4 def y = (byte)1 return x & y")); + assertEquals(0, exec("def x = (short)4 def y = (byte)1 return x & y")); + assertEquals(0, exec("def x = (char)4 def y = (byte)1 return x & y")); + assertEquals(0, exec("def x = (int)4 def y = (byte)1 return x & y")); + assertEquals(0L, exec("def x = (long)4 def y = (byte)1 return x & y")); + assertEquals(0L, exec("def x = (float)4 def y = (byte)1 return x & y")); + assertEquals(0L, exec("def x = (double)4 def y = (byte)1 return x & y")); + + assertEquals(0, exec("def x = (byte)4 def y = (short)1 return x & y")); + assertEquals(0, exec("def x = (short)4 def y = (short)1 return x & y")); + assertEquals(0, exec("def x = (char)4 def y = (short)1 return x & y")); + assertEquals(0, exec("def x = (int)4 def y = (short)1 return x & y")); + assertEquals(0L, exec("def x = (long)4 def y = (short)1 return x & y")); + assertEquals(0L, exec("def x = (float)4 def y = (short)1 return x & y")); + assertEquals(0L, exec("def x = (double)4 def y = (short)1 return x & y")); + + assertEquals(0, exec("def x = (byte)4 def y = (char)1 return x & y")); + assertEquals(0, exec("def x = (short)4 def y = (char)1 return x & y")); + assertEquals(0, exec("def x = (char)4 def y = (char)1 return x & y")); + assertEquals(0, exec("def x = (int)4 def y = (char)1 return x & y")); + assertEquals(0L, exec("def x = (long)4 def y = (char)1 return x & y")); + assertEquals(0L, exec("def x = (float)4 def y = (char)1 return x & y")); + assertEquals(0L, exec("def x = (double)4 def y = (char)1 return x & y")); + + assertEquals(0, exec("def x = (byte)4 def y = (int)1 return x & y")); + assertEquals(0, exec("def x = (short)4 def y = (int)1 return x & y")); + assertEquals(0, exec("def x = (char)4 def y = (int)1 return x & y")); + assertEquals(0, exec("def x = (int)4 def y = (int)1 return x & y")); + assertEquals(0L, exec("def x = (long)4 def y = (int)1 return x & y")); + assertEquals(0L, exec("def x = (float)4 def y = (int)1 return x & y")); + assertEquals(0L, exec("def x = (double)4 def y = (int)1 return x & y")); + + assertEquals(0L, exec("def x = (byte)4 def y = (long)1 return x & y")); + assertEquals(0L, exec("def x = (short)4 def y = (long)1 return x & y")); + assertEquals(0L, exec("def x = (char)4 def y = (long)1 return x & y")); + assertEquals(0L, exec("def x = (int)4 def y = (long)1 return x & y")); + assertEquals(0L, exec("def x = (long)4 def y = (long)1 return x & y")); + assertEquals(0L, exec("def x = (float)4 def y = (long)1 return x & y")); + assertEquals(0L, exec("def x = (double)4 def y = (long)1 return x & y")); + + assertEquals(0L, exec("def x = (byte)4 def y = (float)1 return x & y")); + assertEquals(0L, exec("def x = (short)4 def y = (float)1 return x & y")); + assertEquals(0L, exec("def x = (char)4 def y = (float)1 return x & y")); + assertEquals(0L, exec("def x = (int)4 def y = (float)1 return x & y")); + assertEquals(0L, exec("def x = (long)4 def y = (float)1 return x & y")); + assertEquals(0L, exec("def x = (float)4 def y = (float)1 return x & y")); + assertEquals(0L, exec("def x = (double)4 def y = (float)1 return x & y")); + + assertEquals(0L, exec("def x = (byte)4 def y = (double)1 return x & y")); + assertEquals(0L, exec("def x = (short)4 def y = (double)1 return x & y")); + assertEquals(0L, exec("def x = (char)4 def y = (double)1 return x & y")); + assertEquals(0L, exec("def x = (int)4 def y = (double)1 return x & y")); + assertEquals(0L, exec("def x = (long)4 def y = (double)1 return x & y")); + assertEquals(0L, exec("def x = (float)4 def y = (double)1 return x & y")); + assertEquals(0L, exec("def x = (double)4 def y = (double)1 return x & y")); + + assertEquals(0, exec("def x = (Byte)4 def y = (byte)1 return x & y")); + assertEquals(0, exec("def x = (Short)4 def y = (short)1 return x & y")); + assertEquals(0, exec("def x = (Character)4 def y = (char)1 return x & y")); + assertEquals(0, exec("def x = (Integer)4 def y = (int)1 return x & y")); + assertEquals(0L, exec("def x = (Long)4 def y = (long)1 return x & y")); + assertEquals(0L, exec("def x = (Float)4 def y = (float)1 return x & y")); + assertEquals(0L, exec("def x = (Double)4 def y = (double)1 return x & y")); + } + + public void testXor() { + assertEquals(5, exec("def x = (byte)4 def y = (byte)1 return x ^ y")); + assertEquals(5, exec("def x = (short)4 def y = (byte)1 return x ^ y")); + assertEquals(5, exec("def x = (char)4 def y = (byte)1 return x ^ y")); + assertEquals(5, exec("def x = (int)4 def y = (byte)1 return x ^ y")); + assertEquals(5L, exec("def x = (long)4 def y = (byte)1 return x ^ y")); + assertEquals(5L, exec("def x = (float)4 def y = (byte)1 return x ^ y")); + assertEquals(5L, exec("def x = (double)4 def y = (byte)1 return x ^ y")); + + assertEquals(5, exec("def x = (byte)4 def y = (short)1 return x ^ y")); + assertEquals(5, exec("def x = (short)4 def y = (short)1 return x ^ y")); + assertEquals(5, exec("def x = (char)4 def y = (short)1 return x ^ y")); + assertEquals(5, exec("def x = (int)4 def y = (short)1 return x ^ y")); + assertEquals(5L, exec("def x = (long)4 def y = (short)1 return x ^ y")); + assertEquals(5L, exec("def x = (float)4 def y = (short)1 return x ^ y")); + assertEquals(5L, exec("def x = (double)4 def y = (short)1 return x ^ y")); + + assertEquals(5, exec("def x = (byte)4 def y = (char)1 return x ^ y")); + assertEquals(5, exec("def x = (short)4 def y = (char)1 return x ^ y")); + assertEquals(5, exec("def x = (char)4 def y = (char)1 return x ^ y")); + assertEquals(5, exec("def x = (int)4 def y = (char)1 return x ^ y")); + assertEquals(5L, exec("def x = (long)4 def y = (char)1 return x ^ y")); + assertEquals(5L, exec("def x = (float)4 def y = (char)1 return x ^ y")); + assertEquals(5L, exec("def x = (double)4 def y = (char)1 return x ^ y")); + + assertEquals(5, exec("def x = (byte)4 def y = (int)1 return x ^ y")); + assertEquals(5, exec("def x = (short)4 def y = (int)1 return x ^ y")); + assertEquals(5, exec("def x = (char)4 def y = (int)1 return x ^ y")); + assertEquals(5, exec("def x = (int)4 def y = (int)1 return x ^ y")); + assertEquals(5L, exec("def x = (long)4 def y = (int)1 return x ^ y")); + assertEquals(5L, exec("def x = (float)4 def y = (int)1 return x ^ y")); + assertEquals(5L, exec("def x = (double)4 def y = (int)1 return x ^ y")); + + assertEquals(5L, exec("def x = (byte)4 def y = (long)1 return x ^ y")); + assertEquals(5L, exec("def x = (short)4 def y = (long)1 return x ^ y")); + assertEquals(5L, exec("def x = (char)4 def y = (long)1 return x ^ y")); + assertEquals(5L, exec("def x = (int)4 def y = (long)1 return x ^ y")); + assertEquals(5L, exec("def x = (long)4 def y = (long)1 return x ^ y")); + assertEquals(5L, exec("def x = (float)4 def y = (long)1 return x ^ y")); + assertEquals(5L, exec("def x = (double)4 def y = (long)1 return x ^ y")); + + assertEquals(5L, exec("def x = (byte)4 def y = (float)1 return x ^ y")); + assertEquals(5L, exec("def x = (short)4 def y = (float)1 return x ^ y")); + assertEquals(5L, exec("def x = (char)4 def y = (float)1 return x ^ y")); + assertEquals(5L, exec("def x = (int)4 def y = (float)1 return x ^ y")); + assertEquals(5L, exec("def x = (long)4 def y = (float)1 return x ^ y")); + assertEquals(5L, exec("def x = (float)4 def y = (float)1 return x ^ y")); + assertEquals(5L, exec("def x = (double)4 def y = (float)1 return x ^ y")); + + assertEquals(5L, exec("def x = (byte)4 def y = (double)1 return x ^ y")); + assertEquals(5L, exec("def x = (short)4 def y = (double)1 return x ^ y")); + assertEquals(5L, exec("def x = (char)4 def y = (double)1 return x ^ y")); + assertEquals(5L, exec("def x = (int)4 def y = (double)1 return x ^ y")); + assertEquals(5L, exec("def x = (long)4 def y = (double)1 return x ^ y")); + assertEquals(5L, exec("def x = (float)4 def y = (double)1 return x ^ y")); + assertEquals(5L, exec("def x = (double)4 def y = (double)1 return x ^ y")); + + assertEquals(5, exec("def x = (Byte)4 def y = (byte)1 return x ^ y")); + assertEquals(5, exec("def x = (Short)4 def y = (short)1 return x ^ y")); + assertEquals(5, exec("def x = (Character)4 def y = (char)1 return x ^ y")); + assertEquals(5, exec("def x = (Integer)4 def y = (int)1 return x ^ y")); + assertEquals(5L, exec("def x = (Long)4 def y = (long)1 return x ^ y")); + assertEquals(5L, exec("def x = (Float)4 def y = (float)1 return x ^ y")); + assertEquals(5L, exec("def x = (Double)4 def y = (double)1 return x ^ y")); + } + + public void testOr() { + assertEquals(5, exec("def x = (byte)4 def y = (byte)1 return x | y")); + assertEquals(5, exec("def x = (short)4 def y = (byte)1 return x | y")); + assertEquals(5, exec("def x = (char)4 def y = (byte)1 return x | y")); + assertEquals(5, exec("def x = (int)4 def y = (byte)1 return x | y")); + assertEquals(5L, exec("def x = (long)4 def y = (byte)1 return x | y")); + assertEquals(5L, exec("def x = (float)4 def y = (byte)1 return x | y")); + assertEquals(5L, exec("def x = (double)4 def y = (byte)1 return x | y")); + + assertEquals(5, exec("def x = (byte)4 def y = (short)1 return x | y")); + assertEquals(5, exec("def x = (short)4 def y = (short)1 return x | y")); + assertEquals(5, exec("def x = (char)4 def y = (short)1 return x | y")); + assertEquals(5, exec("def x = (int)4 def y = (short)1 return x | y")); + assertEquals(5L, exec("def x = (long)4 def y = (short)1 return x | y")); + assertEquals(5L, exec("def x = (float)4 def y = (short)1 return x | y")); + assertEquals(5L, exec("def x = (double)4 def y = (short)1 return x | y")); + + assertEquals(5, exec("def x = (byte)4 def y = (char)1 return x | y")); + assertEquals(5, exec("def x = (short)4 def y = (char)1 return x | y")); + assertEquals(5, exec("def x = (char)4 def y = (char)1 return x | y")); + assertEquals(5, exec("def x = (int)4 def y = (char)1 return x | y")); + assertEquals(5L, exec("def x = (long)4 def y = (char)1 return x | y")); + assertEquals(5L, exec("def x = (float)4 def y = (char)1 return x | y")); + assertEquals(5L, exec("def x = (double)4 def y = (char)1 return x | y")); + + assertEquals(5, exec("def x = (byte)4 def y = (int)1 return x | y")); + assertEquals(5, exec("def x = (short)4 def y = (int)1 return x | y")); + assertEquals(5, exec("def x = (char)4 def y = (int)1 return x | y")); + assertEquals(5, exec("def x = (int)4 def y = (int)1 return x | y")); + assertEquals(5L, exec("def x = (long)4 def y = (int)1 return x | y")); + assertEquals(5L, exec("def x = (float)4 def y = (int)1 return x | y")); + assertEquals(5L, exec("def x = (double)4 def y = (int)1 return x | y")); + + assertEquals(5L, exec("def x = (byte)4 def y = (long)1 return x | y")); + assertEquals(5L, exec("def x = (short)4 def y = (long)1 return x | y")); + assertEquals(5L, exec("def x = (char)4 def y = (long)1 return x | y")); + assertEquals(5L, exec("def x = (int)4 def y = (long)1 return x | y")); + assertEquals(5L, exec("def x = (long)4 def y = (long)1 return x | y")); + assertEquals(5L, exec("def x = (float)4 def y = (long)1 return x | y")); + assertEquals(5L, exec("def x = (double)4 def y = (long)1 return x | y")); + + assertEquals(5L, exec("def x = (byte)4 def y = (float)1 return x | y")); + assertEquals(5L, exec("def x = (short)4 def y = (float)1 return x | y")); + assertEquals(5L, exec("def x = (char)4 def y = (float)1 return x | y")); + assertEquals(5L, exec("def x = (int)4 def y = (float)1 return x | y")); + assertEquals(5L, exec("def x = (long)4 def y = (float)1 return x | y")); + assertEquals(5L, exec("def x = (float)4 def y = (float)1 return x | y")); + assertEquals(5L, exec("def x = (double)4 def y = (float)1 return x | y")); + + assertEquals(5L, exec("def x = (byte)4 def y = (double)1 return x | y")); + assertEquals(5L, exec("def x = (short)4 def y = (double)1 return x | y")); + assertEquals(5L, exec("def x = (char)4 def y = (double)1 return x | y")); + assertEquals(5L, exec("def x = (int)4 def y = (double)1 return x | y")); + assertEquals(5L, exec("def x = (long)4 def y = (double)1 return x | y")); + assertEquals(5L, exec("def x = (float)4 def y = (double)1 return x | y")); + assertEquals(5L, exec("def x = (double)4 def y = (double)1 return x | y")); + + assertEquals(5, exec("def x = (Byte)4 def y = (byte)1 return x | y")); + assertEquals(5, exec("def x = (Short)4 def y = (short)1 return x | y")); + assertEquals(5, exec("def x = (Character)4 def y = (char)1 return x | y")); + assertEquals(5, exec("def x = (Integer)4 def y = (int)1 return x | y")); + assertEquals(5L, exec("def x = (Long)4 def y = (long)1 return x | y")); + assertEquals(5L, exec("def x = (Float)4 def y = (float)1 return x | y")); + assertEquals(5L, exec("def x = (Double)4 def y = (double)1 return x | y")); + } + + public void testEq() { + assertEquals(true, exec("def x = (byte)7 def y = (int)7 return x == y")); + assertEquals(true, exec("def x = (short)6 def y = (int)6 return x == y")); + assertEquals(true, exec("def x = (char)5 def y = (int)5 return x == y")); + assertEquals(true, exec("def x = (int)4 def y = (int)4 return x == y")); + assertEquals(false, exec("def x = (long)5 def y = (int)3 return x == y")); + assertEquals(false, exec("def x = (float)6 def y = (int)2 return x == y")); + assertEquals(false, exec("def x = (double)7 def y = (int)1 return x == y")); + + assertEquals(true, exec("def x = (byte)7 def y = (double)7 return x == y")); + assertEquals(true, exec("def x = (short)6 def y = (double)6 return x == y")); + assertEquals(true, exec("def x = (char)5 def y = (double)5 return x == y")); + assertEquals(true, exec("def x = (int)4 def y = (double)4 return x == y")); + assertEquals(false, exec("def x = (long)5 def y = (double)3 return x == y")); + assertEquals(false, exec("def x = (float)6 def y = (double)2 return x == y")); + assertEquals(false, exec("def x = (double)7 def y = (double)1 return x == y")); + + assertEquals(true, exec("def x = new HashMap() def y = new HashMap() return x == y")); + assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x == y")); + assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x == y")); + assertEquals(true, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x == y")); + } + + public void testEqr() { + assertEquals(false, exec("def x = (byte)7 def y = (int)7 return x === y")); + assertEquals(false, exec("def x = (short)6 def y = (int)6 return x === y")); + assertEquals(false, exec("def x = (char)5 def y = (int)5 return x === y")); + assertEquals(true, exec("def x = (int)4 def y = (int)4 return x === y")); + assertEquals(false, exec("def x = (long)5 def y = (int)3 return x === y")); + assertEquals(false, exec("def x = (float)6 def y = (int)2 return x === y")); + assertEquals(false, exec("def x = (double)7 def y = (int)1 return x === y")); + + assertEquals(false, exec("def x = new HashMap() def y = new HashMap() return x === y")); + assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x === y")); + assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x === y")); + assertEquals(true, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x === y")); + } + + public void testNe() { + assertEquals(false, exec("def x = (byte)7 def y = (int)7 return x != y")); + assertEquals(false, exec("def x = (short)6 def y = (int)6 return x != y")); + assertEquals(false, exec("def x = (char)5 def y = (int)5 return x != y")); + assertEquals(false, exec("def x = (int)4 def y = (int)4 return x != y")); + assertEquals(true, exec("def x = (long)5 def y = (int)3 return x != y")); + assertEquals(true, exec("def x = (float)6 def y = (int)2 return x != y")); + assertEquals(true, exec("def x = (double)7 def y = (int)1 return x != y")); + + assertEquals(false, exec("def x = (byte)7 def y = (double)7 return x != y")); + assertEquals(false, exec("def x = (short)6 def y = (double)6 return x != y")); + assertEquals(false, exec("def x = (char)5 def y = (double)5 return x != y")); + assertEquals(false, exec("def x = (int)4 def y = (double)4 return x != y")); + assertEquals(true, exec("def x = (long)5 def y = (double)3 return x != y")); + assertEquals(true, exec("def x = (float)6 def y = (double)2 return x != y")); + assertEquals(true, exec("def x = (double)7 def y = (double)1 return x != y")); + + assertEquals(false, exec("def x = new HashMap() def y = new HashMap() return x != y")); + assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x != y")); + assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x != y")); + assertEquals(false, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x != y")); + } + + public void testNer() { + assertEquals(true, exec("def x = (byte)7 def y = (int)7 return x !== y")); + assertEquals(true, exec("def x = (short)6 def y = (int)6 return x !== y")); + assertEquals(true, exec("def x = (char)5 def y = (int)5 return x !== y")); + assertEquals(false, exec("def x = (int)4 def y = (int)4 return x !== y")); + assertEquals(true, exec("def x = (long)5 def y = (int)3 return x !== y")); + assertEquals(true, exec("def x = (float)6 def y = (int)2 return x !== y")); + assertEquals(true, exec("def x = (double)7 def y = (int)1 return x !== y")); + + assertEquals(true, exec("def x = new HashMap() def y = new HashMap() return x !== y")); + assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x !== y")); + assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x !== y")); + assertEquals(false, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x !== y")); + } + + public void testLt() { + assertEquals(true, exec("def x = (byte)1 def y = (int)7 return x < y")); + assertEquals(true, exec("def x = (short)2 def y = (int)6 return x < y")); + assertEquals(true, exec("def x = (char)3 def y = (int)5 return x < y")); + assertEquals(false, exec("def x = (int)4 def y = (int)4 return x < y")); + assertEquals(false, exec("def x = (long)5 def y = (int)3 return x < y")); + assertEquals(false, exec("def x = (float)6 def y = (int)2 return x < y")); + assertEquals(false, exec("def x = (double)7 def y = (int)1 return x < y")); + + assertEquals(true, exec("def x = (byte)1 def y = (double)7 return x < y")); + assertEquals(true, exec("def x = (short)2 def y = (double)6 return x < y")); + assertEquals(true, exec("def x = (char)3 def y = (double)5 return x < y")); + assertEquals(false, exec("def x = (int)4 def y = (double)4 return x < y")); + assertEquals(false, exec("def x = (long)5 def y = (double)3 return x < y")); + assertEquals(false, exec("def x = (float)6 def y = (double)2 return x < y")); + assertEquals(false, exec("def x = (double)7 def y = (double)1 return x < y")); + } + + public void testLte() { + assertEquals(true, exec("def x = (byte)1 def y = (int)7 return x <= y")); + assertEquals(true, exec("def x = (short)2 def y = (int)6 return x <= y")); + assertEquals(true, exec("def x = (char)3 def y = (int)5 return x <= y")); + assertEquals(true, exec("def x = (int)4 def y = (int)4 return x <= y")); + assertEquals(false, exec("def x = (long)5 def y = (int)3 return x <= y")); + assertEquals(false, exec("def x = (float)6 def y = (int)2 return x <= y")); + assertEquals(false, exec("def x = (double)7 def y = (int)1 return x <= y")); + + assertEquals(true, exec("def x = (byte)1 def y = (double)7 return x <= y")); + assertEquals(true, exec("def x = (short)2 def y = (double)6 return x <= y")); + assertEquals(true, exec("def x = (char)3 def y = (double)5 return x <= y")); + assertEquals(true, exec("def x = (int)4 def y = (double)4 return x <= y")); + assertEquals(false, exec("def x = (long)5 def y = (double)3 return x <= y")); + assertEquals(false, exec("def x = (float)6 def y = (double)2 return x <= y")); + assertEquals(false, exec("def x = (double)7 def y = (double)1 return x <= y")); + } + + public void testGt() { + assertEquals(false, exec("def x = (byte)1 def y = (int)7 return x > y")); + assertEquals(false, exec("def x = (short)2 def y = (int)6 return x > y")); + assertEquals(false, exec("def x = (char)3 def y = (int)5 return x > y")); + assertEquals(false, exec("def x = (int)4 def y = (int)4 return x > y")); + assertEquals(true, exec("def x = (long)5 def y = (int)3 return x > y")); + assertEquals(true, exec("def x = (float)6 def y = (int)2 return x > y")); + assertEquals(true, exec("def x = (double)7 def y = (int)1 return x > y")); + + assertEquals(false, exec("def x = (byte)1 def y = (double)7 return x > y")); + assertEquals(false, exec("def x = (short)2 def y = (double)6 return x > y")); + assertEquals(false, exec("def x = (char)3 def y = (double)5 return x > y")); + assertEquals(false, exec("def x = (int)4 def y = (double)4 return x > y")); + assertEquals(true, exec("def x = (long)5 def y = (double)3 return x > y")); + assertEquals(true, exec("def x = (float)6 def y = (double)2 return x > y")); + assertEquals(true, exec("def x = (double)7 def y = (double)1 return x > y")); + } + + public void testGte() { + assertEquals(false, exec("def x = (byte)1 def y = (int)7 return x >= y")); + assertEquals(false, exec("def x = (short)2 def y = (int)6 return x >= y")); + assertEquals(false, exec("def x = (char)3 def y = (int)5 return x >= y")); + assertEquals(true, exec("def x = (int)4 def y = (int)4 return x >= y")); + assertEquals(true, exec("def x = (long)5 def y = (int)3 return x >= y")); + assertEquals(true, exec("def x = (float)6 def y = (int)2 return x >= y")); + assertEquals(true, exec("def x = (double)7 def y = (int)1 return x >= y")); + + assertEquals(false, exec("def x = (byte)1 def y = (double)7 return x >= y")); + assertEquals(false, exec("def x = (short)2 def y = (double)6 return x >= y")); + assertEquals(false, exec("def x = (char)3 def y = (double)5 return x >= y")); + assertEquals(true, exec("def x = (int)4 def y = (double)4 return x >= y")); + assertEquals(true, exec("def x = (long)5 def y = (double)3 return x >= y")); + assertEquals(true, exec("def x = (float)6 def y = (double)2 return x >= y")); + assertEquals(true, exec("def x = (double)7 def y = (double)1 return x >= y")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/DivisionTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/DivisionTests.java new file mode 100644 index 00000000000..24849fae72b --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/DivisionTests.java @@ -0,0 +1,147 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for division operator across all types */ +//TODO: NaN/Inf/overflow/... +public class DivisionTests extends ScriptTestCase { + + // TODO: byte,short,char + + public void testInt() throws Exception { + assertEquals(1/1, exec("int x = 1; int y = 1; return x/y;")); + assertEquals(2/3, exec("int x = 2; int y = 3; return x/y;")); + assertEquals(5/10, exec("int x = 5; int y = 10; return x/y;")); + assertEquals(10/1/2, exec("int x = 10; int y = 1; int z = 2; return x/y/z;")); + assertEquals((10/1)/2, exec("int x = 10; int y = 1; int z = 2; return (x/y)/z;")); + assertEquals(10/(4/2), exec("int x = 10; int y = 4; int z = 2; return x/(y/z);")); + assertEquals(10/1, exec("int x = 10; int y = 1; return x/y;")); + assertEquals(0/1, exec("int x = 0; int y = 1; return x/y;")); + } + + public void testIntConst() throws Exception { + assertEquals(1/1, exec("return 1/1;")); + assertEquals(2/3, exec("return 2/3;")); + assertEquals(5/10, exec("return 5/10;")); + assertEquals(10/1/2, exec("return 10/1/2;")); + assertEquals((10/1)/2, exec("return (10/1)/2;")); + assertEquals(10/(4/2), exec("return 10/(4/2);")); + assertEquals(10/1, exec("return 10/1;")); + assertEquals(0/1, exec("return 0/1;")); + } + + public void testLong() throws Exception { + assertEquals(1L/1L, exec("long x = 1; long y = 1; return x/y;")); + assertEquals(2L/3L, exec("long x = 2; long y = 3; return x/y;")); + assertEquals(5L/10L, exec("long x = 5; long y = 10; return x/y;")); + assertEquals(10L/1L/2L, exec("long x = 10; long y = 1; long z = 2; return x/y/z;")); + assertEquals((10L/1L)/2L, exec("long x = 10; long y = 1; long z = 2; return (x/y)/z;")); + assertEquals(10L/(4L/2L), exec("long x = 10; long y = 4; long z = 2; return x/(y/z);")); + assertEquals(10L/1L, exec("long x = 10; long y = 1; return x/y;")); + assertEquals(0L/1L, exec("long x = 0; long y = 1; return x/y;")); + } + + public void testLongConst() throws Exception { + assertEquals(1L/1L, exec("return 1L/1L;")); + assertEquals(2L/3L, exec("return 2L/3L;")); + assertEquals(5L/10L, exec("return 5L/10L;")); + assertEquals(10L/1L/2L, exec("return 10L/1L/2L;")); + assertEquals((10L/1L)/2L, exec("return (10L/1L)/2L;")); + assertEquals(10L/(4L/2L), exec("return 10L/(4L/2L);")); + assertEquals(10L/1L, exec("return 10L/1L;")); + assertEquals(0L/1L, exec("return 0L/1L;")); + } + + public void testFloat() throws Exception { + assertEquals(1F/1F, exec("float x = 1; float y = 1; return x/y;")); + assertEquals(2F/3F, exec("float x = 2; float y = 3; return x/y;")); + assertEquals(5F/10F, exec("float x = 5; float y = 10; return x/y;")); + assertEquals(10F/1F/2F, exec("float x = 10; float y = 1; float z = 2; return x/y/z;")); + assertEquals((10F/1F)/2F, exec("float x = 10; float y = 1; float z = 2; return (x/y)/z;")); + assertEquals(10F/(4F/2F), exec("float x = 10; float y = 4; float z = 2; return x/(y/z);")); + assertEquals(10F/1F, exec("float x = 10; float y = 1; return x/y;")); + assertEquals(0F/1F, exec("float x = 0; float y = 1; return x/y;")); + } + + public void testFloatConst() throws Exception { + assertEquals(1F/1F, exec("return 1F/1F;")); + assertEquals(2F/3F, exec("return 2F/3F;")); + assertEquals(5F/10F, exec("return 5F/10F;")); + assertEquals(10F/1F/2F, exec("return 10F/1F/2F;")); + assertEquals((10F/1F)/2F, exec("return (10F/1F)/2F;")); + assertEquals(10F/(4F/2F), exec("return 10F/(4F/2F);")); + assertEquals(10F/1F, exec("return 10F/1F;")); + assertEquals(0F/1F, exec("return 0F/1F;")); + } + + public void testDouble() throws Exception { + assertEquals(1.0/1.0, exec("double x = 1; double y = 1; return x/y;")); + assertEquals(2.0/3.0, exec("double x = 2; double y = 3; return x/y;")); + assertEquals(5.0/10.0, exec("double x = 5; double y = 10; return x/y;")); + assertEquals(10.0/1.0/2.0, exec("double x = 10; double y = 1; double z = 2; return x/y/z;")); + assertEquals((10.0/1.0)/2.0, exec("double x = 10; double y = 1; double z = 2; return (x/y)/z;")); + assertEquals(10.0/(4.0/2.0), exec("double x = 10; double y = 4; double z = 2; return x/(y/z);")); + assertEquals(10.0/1.0, exec("double x = 10; double y = 1; return x/y;")); + assertEquals(0.0/1.0, exec("double x = 0; double y = 1; return x/y;")); + } + + public void testDoubleConst() throws Exception { + assertEquals(1.0/1.0, exec("return 1.0/1.0;")); + assertEquals(2.0/3.0, exec("return 2.0/3.0;")); + assertEquals(5.0/10.0, exec("return 5.0/10.0;")); + assertEquals(10.0/1.0/2.0, exec("return 10.0/1.0/2.0;")); + assertEquals((10.0/1.0)/2.0, exec("return (10.0/1.0)/2.0;")); + assertEquals(10.0/(4.0/2.0), exec("return 10.0/(4.0/2.0);")); + assertEquals(10.0/1.0, exec("return 10.0/1.0;")); + assertEquals(0.0/1.0, exec("return 0.0/1.0;")); + } + + public void testDivideByZero() throws Exception { + try { + exec("int x = 1; int y = 0; return x / y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + + try { + exec("long x = 1L; long y = 0L; return x / y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + } + + public void testDivideByZeroConst() throws Exception { + try { + exec("return 1/0;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + + try { + exec("return 1L/0L;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/EqualsTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/EqualsTests.java new file mode 100644 index 00000000000..db83755aeff --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/EqualsTests.java @@ -0,0 +1,184 @@ +package org.elasticsearch.plan.a; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// TODO: Figure out a way to test autobox caching properly from methods such as Integer.valueOf(int); +public class EqualsTests extends ScriptTestCase { + public void testTypesEquals() { + assertEquals(true, exec("return false === false;")); + assertEquals(true, exec("boolean x = false; boolean y = false; return x === y;")); + assertEquals(false, exec("return (byte)3 === (byte)4;")); + assertEquals(true, exec("byte x = 3; byte y = 3; return x === y;")); + assertEquals(false, exec("return (char)3 === (char)4;")); + assertEquals(true, exec("char x = 3; char y = 3; return x === y;")); + assertEquals(false, exec("return (short)3 === (short)4;")); + assertEquals(true, exec("short x = 3; short y = 3; return x === y;")); + assertEquals(false, exec("return (int)3 === (int)4;")); + assertEquals(true, exec("int x = 3; int y = 3; return x === y;")); + assertEquals(false, exec("return (long)3 === (long)4;")); + assertEquals(true, exec("long x = 3; long y = 3; return x === y;")); + assertEquals(false, exec("return (float)3 === (float)4;")); + assertEquals(true, exec("float x = 3; float y = 3; return x === y;")); + assertEquals(false, exec("return (double)3 === (double)4;")); + assertEquals(true, exec("double x = 3; double y = 3; return x === y;")); + + assertEquals(true, exec("return false == false;")); + assertEquals(true, exec("boolean x = false; boolean y = false; return x == y;")); + assertEquals(false, exec("return (byte)3 == (byte)4;")); + assertEquals(true, exec("byte x = 3; byte y = 3; return x == y;")); + assertEquals(false, exec("return (char)3 == (char)4;")); + assertEquals(true, exec("char x = 3; char y = 3; return x == y;")); + assertEquals(false, exec("return (short)3 == (short)4;")); + assertEquals(true, exec("short x = 3; short y = 3; return x == y;")); + assertEquals(false, exec("return (int)3 == (int)4;")); + assertEquals(true, exec("int x = 3; int y = 3; return x == y;")); + assertEquals(false, exec("return (long)3 == (long)4;")); + assertEquals(true, exec("long x = 3; long y = 3; return x == y;")); + assertEquals(false, exec("return (float)3 == (float)4;")); + assertEquals(true, exec("float x = 3; float y = 3; return x == y;")); + assertEquals(false, exec("return (double)3 == (double)4;")); + assertEquals(true, exec("double x = 3; double y = 3; return x == y;")); + } + + public void testTypesNotEquals() { + assertEquals(false, exec("return true !== true;")); + assertEquals(false, exec("boolean x = false; boolean y = false; return x !== y;")); + assertEquals(true, exec("return (byte)3 !== (byte)4;")); + assertEquals(false, exec("byte x = 3; byte y = 3; return x !== y;")); + assertEquals(true, exec("return (char)3 !== (char)4;")); + assertEquals(false, exec("char x = 3; char y = 3; return x !== y;")); + assertEquals(true, exec("return (short)3 !== (short)4;")); + assertEquals(false, exec("short x = 3; short y = 3; return x !== y;")); + assertEquals(true, exec("return (int)3 !== (int)4;")); + assertEquals(false, exec("int x = 3; int y = 3; return x !== y;")); + assertEquals(true, exec("return (long)3 !== (long)4;")); + assertEquals(false, exec("long x = 3; long y = 3; return x !== y;")); + assertEquals(true, exec("return (float)3 !== (float)4;")); + assertEquals(false, exec("float x = 3; float y = 3; return x !== y;")); + assertEquals(true, exec("return (double)3 !== (double)4;")); + assertEquals(false, exec("double x = 3; double y = 3; return x !== y;")); + + assertEquals(false, exec("return true != true;")); + assertEquals(false, exec("boolean x = false; boolean y = false; return x != y;")); + assertEquals(true, exec("return (byte)3 != (byte)4;")); + assertEquals(false, exec("byte x = 3; byte y = 3; return x != y;")); + assertEquals(true, exec("return (char)3 != (char)4;")); + assertEquals(false, exec("char x = 3; char y = 3; return x != y;")); + assertEquals(true, exec("return (short)3 != (short)4;")); + assertEquals(false, exec("short x = 3; short y = 3; return x != y;")); + assertEquals(true, exec("return (int)3 != (int)4;")); + assertEquals(false, exec("int x = 3; int y = 3; return x != y;")); + assertEquals(true, exec("return (long)3 != (long)4;")); + assertEquals(false, exec("long x = 3; long y = 3; return x != y;")); + assertEquals(true, exec("return (float)3 != (float)4;")); + assertEquals(false, exec("float x = 3; float y = 3; return x != y;")); + assertEquals(true, exec("return (double)3 != (double)4;")); + assertEquals(false, exec("double x = 3; double y = 3; return x != y;")); + } + + public void testEquals() { + assertEquals(true, exec("return new Long(3) == new Long(3);")); + assertEquals(false, exec("return new Long(3) === new Long(3);")); + assertEquals(true, exec("Integer x = new Integer(3); Object y = x; return x == y;")); + assertEquals(true, exec("Integer x = new Integer(3); Object y = x; return x === y;")); + assertEquals(true, exec("Integer x = new Integer(3); Object y = new Integer(3); return x == y;")); + assertEquals(false, exec("Integer x = new Integer(3); Object y = new Integer(3); return x === y;")); + assertEquals(true, exec("Integer x = new Integer(3); int y = 3; return x == y;")); + assertEquals(true, exec("Integer x = new Integer(3); short y = 3; return x == y;")); + assertEquals(true, exec("Integer x = new Integer(3); Short y = (short)3; return x == y;")); + assertEquals(false, exec("Integer x = new Integer(3); int y = 3; return x === y;")); + assertEquals(false, exec("Integer x = new Integer(3); double y = 3; return x === y;")); + assertEquals(true, exec("int[] x = new int[1]; Object y = x; return x == y;")); + assertEquals(true, exec("int[] x = new int[1]; Object y = x; return x === y;")); + assertEquals(false, exec("int[] x = new int[1]; Object y = new int[1]; return x == y;")); + assertEquals(false, exec("int[] x = new int[1]; Object y = new int[1]; return x === y;")); + assertEquals(false, exec("Map x = new HashMap(); List y = new ArrayList(); return x == y;")); + assertEquals(false, exec("Map x = new HashMap(); List y = new ArrayList(); return x === y;")); + } + + public void testNotEquals() { + assertEquals(false, exec("return new Long(3) != new Long(3);")); + assertEquals(true, exec("return new Long(3) !== new Long(3);")); + assertEquals(false, exec("Integer x = new Integer(3); Object y = x; return x != y;")); + assertEquals(false, exec("Integer x = new Integer(3); Object y = x; return x !== y;")); + assertEquals(false, exec("Integer x = new Integer(3); Object y = new Integer(3); return x != y;")); + assertEquals(true, exec("Integer x = new Integer(3); Object y = new Integer(3); return x !== y;")); + assertEquals(true, exec("Integer x = new Integer(3); int y = 3; return x !== y;")); + assertEquals(true, exec("Integer x = new Integer(3); double y = 3; return x !== y;")); + assertEquals(false, exec("int[] x = new int[1]; Object y = x; return x != y;")); + assertEquals(false, exec("int[] x = new int[1]; Object y = x; return x !== y;")); + assertEquals(true, exec("int[] x = new int[1]; Object y = new int[1]; return x != y;")); + assertEquals(true, exec("int[] x = new int[1]; Object y = new int[1]; return x !== y;")); + assertEquals(true, exec("Map x = new HashMap(); List y = new ArrayList(); return x != y;")); + assertEquals(true, exec("Map x = new HashMap(); List y = new ArrayList(); return x !== y;")); + } + + public void testBranchEquals() { + assertEquals(0, exec("Character a = 'a'; Character b = 'b'; if (a == b) return 1; else return 0;")); + assertEquals(1, exec("Character a = 'a'; Character b = 'a'; if (a == b) return 1; else return 0;")); + assertEquals(0, exec("Integer a = new Integer(1); Integer b = 1; if (a === b) return 1; else return 0;")); + assertEquals(0, exec("Character a = 'a'; Character b = new Character('a'); if (a === b) return 1; else return 0;")); + assertEquals(1, exec("Character a = 'a'; Object b = a; if (a === b) return 1; else return 0;")); + assertEquals(1, exec("Integer a = 1; Number b = a; Number c = a; if (c === b) return 1; else return 0;")); + assertEquals(0, exec("Integer a = 1; Character b = 'a'; if (a === (Object)b) return 1; else return 0;")); + } + + public void testBranchNotEquals() { + assertEquals(1, exec("Character a = 'a'; Character b = 'b'; if (a != b) return 1; else return 0;")); + assertEquals(0, exec("Character a = 'a'; Character b = 'a'; if (a != b) return 1; else return 0;")); + assertEquals(1, exec("Integer a = new Integer(1); Integer b = 1; if (a !== b) return 1; else return 0;")); + assertEquals(1, exec("Character a = 'a'; Character b = new Character('a'); if (a !== b) return 1; else return 0;")); + assertEquals(0, exec("Character a = 'a'; Object b = a; if (a !== b) return 1; else return 0;")); + assertEquals(0, exec("Integer a = 1; Number b = a; Number c = a; if (c !== b) return 1; else return 0;")); + assertEquals(1, exec("Integer a = 1; Character b = 'a'; if (a !== (Object)b) return 1; else return 0;")); + } + + public void testRightHandNull() { + assertEquals(false, exec("Character a = 'a'; return a == null;")); + assertEquals(false, exec("Character a = 'a'; return a === null;")); + assertEquals(true, exec("Character a = 'a'; return a != null;")); + assertEquals(true, exec("Character a = 'a'; return a !== null;")); + assertEquals(true, exec("Character a = null; return a == null;")); + assertEquals(false, exec("Character a = null; return a != null;")); + assertEquals(false, exec("Character a = 'a'; Character b = null; return a == b;")); + assertEquals(true, exec("Character a = null; Character b = null; return a === b;")); + assertEquals(true, exec("Character a = 'a'; Character b = null; return a != b;")); + assertEquals(false, exec("Character a = null; Character b = null; return a !== b;")); + assertEquals(false, exec("Integer x = null; double y = 2.0; return x == y;")); + assertEquals(true, exec("Integer x = null; Short y = null; return x == y;")); + } + + public void testLeftHandNull() { + assertEquals(false, exec("Character a = 'a'; return null == a;")); + assertEquals(false, exec("Character a = 'a'; return null === a;")); + assertEquals(true, exec("Character a = 'a'; return null != a;")); + assertEquals(true, exec("Character a = 'a'; return null !== a;")); + assertEquals(true, exec("Character a = null; return null == a;")); + assertEquals(false, exec("Character a = null; return null != a;")); + assertEquals(false, exec("Character a = null; Character b = 'a'; return a == b;")); + assertEquals(true, exec("Character a = null; Character b = null; return a == b;")); + assertEquals(true, exec("Character a = null; Character b = null; return b === a;")); + assertEquals(true, exec("Character a = null; Character b = 'a'; return a != b;")); + assertEquals(false, exec("Character a = null; Character b = null; return b != a;")); + assertEquals(false, exec("Character a = null; Character b = null; return b !== a;")); + assertEquals(false, exec("Integer x = null; double y = 2.0; return y == x;")); + assertEquals(true, exec("Integer x = null; Short y = null; return y == x;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FieldTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FieldTests.java new file mode 100644 index 00000000000..7504ed9d4bc --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FieldTests.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.junit.Before; + +public class FieldTests extends ScriptTestCase { + public static class FieldClass { + public boolean z = false; + public byte b = 0; + public short s = 1; + public char c = 'c'; + public int i = 2; + public int si = -1; + public long j = 3l; + public float f = 4.0f; + public double d = 5.0; + public String t = "s"; + public Object l = new Object(); + + public float test(float a, float b) { + return Math.min(a, b); + } + + public int getSi() { + return si; + } + + public void setSi(final int si) { + this.si = si; + } + } + + public static class FieldDefinition extends Definition { + FieldDefinition() { + super(); + + addStruct("FieldClass", FieldClass.class); + addConstructor("FieldClass", "new", new Type[] {}, null); + addField("FieldClass", "z", null, false, booleanType, null); + addField("FieldClass", "b", null, false, byteType, null); + addField("FieldClass", "s", null, false, shortType, null); + addField("FieldClass", "c", null, false, charType, null); + addField("FieldClass", "i", null, false, intType, null); + addField("FieldClass", "j", null, false, longType, null); + addField("FieldClass", "f", null, false, floatType, null); + addField("FieldClass", "d", null, false, doubleType, null); + addField("FieldClass", "t", null, false, stringType, null); + addField("FieldClass", "l", null, false, objectType, null); + addClass("FieldClass"); + addMethod("FieldClass", "getSi", null, false, intType, new Type[] {}, null, null); + addMethod("FieldClass", "setSi", null, false, voidType, new Type[] {intType}, null, null); + addMethod("FieldClass", "test", null, false, floatType, new Type[] {floatType, floatType}, null, null); + } + } + + @Before + public void setDefinition() { + scriptEngine.setDefinition(new FieldDefinition()); + } + + public void testIntField() { + assertEquals("s5t42", exec("def fc = new FieldClass() return fc.t += 2 + fc.j + \"t\" + 4 + (3 - 1)")); + assertEquals(2.0f, exec("def fc = new FieldClass(); def l = new Double(3) Byte b = new Byte((byte)2) return fc.test(l, b)")); + assertEquals(4, exec("def fc = new FieldClass() fc.i = 4 return fc.i")); + assertEquals(5, exec("FieldClass fc0 = new FieldClass() FieldClass fc1 = new FieldClass() fc0.i = 7 - fc0.i fc1.i = fc0.i return fc1.i")); + assertEquals(8, exec("def fc0 = new FieldClass() def fc1 = new FieldClass() fc0.i += fc1.i fc0.i += fc0.i return fc0.i")); + } + + public void testExplicitShortcut() { + assertEquals(5, exec("FieldClass fc = new FieldClass() fc.setSi(5) return fc.si")); + assertEquals(-1, exec("FieldClass fc = new FieldClass() def x = fc.getSi() x")); + assertEquals(5, exec("FieldClass fc = new FieldClass() fc.si = 5 return fc.si")); + assertEquals(0, exec("FieldClass fc = new FieldClass() fc.si++ return fc.si")); + assertEquals(-1, exec("FieldClass fc = new FieldClass() def x = fc.si++ return x")); + assertEquals(0, exec("FieldClass fc = new FieldClass() def x = ++fc.si return x")); + assertEquals(-2, exec("FieldClass fc = new FieldClass() fc.si *= 2 fc.si")); + assertEquals("-1test", exec("FieldClass fc = new FieldClass() fc.si + \"test\"")); + } + + public void testImplicitShortcut() { + assertEquals(5, exec("def fc = new FieldClass() fc.setSi(5) return fc.si")); + assertEquals(-1, exec("def fc = new FieldClass() def x = fc.getSi() x")); + assertEquals(5, exec("def fc = new FieldClass() fc.si = 5 return fc.si")); + assertEquals(0, exec("def fc = new FieldClass() fc.si++ return fc.si")); + assertEquals(-1, exec("def fc = new FieldClass() def x = fc.si++ return x")); + assertEquals(0, exec("def fc = new FieldClass() def x = ++fc.si return x")); + assertEquals(-2, exec("def fc = new FieldClass() fc.si *= 2 fc.si")); + assertEquals("-1test", exec("def fc = new FieldClass() fc.si + \"test\"")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FloatOverflowDisabledTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FloatOverflowDisabledTests.java new file mode 100644 index 00000000000..94beac0c58c --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FloatOverflowDisabledTests.java @@ -0,0 +1,294 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.elasticsearch.common.settings.Settings; + +/** Tests floating point overflow with numeric overflow disabled */ +public class FloatOverflowDisabledTests extends ScriptTestCase { + + @Override + protected Settings getSettings() { + Settings.Builder builder = Settings.builder(); + builder.put(super.getSettings()); + builder.put(PlanAScriptEngineService.NUMERIC_OVERFLOW, false); + return builder.build(); + } + + public void testAssignmentAdditionOverflow() { + // float + try { + exec("float x = 3.4028234663852886E38f; x += 3.4028234663852886E38f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = -3.4028234663852886E38f; x += -3.4028234663852886E38f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + + // double + try { + exec("double x = 1.7976931348623157E308; x += 1.7976931348623157E308; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = -1.7976931348623157E308; x += -1.7976931348623157E308; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAssignmentSubtractionOverflow() { + // float + try { + exec("float x = 3.4028234663852886E38f; x -= -3.4028234663852886E38f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = -3.4028234663852886E38f; x -= 3.4028234663852886E38f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + + // double + try { + exec("double x = 1.7976931348623157E308; x -= -1.7976931348623157E308; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = -1.7976931348623157E308; x -= 1.7976931348623157E308; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAssignmentMultiplicationOverflow() { + // float + try { + exec("float x = 3.4028234663852886E38f; x *= 3.4028234663852886E38f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = 3.4028234663852886E38f; x *= -3.4028234663852886E38f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + + // double + try { + exec("double x = 1.7976931348623157E308; x *= 1.7976931348623157E308; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.7976931348623157E308; x *= -1.7976931348623157E308; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAssignmentDivisionOverflow() { + // float + try { + exec("float x = 3.4028234663852886E38f; x /= 1.401298464324817E-45f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = 3.4028234663852886E38f; x /= -1.401298464324817E-45f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = 1.0f; x /= 0.0f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + + // double + try { + exec("double x = 1.7976931348623157E308; x /= 4.9E-324; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.7976931348623157E308; x /= -4.9E-324; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.0f; x /= 0.0; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAddition() throws Exception { + try { + exec("float x = 3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x + y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.7976931348623157E308; double y = 1.7976931348623157E308; return x + y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAdditionConst() throws Exception { + try { + exec("return 3.4028234663852886E38f + 3.4028234663852886E38f;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 1.7976931348623157E308 + 1.7976931348623157E308;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testSubtraction() throws Exception { + try { + exec("float x = -3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x - y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = -1.7976931348623157E308; double y = 1.7976931348623157E308; return x - y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testSubtractionConst() throws Exception { + try { + exec("return -3.4028234663852886E38f - 3.4028234663852886E38f;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return -1.7976931348623157E308 - 1.7976931348623157E308;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testMultiplication() throws Exception { + try { + exec("float x = 3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x * y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.7976931348623157E308; double y = 1.7976931348623157E308; return x * y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testMultiplicationConst() throws Exception { + try { + exec("return 3.4028234663852886E38f * 3.4028234663852886E38f;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 1.7976931348623157E308 * 1.7976931348623157E308;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivision() throws Exception { + try { + exec("float x = 3.4028234663852886E38f; float y = 1.401298464324817E-45f; return x / y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = 1.0f; float y = 0.0f; return x / y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.7976931348623157E308; double y = 4.9E-324; return x / y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.0; double y = 0.0; return x / y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivisionConst() throws Exception { + try { + exec("return 3.4028234663852886E38f / 1.401298464324817E-45f;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 1.0f / 0.0f;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 1.7976931348623157E308 / 4.9E-324;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 1.0 / 0.0;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivisionNaN() throws Exception { + // float division, constant division, and assignment + try { + exec("float x = 0f; float y = 0f; return x / y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 0f / 0f;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = 0f; x /= 0f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + + // double division, constant division, and assignment + try { + exec("double x = 0.0; double y = 0.0; return x / y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 0.0 / 0.0;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 0.0; x /= 0.0; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testRemainderNaN() throws Exception { + // float division, constant division, and assignment + try { + exec("float x = 1f; float y = 0f; return x % y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 1f % 0f;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("float x = 1f; x %= 0f; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + + // double division, constant division, and assignment + try { + exec("double x = 1.0; double y = 0.0; return x % y;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("return 1.0 % 0.0;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + try { + exec("double x = 1.0; x %= 0.0; return x;"); + fail("didn't hit expected exception"); + } catch (ArithmeticException expected) {} + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FloatOverflowEnabledTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FloatOverflowEnabledTests.java new file mode 100644 index 00000000000..ff1c315628f --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/FloatOverflowEnabledTests.java @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.elasticsearch.common.settings.Settings; + +/** Tests floating point overflow with numeric overflow enabled */ +public class FloatOverflowEnabledTests extends ScriptTestCase { + + @Override + protected Settings getSettings() { + Settings.Builder builder = Settings.builder(); + builder.put(super.getSettings()); + builder.put(PlanAScriptEngineService.NUMERIC_OVERFLOW, true); + return builder.build(); + } + + public void testAssignmentAdditionOverflow() { + // float + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 3.4028234663852886E38f; x += 3.4028234663852886E38f; return x;")); + assertEquals(Float.NEGATIVE_INFINITY, exec("float x = -3.4028234663852886E38f; x += -3.4028234663852886E38f; return x;")); + + // double + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.7976931348623157E308; x += 1.7976931348623157E308; return x;")); + assertEquals(Double.NEGATIVE_INFINITY, exec("double x = -1.7976931348623157E308; x += -1.7976931348623157E308; return x;")); + } + + public void testAssignmentSubtractionOverflow() { + // float + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 3.4028234663852886E38f; x -= -3.4028234663852886E38f; return x;")); + assertEquals(Float.NEGATIVE_INFINITY, exec("float x = -3.4028234663852886E38f; x -= 3.4028234663852886E38f; return x;")); + + // double + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.7976931348623157E308; x -= -1.7976931348623157E308; return x;")); + assertEquals(Double.NEGATIVE_INFINITY, exec("double x = -1.7976931348623157E308; x -= 1.7976931348623157E308; return x;")); + } + + public void testAssignmentMultiplicationOverflow() { + // float + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 3.4028234663852886E38f; x *= 3.4028234663852886E38f; return x;")); + assertEquals(Float.NEGATIVE_INFINITY, exec("float x = 3.4028234663852886E38f; x *= -3.4028234663852886E38f; return x;")); + + // double + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.7976931348623157E308; x *= 1.7976931348623157E308; return x;")); + assertEquals(Double.NEGATIVE_INFINITY, exec("double x = 1.7976931348623157E308; x *= -1.7976931348623157E308; return x;")); + } + + public void testAssignmentDivisionOverflow() { + // float + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 3.4028234663852886E38f; x /= 1.401298464324817E-45f; return x;")); + assertEquals(Float.NEGATIVE_INFINITY, exec("float x = 3.4028234663852886E38f; x /= -1.401298464324817E-45f; return x;")); + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 1.0f; x /= 0.0f; return x;")); + + // double + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.7976931348623157E308; x /= 4.9E-324; return x;")); + assertEquals(Double.NEGATIVE_INFINITY, exec("double x = 1.7976931348623157E308; x /= -4.9E-324; return x;")); + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.0f; x /= 0.0; return x;")); + } + + public void testAddition() throws Exception { + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x + y;")); + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.7976931348623157E308; double y = 1.7976931348623157E308; return x + y;")); + } + + public void testAdditionConst() throws Exception { + assertEquals(Float.POSITIVE_INFINITY, exec("return 3.4028234663852886E38f + 3.4028234663852886E38f;")); + assertEquals(Double.POSITIVE_INFINITY, exec("return 1.7976931348623157E308 + 1.7976931348623157E308;")); + } + + public void testSubtraction() throws Exception { + assertEquals(Float.NEGATIVE_INFINITY, exec("float x = -3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x - y;")); + assertEquals(Double.NEGATIVE_INFINITY, exec("double x = -1.7976931348623157E308; double y = 1.7976931348623157E308; return x - y;")); + } + + public void testSubtractionConst() throws Exception { + assertEquals(Float.NEGATIVE_INFINITY, exec("return -3.4028234663852886E38f - 3.4028234663852886E38f;")); + assertEquals(Double.NEGATIVE_INFINITY, exec("return -1.7976931348623157E308 - 1.7976931348623157E308;")); + } + + public void testMultiplication() throws Exception { + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x * y;")); + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.7976931348623157E308; double y = 1.7976931348623157E308; return x * y;")); + } + + public void testMultiplicationConst() throws Exception { + assertEquals(Float.POSITIVE_INFINITY, exec("return 3.4028234663852886E38f * 3.4028234663852886E38f;")); + assertEquals(Double.POSITIVE_INFINITY, exec("return 1.7976931348623157E308 * 1.7976931348623157E308;")); + } + + public void testDivision() throws Exception { + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 3.4028234663852886E38f; float y = 1.401298464324817E-45f; return x / y;")); + assertEquals(Float.POSITIVE_INFINITY, exec("float x = 1.0f; float y = 0.0f; return x / y;")); + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.7976931348623157E308; double y = 4.9E-324; return x / y;")); + assertEquals(Double.POSITIVE_INFINITY, exec("double x = 1.0; double y = 0.0; return x / y;")); + } + + public void testDivisionConst() throws Exception { + assertEquals(Float.POSITIVE_INFINITY, exec("return 3.4028234663852886E38f / 1.401298464324817E-45f;")); + assertEquals(Float.POSITIVE_INFINITY, exec("return 1.0f / 0.0f;")); + assertEquals(Double.POSITIVE_INFINITY, exec("return 1.7976931348623157E308 / 4.9E-324;")); + assertEquals(Double.POSITIVE_INFINITY, exec("return 1.0 / 0.0;")); + } + + public void testDivisionNaN() throws Exception { + // float division, constant division, and assignment + assertTrue(Float.isNaN((Float) exec("float x = 0f; float y = 0f; return x / y;"))); + assertTrue(Float.isNaN((Float) exec("return 0f / 0f;"))); + assertTrue(Float.isNaN((Float) exec("float x = 0f; x /= 0f; return x;"))); + + // double division, constant division, and assignment + assertTrue(Double.isNaN((Double) exec("double x = 0.0; double y = 0.0; return x / y;"))); + assertTrue(Double.isNaN((Double) exec("return 0.0 / 0.0;"))); + assertTrue(Double.isNaN((Double) exec("double x = 0.0; x /= 0.0; return x;"))); + } + + public void testRemainderNaN() throws Exception { + // float division, constant division, and assignment + assertTrue(Float.isNaN((Float) exec("float x = 1f; float y = 0f; return x % y;"))); + assertTrue(Float.isNaN((Float) exec("return 1f % 0f;"))); + assertTrue(Float.isNaN((Float) exec("float x = 1f; x %= 0f; return x;"))); + + // double division, constant division, and assignment + assertTrue(Double.isNaN((Double) exec("double x = 1.0; double y = 0.0; return x % y;"))); + assertTrue(Double.isNaN((Double) exec("return 1.0 % 0.0;"))); + assertTrue(Double.isNaN((Double) exec("double x = 1.0; x %= 0.0; return x;"))); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IncrementTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IncrementTests.java new file mode 100644 index 00000000000..ec4ffd0ec1d --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IncrementTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for increment/decrement operators across all data types */ +public class IncrementTests extends ScriptTestCase { + + /** incrementing byte values */ + public void testIncrementByte() { + assertEquals((byte)0, exec("byte x = (byte)0; return x++;")); + assertEquals((byte)0, exec("byte x = (byte)0; return x--;")); + assertEquals((byte)1, exec("byte x = (byte)0; return ++x;")); + assertEquals((byte)-1, exec("byte x = (byte)0; return --x;")); + } + + /** incrementing char values */ + public void testIncrementChar() { + assertEquals((char)0, exec("char x = (char)0; return x++;")); + assertEquals((char)1, exec("char x = (char)1; return x--;")); + assertEquals((char)1, exec("char x = (char)0; return ++x;")); + } + + /** incrementing short values */ + public void testIncrementShort() { + assertEquals((short)0, exec("short x = (short)0; return x++;")); + assertEquals((short)0, exec("short x = (short)0; return x--;")); + assertEquals((short)1, exec("short x = (short)0; return ++x;")); + assertEquals((short)-1, exec("short x = (short)0; return --x;")); + } + + /** incrementing integer values */ + public void testIncrementInt() { + assertEquals(0, exec("int x = 0; return x++;")); + assertEquals(0, exec("int x = 0; return x--;")); + assertEquals(1, exec("int x = 0; return ++x;")); + assertEquals(-1, exec("int x = 0; return --x;")); + } + + /** incrementing long values */ + public void testIncrementLong() { + assertEquals(0L, exec("long x = 0; return x++;")); + assertEquals(0L, exec("long x = 0; return x--;")); + assertEquals(1L, exec("long x = 0; return ++x;")); + assertEquals(-1L, exec("long x = 0; return --x;")); + } + + /** incrementing float values */ + public void testIncrementFloat() { + assertEquals(0F, exec("float x = 0F; return x++;")); + assertEquals(0F, exec("float x = 0F; return x--;")); + assertEquals(1F, exec("float x = 0F; return ++x;")); + assertEquals(-1F, exec("float x = 0F; return --x;")); + } + + /** incrementing double values */ + public void testIncrementDouble() { + assertEquals(0D, exec("double x = 0.0; return x++;")); + assertEquals(0D, exec("double x = 0.0; return x--;")); + assertEquals(1D, exec("double x = 0.0; return ++x;")); + assertEquals(-1D, exec("double x = 0.0; return --x;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IntegerOverflowDisabledTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IntegerOverflowDisabledTests.java new file mode 100644 index 00000000000..279ea0616d9 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IntegerOverflowDisabledTests.java @@ -0,0 +1,445 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.elasticsearch.common.settings.Settings; + +/** Tests integer overflow with numeric overflow disabled */ +public class IntegerOverflowDisabledTests extends ScriptTestCase { + + @Override + protected Settings getSettings() { + Settings.Builder builder = Settings.builder(); + builder.put(super.getSettings()); + builder.put(PlanAScriptEngineService.NUMERIC_OVERFLOW, false); + return builder.build(); + } + + public void testAssignmentAdditionOverflow() { + // byte + try { + exec("byte x = 0; x += 128; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = 0; x += -129; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // short + try { + exec("short x = 0; x += 32768; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = 0; x += -32769; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // char + try { + exec("char x = 0; x += 65536; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("char x = 0; x += -65536; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // int + try { + exec("int x = 1; x += 2147483647; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("int x = -2; x += -2147483647; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // long + try { + exec("long x = 1; x += 9223372036854775807L; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -2; x += -9223372036854775807L; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAssignmentSubtractionOverflow() { + // byte + try { + exec("byte x = 0; x -= -128; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = 0; x -= 129; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // short + try { + exec("short x = 0; x -= -32768; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = 0; x -= 32769; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // char + try { + exec("char x = 0; x -= -65536; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("char x = 0; x -= 65536; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // int + try { + exec("int x = 1; x -= -2147483647; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("int x = -2; x -= 2147483647; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // long + try { + exec("long x = 1; x -= -9223372036854775807L; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -2; x -= 9223372036854775807L; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAssignmentMultiplicationOverflow() { + // byte + try { + exec("byte x = 2; x *= 128; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = 2; x *= -128; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // char + try { + exec("char x = 2; x *= 65536; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("char x = 2; x *= -65536; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // int + try { + exec("int x = 2; x *= 2147483647; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("int x = 2; x *= -2147483647; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // long + try { + exec("long x = 2; x *= 9223372036854775807L; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = 2; x *= -9223372036854775807L; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAssignmentDivisionOverflow() { + // byte + try { + exec("byte x = (byte) -128; x /= -1; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + // short + try { + exec("short x = (short) -32768; x /= -1; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + // cannot happen for char: unsigned + + // int + try { + exec("int x = -2147483647 - 1; x /= -1; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + // long + try { + exec("long x = -9223372036854775807L - 1L; x /=-1L; return x;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testIncrementOverFlow() throws Exception { + // byte + try { + exec("byte x = 127; ++x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = 127; x++; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = (byte) -128; --x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("byte x = (byte) -128; x--; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // short + try { + exec("short x = 32767; ++x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("short x = 32767; x++; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("short x = (short) -32768; --x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("short x = (short) -32768; x--; return x;"); + } catch (ArithmeticException expected) {} + + // char + try { + exec("char x = 65535; ++x; return x;"); + } catch (ArithmeticException expected) {} + + try { + exec("char x = 65535; x++; return x;"); + } catch (ArithmeticException expected) {} + + try { + exec("char x = (char) 0; --x; return x;"); + } catch (ArithmeticException expected) {} + + try { + exec("char x = (char) 0; x--; return x;"); + } catch (ArithmeticException expected) {} + + // int + try { + exec("int x = 2147483647; ++x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("int x = 2147483647; x++; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("int x = (int) -2147483648L; --x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("int x = (int) -2147483648L; x--; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + // long + try { + exec("long x = 9223372036854775807L; ++x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = 9223372036854775807L; x++; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -9223372036854775807L - 1L; --x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -9223372036854775807L - 1L; x--; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAddition() throws Exception { + try { + exec("int x = 2147483647; int y = 2147483647; return x + y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = 9223372036854775807L; long y = 9223372036854775807L; return x + y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testAdditionConst() throws Exception { + try { + exec("return 2147483647 + 2147483647;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("return 9223372036854775807L + 9223372036854775807L;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + + public void testSubtraction() throws Exception { + try { + exec("int x = -10; int y = 2147483647; return x - y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -10L; long y = 9223372036854775807L; return x - y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testSubtractionConst() throws Exception { + try { + exec("return -10 - 2147483647;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("return -10L - 9223372036854775807L;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testMultiplication() throws Exception { + try { + exec("int x = 2147483647; int y = 2147483647; return x * y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = 9223372036854775807L; long y = 9223372036854775807L; return x * y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testMultiplicationConst() throws Exception { + try { + exec("return 2147483647 * 2147483647;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("return 9223372036854775807L * 9223372036854775807L;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivision() throws Exception { + try { + exec("int x = -2147483647 - 1; int y = -1; return x / y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -9223372036854775808L; long y = -1L; return x / y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivisionConst() throws Exception { + try { + exec("return (-2147483648) / -1;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + + try { + exec("return (-9223372036854775808L) / -1L;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) {} + } + + public void testNegationOverflow() throws Exception { + try { + exec("int x = -2147483648; x = -x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -9223372036854775808L; x = -x; return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testNegationOverflowConst() throws Exception { + try { + exec("int x = -(-2147483648); return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + exec("long x = -(-9223372036854775808L); return x;"); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IntegerOverflowEnabledTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IntegerOverflowEnabledTests.java new file mode 100644 index 00000000000..8abd2695915 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/IntegerOverflowEnabledTests.java @@ -0,0 +1,194 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.elasticsearch.common.settings.Settings; + +/** Tests integer overflow with numeric overflow enabled */ +public class IntegerOverflowEnabledTests extends ScriptTestCase { + + @Override + protected Settings getSettings() { + Settings.Builder builder = Settings.builder(); + builder.put(super.getSettings()); + builder.put(PlanAScriptEngineService.NUMERIC_OVERFLOW, true); + return builder.build(); + } + + public void testAssignmentAdditionOverflow() { + // byte + assertEquals((byte)(0 + 128), exec("byte x = 0; x += 128; return x;")); + assertEquals((byte)(0 + -129), exec("byte x = 0; x += -129; return x;")); + + // short + assertEquals((short)(0 + 32768), exec("short x = 0; x += 32768; return x;")); + assertEquals((short)(0 + -32769), exec("short x = 0; x += -32769; return x;")); + + // char + assertEquals((char)(0 + 65536), exec("char x = 0; x += 65536; return x;")); + assertEquals((char)(0 + -65536), exec("char x = 0; x += -65536; return x;")); + + // int + assertEquals(1 + 2147483647, exec("int x = 1; x += 2147483647; return x;")); + assertEquals(-2 + -2147483647, exec("int x = -2; x += -2147483647; return x;")); + + // long + assertEquals(1L + 9223372036854775807L, exec("long x = 1; x += 9223372036854775807L; return x;")); + assertEquals(-2L + -9223372036854775807L, exec("long x = -2; x += -9223372036854775807L; return x;")); + } + + public void testAssignmentSubtractionOverflow() { + // byte + assertEquals((byte)(0 - -128), exec("byte x = 0; x -= -128; return x;")); + assertEquals((byte)(0 - 129), exec("byte x = 0; x -= 129; return x;")); + + // short + assertEquals((short)(0 - -32768), exec("short x = 0; x -= -32768; return x;")); + assertEquals((short)(0 - 32769), exec("short x = 0; x -= 32769; return x;")); + + // char + assertEquals((char)(0 - -65536), exec("char x = 0; x -= -65536; return x;")); + assertEquals((char)(0 - 65536), exec("char x = 0; x -= 65536; return x;")); + + // int + assertEquals(1 - -2147483647, exec("int x = 1; x -= -2147483647; return x;")); + assertEquals(-2 - 2147483647, exec("int x = -2; x -= 2147483647; return x;")); + + // long + assertEquals(1L - -9223372036854775807L, exec("long x = 1; x -= -9223372036854775807L; return x;")); + assertEquals(-2L - 9223372036854775807L, exec("long x = -2; x -= 9223372036854775807L; return x;")); + } + + public void testAssignmentMultiplicationOverflow() { + // byte + assertEquals((byte) (2 * 128), exec("byte x = 2; x *= 128; return x;")); + assertEquals((byte) (2 * -128), exec("byte x = 2; x *= -128; return x;")); + + // char + assertEquals((char) (2 * 65536), exec("char x = 2; x *= 65536; return x;")); + assertEquals((char) (2 * -65536), exec("char x = 2; x *= -65536; return x;")); + + // int + assertEquals(2 * 2147483647, exec("int x = 2; x *= 2147483647; return x;")); + assertEquals(2 * -2147483647, exec("int x = 2; x *= -2147483647; return x;")); + + // long + assertEquals(2L * 9223372036854775807L, exec("long x = 2; x *= 9223372036854775807L; return x;")); + assertEquals(2L * -9223372036854775807L, exec("long x = 2; x *= -9223372036854775807L; return x;")); + } + + public void testAssignmentDivisionOverflow() { + // byte + assertEquals((byte) (-128 / -1), exec("byte x = (byte) -128; x /= -1; return x;")); + + // short + assertEquals((short) (-32768 / -1), exec("short x = (short) -32768; x /= -1; return x;")); + + // cannot happen for char: unsigned + + // int + assertEquals((-2147483647 - 1) / -1, exec("int x = -2147483647 - 1; x /= -1; return x;")); + + // long + assertEquals((-9223372036854775807L - 1L) / -1L, exec("long x = -9223372036854775807L - 1L; x /=-1L; return x;")); + } + + public void testIncrementOverFlow() throws Exception { + // byte + assertEquals((byte) 128, exec("byte x = 127; ++x; return x;")); + assertEquals((byte) 128, exec("byte x = 127; x++; return x;")); + assertEquals((byte) -129, exec("byte x = (byte) -128; --x; return x;")); + assertEquals((byte) -129, exec("byte x = (byte) -128; x--; return x;")); + + // short + assertEquals((short) 32768, exec("short x = 32767; ++x; return x;")); + assertEquals((short) 32768, exec("short x = 32767; x++; return x;")); + assertEquals((short) -32769, exec("short x = (short) -32768; --x; return x;")); + assertEquals((short) -32769, exec("short x = (short) -32768; x--; return x;")); + + // char + assertEquals((char) 65536, exec("char x = 65535; ++x; return x;")); + assertEquals((char) 65536, exec("char x = 65535; x++; return x;")); + assertEquals((char) -1, exec("char x = (char) 0; --x; return x;")); + assertEquals((char) -1, exec("char x = (char) 0; x--; return x;")); + + // int + assertEquals(2147483647 + 1, exec("int x = 2147483647; ++x; return x;")); + assertEquals(2147483647 + 1, exec("int x = 2147483647; x++; return x;")); + assertEquals(-2147483648 - 1, exec("int x = (int) -2147483648L; --x; return x;")); + assertEquals(-2147483648 - 1, exec("int x = (int) -2147483648L; x--; return x;")); + + // long + assertEquals(9223372036854775807L + 1L, exec("long x = 9223372036854775807L; ++x; return x;")); + assertEquals(9223372036854775807L + 1L, exec("long x = 9223372036854775807L; x++; return x;")); + assertEquals(-9223372036854775807L - 1L - 1L, exec("long x = -9223372036854775807L - 1L; --x; return x;")); + assertEquals(-9223372036854775807L - 1L - 1L, exec("long x = -9223372036854775807L - 1L; x--; return x;")); + } + + public void testAddition() throws Exception { + assertEquals(2147483647 + 2147483647, exec("int x = 2147483647; int y = 2147483647; return x + y;")); + assertEquals(9223372036854775807L + 9223372036854775807L, exec("long x = 9223372036854775807L; long y = 9223372036854775807L; return x + y;")); + } + + public void testAdditionConst() throws Exception { + assertEquals(2147483647 + 2147483647, exec("return 2147483647 + 2147483647;")); + assertEquals(9223372036854775807L + 9223372036854775807L, exec("return 9223372036854775807L + 9223372036854775807L;")); + } + + public void testSubtraction() throws Exception { + assertEquals(-10 - 2147483647, exec("int x = -10; int y = 2147483647; return x - y;")); + assertEquals(-10L - 9223372036854775807L, exec("long x = -10L; long y = 9223372036854775807L; return x - y;")); + } + + public void testSubtractionConst() throws Exception { + assertEquals(-10 - 2147483647, exec("return -10 - 2147483647;")); + assertEquals(-10L - 9223372036854775807L, exec("return -10L - 9223372036854775807L;")); + } + + public void testMultiplication() throws Exception { + assertEquals(2147483647 * 2147483647, exec("int x = 2147483647; int y = 2147483647; return x * y;")); + assertEquals(9223372036854775807L * 9223372036854775807L, exec("long x = 9223372036854775807L; long y = 9223372036854775807L; return x * y;")); + } + + public void testMultiplicationConst() throws Exception { + assertEquals(2147483647 * 2147483647, exec("return 2147483647 * 2147483647;")); + assertEquals(9223372036854775807L * 9223372036854775807L, exec("return 9223372036854775807L * 9223372036854775807L;")); + } + + public void testDivision() throws Exception { + assertEquals((-2147483647 - 1) / -1, exec("int x = -2147483648; int y = -1; return x / y;")); + assertEquals((-9223372036854775807L - 1L) / -1L, exec("long x = -9223372036854775808L; long y = -1L; return x / y;")); + } + + public void testDivisionConst() throws Exception { + assertEquals((-2147483647 - 1) / -1, exec("return (-2147483648) / -1;")); + assertEquals((-9223372036854775807L - 1L) / -1L, exec("return (-9223372036854775808L) / -1L;")); + } + + public void testNegationOverflow() throws Exception { + assertEquals(-(-2147483647 - 1), exec("int x = -2147483648; x = -x; return x;")); + assertEquals(-(-9223372036854775807L - 1L), exec("long x = -9223372036854775808L; x = -x; return x;")); + } + + public void testNegationOverflowConst() throws Exception { + assertEquals(-(-2147483647 - 1), exec("int x = -(-2147483648); return x;")); + assertEquals(-(-9223372036854775807L - 1L), exec("long x = -(-9223372036854775808L); return x;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/MultiplicationTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/MultiplicationTests.java new file mode 100644 index 00000000000..c5fde3b6ff1 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/MultiplicationTests.java @@ -0,0 +1,126 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for multiplication operator across all types */ +//TODO: NaN/Inf/overflow/... +public class MultiplicationTests extends ScriptTestCase { + + // TODO: short,byte,char + + public void testInt() throws Exception { + assertEquals(1*1, exec("int x = 1; int y = 1; return x*y;")); + assertEquals(2*3, exec("int x = 2; int y = 3; return x*y;")); + assertEquals(5*10, exec("int x = 5; int y = 10; return x*y;")); + assertEquals(1*1*2, exec("int x = 1; int y = 1; int z = 2; return x*y*z;")); + assertEquals((1*1)*2, exec("int x = 1; int y = 1; int z = 2; return (x*y)*z;")); + assertEquals(1*(1*2), exec("int x = 1; int y = 1; int z = 2; return x*(y*z);")); + assertEquals(10*0, exec("int x = 10; int y = 0; return x*y;")); + assertEquals(0*0, exec("int x = 0; int y = 0; return x*x;")); + } + + public void testIntConst() throws Exception { + assertEquals(1*1, exec("return 1*1;")); + assertEquals(2*3, exec("return 2*3;")); + assertEquals(5*10, exec("return 5*10;")); + assertEquals(1*1*2, exec("return 1*1*2;")); + assertEquals((1*1)*2, exec("return (1*1)*2;")); + assertEquals(1*(1*2), exec("return 1*(1*2);")); + assertEquals(10*0, exec("return 10*0;")); + assertEquals(0*0, exec("return 0*0;")); + } + + public void testByte() throws Exception { + assertEquals((byte)1*(byte)1, exec("byte x = 1; byte y = 1; return x*y;")); + assertEquals((byte)2*(byte)3, exec("byte x = 2; byte y = 3; return x*y;")); + assertEquals((byte)5*(byte)10, exec("byte x = 5; byte y = 10; return x*y;")); + assertEquals((byte)1*(byte)1*(byte)2, exec("byte x = 1; byte y = 1; byte z = 2; return x*y*z;")); + assertEquals(((byte)1*(byte)1)*(byte)2, exec("byte x = 1; byte y = 1; byte z = 2; return (x*y)*z;")); + assertEquals((byte)1*((byte)1*(byte)2), exec("byte x = 1; byte y = 1; byte z = 2; return x*(y*z);")); + assertEquals((byte)10*(byte)0, exec("byte x = 10; byte y = 0; return x*y;")); + assertEquals((byte)0*(byte)0, exec("byte x = 0; byte y = 0; return x*x;")); + } + + public void testLong() throws Exception { + assertEquals(1L*1L, exec("long x = 1; long y = 1; return x*y;")); + assertEquals(2L*3L, exec("long x = 2; long y = 3; return x*y;")); + assertEquals(5L*10L, exec("long x = 5; long y = 10; return x*y;")); + assertEquals(1L*1L*2L, exec("long x = 1; long y = 1; int z = 2; return x*y*z;")); + assertEquals((1L*1L)*2L, exec("long x = 1; long y = 1; int z = 2; return (x*y)*z;")); + assertEquals(1L*(1L*2L), exec("long x = 1; long y = 1; int z = 2; return x*(y*z);")); + assertEquals(10L*0L, exec("long x = 10; long y = 0; return x*y;")); + assertEquals(0L*0L, exec("long x = 0; long y = 0; return x*x;")); + } + + public void testLongConst() throws Exception { + assertEquals(1L*1L, exec("return 1L*1L;")); + assertEquals(2L*3L, exec("return 2L*3L;")); + assertEquals(5L*10L, exec("return 5L*10L;")); + assertEquals(1L*1L*2L, exec("return 1L*1L*2L;")); + assertEquals((1L*1L)*2L, exec("return (1L*1L)*2L;")); + assertEquals(1L*(1L*2L), exec("return 1L*(1L*2L);")); + assertEquals(10L*0L, exec("return 10L*0L;")); + assertEquals(0L*0L, exec("return 0L*0L;")); + } + + public void testFloat() throws Exception { + assertEquals(1F*1F, exec("float x = 1; float y = 1; return x*y;")); + assertEquals(2F*3F, exec("float x = 2; float y = 3; return x*y;")); + assertEquals(5F*10F, exec("float x = 5; float y = 10; return x*y;")); + assertEquals(1F*1F*2F, exec("float x = 1; float y = 1; float z = 2; return x*y*z;")); + assertEquals((1F*1F)*2F, exec("float x = 1; float y = 1; float z = 2; return (x*y)*z;")); + assertEquals(1F*(1F*2F), exec("float x = 1; float y = 1; float z = 2; return x*(y*z);")); + assertEquals(10F*0F, exec("float x = 10; float y = 0; return x*y;")); + assertEquals(0F*0F, exec("float x = 0; float y = 0; return x*x;")); + } + + public void testFloatConst() throws Exception { + assertEquals(1F*1F, exec("return 1F*1F;")); + assertEquals(2F*3F, exec("return 2F*3F;")); + assertEquals(5F*10F, exec("return 5F*10F;")); + assertEquals(1F*1F*2F, exec("return 1F*1F*2F;")); + assertEquals((1F*1F)*2F, exec("return (1F*1F)*2F;")); + assertEquals(1F*(1F*2F), exec("return 1F*(1F*2F);")); + assertEquals(10F*0F, exec("return 10F*0F;")); + assertEquals(0F*0F, exec("return 0F*0F;")); + } + + public void testDouble() throws Exception { + assertEquals(1D*1D, exec("double x = 1; double y = 1; return x*y;")); + assertEquals(2D*3D, exec("double x = 2; double y = 3; return x*y;")); + assertEquals(5D*10D, exec("double x = 5; double y = 10; return x*y;")); + assertEquals(1D*1D*2D, exec("double x = 1; double y = 1; double z = 2; return x*y*z;")); + assertEquals((1D*1D)*2D, exec("double x = 1; double y = 1; double z = 2; return (x*y)*z;")); + assertEquals(1D*(1D*2D), exec("double x = 1; double y = 1; double z = 2; return x*(y*z);")); + assertEquals(10D*0D, exec("double x = 10; float y = 0; return x*y;")); + assertEquals(0D*0D, exec("double x = 0; float y = 0; return x*x;")); + } + + public void testDoubleConst() throws Exception { + assertEquals(1.0*1.0, exec("return 1.0*1.0;")); + assertEquals(2.0*3.0, exec("return 2.0*3.0;")); + assertEquals(5.0*10.0, exec("return 5.0*10.0;")); + assertEquals(1.0*1.0*2.0, exec("return 1.0*1.0*2.0;")); + assertEquals((1.0*1.0)*2.0, exec("return (1.0*1.0)*2.0;")); + assertEquals(1.0*(1.0*2.0), exec("return 1.0*(1.0*2.0);")); + assertEquals(10.0*0.0, exec("return 10.0*0.0;")); + assertEquals(0.0*0.0, exec("return 0.0*0.0;")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/NoSemiColonTest.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/NoSemiColonTest.java new file mode 100644 index 00000000000..ff56ee3f07e --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/NoSemiColonTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import java.util.HashMap; +import java.util.Map; + +public class NoSemiColonTest extends ScriptTestCase { + + public void testIfStatement() { + assertEquals(1, exec("int x = 5 if (x == 5) return 1 return 0")); + assertEquals(0, exec("int x = 4 if (x == 5) return 1 else return 0")); + assertEquals(2, exec("int x = 4 if (x == 5) return 1 else if (x == 4) return 2 else return 0")); + assertEquals(1, exec("int x = 4 if (x == 5) return 1 else if (x == 4) return 1 else return 0")); + + assertEquals(3, exec( + "int x = 5\n" + + "if (x == 5) {\n" + + " int y = 2\n" + + " \n" + + " if (y == 2) {\n" + + " x = 3\n" + + " }\n" + + " \n" + + "}\n" + + "\n" + + "return x\n")); + } + + public void testWhileStatement() { + + assertEquals("aaaaaa", exec("String c = \"a\" int x while (x < 5) { c ..= \"a\" ++x } return c")); + + Object value = exec( + " byte[][] b = new byte[5][5] \n" + + " byte x = 0, y \n" + + " \n" + + " while (x < 5) { \n" + + " y = 0 \n" + + " \n" + + " while (y < 5) { \n" + + " b[x][y] = (byte)(x*y) \n" + + " ++y \n" + + " } \n" + + " \n" + + " ++x \n" + + " } \n" + + " \n" + + " return b \n"); + + byte[][] b = (byte[][])value; + + for (byte x = 0; x < 5; ++x) { + for (byte y = 0; y < 5; ++y) { + assertEquals(x*y, b[x][y]); + } + } + } + + public void testDoWhileStatement() { + assertEquals("aaaaaa", exec("String c = \"a\" int x do { c ..= \"a\" ++x } while (x < 5) return c")); + + Object value = exec( + " long[][] l = new long[5][5] \n" + + " long x = 0, y \n" + + " \n" + + " do { \n" + + " y = 0 \n" + + " \n" + + " do { \n" + + " l[(int)x][(int)y] = x*y \n" + + " ++y \n" + + " } while (y < 5) \n" + + " \n" + + " ++x \n" + + " } while (x < 5) \n" + + " \n" + + " return l \n"); + + long[][] l = (long[][])value; + + for (long x = 0; x < 5; ++x) { + for (long y = 0; y < 5; ++y) { + assertEquals(x*y, l[(int)x][(int)y]); + } + } + } + + public void testForStatement() { + assertEquals("aaaaaa", exec("String c = \"a\" for (int x = 0; x < 5; ++x) c ..= \"a\" return c")); + + Object value = exec( + " int[][] i = new int[5][5] \n" + + " for (int x = 0; x < 5; ++x) { \n" + + " for (int y = 0; y < 5; ++y) { \n" + + " i[x][y] = x*y \n" + + " } \n" + + " } \n" + + " \n" + + " return i \n"); + + int[][] i = (int[][])value; + + for (int x = 0; x < 5; ++x) { + for (int y = 0; y < 5; ++y) { + assertEquals(x*y, i[x][y]); + } + } + } + + public void testDeclarationStatement() { + assertEquals((byte)2, exec("byte a = 2 return a")); + assertEquals((short)2, exec("short a = 2 return a")); + assertEquals((char)2, exec("char a = 2 return a")); + assertEquals(2, exec("int a = 2 return a")); + assertEquals(2L, exec("long a = 2 return a")); + assertEquals(2F, exec("float a = 2 return a")); + assertEquals(2.0, exec("double a = 2 return a")); + assertEquals(false, exec("boolean a = false return a")); + assertEquals("string", exec("String a = \"string\" return a")); + assertEquals(HashMap.class, exec("Map a = new HashMap() return a").getClass()); + + assertEquals(byte[].class, exec("byte[] a = new byte[1] return a").getClass()); + assertEquals(short[].class, exec("short[] a = new short[1] return a").getClass()); + assertEquals(char[].class, exec("char[] a = new char[1] return a").getClass()); + assertEquals(int[].class, exec("int[] a = new int[1] return a").getClass()); + assertEquals(long[].class, exec("long[] a = new long[1] return a").getClass()); + assertEquals(float[].class, exec("float[] a = new float[1] return a").getClass()); + assertEquals(double[].class, exec("double[] a = new double[1] return a").getClass()); + assertEquals(boolean[].class, exec("boolean[] a = new boolean[1] return a").getClass()); + assertEquals(String[].class, exec("String[] a = new String[1] return a").getClass()); + assertEquals(Map[].class, exec("Map[] a = new Map[1] return a").getClass()); + + assertEquals(byte[][].class, exec("byte[][] a = new byte[1][2] return a").getClass()); + assertEquals(short[][][].class, exec("short[][][] a = new short[1][2][3] return a").getClass()); + assertEquals(char[][][][].class, exec("char[][][][] a = new char[1][2][3][4] return a").getClass()); + assertEquals(int[][][][][].class, exec("int[][][][][] a = new int[1][2][3][4][5] return a").getClass()); + assertEquals(long[][].class, exec("long[][] a = new long[1][2] return a").getClass()); + assertEquals(float[][][].class, exec("float[][][] a = new float[1][2][3] return a").getClass()); + assertEquals(double[][][][].class, exec("double[][][][] a = new double[1][2][3][4] return a").getClass()); + assertEquals(boolean[][][][][].class, exec("boolean[][][][][] a = new boolean[1][2][3][4][5] return a").getClass()); + assertEquals(String[][].class, exec("String[][] a = new String[1][2] return a").getClass()); + assertEquals(Map[][][].class, exec("Map[][][] a = new Map[1][2][3] return a").getClass()); + } + + public void testContinueStatement() { + assertEquals(9, exec("int x = 0, y = 0 while (x < 10) { ++x if (x == 1) continue ++y } return y")); + } + + public void testBreakStatement() { + assertEquals(4, exec("int x = 0, y = 0 while (x < 10) { ++x if (x == 5) break ++y } return y")); + } + + public void testReturnStatement() { + assertEquals(10, exec("return 10")); + assertEquals(5, exec("int x = 5 return x")); + assertEquals(4, exec("int[] x = new int[2] x[1] = 4 return x[1]")); + assertEquals(5, ((short[])exec("short[] s = new short[3] s[1] = 5 return s"))[1]); + assertEquals(10, ((Map)exec("Map s = new HashMap< String,Object>() s.put(\"x\", 10) return s")).get("x")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/OrTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/OrTests.java new file mode 100644 index 00000000000..f3ba0c88fc1 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/OrTests.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for or operator across all types */ +public class OrTests extends ScriptTestCase { + + public void testInt() throws Exception { + assertEquals(5 | 12, exec("int x = 5; int y = 12; return x | y;")); + assertEquals(5 | -12, exec("int x = 5; int y = -12; return x | y;")); + assertEquals(7 | 15 | 3, exec("int x = 7; int y = 15; int z = 3; return x | y | z;")); + } + + public void testIntConst() throws Exception { + assertEquals(5 | 12, exec("return 5 | 12;")); + assertEquals(5 | -12, exec("return 5 | -12;")); + assertEquals(7 | 15 | 3, exec("return 7 | 15 | 3;")); + } + + public void testLong() throws Exception { + assertEquals(5L | 12L, exec("long x = 5; long y = 12; return x | y;")); + assertEquals(5L | -12L, exec("long x = 5; long y = -12; return x | y;")); + assertEquals(7L | 15L | 3L, exec("long x = 7; long y = 15; long z = 3; return x | y | z;")); + } + + public void testLongConst() throws Exception { + assertEquals(5L | 12L, exec("return 5L | 12L;")); + assertEquals(5L | -12L, exec("return 5L | -12L;")); + assertEquals(7L | 15L | 3L, exec("return 7L | 15L | 3L;")); + } +} diff --git a/core/src/test/java/org/elasticsearch/test/rest/Rest3IT.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/PlanARestIT.java similarity index 68% rename from core/src/test/java/org/elasticsearch/test/rest/Rest3IT.java rename to plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/PlanARestIT.java index 7cbc974de0d..c2c19ccb03a 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/Rest3IT.java +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/PlanARestIT.java @@ -17,22 +17,33 @@ * under the License. */ -package org.elasticsearch.test.rest; +package org.elasticsearch.plan.a; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; -/** Rest API tests subset 3 */ -public class Rest3IT extends ESRestTestCase { - public Rest3IT(@Name("yaml") RestTestCandidate testCandidate) { +/** Runs yaml rest tests */ +public class PlanARestIT extends ESRestTestCase { + + @Override + protected Collection> nodePlugins() { + return pluginList(PlanAPlugin.class); + } + + public PlanARestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } + @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(3, 8); + return ESRestTestCase.createParameters(0, 1); } } + diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/RemainderTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/RemainderTests.java new file mode 100644 index 00000000000..c7b6f7b1e3f --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/RemainderTests.java @@ -0,0 +1,147 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for division operator across all types */ +//TODO: NaN/Inf/overflow/... +public class RemainderTests extends ScriptTestCase { + + // TODO: byte,short,char + + public void testInt() throws Exception { + assertEquals(1%1, exec("int x = 1; int y = 1; return x%y;")); + assertEquals(2%3, exec("int x = 2; int y = 3; return x%y;")); + assertEquals(5%10, exec("int x = 5; int y = 10; return x%y;")); + assertEquals(10%1%2, exec("int x = 10; int y = 1; int z = 2; return x%y%z;")); + assertEquals((10%1)%2, exec("int x = 10; int y = 1; int z = 2; return (x%y)%z;")); + assertEquals(10%(4%3), exec("int x = 10; int y = 4; int z = 3; return x%(y%z);")); + assertEquals(10%1, exec("int x = 10; int y = 1; return x%y;")); + assertEquals(0%1, exec("int x = 0; int y = 1; return x%y;")); + } + + public void testIntConst() throws Exception { + assertEquals(1%1, exec("return 1%1;")); + assertEquals(2%3, exec("return 2%3;")); + assertEquals(5%10, exec("return 5%10;")); + assertEquals(10%1%2, exec("return 10%1%2;")); + assertEquals((10%1)%2, exec("return (10%1)%2;")); + assertEquals(10%(4%3), exec("return 10%(4%3);")); + assertEquals(10%1, exec("return 10%1;")); + assertEquals(0%1, exec("return 0%1;")); + } + + public void testLong() throws Exception { + assertEquals(1L%1L, exec("long x = 1; long y = 1; return x%y;")); + assertEquals(2L%3L, exec("long x = 2; long y = 3; return x%y;")); + assertEquals(5L%10L, exec("long x = 5; long y = 10; return x%y;")); + assertEquals(10L%1L%2L, exec("long x = 10; long y = 1; long z = 2; return x%y%z;")); + assertEquals((10L%1L)%2L, exec("long x = 10; long y = 1; long z = 2; return (x%y)%z;")); + assertEquals(10L%(4L%3L), exec("long x = 10; long y = 4; long z = 3; return x%(y%z);")); + assertEquals(10L%1L, exec("long x = 10; long y = 1; return x%y;")); + assertEquals(0L%1L, exec("long x = 0; long y = 1; return x%y;")); + } + + public void testLongConst() throws Exception { + assertEquals(1L%1L, exec("return 1L%1L;")); + assertEquals(2L%3L, exec("return 2L%3L;")); + assertEquals(5L%10L, exec("return 5L%10L;")); + assertEquals(10L%1L%2L, exec("return 10L%1L%2L;")); + assertEquals((10L%1L)%2L, exec("return (10L%1L)%2L;")); + assertEquals(10L%(4L%3L), exec("return 10L%(4L%3L);")); + assertEquals(10L%1L, exec("return 10L%1L;")); + assertEquals(0L%1L, exec("return 0L%1L;")); + } + + public void testFloat() throws Exception { + assertEquals(1F%1F, exec("float x = 1; float y = 1; return x%y;")); + assertEquals(2F%3F, exec("float x = 2; float y = 3; return x%y;")); + assertEquals(5F%10F, exec("float x = 5; float y = 10; return x%y;")); + assertEquals(10F%1F%2F, exec("float x = 10; float y = 1; float z = 2; return x%y%z;")); + assertEquals((10F%1F)%2F, exec("float x = 10; float y = 1; float z = 2; return (x%y)%z;")); + assertEquals(10F%(4F%3F), exec("float x = 10; float y = 4; float z = 3; return x%(y%z);")); + assertEquals(10F%1F, exec("float x = 10; float y = 1; return x%y;")); + assertEquals(0F%1F, exec("float x = 0; float y = 1; return x%y;")); + } + + public void testFloatConst() throws Exception { + assertEquals(1F%1F, exec("return 1F%1F;")); + assertEquals(2F%3F, exec("return 2F%3F;")); + assertEquals(5F%10F, exec("return 5F%10F;")); + assertEquals(10F%1F%2F, exec("return 10F%1F%2F;")); + assertEquals((10F%1F)%2F, exec("return (10F%1F)%2F;")); + assertEquals(10F%(4F%3F), exec("return 10F%(4F%3F);")); + assertEquals(10F%1F, exec("return 10F%1F;")); + assertEquals(0F%1F, exec("return 0F%1F;")); + } + + public void testDouble() throws Exception { + assertEquals(1.0%1.0, exec("double x = 1; double y = 1; return x%y;")); + assertEquals(2.0%3.0, exec("double x = 2; double y = 3; return x%y;")); + assertEquals(5.0%10.0, exec("double x = 5; double y = 10; return x%y;")); + assertEquals(10.0%1.0%2.0, exec("double x = 10; double y = 1; double z = 2; return x%y%z;")); + assertEquals((10.0%1.0)%2.0, exec("double x = 10; double y = 1; double z = 2; return (x%y)%z;")); + assertEquals(10.0%(4.0%3.0), exec("double x = 10; double y = 4; double z = 3; return x%(y%z);")); + assertEquals(10.0%1.0, exec("double x = 10; double y = 1; return x%y;")); + assertEquals(0.0%1.0, exec("double x = 0; double y = 1; return x%y;")); + } + + public void testDoubleConst() throws Exception { + assertEquals(1.0%1.0, exec("return 1.0%1.0;")); + assertEquals(2.0%3.0, exec("return 2.0%3.0;")); + assertEquals(5.0%10.0, exec("return 5.0%10.0;")); + assertEquals(10.0%1.0%2.0, exec("return 10.0%1.0%2.0;")); + assertEquals((10.0%1.0)%2.0, exec("return (10.0%1.0)%2.0;")); + assertEquals(10.0%(4.0%3.0), exec("return 10.0%(4.0%3.0);")); + assertEquals(10.0%1.0, exec("return 10.0%1.0;")); + assertEquals(0.0%1.0, exec("return 0.0%1.0;")); + } + + public void testDivideByZero() throws Exception { + try { + exec("int x = 1; int y = 0; return x % y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + + try { + exec("long x = 1L; long y = 0L; return x % y;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + } + + public void testDivideByZeroConst() throws Exception { + try { + exec("return 1%0;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + + try { + exec("return 1L%0L;"); + fail("should have hit exception"); + } catch (ArithmeticException expected) { + // divide by zero + } + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ScriptEngineTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ScriptEngineTests.java new file mode 100644 index 00000000000..d2bbe02a625 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ScriptEngineTests.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptService; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class ScriptEngineTests extends ScriptTestCase { + + public void testSimpleEquation() { + final Object value = exec("return 1 + 2;"); + assertEquals(3, ((Number)value).intValue()); + } + + public void testMapAccess() { + Map vars = new HashMap<>(); + Map obj2 = new HashMap<>(); + obj2.put("prop2", "value2"); + Map obj1 = new HashMap<>(); + obj1.put("prop1", "value1"); + obj1.put("obj2", obj2); + obj1.put("l", Arrays.asList("2", "1")); + vars.put("obj1", obj1); + + Object value = exec("return input.get(\"obj1\");", vars); + obj1 = (Map)value; + assertEquals("value1", obj1.get("prop1")); + assertEquals("value2", ((Map) obj1.get("obj2")).get("prop2")); + + value = exec("return ((List)((Map)input.get(\"obj1\")).get(\"l\")).get(0);", vars); + assertEquals("2", value); + } + + public void testAccessListInScript() { + Map vars = new HashMap<>(); + Map obj2 = new HashMap<>(); + obj2.put("prop2", "value2"); + Map obj1 = new HashMap<>(); + obj1.put("prop1", "value1"); + obj1.put("obj2", obj2); + vars.put("l", Arrays.asList("1", "2", "3", obj1)); + + assertEquals(4, exec("return ((List)input.get(\"l\")).size();", vars)); + assertEquals("1", exec("return ((List)input.get(\"l\")).get(0);", vars)); + + Object value = exec("return ((List)input.get(\"l\")).get(3);", vars); + obj1 = (Map)value; + assertEquals("value1", obj1.get("prop1")); + assertEquals("value2", ((Map)obj1.get("obj2")).get("prop2")); + + assertEquals("value1", exec("return ((Map)((List)input.get(\"l\")).get(3)).get(\"prop1\");", vars)); + } + + public void testChangingVarsCrossExecution1() { + Map vars = new HashMap<>(); + Map ctx = new HashMap<>(); + vars.put("ctx", ctx); + + Object compiledScript = scriptEngine.compile("return ((Map)input.get(\"ctx\")).get(\"value\");"); + ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE, + "testChangingVarsCrossExecution1", "plan-a", compiledScript), vars); + + ctx.put("value", 1); + Object o = script.run(); + assertEquals(1, ((Number) o).intValue()); + + ctx.put("value", 2); + o = script.run(); + assertEquals(2, ((Number) o).intValue()); + } + + public void testChangingVarsCrossExecution2() { + Map vars = new HashMap<>(); + Object compiledScript = scriptEngine.compile("return input.get(\"value\");"); + + ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE, + "testChangingVarsCrossExecution2", "plan-a", compiledScript), vars); + + script.setNextVar("value", 1); + Object value = script.run(); + assertEquals(1, ((Number)value).intValue()); + + script.setNextVar("value", 2); + value = script.run(); + assertEquals(2, ((Number)value).intValue()); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ScriptTestCase.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ScriptTestCase.java new file mode 100644 index 00000000000..253e37183f3 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/ScriptTestCase.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.Map; + +/** + * Base test case for scripting unit tests. + *

    + * Typically just asserts the output of {@code exec()} + */ +public abstract class ScriptTestCase extends ESTestCase { + protected PlanAScriptEngineService scriptEngine; + + /** Override to provide different compiler settings */ + protected Settings getSettings() { + Settings.Builder builder = Settings.builder(); + builder.put(PlanAScriptEngineService.NUMERIC_OVERFLOW, random().nextBoolean()); + return builder.build(); + } + + @Before + public void setup() { + scriptEngine = new PlanAScriptEngineService(getSettings()); + } + + /** Compiles and returns the result of {@code script} */ + public Object exec(String script) { + return exec(script, null); + } + + /** Compiles and returns the result of {@code script} with access to {@code vars} */ + public Object exec(String script, Map vars) { + Object object = scriptEngine.compile(script); + CompiledScript compiled = new CompiledScript(ScriptService.ScriptType.INLINE, getTestName(), "plan-a", object); + return scriptEngine.executable(compiled, vars).run(); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/StringTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/StringTests.java new file mode 100644 index 00000000000..0fbcaa1e6d3 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/StringTests.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +public class StringTests extends ScriptTestCase { + + public void testAppend() { + // boolean + assertEquals("cat" + true, exec("String s = \"cat\"; return s + true;")); + // byte + assertEquals("cat" + (byte)3, exec("String s = \"cat\"; return s + (byte)3;")); + // short + assertEquals("cat" + (short)3, exec("String s = \"cat\"; return s + (short)3;")); + // char + assertEquals("cat" + 't', exec("String s = \"cat\"; return s + 't';")); + assertEquals("cat" + (char)40, exec("String s = \"cat\"; return s + (char)40;")); + // int + assertEquals("cat" + 2, exec("String s = \"cat\"; return s + 2;")); + // long + assertEquals("cat" + 2L, exec("String s = \"cat\"; return s + 2L;")); + // float + assertEquals("cat" + 2F, exec("String s = \"cat\"; return s + 2F;")); + // double + assertEquals("cat" + 2.0, exec("String s = \"cat\"; return s + 2.0;")); + // String + assertEquals("cat" + "cat", exec("String s = \"cat\"; return s + s;")); + } + + public void testStringAPI() { + assertEquals("", exec("return new String();")); + assertEquals('x', exec("String s = \"x\"; return s.charAt(0);")); + assertEquals(120, exec("String s = \"x\"; return s.codePointAt(0);")); + assertEquals(0, exec("String s = \"x\"; return s.compareTo(\"x\");")); + assertEquals("xx", exec("String s = \"x\"; return s.concat(\"x\");")); + assertEquals(true, exec("String s = \"xy\"; return s.endsWith(\"y\");")); + assertEquals(2, exec("String t = \"abcde\"; return t.indexOf(\"cd\", 1);")); + assertEquals(false, exec("String t = \"abcde\"; return t.isEmpty();")); + assertEquals(5, exec("String t = \"abcde\"; return t.length();")); + assertEquals("cdcde", exec("String t = \"abcde\"; return t.replace(\"ab\", \"cd\");")); + assertEquals(false, exec("String s = \"xy\"; return s.startsWith(\"y\");")); + assertEquals("e", exec("String t = \"abcde\"; return t.substring(4, 5);")); + assertEquals(97, ((char[])exec("String s = \"a\"; return s.toCharArray();"))[0]); + assertEquals("a", exec("String s = \" a \"; return s.trim();")); + assertEquals('x', exec("return \"x\".charAt(0);")); + assertEquals(120, exec("return \"x\".codePointAt(0);")); + assertEquals(0, exec("return \"x\".compareTo(\"x\");")); + assertEquals("xx", exec("return \"x\".concat(\"x\");")); + assertEquals(true, exec("return \"xy\".endsWith(\"y\");")); + assertEquals(2, exec("return \"abcde\".indexOf(\"cd\", 1);")); + assertEquals(false, exec("return \"abcde\".isEmpty();")); + assertEquals(5, exec("return \"abcde\".length();")); + assertEquals("cdcde", exec("return \"abcde\".replace(\"ab\", \"cd\");")); + assertEquals(false, exec("return \"xy\".startsWith(\"y\");")); + assertEquals("e", exec("return \"abcde\".substring(4, 5);")); + assertEquals(97, ((char[])exec("return \"a\".toCharArray();"))[0]); + assertEquals("a", exec("return \" a \".trim();")); + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/SubtractionTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/SubtractionTests.java new file mode 100644 index 00000000000..1acd0458b52 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/SubtractionTests.java @@ -0,0 +1,179 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for subtraction operator across all types */ +//TODO: NaN/Inf/overflow/... +public class SubtractionTests extends ScriptTestCase { + + public void testInt() throws Exception { + assertEquals(1-1, exec("int x = 1; int y = 1; return x-y;")); + assertEquals(2-3, exec("int x = 2; int y = 3; return x-y;")); + assertEquals(5-10, exec("int x = 5; int y = 10; return x-y;")); + assertEquals(1-1-2, exec("int x = 1; int y = 1; int z = 2; return x-y-z;")); + assertEquals((1-1)-2, exec("int x = 1; int y = 1; int z = 2; return (x-y)-z;")); + assertEquals(1-(1-2), exec("int x = 1; int y = 1; int z = 2; return x-(y-z);")); + assertEquals(10-0, exec("int x = 10; int y = 0; return x-y;")); + assertEquals(0-0, exec("int x = 0; int y = 0; return x-x;")); + } + + public void testIntConst() throws Exception { + assertEquals(1-1, exec("return 1-1;")); + assertEquals(2-3, exec("return 2-3;")); + assertEquals(5-10, exec("return 5-10;")); + assertEquals(1-1-2, exec("return 1-1-2;")); + assertEquals((1-1)-2, exec("return (1-1)-2;")); + assertEquals(1-(1-2), exec("return 1-(1-2);")); + assertEquals(10-0, exec("return 10-0;")); + assertEquals(0-0, exec("return 0-0;")); + } + + public void testByte() throws Exception { + assertEquals((byte)1-(byte)1, exec("byte x = 1; byte y = 1; return x-y;")); + assertEquals((byte)2-(byte)3, exec("byte x = 2; byte y = 3; return x-y;")); + assertEquals((byte)5-(byte)10, exec("byte x = 5; byte y = 10; return x-y;")); + assertEquals((byte)1-(byte)1-(byte)2, exec("byte x = 1; byte y = 1; byte z = 2; return x-y-z;")); + assertEquals(((byte)1-(byte)1)-(byte)2, exec("byte x = 1; byte y = 1; byte z = 2; return (x-y)-z;")); + assertEquals((byte)1-((byte)1-(byte)2), exec("byte x = 1; byte y = 1; byte z = 2; return x-(y-z);")); + assertEquals((byte)10-(byte)1, exec("byte x = 10; byte y = 1; return x-y;")); + assertEquals((byte)0-(byte)0, exec("byte x = 0; byte y = 0; return x-y;")); + } + + public void testByteConst() throws Exception { + assertEquals((byte)1-(byte)1, exec("return (byte)1-(byte)1;")); + assertEquals((byte)2-(byte)3, exec("return (byte)2-(byte)3;")); + assertEquals((byte)5-(byte)10, exec("return (byte)5-(byte)10;")); + assertEquals((byte)1-(byte)1-(byte)2, exec("return (byte)1-(byte)1-(byte)2;")); + assertEquals(((byte)1-(byte)1)-(byte)2, exec("return ((byte)1-(byte)1)-(byte)2;")); + assertEquals((byte)1-((byte)1-(byte)2), exec("return (byte)1-((byte)1-(byte)2);")); + assertEquals((byte)10-(byte)1, exec("return (byte)10-(byte)1;")); + assertEquals((byte)0-(byte)0, exec("return (byte)0-(byte)0;")); + } + + public void testChar() throws Exception { + assertEquals((char)1-(char)1, exec("char x = 1; char y = 1; return x-y;")); + assertEquals((char)2-(char)3, exec("char x = 2; char y = 3; return x-y;")); + assertEquals((char)5-(char)10, exec("char x = 5; char y = 10; return x-y;")); + assertEquals((char)1-(char)1-(char)2, exec("char x = 1; char y = 1; char z = 2; return x-y-z;")); + assertEquals(((char)1-(char)1)-(char)2, exec("char x = 1; char y = 1; char z = 2; return (x-y)-z;")); + assertEquals((char)1-((char)1-(char)2), exec("char x = 1; char y = 1; char z = 2; return x-(y-z);")); + assertEquals((char)10-(char)1, exec("char x = 10; char y = 1; return x-y;")); + assertEquals((char)0-(char)0, exec("char x = 0; char y = 0; return x-y;")); + } + + public void testCharConst() throws Exception { + assertEquals((char)1-(char)1, exec("return (char)1-(char)1;")); + assertEquals((char)2-(char)3, exec("return (char)2-(char)3;")); + assertEquals((char)5-(char)10, exec("return (char)5-(char)10;")); + assertEquals((char)1-(char)1-(char)2, exec("return (char)1-(char)1-(char)2;")); + assertEquals(((char)1-(char)1)-(char)2, exec("return ((char)1-(char)1)-(char)2;")); + assertEquals((char)1-((char)1-(char)2), exec("return (char)1-((char)1-(char)2);")); + assertEquals((char)10-(char)1, exec("return (char)10-(char)1;")); + assertEquals((char)0-(char)0, exec("return (char)0-(char)0;")); + } + + public void testShort() throws Exception { + assertEquals((short)1-(short)1, exec("short x = 1; short y = 1; return x-y;")); + assertEquals((short)2-(short)3, exec("short x = 2; short y = 3; return x-y;")); + assertEquals((short)5-(short)10, exec("short x = 5; short y = 10; return x-y;")); + assertEquals((short)1-(short)1-(short)2, exec("short x = 1; short y = 1; short z = 2; return x-y-z;")); + assertEquals(((short)1-(short)1)-(short)2, exec("short x = 1; short y = 1; short z = 2; return (x-y)-z;")); + assertEquals((short)1-((short)1-(short)2), exec("short x = 1; short y = 1; short z = 2; return x-(y-z);")); + assertEquals((short)10-(short)1, exec("short x = 10; short y = 1; return x-y;")); + assertEquals((short)0-(short)0, exec("short x = 0; short y = 0; return x-y;")); + } + + public void testShortConst() throws Exception { + assertEquals((short)1-(short)1, exec("return (short)1-(short)1;")); + assertEquals((short)2-(short)3, exec("return (short)2-(short)3;")); + assertEquals((short)5-(short)10, exec("return (short)5-(short)10;")); + assertEquals((short)1-(short)1-(short)2, exec("return (short)1-(short)1-(short)2;")); + assertEquals(((short)1-(short)1)-(short)2, exec("return ((short)1-(short)1)-(short)2;")); + assertEquals((short)1-((short)1-(short)2), exec("return (short)1-((short)1-(short)2);")); + assertEquals((short)10-(short)1, exec("return (short)10-(short)1;")); + assertEquals((short)0-(short)0, exec("return (short)0-(short)0;")); + } + + public void testLong() throws Exception { + assertEquals(1L-1L, exec("long x = 1; long y = 1; return x-y;")); + assertEquals(2L-3L, exec("long x = 2; long y = 3; return x-y;")); + assertEquals(5L-10L, exec("long x = 5; long y = 10; return x-y;")); + assertEquals(1L-1L-2L, exec("long x = 1; long y = 1; int z = 2; return x-y-z;")); + assertEquals((1L-1L)-2L, exec("long x = 1; long y = 1; int z = 2; return (x-y)-z;")); + assertEquals(1L-(1L-2L), exec("long x = 1; long y = 1; int z = 2; return x-(y-z);")); + assertEquals(10L-0L, exec("long x = 10; long y = 0; return x-y;")); + assertEquals(0L-0L, exec("long x = 0; long y = 0; return x-x;")); + } + + public void testLongConst() throws Exception { + assertEquals(1L-1L, exec("return 1L-1L;")); + assertEquals(2L-3L, exec("return 2L-3L;")); + assertEquals(5L-10L, exec("return 5L-10L;")); + assertEquals(1L-1L-2L, exec("return 1L-1L-2L;")); + assertEquals((1L-1L)-2L, exec("return (1L-1L)-2L;")); + assertEquals(1L-(1L-2L), exec("return 1L-(1L-2L);")); + assertEquals(10L-0L, exec("return 10L-0L;")); + assertEquals(0L-0L, exec("return 0L-0L;")); + } + + public void testFloat() throws Exception { + assertEquals(1F-1F, exec("float x = 1; float y = 1; return x-y;")); + assertEquals(2F-3F, exec("float x = 2; float y = 3; return x-y;")); + assertEquals(5F-10F, exec("float x = 5; float y = 10; return x-y;")); + assertEquals(1F-1F-2F, exec("float x = 1; float y = 1; float z = 2; return x-y-z;")); + assertEquals((1F-1F)-2F, exec("float x = 1; float y = 1; float z = 2; return (x-y)-z;")); + assertEquals(1F-(1F-2F), exec("float x = 1; float y = 1; float z = 2; return x-(y-z);")); + assertEquals(10F-0F, exec("float x = 10; float y = 0; return x-y;")); + assertEquals(0F-0F, exec("float x = 0; float y = 0; return x-x;")); + } + + public void testFloatConst() throws Exception { + assertEquals(1F-1F, exec("return 1F-1F;")); + assertEquals(2F-3F, exec("return 2F-3F;")); + assertEquals(5F-10F, exec("return 5F-10F;")); + assertEquals(1F-1F-2F, exec("return 1F-1F-2F;")); + assertEquals((1F-1F)-2F, exec("return (1F-1F)-2F;")); + assertEquals(1F-(1F-2F), exec("return 1F-(1F-2F);")); + assertEquals(10F-0F, exec("return 10F-0F;")); + assertEquals(0F-0F, exec("return 0F-0F;")); + } + + public void testDouble() throws Exception { + assertEquals(1D-1D, exec("double x = 1; double y = 1; return x-y;")); + assertEquals(2D-3D, exec("double x = 2; double y = 3; return x-y;")); + assertEquals(5D-10D, exec("double x = 5; double y = 10; return x-y;")); + assertEquals(1D-1D-2D, exec("double x = 1; double y = 1; double z = 2; return x-y-z;")); + assertEquals((1D-1D)-2D, exec("double x = 1; double y = 1; double z = 2; return (x-y)-z;")); + assertEquals(1D-(1D-2D), exec("double x = 1; double y = 1; double z = 2; return x-(y-z);")); + assertEquals(10D-0D, exec("double x = 10; float y = 0; return x-y;")); + assertEquals(0D-0D, exec("double x = 0; float y = 0; return x-x;")); + } + + public void testyDoubleConst() throws Exception { + assertEquals(1.0-1.0, exec("return 1.0-1.0;")); + assertEquals(2.0-3.0, exec("return 2.0-3.0;")); + assertEquals(5.0-10.0, exec("return 5.0-10.0;")); + assertEquals(1.0-1.0-2.0, exec("return 1.0-1.0-2.0;")); + assertEquals((1.0-1.0)-2.0, exec("return (1.0-1.0)-2.0;")); + assertEquals(1.0-(1.0-2.0), exec("return 1.0-(1.0-2.0);")); + assertEquals(10.0-0.0, exec("return 10.0-0.0;")); + assertEquals(0.0-0.0, exec("return 0.0-0.0;")); + } +} diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3IndexModule.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/UnaryTests.java similarity index 52% rename from plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3IndexModule.java rename to plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/UnaryTests.java index 51054d774bd..c0199ffadd5 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3IndexModule.java +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/UnaryTests.java @@ -17,15 +17,26 @@ * under the License. */ -package org.elasticsearch.plugin.mapper; +package org.elasticsearch.plan.a; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.mapper.murmur3.RegisterMurmur3FieldMapper; +/** Tests for unary operators across different types */ +public class UnaryTests extends ScriptTestCase { -public class MapperMurmur3IndexModule extends AbstractModule { + /** basic tests */ + public void testBasics() { + assertEquals(false, exec("return !true;")); + assertEquals(true, exec("boolean x = false; return !x;")); + assertEquals(-2, exec("return ~1;")); + assertEquals(-2, exec("byte x = 1; return ~x;")); + assertEquals(1, exec("return +1;")); + assertEquals(1.0, exec("double x = 1; return +x;")); + assertEquals(-1, exec("return -1;")); + assertEquals(-2, exec("short x = 2; return -x;")); + } - @Override - protected void configure() { - bind(RegisterMurmur3FieldMapper.class).asEagerSingleton(); + public void testNegationInt() throws Exception { + assertEquals(-1, exec("return -1;")); + assertEquals(1, exec("return -(-1);")); + assertEquals(0, exec("return -0;")); } } diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/UtilityTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/UtilityTests.java new file mode 100644 index 00000000000..5c9fe20d1a7 --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/UtilityTests.java @@ -0,0 +1,250 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +import org.elasticsearch.test.ESTestCase; + +/** + * Tests utility methods (typically built-ins) + */ +public class UtilityTests extends ESTestCase { + + public void testDivideWithoutOverflowInt() { + assertEquals(5 / 2, Utility.divideWithoutOverflow(5, 2)); + + try { + Utility.divideWithoutOverflow(Integer.MIN_VALUE, -1); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.divideWithoutOverflow(5, 0); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivideWithoutOverflowLong() { + assertEquals(5L / 2L, Utility.divideWithoutOverflow(5L, 2L)); + + try { + Utility.divideWithoutOverflow(Long.MIN_VALUE, -1L); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.divideWithoutOverflow(5L, 0L); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testToByteExact() { + for (int b = Byte.MIN_VALUE; b < Byte.MAX_VALUE; b++) { + assertEquals((byte)b, Utility.toByteExact(b)); + } + + try { + Utility.toByteExact(Byte.MIN_VALUE - 1); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.toByteExact(Byte.MAX_VALUE + 1); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testToShortExact() { + for (int s = Short.MIN_VALUE; s < Short.MAX_VALUE; s++) { + assertEquals((short)s, Utility.toShortExact(s)); + } + + try { + Utility.toShortExact(Short.MIN_VALUE - 1); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.toShortExact(Short.MAX_VALUE + 1); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testToCharExact() { + for (int c = Character.MIN_VALUE; c < Character.MAX_VALUE; c++) { + assertEquals((char)c, Utility.toCharExact(c)); + } + + try { + Utility.toCharExact(Character.MIN_VALUE - 1); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.toCharExact(Character.MAX_VALUE + 1); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAddWithoutOverflowFloat() { + assertEquals(10F, Utility.addWithoutOverflow(5F, 5F), 0F); + assertTrue(Float.isNaN(Utility.addWithoutOverflow(5F, Float.NaN))); + assertTrue(Float.isNaN(Utility.addWithoutOverflow(Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY))); + + try { + Utility.addWithoutOverflow(Float.MAX_VALUE, Float.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.addWithoutOverflow(-Float.MAX_VALUE, -Float.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testAddWithoutOverflowDouble() { + assertEquals(10D, Utility.addWithoutOverflow(5D, 5D), 0D); + assertTrue(Double.isNaN(Utility.addWithoutOverflow(5D, Double.NaN))); + assertTrue(Double.isNaN(Utility.addWithoutOverflow(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY))); + + try { + Utility.addWithoutOverflow(Double.MAX_VALUE, Double.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.addWithoutOverflow(-Double.MAX_VALUE, -Double.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testSubtractWithoutOverflowFloat() { + assertEquals(5F, Utility.subtractWithoutOverflow(10F, 5F), 0F); + assertTrue(Float.isNaN(Utility.subtractWithoutOverflow(5F, Float.NaN))); + assertTrue(Float.isNaN(Utility.subtractWithoutOverflow(Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY))); + + try { + Utility.subtractWithoutOverflow(Float.MAX_VALUE, -Float.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.subtractWithoutOverflow(-Float.MAX_VALUE, Float.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testSubtractWithoutOverflowDouble() { + assertEquals(5D, Utility.subtractWithoutOverflow(10D, 5D), 0D); + assertTrue(Double.isNaN(Utility.subtractWithoutOverflow(5D, Double.NaN))); + assertTrue(Double.isNaN(Utility.subtractWithoutOverflow(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY))); + + try { + Utility.subtractWithoutOverflow(Double.MAX_VALUE, -Double.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.subtractWithoutOverflow(-Double.MAX_VALUE, Double.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testMultiplyWithoutOverflowFloat() { + assertEquals(25F, Utility.multiplyWithoutOverflow(5F, 5F), 0F); + assertTrue(Float.isNaN(Utility.multiplyWithoutOverflow(5F, Float.NaN))); + assertEquals(Float.POSITIVE_INFINITY, Utility.multiplyWithoutOverflow(5F, Float.POSITIVE_INFINITY), 0F); + + try { + Utility.multiplyWithoutOverflow(Float.MAX_VALUE, Float.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testMultiplyWithoutOverflowDouble() { + assertEquals(25D, Utility.multiplyWithoutOverflow(5D, 5D), 0D); + assertTrue(Double.isNaN(Utility.multiplyWithoutOverflow(5D, Double.NaN))); + assertEquals(Double.POSITIVE_INFINITY, Utility.multiplyWithoutOverflow(5D, Double.POSITIVE_INFINITY), 0D); + + try { + Utility.multiplyWithoutOverflow(Double.MAX_VALUE, Double.MAX_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivideWithoutOverflowFloat() { + assertEquals(5F, Utility.divideWithoutOverflow(25F, 5F), 0F); + assertTrue(Float.isNaN(Utility.divideWithoutOverflow(5F, Float.NaN))); + assertEquals(Float.POSITIVE_INFINITY, Utility.divideWithoutOverflow(Float.POSITIVE_INFINITY, 5F), 0F); + + try { + Utility.divideWithoutOverflow(Float.MAX_VALUE, Float.MIN_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.divideWithoutOverflow(0F, 0F); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.divideWithoutOverflow(5F, 0F); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testDivideWithoutOverflowDouble() { + assertEquals(5D, Utility.divideWithoutOverflow(25D, 5D), 0D); + assertTrue(Double.isNaN(Utility.divideWithoutOverflow(5D, Double.NaN))); + assertEquals(Double.POSITIVE_INFINITY, Utility.divideWithoutOverflow(Double.POSITIVE_INFINITY, 5D), 0D); + + try { + Utility.divideWithoutOverflow(Double.MAX_VALUE, Double.MIN_VALUE); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.divideWithoutOverflow(0D, 0D); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + + try { + Utility.divideWithoutOverflow(5D, 0D); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testRemainderWithoutOverflowFloat() { + assertEquals(1F, Utility.remainderWithoutOverflow(25F, 4F), 0F); + + try { + Utility.remainderWithoutOverflow(5F, 0F); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } + + public void testRemainderWithoutOverflowDouble() { + assertEquals(1D, Utility.remainderWithoutOverflow(25D, 4D), 0D); + + try { + Utility.remainderWithoutOverflow(5D, 0D); + fail("did not get expected exception"); + } catch (ArithmeticException expected) {} + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/WhenThingsGoWrongTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/WhenThingsGoWrongTests.java new file mode 100644 index 00000000000..de2c1c9ea3e --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/WhenThingsGoWrongTests.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +public class WhenThingsGoWrongTests extends ScriptTestCase { + public void testNullPointer() { + try { + exec("int x = (int) ((Map) input).get(\"missing\"); return x;"); + fail("should have hit npe"); + } catch (NullPointerException expected) {} + } + + public void testInvalidShift() { + try { + exec("float x = 15F; x <<= 2; return x;"); + fail("should have hit cce"); + } catch (ClassCastException expected) {} + + try { + exec("double x = 15F; x <<= 2; return x;"); + fail("should have hit cce"); + } catch (ClassCastException expected) {} + } +} diff --git a/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/XorTests.java b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/XorTests.java new file mode 100644 index 00000000000..f10477dcd0a --- /dev/null +++ b/plugins/lang-plan-a/src/test/java/org/elasticsearch/plan/a/XorTests.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plan.a; + +/** Tests for xor operator across all types */ +public class XorTests extends ScriptTestCase { + + public void testInt() throws Exception { + assertEquals(5 ^ 12, exec("int x = 5; int y = 12; return x ^ y;")); + assertEquals(5 ^ -12, exec("int x = 5; int y = -12; return x ^ y;")); + assertEquals(7 ^ 15 ^ 3, exec("int x = 7; int y = 15; int z = 3; return x ^ y ^ z;")); + } + + public void testIntConst() throws Exception { + assertEquals(5 ^ 12, exec("return 5 ^ 12;")); + assertEquals(5 ^ -12, exec("return 5 ^ -12;")); + assertEquals(7 ^ 15 ^ 3, exec("return 7 ^ 15 ^ 3;")); + } + + public void testLong() throws Exception { + assertEquals(5L ^ 12L, exec("long x = 5; long y = 12; return x ^ y;")); + assertEquals(5L ^ -12L, exec("long x = 5; long y = -12; return x ^ y;")); + assertEquals(7L ^ 15L ^ 3L, exec("long x = 7; long y = 15; long z = 3; return x ^ y ^ z;")); + } + + public void testLongConst() throws Exception { + assertEquals(5L ^ 12L, exec("return 5L ^ 12L;")); + assertEquals(5L ^ -12L, exec("return 5L ^ -12L;")); + assertEquals(7L ^ 15L ^ 3L, exec("return 7L ^ 15L ^ 3L;")); + } + + public void testBool() throws Exception { + assertEquals(false, exec("boolean x = true; boolean y = true; return x ^ y;")); + assertEquals(true, exec("boolean x = true; boolean y = false; return x ^ y;")); + assertEquals(true, exec("boolean x = false; boolean y = true; return x ^ y;")); + assertEquals(false, exec("boolean x = false; boolean y = false; return x ^ y;")); + } + + public void testBoolConst() throws Exception { + assertEquals(false, exec("return true ^ true;")); + assertEquals(true, exec("return true ^ false;")); + assertEquals(true, exec("return false ^ true;")); + assertEquals(false, exec("return false ^ false;")); + } +} diff --git a/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/10_basic.yaml b/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/10_basic.yaml new file mode 100644 index 00000000000..6259780bfb4 --- /dev/null +++ b/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/10_basic.yaml @@ -0,0 +1,14 @@ +# Integration tests for Plan A Plugin +# +"Plan A plugin loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.plugins.0.name: lang-plan-a } + - match: { nodes.$master.plugins.0.jvm: true } diff --git a/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml b/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml new file mode 100644 index 00000000000..0a5a3a4a8d4 --- /dev/null +++ b/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml @@ -0,0 +1,27 @@ +# Integration tests for using a scripted field +# +setup: + - do: + index: + index: test + type: test + id: 1 + body: { "foo": "aaa" } + - do: + indices.refresh: {} + +--- + +"Scripted Field": + - do: + search: + body: + script_fields: + bar: + script: + inline: "input.doc.foo.0 + input.x;" + lang: plan-a + params: + x: "bbb" + + - match: { hits.hits.0.fields.bar.0: "aaabbb"} diff --git a/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml new file mode 100644 index 00000000000..a8d96a0d6fa --- /dev/null +++ b/plugins/lang-plan-a/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml @@ -0,0 +1,97 @@ +# Integration tests for Plan-A search scripting +# +"Plan-A Query": + - do: + index: + index: test + type: test + id: 1 + body: { "test": "value beck", "num1": 1.0 } + - do: + index: + index: test + type: test + id: 2 + body: { "test": "value beck", "num1": 2.0 } + - do: + index: + index: test + type: test + id: 3 + body: { "test": "value beck", "num1": 3.0 } + - do: + indices.refresh: {} + + - do: + index: test + search: + body: + query: + script: + script: + inline: "input.doc.num1.0 > 1;" + lang: plan-a + script_fields: + sNum1: + script: + inline: "input.doc.num1.0;" + lang: plan-a + sort: + num1: + order: asc + + - match: { hits.total: 2 } + - match: { hits.hits.0.fields.sNum1.0: 2.0 } + - match: { hits.hits.1.fields.sNum1.0: 3.0 } + + - do: + index: test + search: + body: + query: + script: + script: + inline: "input.doc.num1.0 > input.param1;" + lang: plan-a + params: + param1: 1 + + script_fields: + sNum1: + script: + inline: "return input.doc.num1.0;" + lang: plan-a + sort: + num1: + order: asc + + - match: { hits.total: 2 } + - match: { hits.hits.0.fields.sNum1.0: 2.0 } + - match: { hits.hits.1.fields.sNum1.0: 3.0 } + + - do: + index: test + search: + body: + query: + script: + script: + inline: "input.doc.num1.0 > input.param1;" + lang: plan-a + params: + param1: -1 + + script_fields: + sNum1: + script: + inline: "input.doc.num1.0;" + lang: plan-a + sort: + num1: + order: asc + + - match: { hits.total: 3 } + - match: { hits.hits.0.fields.sNum1.0: 1.0 } + - match: { hits.hits.1.fields.sNum1.0: 2.0 } + - match: { hits.hits.2.fields.sNum1.0: 3.0 } + diff --git a/plugins/lang-python/build.gradle b/plugins/lang-python/build.gradle new file mode 100644 index 00000000000..269a3249386 --- /dev/null +++ b/plugins/lang-python/build.gradle @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Python language plugin allows to have python as the language of scripts to execute.' + classname 'org.elasticsearch.plugin.python.PythonPlugin' +} + +dependencies { + compile 'org.python:jython-standalone:2.7.0' +} + +compileJava.options.compilerArgs << "-Xlint:-unchecked" +compileTestJava.options.compilerArgs << "-Xlint:-unchecked" + +integTest { + cluster { + systemProperty 'es.script.inline', 'on' + systemProperty 'es.script.indexed', 'on' + } +} + diff --git a/plugins/lang-python/pom.xml b/plugins/lang-python/pom.xml deleted file mode 100644 index 73742e2b5e3..00000000000 --- a/plugins/lang-python/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - lang-python - Plugin: Language: Python - The Python language plugin allows to have python as the language of scripts to execute. - - - org.elasticsearch.plugin.python.PythonPlugin - lang_python - false - -Xlint:-unchecked - - - - - - org.python - jython-standalone - 2.7.0 - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 3dfa4bcd0f9..1930f530671 100644 --- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -25,7 +25,11 @@ import java.security.AccessController; import java.security.Permissions; import java.security.PrivilegedAction; import java.security.ProtectionDomain; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; @@ -34,6 +38,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ClassPermission; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.LeafSearchScript; @@ -55,20 +60,36 @@ import org.python.util.PythonInterpreter; public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService { private final PythonInterpreter interp; - + @Inject public PythonScriptEngineService(Settings settings) { super(settings); // classloader created here - SecurityManager sm = System.getSecurityManager(); + final SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new SpecialPermission()); } this.interp = AccessController.doPrivileged(new PrivilegedAction () { @Override public PythonInterpreter run() { - return PythonInterpreter.threadLocalStateInterpreter(null); + // snapshot our context here for checks, as the script has no permissions + final AccessControlContext engineContext = AccessController.getContext(); + PythonInterpreter interp = PythonInterpreter.threadLocalStateInterpreter(null); + if (sm != null) { + interp.getSystemState().setClassLoader(new ClassLoader(getClass().getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + return super.loadClass(name, resolve); + } + }); + } + return interp; } }); } diff --git a/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy b/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy index e45c1b86ceb..86f4df64db4 100644 --- a/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy @@ -20,4 +20,8 @@ grant { // needed to generate runtime classes permission java.lang.RuntimePermission "createClassLoader"; + // needed by PySystemState init (TODO: see if we can avoid this) + permission java.lang.RuntimePermission "getClassLoader"; + // Standard set of classes + permission org.elasticsearch.script.ClassPermission "<>"; }; diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java index 979da657c5f..e713bd67c92 100644 --- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.Arrays; import java.util.HashMap; @@ -40,7 +39,6 @@ import static org.hamcrest.Matchers.instanceOf; * */ public class PythonScriptEngineTests extends ESTestCase { - private PythonScriptEngineService se; @Before @@ -50,20 +48,15 @@ public class PythonScriptEngineTests extends ESTestCase { @After public void close() { - // We need to clear some system properties - System.clearProperty("python.cachedir.skip"); - System.clearProperty("python.console.encoding"); se.close(); } - @Test public void testSimpleEquation() { Map vars = new HashMap(); Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "python", se.compile("1 + 2")), vars).run(); assertThat(((Number) o).intValue(), equalTo(3)); } - @Test public void testMapAccess() { Map vars = new HashMap(); @@ -80,7 +73,6 @@ public class PythonScriptEngineTests extends ESTestCase { assertThat(((String) o), equalTo("2")); } - @Test public void testObjectMapInter() { Map vars = new HashMap(); Map ctx = new HashMap(); @@ -99,9 +91,7 @@ public class PythonScriptEngineTests extends ESTestCase { assertThat((String) ((Map) ctx.get("obj2")).get("prop2"), equalTo("value2")); } - @Test public void testAccessListInScript() { - Map vars = new HashMap(); Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).map(); @@ -122,7 +112,6 @@ public class PythonScriptEngineTests extends ESTestCase { assertThat(((String) o), equalTo("value1")); } - @Test public void testChangingVarsCrossExecution1() { Map vars = new HashMap(); Map ctx = new HashMap(); @@ -139,7 +128,6 @@ public class PythonScriptEngineTests extends ESTestCase { assertThat(((Number) o).intValue(), equalTo(2)); } - @Test public void testChangingVarsCrossExecution2() { Map vars = new HashMap(); Map ctx = new HashMap(); diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java index 81ebf69b62a..7b9663f6b6a 100644 --- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java @@ -24,8 +24,6 @@ import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; -import org.junit.After; -import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -41,14 +39,6 @@ import static org.hamcrest.Matchers.equalTo; */ public class PythonScriptMultiThreadedTests extends ESTestCase { - @After - public void close() { - // We need to clear some system properties - System.clearProperty("python.cachedir.skip"); - System.clearProperty("python.console.encoding"); - } - - @Test public void testExecutableNoRuntimeParams() throws Exception { final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); final Object compiled = se.compile("x + y"); @@ -93,7 +83,7 @@ public class PythonScriptMultiThreadedTests extends ESTestCase { } -// @Test public void testExecutableWithRuntimeParams() throws Exception { +// public void testExecutableWithRuntimeParams() throws Exception { // final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); // final Object compiled = se.compile("x + y"); // final AtomicBoolean failed = new AtomicBoolean(); @@ -135,7 +125,6 @@ public class PythonScriptMultiThreadedTests extends ESTestCase { // assertThat(failed.get(), equalTo(false)); // } - @Test public void testExecute() throws Exception { final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); final Object compiled = se.compile("x + y"); diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java index fd60607e2e6..e90ac503f13 100644 --- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java @@ -23,11 +23,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; -import org.junit.After; -import org.junit.Before; import org.python.core.PyException; +import java.text.DecimalFormatSymbols; import java.util.HashMap; +import java.util.Locale; import java.util.Map; /** @@ -37,17 +37,18 @@ public class PythonSecurityTests extends ESTestCase { private PythonScriptEngineService se; - @Before - public void setup() { + @Override + public void setUp() throws Exception { + super.setUp(); se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + // otherwise will exit your VM and other bad stuff + assumeTrue("test requires security manager to be enabled", System.getSecurityManager() != null); } - @After - public void close() { - // We need to clear some system properties - System.clearProperty("python.cachedir.skip"); - System.clearProperty("python.console.encoding"); + @Override + public void tearDown() throws Exception { se.close(); + super.tearDown(); } /** runs a script */ @@ -67,12 +68,12 @@ public class PythonSecurityTests extends ESTestCase { doTest(script); fail("did not get expected exception"); } catch (PyException expected) { - Throwable cause = expected.getCause(); // TODO: fix jython localization bugs: https://github.com/elastic/elasticsearch/issues/13967 - // this is the correct assert: - // assertNotNull("null cause for exception: " + expected, cause); - assertNotNull("null cause for exception", cause); - assertTrue("unexpected exception: " + cause, cause instanceof SecurityException); + // we do a gross hack for now + DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(Locale.getDefault()); + if (symbols.getZeroDigit() == '0') { + assertTrue(expected.toString().contains("cannot import")); + } } } @@ -92,4 +93,16 @@ public class PythonSecurityTests extends ESTestCase { // no files assertFailure("from java.io import File\nFile.createTempFile(\"test\", \"tmp\")"); } + + /** Test again from a new thread, python has complex threadlocal configuration */ + public void testNotOKFromSeparateThread() throws Exception { + Thread t = new Thread() { + @Override + public void run() { + assertFailure("from java.lang import Runtime\nRuntime.availableProcessors()"); + } + }; + t.start(); + t.join(); + } } diff --git a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml index ba7b733e806..4a811d1f26a 100644 --- a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml +++ b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/10_basic.yaml @@ -18,9 +18,10 @@ setup: body: script_fields: bar: - lang: python - script: "doc['foo'].value + x" - params: - x: "bbb" + script: + inline: "doc['foo'].value + x" + lang: python + params: + x: "bbb" - match: { hits.hits.0.fields.bar.0: "aaabbb"} diff --git a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml index d19561a549f..b0f18e17748 100644 --- a/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml +++ b/plugins/lang-python/src/test/resources/rest-api-spec/test/lang_python/20_search.yaml @@ -33,8 +33,9 @@ lang: python script_fields: sNum1: - lang: python - script: "doc['num1'].value" + script: + inline: "doc['num1'].value" + lang: python sort: num1: order: asc @@ -57,8 +58,9 @@ script_fields: sNum1: - lang: python - script: "doc['num1'].value" + script: + inline: "doc['num1'].value" + lang: python sort: num1: order: asc @@ -81,8 +83,9 @@ script_fields: sNum1: - lang: python - script: "doc['num1'].value" + script: + inline: "doc['num1'].value" + lang: python sort: num1: order: asc @@ -118,17 +121,21 @@ body: script_fields: s_obj1: - lang: python - script: "_source['obj1']" + script: + inline: "_source['obj1']" + lang: python s_obj1_test: - lang: python - script: "_source['obj1']['test']" + script: + inline: "_source['obj1']['test']" + lang: python s_obj2: - lang: python - script: "_source['obj2']" + script: + inline: "_source['obj2']" + lang: python s_obj2_arr2: - lang: python - script: "_source['obj2']['arr2']" + script: + inline: "_source['obj2']['arr2']" + lang: python - match: { hits.total: 1 } - match: { hits.hits.0.fields.s_obj1.0.test: something } diff --git a/plugins/mapper-attachments/build.gradle b/plugins/mapper-attachments/build.gradle new file mode 100644 index 00000000000..e14cf543043 --- /dev/null +++ b/plugins/mapper-attachments/build.gradle @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika.' + classname 'org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin' +} + +versions << [ + 'tika': '1.11', + 'pdfbox': '1.8.10', + 'bouncycastle': '1.52', + 'poi': '3.13' +] + +dependencies { + // mandatory for tika + compile "org.apache.tika:tika-core:${versions.tika}" + compile "org.apache.tika:tika-parsers:${versions.tika}" + compile 'commons-io:commons-io:2.4' + + // character set detection + compile 'com.googlecode.juniversalchardet:juniversalchardet:1.0.3' + + // external parser libraries + // HTML + compile 'org.ccil.cowan.tagsoup:tagsoup:1.2.1' + // Adobe PDF + compile "org.apache.pdfbox:pdfbox:${versions.pdfbox}" + compile "org.apache.pdfbox:fontbox:${versions.pdfbox}" + compile "org.apache.pdfbox:jempbox:${versions.pdfbox}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}" + compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}" + compile "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}" + // OpenOffice + compile "org.apache.poi:poi-ooxml:${versions.poi}" + compile "org.apache.poi:poi:${versions.poi}" + compile "org.apache.poi:poi-ooxml-schemas:${versions.poi}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile 'org.apache.xmlbeans:xmlbeans:2.6.0' + compile 'stax:stax-api:1.0.1' + // MS Office + compile "org.apache.poi:poi-scratchpad:${versions.poi}" + // Apple iWork + compile 'org.apache.commons:commons-compress:1.10' +} + +compileJava.options.compilerArgs << '-Xlint:-cast,-deprecation,-rawtypes' + +forbiddenPatterns { + exclude '**/*.docx' + exclude '**/*.pdf' + exclude '**/*.epub' +} diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1 new file mode 100644 index 00000000000..de084c948f4 --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1 @@ -0,0 +1 @@ +4995a870400e1554d1c7ed2afcb5d198fae12db9 diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-LICENSE.txt b/plugins/mapper-attachments/licenses/bcmail-jdk15on-LICENSE.txt new file mode 100644 index 00000000000..dbba1dd7829 --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcmail-jdk15on-LICENSE.txt @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. + (http://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-NOTICE.txt b/plugins/mapper-attachments/licenses/bcmail-jdk15on-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1 new file mode 100644 index 00000000000..489ceeaaf36 --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1 @@ -0,0 +1 @@ +b8ffac2bbc6626f86909589c8cc63637cc936504 diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-LICENSE.txt b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-LICENSE.txt new file mode 100644 index 00000000000..e1fc4a1506d --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-LICENSE.txt @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. + (http://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-NOTICE.txt b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1 new file mode 100644 index 00000000000..14ecc1be40b --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1 @@ -0,0 +1 @@ +88a941faf9819d371e3174b5ed56a3f3f7d73269 diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-LICENSE.txt b/plugins/mapper-attachments/licenses/bcprov-jdk15on-LICENSE.txt new file mode 100644 index 00000000000..e1fc4a1506d --- /dev/null +++ b/plugins/mapper-attachments/licenses/bcprov-jdk15on-LICENSE.txt @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. + (http://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-NOTICE.txt b/plugins/mapper-attachments/licenses/bcprov-jdk15on-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/commons-codec-1.10.jar.sha1 b/plugins/mapper-attachments/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/commons-codec-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-codec-LICENSE.txt new file mode 100644 index 00000000000..57bc88a15a0 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-codec-LICENSE.txt @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/plugins/mapper-attachments/licenses/commons-codec-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-codec-NOTICE.txt new file mode 100644 index 00000000000..72eb32a9024 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-codec-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons CLI +Copyright 2001-2009 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/mapper-attachments/licenses/commons-compress-1.10.jar.sha1 b/plugins/mapper-attachments/licenses/commons-compress-1.10.jar.sha1 new file mode 100644 index 00000000000..65c74b9a88f --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-compress-1.10.jar.sha1 @@ -0,0 +1 @@ +5eeb27c57eece1faf2d837868aeccc94d84dcc9a \ No newline at end of file diff --git a/distribution/licenses/snakeyaml-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-compress-LICENSE.txt similarity index 89% rename from distribution/licenses/snakeyaml-LICENSE.txt rename to plugins/mapper-attachments/licenses/commons-compress-LICENSE.txt index d9a10c0d8e8..261eeb9e9f8 100644 --- a/distribution/licenses/snakeyaml-LICENSE.txt +++ b/plugins/mapper-attachments/licenses/commons-compress-LICENSE.txt @@ -174,3 +174,28 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/commons-compress-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-compress-NOTICE.txt new file mode 100644 index 00000000000..edd2f2c78ee --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-compress-NOTICE.txt @@ -0,0 +1,11 @@ +Apache Commons Compress +Copyright 2002-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The files in the package org.apache.commons.compress.archivers.sevenz +were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), +which has been placed in the public domain: + +"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) diff --git a/plugins/mapper-attachments/licenses/commons-io-2.4.jar.sha1 b/plugins/mapper-attachments/licenses/commons-io-2.4.jar.sha1 new file mode 100644 index 00000000000..688318c938c --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-io-2.4.jar.sha1 @@ -0,0 +1 @@ +b1b6ea3b7e4aa4f492509a4952029cd8e48019ad diff --git a/plugins/mapper-attachments/licenses/commons-io-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-io-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-io-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/commons-io-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-io-NOTICE.txt new file mode 100644 index 00000000000..a6b77d1eb60 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-io-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons IO +Copyright 2002-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/mapper-attachments/licenses/commons-logging-1.1.3.jar.sha1 b/plugins/mapper-attachments/licenses/commons-logging-1.1.3.jar.sha1 new file mode 100644 index 00000000000..5b8f029e582 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-logging-1.1.3.jar.sha1 @@ -0,0 +1 @@ +f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/commons-logging-LICENSE.txt b/plugins/mapper-attachments/licenses/commons-logging-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-logging-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/commons-logging-NOTICE.txt b/plugins/mapper-attachments/licenses/commons-logging-NOTICE.txt new file mode 100644 index 00000000000..d3d6e140ce4 --- /dev/null +++ b/plugins/mapper-attachments/licenses/commons-logging-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons Logging +Copyright 2003-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1 new file mode 100644 index 00000000000..ce7f9f5d49c --- /dev/null +++ b/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1 @@ -0,0 +1 @@ +41776c7713e3f3a1ce688bd96459fc597298c340 diff --git a/plugins/mapper-attachments/licenses/fontbox-LICENSE.txt b/plugins/mapper-attachments/licenses/fontbox-LICENSE.txt new file mode 100644 index 00000000000..97553f24a43 --- /dev/null +++ b/plugins/mapper-attachments/licenses/fontbox-LICENSE.txt @@ -0,0 +1,344 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +EXTERNAL COMPONENTS + +Apache PDFBox includes a number of components with separate copyright notices +and license terms. Your use of these components is subject to the terms and +conditions of the following licenses. + +Contributions made to the original PDFBox and FontBox projects: + + Copyright (c) 2002-2007, www.pdfbox.org + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of pdfbox; nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Adobe Font Metrics (AFM) for PDF Core 14 Fonts + + This file and the 14 PostScript(R) AFM files it accompanies may be used, + copied, and distributed for any purpose and without charge, with or without + modification, provided that all copyright notices are retained; that the + AFM files are not distributed without this file; that all modifications + to this file or any of the AFM files are prominently noted in the modified + file(s); and that this paragraph is not modified. Adobe Systems has no + responsibility or obligation to support the use of the AFM files. + +CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads) + + Copyright 1990-2009 Adobe Systems Incorporated. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Adobe Systems Incorporated nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +PaDaF PDF/A preflight (http://sourceforge.net/projects/padaf) + + Copyright 2010 Atos Worldline SAS + + Licensed by Atos Worldline SAS under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + Atos Worldline SAS licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +OSXAdapter + + Version: 2.0 + + Disclaimer: IMPORTANT: This Apple software is supplied to you by + Apple Inc. ("Apple") in consideration of your agreement to the + following terms, and your use, installation, modification or + redistribution of this Apple software constitutes acceptance of these + terms. If you do not agree with these terms, please do not use, + install, modify or redistribute this Apple software. + + In consideration of your agreement to abide by the following terms, and + subject to these terms, Apple grants you a personal, non-exclusive + license, under Apple's copyrights in this original Apple software (the + "Apple Software"), to use, reproduce, modify and redistribute the Apple + Software, with or without modifications, in source and/or binary forms; + provided that if you redistribute the Apple Software in its entirety and + without modifications, you must retain this notice and the following + text and disclaimers in all such redistributions of the Apple Software. + Neither the name, trademarks, service marks or logos of Apple Inc. + may be used to endorse or promote products derived from the Apple + Software without specific prior written permission from Apple. Except + as expressly stated in this notice, no other rights or licenses, express + or implied, are granted by Apple herein, including but not limited to + any patent rights that may be infringed by your derivative works or by + other works in which the Apple Software may be incorporated. + + The Apple Software is provided by Apple on an "AS IS" basis. APPLE + MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION + THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND + OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS. + + IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, + MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED + AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE), + STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/fontbox-NOTICE.txt b/plugins/mapper-attachments/licenses/fontbox-NOTICE.txt new file mode 100644 index 00000000000..3c857082561 --- /dev/null +++ b/plugins/mapper-attachments/licenses/fontbox-NOTICE.txt @@ -0,0 +1,22 @@ +Apache PDFBox +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Based on source code originally developed in the PDFBox and +FontBox projects. + +Copyright (c) 2002-2007, www.pdfbox.org + +Based on source code originally developed in the PaDaF project. +Copyright (c) 2010 Atos Worldline SAS + +Includes the Adobe Glyph List +Copyright 1997, 1998, 2002, 2007, 2010 Adobe Systems Incorporated. + +Includes the Zapf Dingbats Glyph List +Copyright 2002, 2010 Adobe Systems Incorporated. + +Includes OSXAdapter +Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1 new file mode 100644 index 00000000000..5a7b1997208 --- /dev/null +++ b/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1 @@ -0,0 +1 @@ +40df4e4ca884aadc20b82d5abd0a3679774c55a6 diff --git a/plugins/mapper-attachments/licenses/jempbox-LICENSE.txt b/plugins/mapper-attachments/licenses/jempbox-LICENSE.txt new file mode 100644 index 00000000000..1cf412f9c57 --- /dev/null +++ b/plugins/mapper-attachments/licenses/jempbox-LICENSE.txt @@ -0,0 +1,25 @@ +Copyright (c) 2006-2007, www.jempbox.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of fontbox; nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plugins/mapper-attachments/licenses/jempbox-NOTICE.txt b/plugins/mapper-attachments/licenses/jempbox-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/juniversalchardet-1.0.3.jar.sha1 b/plugins/mapper-attachments/licenses/juniversalchardet-1.0.3.jar.sha1 new file mode 100644 index 00000000000..6b06952678f --- /dev/null +++ b/plugins/mapper-attachments/licenses/juniversalchardet-1.0.3.jar.sha1 @@ -0,0 +1 @@ +cd49678784c46aa8789c060538e0154013bb421b diff --git a/plugins/mapper-attachments/licenses/juniversalchardet-LICENSE.txt b/plugins/mapper-attachments/licenses/juniversalchardet-LICENSE.txt new file mode 100644 index 00000000000..06f965147a8 --- /dev/null +++ b/plugins/mapper-attachments/licenses/juniversalchardet-LICENSE.txt @@ -0,0 +1,469 @@ + MOZILLA PUBLIC LICENSE + Version 1.1 + + --------------- + +1. Definitions. + + 1.0.1. "Commercial Use" means distribution or otherwise making the + Covered Code available to a third party. + + 1.1. "Contributor" means each entity that creates or contributes to + the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Code, prior Modifications used by a Contributor, and the Modifications + made by that particular Contributor. + + 1.3. "Covered Code" means the Original Code or Modifications or the + combination of the Original Code and Modifications, in each case + including portions thereof. + + 1.4. "Electronic Distribution Mechanism" means a mechanism generally + accepted in the software development community for the electronic + transfer of data. + + 1.5. "Executable" means Covered Code in any form other than Source + Code. + + 1.6. "Initial Developer" means the individual or entity identified + as the Initial Developer in the Source Code notice required by Exhibit + A. + + 1.7. "Larger Work" means a work which combines Covered Code or + portions thereof with code not governed by the terms of this License. + + 1.8. "License" means this document. + + 1.8.1. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means any addition to or deletion from the + substance or structure of either the Original Code or any previous + Modifications. When Covered Code is released as a series of files, a + Modification is: + A. Any addition to or deletion from the contents of a file + containing Original Code or previous Modifications. + + B. Any new file that contains any part of the Original Code or + previous Modifications. + + 1.10. "Original Code" means Source Code of computer software code + which is described in the Source Code notice required by Exhibit A as + Original Code, and which, at the time of its release under this + License is not already Covered Code governed by this License. + + 1.10.1. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.11. "Source Code" means the preferred form of the Covered Code for + making modifications to it, including all modules it contains, plus + any associated interface definition files, scripts used to control + compilation and installation of an Executable, or source code + differential comparisons against either the Original Code or another + well known, available Covered Code of the Contributor's choice. The + Source Code can be in a compressed or archival form, provided the + appropriate decompression or de-archiving software is widely available + for no charge. + + 1.12. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, this + License or a future version of this License issued under Section 6.1. + For legal entities, "You" includes any entity which controls, is + controlled by, or is under common control with You. For purposes of + this definition, "control" means (a) the power, direct or indirect, + to cause the direction or management of such entity, whether by + contract or otherwise, or (b) ownership of more than fifty percent + (50%) of the outstanding shares or beneficial ownership of such + entity. + +2. Source Code License. + + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, + non-exclusive license, subject to third party intellectual property + claims: + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Code (or portions thereof) with or without Modifications, and/or + as part of a Larger Work; and + + (b) under Patents Claims infringed by the making, using or + selling of Original Code, to make, have made, use, practice, + sell, and offer for sale, and/or otherwise dispose of the + Original Code (or portions thereof). + + (c) the licenses granted in this Section 2.1(a) and (b) are + effective on the date Initial Developer first distributes + Original Code under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: 1) for code that You delete from the Original Code; 2) + separate from the Original Code; or 3) for infringements caused + by: i) the modification of the Original Code or ii) the + combination of the Original Code with other software or devices. + + 2.2. Contributor Grant. + Subject to third party intellectual property claims, each Contributor + hereby grants You a world-wide, royalty-free, non-exclusive license + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor, to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof) either on an + unmodified basis, with other Modifications, as Covered Code + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or + selling of Modifications made by that Contributor either alone + and/or in combination with its Contributor Version (or portions + of such combination), to make, use, sell, offer for sale, have + made, and/or otherwise dispose of: 1) Modifications made by that + Contributor (or portions thereof); and 2) the combination of + Modifications made by that Contributor with its Contributor + Version (or portions of such combination). + + (c) the licenses granted in Sections 2.2(a) and 2.2(b) are + effective on the date Contributor first makes Commercial Use of + the Covered Code. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: 1) for any code that Contributor has deleted from the + Contributor Version; 2) separate from the Contributor Version; + 3) for infringements caused by: i) third party modifications of + Contributor Version or ii) the combination of Modifications made + by that Contributor with other software (except as part of the + Contributor Version) or other devices; or 4) under Patent Claims + infringed by Covered Code in the absence of Modifications made by + that Contributor. + +3. Distribution Obligations. + + 3.1. Application of License. + The Modifications which You create or to which You contribute are + governed by the terms of this License, including without limitation + Section 2.2. The Source Code version of Covered Code may be + distributed only under the terms of this License or a future version + of this License released under Section 6.1, and You must include a + copy of this License with every copy of the Source Code You + distribute. You may not offer or impose any terms on any Source Code + version that alters or restricts the applicable version of this + License or the recipients' rights hereunder. However, You may include + an additional document offering the additional rights described in + Section 3.5. + + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be + made available in Source Code form under the terms of this License + either on the same media as an Executable version or via an accepted + Electronic Distribution Mechanism to anyone to whom you made an + Executable version available; and if made available via Electronic + Distribution Mechanism, must remain available for at least twelve (12) + months after the date it initially became available, or at least six + (6) months after a subsequent version of that particular Modification + has been made available to such recipients. You are responsible for + ensuring that the Source Code version remains available even if the + Electronic Distribution Mechanism is maintained by a third party. + + 3.3. Description of Modifications. + You must cause all Covered Code to which You contribute to contain a + file documenting the changes You made to create that Covered Code and + the date of any change. You must include a prominent statement that + the Modification is derived, directly or indirectly, from Original + Code provided by the Initial Developer and including the name of the + Initial Developer in (a) the Source Code, and (b) in any notice in an + Executable version or related documentation in which You describe the + origin or ownership of the Covered Code. + + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If Contributor has knowledge that a license under a third party's + intellectual property rights is required to exercise the rights + granted by such Contributor under Sections 2.1 or 2.2, + Contributor must include a text file with the Source Code + distribution titled "LEGAL" which describes the claim and the + party making the claim in sufficient detail that a recipient will + know whom to contact. If Contributor obtains such knowledge after + the Modification is made available as described in Section 3.2, + Contributor shall promptly modify the LEGAL file in all copies + Contributor makes available thereafter and shall take other steps + (such as notifying appropriate mailing lists or newsgroups) + reasonably calculated to inform those who received the Covered + Code that new knowledge has been obtained. + + (b) Contributor APIs. + If Contributor's Modifications include an application programming + interface and Contributor has knowledge of patent licenses which + are reasonably necessary to implement that API, Contributor must + also include this information in the LEGAL file. + + (c) Representations. + Contributor represents that, except as disclosed pursuant to + Section 3.4(a) above, Contributor believes that Contributor's + Modifications are Contributor's original creation(s) and/or + Contributor has sufficient rights to grant the rights conveyed by + this License. + + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source + Code. If it is not possible to put such notice in a particular Source + Code file due to its structure, then You must include such notice in a + location (such as a relevant directory) where a user would be likely + to look for such a notice. If You created one or more Modification(s) + You may add your name as a Contributor to the notice described in + Exhibit A. You must also duplicate this License in any documentation + for the Source Code where You describe recipients' rights or ownership + rights relating to Covered Code. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Code. However, You + may do so only on Your own behalf, and not on behalf of the Initial + Developer or any Contributor. You must make it absolutely clear than + any such warranty, support, indemnity or liability obligation is + offered by You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred by the + Initial Developer or such Contributor as a result of warranty, + support, indemnity or liability terms You offer. + + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the + requirements of Section 3.1-3.5 have been met for that Covered Code, + and if You include a notice stating that the Source Code version of + the Covered Code is available under the terms of this License, + including a description of how and where You have fulfilled the + obligations of Section 3.2. The notice must be conspicuously included + in any notice in an Executable version, related documentation or + collateral in which You describe recipients' rights relating to the + Covered Code. You may distribute the Executable version of Covered + Code or ownership rights under a license of Your choice, which may + contain terms different from this License, provided that You are in + compliance with the terms of this License and that the license for the + Executable version does not attempt to limit or alter the recipient's + rights in the Source Code version from the rights set forth in this + License. If You distribute the Executable version under a different + license You must make it absolutely clear that any terms which differ + from this License are offered by You alone, not by the Initial + Developer or any Contributor. You hereby agree to indemnify the + Initial Developer and every Contributor for any liability incurred by + the Initial Developer or such Contributor as a result of any such + terms You offer. + + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code + not governed by the terms of this License and distribute the Larger + Work as a single product. In such a case, You must make sure the + requirements of this License are fulfilled for the Covered Code. + +4. Inability to Comply Due to Statute or Regulation. + + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Code due to + statute, judicial order, or regulation then You must: (a) comply with + the terms of this License to the maximum extent possible; and (b) + describe the limitations and the code they affect. Such description + must be included in the LEGAL file described in Section 3.4 and must + be included with all distributions of the Source Code. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Application of this License. + + This License applies to code to which the Initial Developer has + attached the notice in Exhibit A and to related Covered Code. + +6. Versions of the License. + + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised + and/or new versions of the License from time to time. Each version + will be given a distinguishing version number. + + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the + License, You may always continue to use it under the terms of that + version. You may also choose to use such Covered Code under the terms + of any subsequent version of the License published by Netscape. No one + other than Netscape has the right to modify the terms applicable to + Covered Code created under this License. + + 6.3. Derivative Works. + If You create or use a modified version of this License (which you may + only do in order to apply it to code which is not already Covered Code + governed by this License), You must (a) rename Your license so that + the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", + "MPL", "NPL" or any confusingly similar phrase do not appear in your + license (except to note that your license differs from this License) + and (b) otherwise make it clear that Your version of the license + contains terms which differ from the Mozilla Public License and + Netscape Public License. (Filling in the name of the Initial + Developer, Original Code or Contributor in the notice described in + Exhibit A shall not of themselves be deemed to be modifications of + this License.) + +7. DISCLAIMER OF WARRANTY. + + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, + WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF + DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. + THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE + IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, + YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE + COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER + OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +8. TERMINATION. + + 8.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to cure + such breach within 30 days of becoming aware of the breach. All + sublicenses to the Covered Code which are properly granted shall + survive any termination of this License. Provisions which, by their + nature, must remain in effect beyond the termination of this License + shall survive. + + 8.2. If You initiate litigation by asserting a patent infringement + claim (excluding declatory judgment actions) against Initial Developer + or a Contributor (the Initial Developer or Contributor against whom + You file such action is referred to as "Participant") alleging that: + + (a) such Participant's Contributor Version directly or indirectly + infringes any patent, then any and all rights granted by such + Participant to You under Sections 2.1 and/or 2.2 of this License + shall, upon 60 days notice from Participant terminate prospectively, + unless if within 60 days after receipt of notice You either: (i) + agree in writing to pay Participant a mutually agreeable reasonable + royalty for Your past and future use of Modifications made by such + Participant, or (ii) withdraw Your litigation claim with respect to + the Contributor Version against such Participant. If within 60 days + of notice, a reasonable royalty and payment arrangement are not + mutually agreed upon in writing by the parties or the litigation claim + is not withdrawn, the rights granted by Participant to You under + Sections 2.1 and/or 2.2 automatically terminate at the expiration of + the 60 day notice period specified above. + + (b) any software, hardware, or device, other than such Participant's + Contributor Version, directly or indirectly infringes any patent, then + any rights granted to You by such Participant under Sections 2.1(b) + and 2.2(b) are revoked effective as of the date You first made, used, + sold, distributed, or had made, Modifications made by that + Participant. + + 8.3. If You assert a patent infringement claim against Participant + alleging that such Participant's Contributor Version directly or + indirectly infringes any patent where such claim is resolved (such as + by license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 8.4. In the event of termination under Sections 8.1 or 8.2 above, + all end user license agreements (excluding distributors and resellers) + which have been validly granted by You or any distributor hereunder + prior to termination shall survive termination. + +9. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL + DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, + OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR + ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY + CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, + WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY + RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW + PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE + EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO + THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +10. U.S. GOVERNMENT END USERS. + + The Covered Code is a "commercial item," as that term is defined in + 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" and "commercial computer software documentation," as such + terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 + C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), + all U.S. Government End Users acquire Covered Code with only those + rights set forth herein. + +11. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + California law provisions (except to the extent applicable law, if + any, provides otherwise), excluding its conflict-of-law provisions. + With respect to disputes in which at least one party is a citizen of, + or an entity chartered or registered to do business in the United + States of America, any litigation relating to this License shall be + subject to the jurisdiction of the Federal Courts of the Northern + District of California, with venue lying in Santa Clara County, + California, with the losing party responsible for costs, including + without limitation, court costs and reasonable attorneys' fees and + expenses. The application of the United Nations Convention on + Contracts for the International Sale of Goods is expressly excluded. + Any law or regulation which provides that the language of a contract + shall be construed against the drafter shall not apply to this + License. + +12. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. + +13. MULTIPLE-LICENSED CODE. + + Initial Developer may designate portions of the Covered Code as + "Multiple-Licensed". "Multiple-Licensed" means that the Initial + Developer permits you to utilize portions of the Covered Code under + Your choice of the NPL or the alternative licenses, if any, specified + by the Initial Developer in the file described in Exhibit A. + +EXHIBIT A -Mozilla Public License. + + ``The contents of this file are subject to the Mozilla Public License + Version 1.1 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at + http://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + License for the specific language governing rights and limitations + under the License. + + The Original Code is ______________________________________. + + The Initial Developer of the Original Code is ________________________. + Portions created by ______________________ are Copyright (C) ______ + _______________________. All Rights Reserved. + + Contributor(s): ______________________________________. + + Alternatively, the contents of this file may be used under the terms + of the _____ license (the "[___] License"), in which case the + provisions of [______] License are applicable instead of those + above. If you wish to allow use of your version of this file only + under the terms of the [____] License and not to allow others to use + your version of this file under the MPL, indicate your decision by + deleting the provisions above and replace them with the notice and + other provisions required by the [___] License. If you do not delete + the provisions above, a recipient may use your version of this file + under either the MPL or the [___] License." + + [NOTE: The text of this Exhibit A may differ slightly from the text of + the notices in the Source Code files of the Original Code. You should + use the text of this Exhibit A rather than the text found in the + Original Code Source Code for Your Modifications.] diff --git a/plugins/mapper-attachments/licenses/juniversalchardet-NOTICE.txt b/plugins/mapper-attachments/licenses/juniversalchardet-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1 new file mode 100644 index 00000000000..98ce1f9d98c --- /dev/null +++ b/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1 @@ -0,0 +1 @@ +bc5d1254495be36d0a3b3d6c35f88d05200b9311 diff --git a/plugins/mapper-attachments/licenses/pdfbox-LICENSE.txt b/plugins/mapper-attachments/licenses/pdfbox-LICENSE.txt new file mode 100644 index 00000000000..97553f24a43 --- /dev/null +++ b/plugins/mapper-attachments/licenses/pdfbox-LICENSE.txt @@ -0,0 +1,344 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +EXTERNAL COMPONENTS + +Apache PDFBox includes a number of components with separate copyright notices +and license terms. Your use of these components is subject to the terms and +conditions of the following licenses. + +Contributions made to the original PDFBox and FontBox projects: + + Copyright (c) 2002-2007, www.pdfbox.org + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of pdfbox; nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Adobe Font Metrics (AFM) for PDF Core 14 Fonts + + This file and the 14 PostScript(R) AFM files it accompanies may be used, + copied, and distributed for any purpose and without charge, with or without + modification, provided that all copyright notices are retained; that the + AFM files are not distributed without this file; that all modifications + to this file or any of the AFM files are prominently noted in the modified + file(s); and that this paragraph is not modified. Adobe Systems has no + responsibility or obligation to support the use of the AFM files. + +CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads) + + Copyright 1990-2009 Adobe Systems Incorporated. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Adobe Systems Incorporated nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +PaDaF PDF/A preflight (http://sourceforge.net/projects/padaf) + + Copyright 2010 Atos Worldline SAS + + Licensed by Atos Worldline SAS under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + Atos Worldline SAS licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +OSXAdapter + + Version: 2.0 + + Disclaimer: IMPORTANT: This Apple software is supplied to you by + Apple Inc. ("Apple") in consideration of your agreement to the + following terms, and your use, installation, modification or + redistribution of this Apple software constitutes acceptance of these + terms. If you do not agree with these terms, please do not use, + install, modify or redistribute this Apple software. + + In consideration of your agreement to abide by the following terms, and + subject to these terms, Apple grants you a personal, non-exclusive + license, under Apple's copyrights in this original Apple software (the + "Apple Software"), to use, reproduce, modify and redistribute the Apple + Software, with or without modifications, in source and/or binary forms; + provided that if you redistribute the Apple Software in its entirety and + without modifications, you must retain this notice and the following + text and disclaimers in all such redistributions of the Apple Software. + Neither the name, trademarks, service marks or logos of Apple Inc. + may be used to endorse or promote products derived from the Apple + Software without specific prior written permission from Apple. Except + as expressly stated in this notice, no other rights or licenses, express + or implied, are granted by Apple herein, including but not limited to + any patent rights that may be infringed by your derivative works or by + other works in which the Apple Software may be incorporated. + + The Apple Software is provided by Apple on an "AS IS" basis. APPLE + MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION + THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND + OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS. + + IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, + MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED + AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE), + STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/pdfbox-NOTICE.txt b/plugins/mapper-attachments/licenses/pdfbox-NOTICE.txt new file mode 100644 index 00000000000..3c857082561 --- /dev/null +++ b/plugins/mapper-attachments/licenses/pdfbox-NOTICE.txt @@ -0,0 +1,22 @@ +Apache PDFBox +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Based on source code originally developed in the PDFBox and +FontBox projects. + +Copyright (c) 2002-2007, www.pdfbox.org + +Based on source code originally developed in the PaDaF project. +Copyright (c) 2010 Atos Worldline SAS + +Includes the Adobe Glyph List +Copyright 1997, 1998, 2002, 2007, 2010 Adobe Systems Incorporated. + +Includes the Zapf Dingbats Glyph List +Copyright 2002, 2010 Adobe Systems Incorporated. + +Includes OSXAdapter +Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1 new file mode 100644 index 00000000000..09063c1e5e0 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1 @@ -0,0 +1 @@ +0f59f504ba8c521e61e25f417ec652fd485010f3 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1 new file mode 100644 index 00000000000..16784299855 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1 @@ -0,0 +1 @@ +c364a8f5422d613e3a56db3b4b889f2989d7ee73 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1 new file mode 100644 index 00000000000..b5a3a05c489 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1 @@ -0,0 +1 @@ +56fb0b9f3ffc3d7f7fc9b59e17b5fa2c3ab921e7 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1 new file mode 100644 index 00000000000..cc61780e2a5 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1 @@ -0,0 +1 @@ +09d763275e6c7fa05d47e2581606748669e88c55 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-LICENSE.txt b/plugins/mapper-attachments/licenses/poi-scratchpad-LICENSE.txt new file mode 100644 index 00000000000..dd2cbd5fbc1 --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-scratchpad-LICENSE.txt @@ -0,0 +1,463 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE POI SUBCOMPONENTS: + +Apache POI includes subcomponents with separate copyright notices and +license terms. Your use of these subcomponents is subject to the terms +and conditions of the following licenses: + + +Office Open XML schemas (ooxml-schemas-1.1.jar) + + The Office Open XML schema definitions used by Apache POI are + a part of the Office Open XML ECMA Specification (ECMA-376, [1]). + As defined in section 9.4 of the ECMA bylaws [2], this specification + is available to all interested parties without restriction: + + 9.4 All documents when approved shall be made available to + all interested parties without restriction. + + Furthermore, both Microsoft and Adobe have granted patent licenses + to this work [3,4,5]. + + [1] http://www.ecma-international.org/publications/standards/Ecma-376.htm + [2] http://www.ecma-international.org/memento/Ecmabylaws.htm + [3] http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx + [4] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf + [5] http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf + + +JUnit test library (junit-4.11.jar) + + Common Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from + and are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents " mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license + to reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, in + source code and object code form. This patent license shall apply to + the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the rights + and licenses granted hereunder, each Recipient hereby assumes sole + responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license before + distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties + or conditions of merchantability and fitness for a particular + purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a + reasonable manner on or through a medium customarily used for + software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of + the Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product offering + should do so in a manner which does not create potential liability for + other Contributors. Therefore, if a Contributor includes the Program + in a commercial product offering, such Contributor ("Commercial + Contributor") hereby agrees to defend and indemnify every other + Contributor ("Indemnified Contributor") against any losses, damages + and costs (collectively "Losses") arising from claims, lawsuits and + other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the + Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual + or alleged intellectual property infringement. In order to qualify, + an Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The Indemnified + Contributor may participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED + ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR + CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR + A PARTICULAR PURPOSE. Each Recipient is solely responsible for + determining the appropriateness of using and distributing the Program + and assumes all risks associated with its exercise of rights under this + Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, + programs or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against a Contributor with + respect to a patent applicable to software (including a cross-claim or + counterclaim in a lawsuit), then any patent licenses granted by that + Contributor to such Recipient under this Agreement shall terminate as of + the date such litigation is filed. In addition, if Recipient institutes + patent litigation against any entity (including a cross-claim or + counterclaim in a lawsuit) alleging that the Program itself (excluding + combinations of the Program with other software or hardware) infringes + such Recipient's patent(s), then such Recipient's rights granted under + Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this Agreement + and does not cure such failure in a reasonable period of time after + becoming aware of such noncompliance. If all Recipient's rights under + this Agreement terminate, Recipient agrees to cease use and distribution + of the Program as soon as reasonably practicable. However, Recipient's + obligations under this Agreement and any licenses granted by Recipient + relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and may + only be modified in the following manner. The Agreement Steward reserves + the right to publish new versions (including revisions) of this Agreement + from time to time. No one other than the Agreement Steward has the right + to modify this Agreement. IBM is the initial Agreement Steward. IBM may + assign the responsibility to serve as the Agreement Steward to a suitable + separate entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) may + always be distributed subject to the version of the Agreement under which + it was received. In addition, after a new version of the Agreement is + published, Contributor may elect to distribute the Program (including + its Contributions) under the new version. Except as expressly stated in + Sections 2(a) and 2(b) above, Recipient receives no rights or licenses + to the intellectual property of any Contributor under this Agreement, + whether expressly, by implication, estoppel or otherwise. All rights in + the Program not expressly granted under this Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-NOTICE.txt b/plugins/mapper-attachments/licenses/poi-scratchpad-NOTICE.txt new file mode 100644 index 00000000000..12ff265290d --- /dev/null +++ b/plugins/mapper-attachments/licenses/poi-scratchpad-NOTICE.txt @@ -0,0 +1,23 @@ +Apache POI +Copyright 2003-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product contains parts that were originally based on software from BEA. +Copyright (c) 2000-2003, BEA Systems, . + +This product contains W3C XML Schema documents. Copyright 2001-2003 (c) +World Wide Web Consortium (Massachusetts Institute of Technology, European +Research Consortium for Informatics and Mathematics, Keio University) + +This product contains the Piccolo XML Parser for Java +(http://piccolo.sourceforge.net/). Copyright 2002 Yuval Oren. + +This product contains the chunks_parse_cmds.tbl file from the vsdump program. +Copyright (C) 2006-2007 Valek Filippov (frob@df.ru) + +This product contains parts of the eID Applet project +(http://eid-applet.googlecode.com). Copyright (c) 2009-2014 +FedICT (federal ICT department of Belgium), e-Contract.be BVBA (https://www.e-contract.be), +Bart Hanssens from FedICT diff --git a/plugins/mapper-attachments/licenses/stax-api-1.0.1.jar.sha1 b/plugins/mapper-attachments/licenses/stax-api-1.0.1.jar.sha1 new file mode 100644 index 00000000000..4426e34685d --- /dev/null +++ b/plugins/mapper-attachments/licenses/stax-api-1.0.1.jar.sha1 @@ -0,0 +1 @@ +49c100caf72d658aca8e58bd74a4ba90fa2b0d70 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/stax-api-LICENSE.txt b/plugins/mapper-attachments/licenses/stax-api-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/mapper-attachments/licenses/stax-api-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/stax-api-NOTICE.txt b/plugins/mapper-attachments/licenses/stax-api-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/tagsoup-1.2.1.jar.sha1 b/plugins/mapper-attachments/licenses/tagsoup-1.2.1.jar.sha1 new file mode 100644 index 00000000000..5d227b11a0f --- /dev/null +++ b/plugins/mapper-attachments/licenses/tagsoup-1.2.1.jar.sha1 @@ -0,0 +1 @@ +5584627487e984c03456266d3f8802eb85a9ce97 diff --git a/plugins/mapper-attachments/licenses/tagsoup-LICENSE.txt b/plugins/mapper-attachments/licenses/tagsoup-LICENSE.txt new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tagsoup-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/tagsoup-NOTICE.txt b/plugins/mapper-attachments/licenses/tagsoup-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1 b/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1 new file mode 100644 index 00000000000..a6dfd778a9c --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1 @@ -0,0 +1 @@ +d37a6b9080c8361e47b2050f69833fd61501ede9 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/tika-core-LICENSE.txt b/plugins/mapper-attachments/licenses/tika-core-LICENSE.txt new file mode 100644 index 00000000000..9537d733ea9 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-core-LICENSE.txt @@ -0,0 +1,372 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +APACHE TIKA SUBCOMPONENTS + +Apache Tika includes a number of subcomponents with separate copyright notices +and license terms. Your use of these subcomponents is subject to the terms and +conditions of the following licenses. + +MIME type information from file-4.26.tar.gz (http://www.darwinsys.com/file/) + + Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995. + Software written by Ian F. Darwin and others; + maintained 1994- Christos Zoulas. + + This software is not subject to any export provision of the United States + Department of Commerce, and may be exported to any country or planet. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice immediately at the beginning of the file, without modification, + this list of conditions, and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Charset detection code from ICU4J (http://site.icu-project.org/) + + Copyright (c) 1995-2009 International Business Machines Corporation + and others + + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, and/or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, provided that the above + copyright notice(s) and this permission notice appear in all copies + of the Software and that both the above copyright notice(s) and this + permission notice appear in supporting documentation. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE + BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, + OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, + ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS + SOFTWARE. + + Except as contained in this notice, the name of a copyright holder shall + not be used in advertising or otherwise to promote the sale, use or other + dealings in this Software without prior written authorization of the + copyright holder. + + +Parsing functionality provided by the NetCDF Java Library (http://www.unidata.ucar.edu/software/netcdf-java/) + + Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata + + Portions of this software were developed by the Unidata Program at the University + Corporation for Atmospheric Research. + + Access and use of this software shall impose the following obligations and understandings + on the user. The user is granted the right, without any fee or cost, to use, copy, modify, + alter, enhance and distribute this software, and any derivative works thereof, and its + supporting documentation for any purpose whatsoever, provided that this entire notice + appears in all copies of the software, derivative works and supporting documentation. Further, + UCAR requests that the user credit UCAR/Unidata in any publications that result from the use + of this software or in any product that includes this software, although this is not an obligation. + The names UCAR and/or Unidata, however, may not be used in any advertising or publicity to endorse + or promote any products or commercial entity unless specific written permission is obtained from + UCAR/Unidata. The user also understands that UCAR/Unidata is not obligated to provide the user with + any support, consulting, training or assistance of any kind with regard to the use, operation and + performance of this software nor to provide the user with any updates, revisions, new versions or + "bug fixes." + + THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, + BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE ACCESS, + USE OR PERFORMANCE OF THIS SOFTWARE. + + +IPTC Photo Metadata descriptions are taken from the IPTC Photo Metadata +Standard, July 2010, Copyright 2010 International Press Telecommunications +Council. + + 1. The Specifications and Materials are licensed for use only on the condition that you agree to be bound by the terms of this license. Subject to this and other licensing requirements contained herein, you may, on a non-exclusive basis, use the Specifications and Materials. + 2. The IPTC openly provides the Specifications and Materials for voluntary use by individuals, partnerships, companies, corporations, organizations and any other entity for use at the entity's own risk. This disclaimer, license and release is intended to apply to the IPTC, its officers, directors, agents, representatives, members, contributors, affiliates, contractors, or co-venturers acting jointly or severally. + 3. The Document and translations thereof may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the copyright and license notices and references to the IPTC appearing in the Document and the terms of this Specifications License Agreement are included on all such copies and derivative works. Further, upon the receipt of written permission from the IPTC, the Document may be modified for the purpose of developing applications that use IPTC Specifications or as required to translate the Document into languages other than English. + 4. Any use, duplication, distribution, or exploitation of the Document and Specifications and Materials in any manner is at your own risk. + 5. NO WARRANTY, EXPRESSED OR IMPLIED, IS MADE REGARDING THE ACCURACY, ADEQUACY, COMPLETENESS, LEGALITY, RELIABILITY OR USEFULNESS OF ANY INFORMATION CONTAINED IN THE DOCUMENT OR IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE PRODUCED OR SPONSORED BY THE IPTC. THE DOCUMENT AND THE INFORMATION CONTAINED HEREIN AND INCLUDED IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE OF THE IPTC IS PROVIDED ON AN "AS IS" BASIS. THE IPTC DISCLAIMS ALL WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY ACTUAL OR ASSERTED WARRANTY OF NON-INFRINGEMENT OF PROPRIETARY RIGHTS, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. NEITHER THE IPTC NOR ITS CONTRIBUTORS SHALL BE HELD LIABLE FOR ANY IMPROPER OR INCORRECT USE OF INFORMATION. NEITHER THE IPTC NOR ITS CONTRIBUTORS ASSUME ANY RESPONSIBILITY FOR ANYONE'S USE OF INFORMATION PROVIDED BY THE IPTC. IN NO EVENT SHALL THE IPTC OR ITS CONTRIBUTORS BE LIABLE TO ANYONE FOR DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO, COMPENSATORY DAMAGES, LOST PROFITS, LOST DATA OR ANY FORM OF SPECIAL, INCIDENTAL, INDIRECT, CONSEQUENTIAL OR PUNITIVE DAMAGES OF ANY KIND WHETHER BASED ON BREACH OF CONTRACT OR WARRANTY, TORT, PRODUCT LIABILITY OR OTHERWISE. + 6. The IPTC takes no position regarding the validity or scope of any Intellectual Property or other rights that might be claimed to pertain to the implementation or use of the technology described in the Document or the extent to which any license under such rights might or might not be available. The IPTC does not represent that it has made any effort to identify any such rights. Copies of claims of rights made available for publication, assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of the Specifications and Materials, can be obtained from the Managing Director of the IPTC. + 7. By using the Specifications and Materials including the Document in any manner or for any purpose, you release the IPTC from all liabilities, claims, causes of action, allegations, losses, injuries, damages, or detriments of any nature arising from or relating to the use of the Specifications, Materials or any portion thereof. You further agree not to file a lawsuit, make a claim, or take any other formal or informal legal action against the IPTC, resulting from your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. Finally, you hereby agree that the IPTC is not liable for any direct, indirect, special or consequential damages arising from or relating to your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. + 8. Specifications and Materials may be downloaded or copied provided that ALL copies retain the ownership, copyright and license notices. + 9. Materials may not be edited, modified, or presented in a context that creates a misleading or false impression or statement as to the positions, actions, or statements of the IPTC. + 10. The name and trademarks of the IPTC may not be used in advertising, publicity, or in relation to products or services and their names without the specific, written prior permission of the IPTC. Any permitted use of the trademarks of the IPTC, whether registered or not, shall be accompanied by an appropriate mark and attribution, as agreed with the IPTC. + 11. Specifications may be extended by both members and non-members to provide additional functionality (Extension Specifications) provided that there is a clear recognition of the IPTC IP and its ownership in the Extension Specifications and the related documentation and provided that the extensions are clearly identified and provided that a perpetual license is granted by the creator of the Extension Specifications for other members and non-members to use the Extension Specifications and to continue extensions of the Extension Specifications. The IPTC does not waive any of its rights in the Specifications and Materials in this context. The Extension Specifications may be considered the intellectual property of their creator. The IPTC expressly disclaims any responsibility for damage caused by an extension to the Specifications. + 12. Specifications and Materials may be included in derivative work of both members and non-members provided that there is a clear recognition of the IPTC IP and its ownership in the derivative work and its related documentation. The IPTC does not waive any of its rights in the Specifications and Materials in this context. Derivative work in its entirety may be considered the intellectual property of the creator of the work .The IPTC expressly disclaims any responsibility for damage caused when its IP is used in a derivative context. + 13. This Specifications License Agreement is perpetual subject to your conformance to the terms of this Agreement. The IPTC may terminate this Specifications License Agreement immediately upon your breach of this Agreement and, upon such termination you will cease all use, duplication, distribution, and/or exploitation in any manner of the Specifications and Materials. + 14. This Specifications License Agreement reflects the entire agreement of the parties regarding the subject matter hereof and supersedes all prior agreements or representations regarding such matters, whether written or oral. To the extent any portion or provision of this Specifications License Agreement is found to be illegal or unenforceable, then the remaining provisions of this Specifications License Agreement will remain in full force and effect and the illegal or unenforceable provision will be construed to give it such effect as it may properly have that is consistent with the intentions of the parties. + 15. This Specifications License Agreement may only be modified in writing signed by an authorized representative of the IPTC. + 16. This Specifications License Agreement is governed by the law of United Kingdom, as such law is applied to contracts made and fully performed in the United Kingdom. Any disputes arising from or relating to this Specifications License Agreement will be resolved in the courts of the United Kingdom. You consent to the jurisdiction of such courts over you and covenant not to assert before such courts any objection to proceeding in such forums. + + +JUnRAR (https://github.com/edmund-wagner/junrar/) + + JUnRAR is based on the UnRAR tool, and covered by the same license + It was formerly available from http://java-unrar.svn.sourceforge.net/ + + ****** ***** ****** UnRAR - free utility for RAR archives + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ****** ******* ****** License for use and distribution of + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ** ** ** ** ** ** FREE portable version + ~~~~~~~~~~~~~~~~~~~~~ + + The source code of UnRAR utility is freeware. This means: + + 1. All copyrights to RAR and the utility UnRAR are exclusively + owned by the author - Alexander Roshal. + + 2. The UnRAR sources may be used in any software to handle RAR + archives without limitations free of charge, but cannot be used + to re-create the RAR compression algorithm, which is proprietary. + Distribution of modified UnRAR sources in separate form or as a + part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + 3. The UnRAR utility may be freely distributed. It is allowed + to distribute UnRAR inside of other software packages. + + 4. THE RAR ARCHIVER AND THE UnRAR UTILITY ARE DISTRIBUTED "AS IS". + NO WARRANTY OF ANY KIND IS EXPRESSED OR IMPLIED. YOU USE AT + YOUR OWN RISK. THE AUTHOR WILL NOT BE LIABLE FOR DATA LOSS, + DAMAGES, LOSS OF PROFITS OR ANY OTHER KIND OF LOSS WHILE USING + OR MISUSING THIS SOFTWARE. + + 5. Installing and using the UnRAR utility signifies acceptance of + these terms and conditions of the license. + + 6. If you don't agree with terms of the license you must remove + UnRAR files from your storage devices and cease to use the + utility. + + Thank you for your interest in RAR and UnRAR. Alexander L. Roshal + +Sqlite (bundled in org.xerial's sqlite-jdbc) + This product bundles Sqlite, which is in the Public Domain. For details + see: https://www.sqlite.org/copyright.html diff --git a/plugins/mapper-attachments/licenses/tika-core-NOTICE.txt b/plugins/mapper-attachments/licenses/tika-core-NOTICE.txt new file mode 100644 index 00000000000..8e94f644b81 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-core-NOTICE.txt @@ -0,0 +1,17 @@ +Apache Tika +Copyright 2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata +This software contains code derived from UCAR/Unidata's NetCDF library. + +Tika-server component uses CDDL-licensed dependencies: jersey (http://jersey.java.net/) and +Grizzly (http://grizzly.java.net/) + +Tika-parsers component uses CDDL/LGPL dual-licensed dependency: jhighlight (https://github.com/codelibs/jhighlight) + +OpenCSV: Copyright 2005 Bytecode Pty Ltd. Licensed under the Apache License, Version 2.0 + +IPTC Photo Metadata descriptions Copyright 2010 International Press Telecommunications Council. diff --git a/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1 b/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1 new file mode 100644 index 00000000000..fbbd59efaf9 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1 @@ -0,0 +1 @@ +355dc05d842ed223fc682da472229473ba706d68 \ No newline at end of file diff --git a/plugins/mapper-attachments/licenses/tika-parsers-LICENSE.txt b/plugins/mapper-attachments/licenses/tika-parsers-LICENSE.txt new file mode 100644 index 00000000000..9537d733ea9 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-parsers-LICENSE.txt @@ -0,0 +1,372 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +APACHE TIKA SUBCOMPONENTS + +Apache Tika includes a number of subcomponents with separate copyright notices +and license terms. Your use of these subcomponents is subject to the terms and +conditions of the following licenses. + +MIME type information from file-4.26.tar.gz (http://www.darwinsys.com/file/) + + Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995. + Software written by Ian F. Darwin and others; + maintained 1994- Christos Zoulas. + + This software is not subject to any export provision of the United States + Department of Commerce, and may be exported to any country or planet. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice immediately at the beginning of the file, without modification, + this list of conditions, and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Charset detection code from ICU4J (http://site.icu-project.org/) + + Copyright (c) 1995-2009 International Business Machines Corporation + and others + + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, and/or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, provided that the above + copyright notice(s) and this permission notice appear in all copies + of the Software and that both the above copyright notice(s) and this + permission notice appear in supporting documentation. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE + BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, + OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, + ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS + SOFTWARE. + + Except as contained in this notice, the name of a copyright holder shall + not be used in advertising or otherwise to promote the sale, use or other + dealings in this Software without prior written authorization of the + copyright holder. + + +Parsing functionality provided by the NetCDF Java Library (http://www.unidata.ucar.edu/software/netcdf-java/) + + Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata + + Portions of this software were developed by the Unidata Program at the University + Corporation for Atmospheric Research. + + Access and use of this software shall impose the following obligations and understandings + on the user. The user is granted the right, without any fee or cost, to use, copy, modify, + alter, enhance and distribute this software, and any derivative works thereof, and its + supporting documentation for any purpose whatsoever, provided that this entire notice + appears in all copies of the software, derivative works and supporting documentation. Further, + UCAR requests that the user credit UCAR/Unidata in any publications that result from the use + of this software or in any product that includes this software, although this is not an obligation. + The names UCAR and/or Unidata, however, may not be used in any advertising or publicity to endorse + or promote any products or commercial entity unless specific written permission is obtained from + UCAR/Unidata. The user also understands that UCAR/Unidata is not obligated to provide the user with + any support, consulting, training or assistance of any kind with regard to the use, operation and + performance of this software nor to provide the user with any updates, revisions, new versions or + "bug fixes." + + THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, + BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE ACCESS, + USE OR PERFORMANCE OF THIS SOFTWARE. + + +IPTC Photo Metadata descriptions are taken from the IPTC Photo Metadata +Standard, July 2010, Copyright 2010 International Press Telecommunications +Council. + + 1. The Specifications and Materials are licensed for use only on the condition that you agree to be bound by the terms of this license. Subject to this and other licensing requirements contained herein, you may, on a non-exclusive basis, use the Specifications and Materials. + 2. The IPTC openly provides the Specifications and Materials for voluntary use by individuals, partnerships, companies, corporations, organizations and any other entity for use at the entity's own risk. This disclaimer, license and release is intended to apply to the IPTC, its officers, directors, agents, representatives, members, contributors, affiliates, contractors, or co-venturers acting jointly or severally. + 3. The Document and translations thereof may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the copyright and license notices and references to the IPTC appearing in the Document and the terms of this Specifications License Agreement are included on all such copies and derivative works. Further, upon the receipt of written permission from the IPTC, the Document may be modified for the purpose of developing applications that use IPTC Specifications or as required to translate the Document into languages other than English. + 4. Any use, duplication, distribution, or exploitation of the Document and Specifications and Materials in any manner is at your own risk. + 5. NO WARRANTY, EXPRESSED OR IMPLIED, IS MADE REGARDING THE ACCURACY, ADEQUACY, COMPLETENESS, LEGALITY, RELIABILITY OR USEFULNESS OF ANY INFORMATION CONTAINED IN THE DOCUMENT OR IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE PRODUCED OR SPONSORED BY THE IPTC. THE DOCUMENT AND THE INFORMATION CONTAINED HEREIN AND INCLUDED IN ANY SPECIFICATION OR OTHER PRODUCT OR SERVICE OF THE IPTC IS PROVIDED ON AN "AS IS" BASIS. THE IPTC DISCLAIMS ALL WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY ACTUAL OR ASSERTED WARRANTY OF NON-INFRINGEMENT OF PROPRIETARY RIGHTS, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. NEITHER THE IPTC NOR ITS CONTRIBUTORS SHALL BE HELD LIABLE FOR ANY IMPROPER OR INCORRECT USE OF INFORMATION. NEITHER THE IPTC NOR ITS CONTRIBUTORS ASSUME ANY RESPONSIBILITY FOR ANYONE'S USE OF INFORMATION PROVIDED BY THE IPTC. IN NO EVENT SHALL THE IPTC OR ITS CONTRIBUTORS BE LIABLE TO ANYONE FOR DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO, COMPENSATORY DAMAGES, LOST PROFITS, LOST DATA OR ANY FORM OF SPECIAL, INCIDENTAL, INDIRECT, CONSEQUENTIAL OR PUNITIVE DAMAGES OF ANY KIND WHETHER BASED ON BREACH OF CONTRACT OR WARRANTY, TORT, PRODUCT LIABILITY OR OTHERWISE. + 6. The IPTC takes no position regarding the validity or scope of any Intellectual Property or other rights that might be claimed to pertain to the implementation or use of the technology described in the Document or the extent to which any license under such rights might or might not be available. The IPTC does not represent that it has made any effort to identify any such rights. Copies of claims of rights made available for publication, assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of the Specifications and Materials, can be obtained from the Managing Director of the IPTC. + 7. By using the Specifications and Materials including the Document in any manner or for any purpose, you release the IPTC from all liabilities, claims, causes of action, allegations, losses, injuries, damages, or detriments of any nature arising from or relating to the use of the Specifications, Materials or any portion thereof. You further agree not to file a lawsuit, make a claim, or take any other formal or informal legal action against the IPTC, resulting from your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. Finally, you hereby agree that the IPTC is not liable for any direct, indirect, special or consequential damages arising from or relating to your acquisition, use, duplication, distribution, or exploitation of the Specifications, Materials or any portion thereof. + 8. Specifications and Materials may be downloaded or copied provided that ALL copies retain the ownership, copyright and license notices. + 9. Materials may not be edited, modified, or presented in a context that creates a misleading or false impression or statement as to the positions, actions, or statements of the IPTC. + 10. The name and trademarks of the IPTC may not be used in advertising, publicity, or in relation to products or services and their names without the specific, written prior permission of the IPTC. Any permitted use of the trademarks of the IPTC, whether registered or not, shall be accompanied by an appropriate mark and attribution, as agreed with the IPTC. + 11. Specifications may be extended by both members and non-members to provide additional functionality (Extension Specifications) provided that there is a clear recognition of the IPTC IP and its ownership in the Extension Specifications and the related documentation and provided that the extensions are clearly identified and provided that a perpetual license is granted by the creator of the Extension Specifications for other members and non-members to use the Extension Specifications and to continue extensions of the Extension Specifications. The IPTC does not waive any of its rights in the Specifications and Materials in this context. The Extension Specifications may be considered the intellectual property of their creator. The IPTC expressly disclaims any responsibility for damage caused by an extension to the Specifications. + 12. Specifications and Materials may be included in derivative work of both members and non-members provided that there is a clear recognition of the IPTC IP and its ownership in the derivative work and its related documentation. The IPTC does not waive any of its rights in the Specifications and Materials in this context. Derivative work in its entirety may be considered the intellectual property of the creator of the work .The IPTC expressly disclaims any responsibility for damage caused when its IP is used in a derivative context. + 13. This Specifications License Agreement is perpetual subject to your conformance to the terms of this Agreement. The IPTC may terminate this Specifications License Agreement immediately upon your breach of this Agreement and, upon such termination you will cease all use, duplication, distribution, and/or exploitation in any manner of the Specifications and Materials. + 14. This Specifications License Agreement reflects the entire agreement of the parties regarding the subject matter hereof and supersedes all prior agreements or representations regarding such matters, whether written or oral. To the extent any portion or provision of this Specifications License Agreement is found to be illegal or unenforceable, then the remaining provisions of this Specifications License Agreement will remain in full force and effect and the illegal or unenforceable provision will be construed to give it such effect as it may properly have that is consistent with the intentions of the parties. + 15. This Specifications License Agreement may only be modified in writing signed by an authorized representative of the IPTC. + 16. This Specifications License Agreement is governed by the law of United Kingdom, as such law is applied to contracts made and fully performed in the United Kingdom. Any disputes arising from or relating to this Specifications License Agreement will be resolved in the courts of the United Kingdom. You consent to the jurisdiction of such courts over you and covenant not to assert before such courts any objection to proceeding in such forums. + + +JUnRAR (https://github.com/edmund-wagner/junrar/) + + JUnRAR is based on the UnRAR tool, and covered by the same license + It was formerly available from http://java-unrar.svn.sourceforge.net/ + + ****** ***** ****** UnRAR - free utility for RAR archives + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ****** ******* ****** License for use and distribution of + ** ** ** ** ** ** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ** ** ** ** ** ** FREE portable version + ~~~~~~~~~~~~~~~~~~~~~ + + The source code of UnRAR utility is freeware. This means: + + 1. All copyrights to RAR and the utility UnRAR are exclusively + owned by the author - Alexander Roshal. + + 2. The UnRAR sources may be used in any software to handle RAR + archives without limitations free of charge, but cannot be used + to re-create the RAR compression algorithm, which is proprietary. + Distribution of modified UnRAR sources in separate form or as a + part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + 3. The UnRAR utility may be freely distributed. It is allowed + to distribute UnRAR inside of other software packages. + + 4. THE RAR ARCHIVER AND THE UnRAR UTILITY ARE DISTRIBUTED "AS IS". + NO WARRANTY OF ANY KIND IS EXPRESSED OR IMPLIED. YOU USE AT + YOUR OWN RISK. THE AUTHOR WILL NOT BE LIABLE FOR DATA LOSS, + DAMAGES, LOSS OF PROFITS OR ANY OTHER KIND OF LOSS WHILE USING + OR MISUSING THIS SOFTWARE. + + 5. Installing and using the UnRAR utility signifies acceptance of + these terms and conditions of the license. + + 6. If you don't agree with terms of the license you must remove + UnRAR files from your storage devices and cease to use the + utility. + + Thank you for your interest in RAR and UnRAR. Alexander L. Roshal + +Sqlite (bundled in org.xerial's sqlite-jdbc) + This product bundles Sqlite, which is in the Public Domain. For details + see: https://www.sqlite.org/copyright.html diff --git a/plugins/mapper-attachments/licenses/tika-parsers-NOTICE.txt b/plugins/mapper-attachments/licenses/tika-parsers-NOTICE.txt new file mode 100644 index 00000000000..8e94f644b81 --- /dev/null +++ b/plugins/mapper-attachments/licenses/tika-parsers-NOTICE.txt @@ -0,0 +1,17 @@ +Apache Tika +Copyright 2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Copyright 1993-2010 University Corporation for Atmospheric Research/Unidata +This software contains code derived from UCAR/Unidata's NetCDF library. + +Tika-server component uses CDDL-licensed dependencies: jersey (http://jersey.java.net/) and +Grizzly (http://grizzly.java.net/) + +Tika-parsers component uses CDDL/LGPL dual-licensed dependency: jhighlight (https://github.com/codelibs/jhighlight) + +OpenCSV: Copyright 2005 Bytecode Pty Ltd. Licensed under the Apache License, Version 2.0 + +IPTC Photo Metadata descriptions Copyright 2010 International Press Telecommunications Council. diff --git a/plugins/mapper-attachments/licenses/xmlbeans-2.6.0.jar.sha1 b/plugins/mapper-attachments/licenses/xmlbeans-2.6.0.jar.sha1 new file mode 100644 index 00000000000..d27c56f66cb --- /dev/null +++ b/plugins/mapper-attachments/licenses/xmlbeans-2.6.0.jar.sha1 @@ -0,0 +1 @@ +29e80d2dd51f9dcdef8f9ffaee0d4dc1c9bbfc87 diff --git a/plugins/mapper-attachments/licenses/xmlbeans-LICENSE.txt b/plugins/mapper-attachments/licenses/xmlbeans-LICENSE.txt new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/plugins/mapper-attachments/licenses/xmlbeans-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/mapper-attachments/licenses/xmlbeans-NOTICE.txt b/plugins/mapper-attachments/licenses/xmlbeans-NOTICE.txt new file mode 100644 index 00000000000..906cc4c9684 --- /dev/null +++ b/plugins/mapper-attachments/licenses/xmlbeans-NOTICE.txt @@ -0,0 +1,29 @@ + ========================================================================= + == NOTICE file corresponding to section 4(d) of the Apache License, == + == Version 2.0, in this case for the Apache XmlBeans distribution. == + ========================================================================= + + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were originally based on the following: + - software copyright (c) 2000-2003, BEA Systems, . + + Aside from contributions to the Apache XMLBeans project, this + software also includes: + + - one or more source files from the Apache Xerces-J and Apache Axis + products, Copyright (c) 1999-2003 Apache Software Foundation + + - W3C XML Schema documents Copyright 2001-2003 (c) World Wide Web + Consortium (Massachusetts Institute of Technology, European Research + Consortium for Informatics and Mathematics, Keio University) + + - resolver.jar from Apache Xml Commons project, + Copyright (c) 2001-2003 Apache Software Foundation + + - Piccolo XML Parser for Java from http://piccolo.sourceforge.net/, + Copyright 2002 Yuval Oren under the terms of the Apache Software License 2.0 + + - JSR-173 Streaming API for XML from http://sourceforge.net/projects/xmlpullparser/, + Copyright 2005 BEA under the terms of the Apache Software License 2.0 diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java new file mode 100644 index 00000000000..eb0e143c946 --- /dev/null +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java @@ -0,0 +1,655 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; +import org.apache.tika.language.LanguageIdentifier; +import org.apache.tika.metadata.Metadata; +import org.elasticsearch.Version; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.*; + +import java.io.IOException; +import java.util.*; + +import static org.elasticsearch.index.mapper.MapperBuilders.*; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parsePathType; + +/** + * 

    + *      "field1" : "..."
+ *
    + *

    Or: + * 

    + * {
+ *      "file1" : {
+ *          "_content_type" : "application/pdf",
+ *          "_content_length" : "500000000",
+ *          "_name" : "..../something.pdf",
+ *          "_content" : ""
+ *      }
+ * }
+ *
    + *

    + * _content_length = Specify the maximum amount of characters to extract from the attachment. If not specified, then the default for + * tika is 100,000 characters. Caution is required when setting large values as this can cause memory issues. + */ +public class AttachmentMapper extends FieldMapper { + + private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment"); + + public static final String CONTENT_TYPE = "attachment"; + + public static class Defaults { + public static final ContentPath.Type PATH_TYPE = ContentPath.Type.FULL; + + public static final AttachmentFieldType FIELD_TYPE = new AttachmentFieldType(); + static { + FIELD_TYPE.freeze(); + } + } + + public static class FieldNames { + public static final String CONTENT = "content"; + public static final String TITLE = "title"; + public static final String NAME = "name"; + public static final String AUTHOR = "author"; + public static final String KEYWORDS = "keywords"; + public static final String DATE = "date"; + public static final String CONTENT_TYPE = "content_type"; + public static final String CONTENT_LENGTH = "content_length"; + public static final String LANGUAGE = "language"; + } + + static final class AttachmentFieldType extends MappedFieldType { + public AttachmentFieldType() {} + + protected AttachmentFieldType(AttachmentMapper.AttachmentFieldType ref) { + super(ref); + } + + public AttachmentMapper.AttachmentFieldType clone() { + return new AttachmentMapper.AttachmentFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + public String value(Object value) { + return value == null?null:value.toString(); + } + } + + public static class Builder extends FieldMapper.Builder { + + private ContentPath.Type pathType = Defaults.PATH_TYPE; + + private Boolean ignoreErrors = null; + + private Integer defaultIndexedChars = null; + + private Boolean langDetect = null; + + private Mapper.Builder contentBuilder; + + private Mapper.Builder titleBuilder = stringField(FieldNames.TITLE); + + private Mapper.Builder nameBuilder = stringField(FieldNames.NAME); + + private Mapper.Builder authorBuilder = stringField(FieldNames.AUTHOR); + + private Mapper.Builder keywordsBuilder = stringField(FieldNames.KEYWORDS); + + private Mapper.Builder dateBuilder = dateField(FieldNames.DATE); + + private Mapper.Builder contentTypeBuilder = stringField(FieldNames.CONTENT_TYPE); + + private Mapper.Builder contentLengthBuilder = integerField(FieldNames.CONTENT_LENGTH); + + private Mapper.Builder languageBuilder = stringField(FieldNames.LANGUAGE); + + public Builder(String name) { + super(name, new AttachmentFieldType()); + this.builder = this; + this.contentBuilder = stringField(FieldNames.CONTENT); + } + + public Builder pathType(ContentPath.Type pathType) { + this.pathType = pathType; + return this; + } + + public Builder content(Mapper.Builder content) { + this.contentBuilder = content; + return this; + } + + public Builder date(Mapper.Builder date) { + this.dateBuilder = date; + return this; + } + + public Builder author(Mapper.Builder author) { + this.authorBuilder = author; + return this; + } + + public Builder title(Mapper.Builder title) { + this.titleBuilder = title; + return this; + } + + public Builder name(Mapper.Builder name) { + this.nameBuilder = name; + return this; + } + + public Builder keywords(Mapper.Builder keywords) { + this.keywordsBuilder = keywords; + return this; + } + + public Builder contentType(Mapper.Builder contentType) { + this.contentTypeBuilder = contentType; + return this; + } + + public Builder contentLength(Mapper.Builder contentType) { + this.contentLengthBuilder = contentType; + return this; + } + + public Builder language(Mapper.Builder language) { + this.languageBuilder = language; + return this; + } + + @Override + public AttachmentMapper build(BuilderContext context) { + ContentPath.Type origPathType = context.path().pathType(); + context.path().pathType(pathType); + + FieldMapper contentMapper; + if (context.indexCreatedVersion().before(Version.V_2_0_0_beta1)) { + // old behavior, we need the content to be indexed under the attachment field name + if (contentBuilder instanceof FieldMapper.Builder == false) { + throw new IllegalStateException("content field for attachment must be a field mapper"); + } + ((FieldMapper.Builder)contentBuilder).indexName(name); + contentBuilder.name = name + "." + FieldNames.CONTENT; + contentMapper = (FieldMapper) contentBuilder.build(context); + context.path().add(name); + } else { + context.path().add(name); + contentMapper = (FieldMapper) contentBuilder.build(context); + } + + FieldMapper dateMapper = (FieldMapper) dateBuilder.build(context); + FieldMapper authorMapper = (FieldMapper) authorBuilder.build(context); + FieldMapper titleMapper = (FieldMapper) titleBuilder.build(context); + FieldMapper nameMapper = (FieldMapper) nameBuilder.build(context); + FieldMapper keywordsMapper = (FieldMapper) keywordsBuilder.build(context); + FieldMapper contentTypeMapper = (FieldMapper) contentTypeBuilder.build(context); + FieldMapper contentLength = (FieldMapper) contentLengthBuilder.build(context); + FieldMapper language = (FieldMapper) languageBuilder.build(context); + context.path().remove(); + + context.path().pathType(origPathType); + + if (defaultIndexedChars == null && context.indexSettings() != null) { + defaultIndexedChars = context.indexSettings().getAsInt("index.mapping.attachment.indexed_chars", 100000); + } + if (defaultIndexedChars == null) { + defaultIndexedChars = 100000; + } + + if (ignoreErrors == null && context.indexSettings() != null) { + ignoreErrors = context.indexSettings().getAsBoolean("index.mapping.attachment.ignore_errors", Boolean.TRUE); + } + if (ignoreErrors == null) { + ignoreErrors = Boolean.TRUE; + } + + if (langDetect == null && context.indexSettings() != null) { + langDetect = context.indexSettings().getAsBoolean("index.mapping.attachment.detect_language", Boolean.FALSE); + } + if (langDetect == null) { + langDetect = Boolean.FALSE; + } + MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone(); + if(this.fieldType.indexOptions() != IndexOptions.NONE && !this.fieldType.tokenized()) { + defaultFieldType.setOmitNorms(true); + defaultFieldType.setIndexOptions(IndexOptions.DOCS); + if(!this.omitNormsSet && this.fieldType.boost() == 1.0F) { + this.fieldType.setOmitNorms(true); + } + + if(!this.indexOptionsSet) { + this.fieldType.setIndexOptions(IndexOptions.DOCS); + } + } + + defaultFieldType.freeze(); + this.setupFieldType(context); + return new AttachmentMapper(name, fieldType, defaultFieldType, pathType, defaultIndexedChars, ignoreErrors, langDetect, contentMapper, + dateMapper, titleMapper, nameMapper, authorMapper, keywordsMapper, contentTypeMapper, contentLength, + language, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + } + } + + /** + * 

    +     *  field1 : { type : "attachment" }
+     *
    + * Or: + * 
    +     *  field1 : {
+     *      type : "attachment",
+     *      fields : {
+     *          content : {type : "binary"},
+     *          title : {store : "yes"},
+     *          date : {store : "yes"},
+     *          name : {store : "yes"},
+     *          author : {store : "yes"},
+     *          keywords : {store : "yes"},
+     *          content_type : {store : "yes"},
+     *          content_length : {store : "yes"}
+     *      }
+     * }
+     *
    + */ + public static class TypeParser implements Mapper.TypeParser { + + private Mapper.Builder findMapperBuilder(Map propNode, String propName, ParserContext parserContext) { + String type; + Object typeNode = propNode.get("type"); + if (typeNode != null) { + type = typeNode.toString(); + } else { + type = "string"; + } + Mapper.TypeParser typeParser = parserContext.typeParser(type); + Mapper.Builder mapperBuilder = typeParser.parse(propName, (Map) propNode, parserContext); + + return mapperBuilder; + } + + @SuppressWarnings({"unchecked"}) + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + AttachmentMapper.Builder builder = new AttachmentMapper.Builder(name); + + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + Object fieldNode = entry.getValue(); + if (fieldName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + builder.pathType(parsePathType(name, fieldNode.toString())); + iterator.remove(); + } else if (fieldName.equals("fields")) { + Map fieldsNode = (Map) fieldNode; + for (Iterator> fieldsIterator = fieldsNode.entrySet().iterator(); fieldsIterator.hasNext();) { + Map.Entry entry1 = fieldsIterator.next(); + String propName = entry1.getKey(); + Map propNode = (Map) entry1.getValue(); + + Mapper.Builder mapperBuilder = findMapperBuilder(propNode, propName, parserContext); + if (parseMultiField((FieldMapper.Builder) mapperBuilder, fieldName, parserContext, propName, propNode)) { + fieldsIterator.remove(); + } else if (propName.equals(name) && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + builder.content(mapperBuilder); + fieldsIterator.remove(); + } else { + switch (propName) { + case FieldNames.CONTENT: + builder.content(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.DATE: + builder.date(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.AUTHOR: + builder.author(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.CONTENT_LENGTH: + builder.contentLength(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.CONTENT_TYPE: + builder.contentType(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.KEYWORDS: + builder.keywords(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.LANGUAGE: + builder.language(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.TITLE: + builder.title(mapperBuilder); + fieldsIterator.remove(); + break; + case FieldNames.NAME: + builder.name(mapperBuilder); + fieldsIterator.remove(); + break; + } + } + } + DocumentMapperParser.checkNoRemainingFields(fieldName, fieldsNode, parserContext.indexVersionCreated()); + iterator.remove(); + } + } + + return builder; + } + } + + private final ContentPath.Type pathType; + + private final int defaultIndexedChars; + + private final boolean ignoreErrors; + + private final boolean defaultLangDetect; + + private final FieldMapper contentMapper; + + private final FieldMapper dateMapper; + + private final FieldMapper authorMapper; + + private final FieldMapper titleMapper; + + private final FieldMapper nameMapper; + + private final FieldMapper keywordsMapper; + + private final FieldMapper contentTypeMapper; + + private final FieldMapper contentLengthMapper; + + private final FieldMapper languageMapper; + + public AttachmentMapper(String simpleName, MappedFieldType type, MappedFieldType defaultFieldType, ContentPath.Type pathType, int defaultIndexedChars, Boolean ignoreErrors, + Boolean defaultLangDetect, FieldMapper contentMapper, + FieldMapper dateMapper, FieldMapper titleMapper, FieldMapper nameMapper, FieldMapper authorMapper, + FieldMapper keywordsMapper, FieldMapper contentTypeMapper, FieldMapper contentLengthMapper, + FieldMapper languageMapper, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { + super(simpleName, type, defaultFieldType, indexSettings, multiFields, copyTo); + this.pathType = pathType; + this.defaultIndexedChars = defaultIndexedChars; + this.ignoreErrors = ignoreErrors; + this.defaultLangDetect = defaultLangDetect; + this.contentMapper = contentMapper; + this.dateMapper = dateMapper; + this.titleMapper = titleMapper; + this.nameMapper = nameMapper; + this.authorMapper = authorMapper; + this.keywordsMapper = keywordsMapper; + this.contentTypeMapper = contentTypeMapper; + this.contentLengthMapper = contentLengthMapper; + this.languageMapper = languageMapper; + } + + @Override + public Mapper parse(ParseContext context) throws IOException { + byte[] content = null; + String contentType = null; + int indexedChars = defaultIndexedChars; + boolean langDetect = defaultLangDetect; + String name = null; + String language = null; + + XContentParser parser = context.parser(); + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.VALUE_STRING) { + content = parser.binaryValue(); + } else { + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.VALUE_STRING) { + if ("_content".equals(currentFieldName)) { + content = parser.binaryValue(); + } else if ("_content_type".equals(currentFieldName)) { + contentType = parser.text(); + } else if ("_name".equals(currentFieldName)) { + name = parser.text(); + } else if ("_language".equals(currentFieldName)) { + language = parser.text(); + } + } else if (token == XContentParser.Token.VALUE_NUMBER) { + if ("_indexed_chars".equals(currentFieldName) || "_indexedChars".equals(currentFieldName)) { + indexedChars = parser.intValue(); + } + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + if ("_detect_language".equals(currentFieldName) || "_detectLanguage".equals(currentFieldName)) { + langDetect = parser.booleanValue(); + } + } + } + } + + // Throw clean exception when no content is provided Fix #23 + if (content == null) { + throw new MapperParsingException("No content is provided."); + } + + Metadata metadata = new Metadata(); + if (contentType != null) { + metadata.add(Metadata.CONTENT_TYPE, contentType); + } + if (name != null) { + metadata.add(Metadata.RESOURCE_NAME_KEY, name); + } + + String parsedContent; + try { + parsedContent = TikaImpl.parse(content, metadata, indexedChars); + } catch (Throwable e) { + // #18: we could ignore errors when Tika does not parse data + if (!ignoreErrors) { + logger.trace("exception caught", e); + throw new MapperParsingException("Failed to extract [" + indexedChars + "] characters of text for [" + name + "] : " + + e.getMessage(), e); + } else { + logger.debug("Failed to extract [{}] characters of text for [{}]: [{}]", indexedChars, name, e.getMessage()); + logger.trace("exception caught", e); + } + return null; + } + + context = context.createExternalValueContext(parsedContent); + contentMapper.parse(context); + + if (langDetect) { + try { + if (language != null) { + metadata.add(Metadata.CONTENT_LANGUAGE, language); + } else { + LanguageIdentifier identifier = new LanguageIdentifier(parsedContent); + language = identifier.getLanguage(); + } + context = context.createExternalValueContext(language); + languageMapper.parse(context); + } catch(Throwable t) { + logger.debug("Cannot detect language: [{}]", t.getMessage()); + } + } + + if (name != null) { + try { + context = context.createExternalValueContext(name); + nameMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing name: [{}]", + e.getMessage()); + } + } + + if (metadata.get(Metadata.DATE) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.DATE)); + dateMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing date: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (metadata.get(Metadata.TITLE) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.TITLE)); + titleMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing title: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (metadata.get(Metadata.AUTHOR) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.AUTHOR)); + authorMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing author: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (metadata.get(Metadata.KEYWORDS) != null) { + try { + context = context.createExternalValueContext(metadata.get(Metadata.KEYWORDS)); + keywordsMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing keywords: [{}]: [{}]", + e.getMessage(), context.externalValue()); + } + } + + if (contentType == null) { + contentType = metadata.get(Metadata.CONTENT_TYPE); + } + if (contentType != null) { + try { + context = context.createExternalValueContext(contentType); + contentTypeMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing content_type: [{}]: [{}]", e.getMessage(), context.externalValue()); + } + } + + int length = content.length; + // If we have CONTENT_LENGTH from Tika we use it + if (metadata.get(Metadata.CONTENT_LENGTH) != null) { + length = Integer.parseInt(metadata.get(Metadata.CONTENT_LENGTH)); + } + + try { + context = context.createExternalValueContext(length); + contentLengthMapper.parse(context); + } catch(MapperParsingException e){ + if (!ignoreErrors) throw e; + if (logger.isDebugEnabled()) logger.debug("Ignoring MapperParsingException catch while parsing content_length: [{}]: [{}]", e.getMessage(), context.externalValue()); + } + +// multiFields.parse(this, context); + + return null; + } + + @Override + protected void parseCreateField(ParseContext parseContext, List fields) throws IOException { + + } + + @Override + public void merge(Mapper mergeWith, MergeResult mergeResult) { + // ignore this for now + } + + @Override + @SuppressWarnings("unchecked") + public Iterator iterator() { + List extras = Arrays.asList( + contentMapper, + dateMapper, + titleMapper, + nameMapper, + authorMapper, + keywordsMapper, + contentTypeMapper, + contentLengthMapper, + languageMapper); + return Iterators.concat(super.iterator(), extras.iterator()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(simpleName()); + builder.field("type", CONTENT_TYPE); + if (indexCreatedBefore2x) { + builder.field("path", pathType.name().toLowerCase(Locale.ROOT)); + } + + builder.startObject("fields"); + contentMapper.toXContent(builder, params); + authorMapper.toXContent(builder, params); + titleMapper.toXContent(builder, params); + nameMapper.toXContent(builder, params); + dateMapper.toXContent(builder, params); + keywordsMapper.toXContent(builder, params); + contentTypeMapper.toXContent(builder, params); + contentLengthMapper.toXContent(builder, params); + languageMapper.toXContent(builder, params); + multiFields.toXContent(builder, params); + builder.endObject(); + + multiFields.toXContent(builder, params); + builder.endObject(); + return builder; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalysisBinderProcessor.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java similarity index 60% rename from plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalysisBinderProcessor.java rename to plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java index 408d80cd983..9b640f98d16 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalysisBinderProcessor.java +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java @@ -17,20 +17,24 @@ * under the License. */ -package org.elasticsearch.index.analysis.pl; +package org.elasticsearch.mapper.attachments; -import org.elasticsearch.index.analysis.AnalysisModule; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.Plugin; -/** - */ -public class PolishAnalysisBinderProcessor extends AnalysisModule.AnalysisBinderProcessor { +public class MapperAttachmentsPlugin extends Plugin { @Override - public void processAnalyzers(AnalyzersBindings analyzersBindings) { - analyzersBindings.processAnalyzer("polish", PolishAnalyzerProvider.class); + public String name() { + return "mapper-attachments"; } + @Override - public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) { - tokenFiltersBindings.processTokenFilter("polish_stem", PolishStemTokenFilterFactory.class); - } + public String description() { + return "Adds the attachment type allowing to parse difference attachment formats"; + } + + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMapper("attachment", new AttachmentMapper.TypeParser()); + } } diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java new file mode 100644 index 00000000000..38e292725a5 --- /dev/null +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java @@ -0,0 +1,159 @@ +package org.elasticsearch.mapper.attachments; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.ByteArrayInputStream; +import java.io.FilePermission; +import java.io.IOException; +import java.lang.reflect.ReflectPermission; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.security.ProtectionDomain; +import java.security.SecurityPermission; +import java.util.PropertyPermission; + +import org.apache.tika.Tika; +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.parser.Parser; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; + +/** + * Runs tika with limited parsers and limited permissions. + *

    + * Do NOT make public + */ +final class TikaImpl { + + /** subset of parsers for types we support */ + private static final Parser PARSERS[] = new Parser[] { + // documents + new org.apache.tika.parser.html.HtmlParser(), + new org.apache.tika.parser.rtf.RTFParser(), + new org.apache.tika.parser.pdf.PDFParser(), + new org.apache.tika.parser.txt.TXTParser(), + new org.apache.tika.parser.microsoft.OfficeParser(), + new org.apache.tika.parser.microsoft.OldExcelParser(), + new org.apache.tika.parser.microsoft.ooxml.OOXMLParser(), + new org.apache.tika.parser.odf.OpenDocumentParser(), + new org.apache.tika.parser.iwork.IWorkPackageParser(), + new org.apache.tika.parser.xml.DcXMLParser(), + new org.apache.tika.parser.epub.EpubParser(), + }; + + /** autodetector based on this subset */ + private static final AutoDetectParser PARSER_INSTANCE = new AutoDetectParser(PARSERS); + + /** singleton tika instance */ + private static final Tika TIKA_INSTANCE = new Tika(PARSER_INSTANCE.getDetector(), PARSER_INSTANCE); + + /** + * parses with tika, throwing any exception hit while parsing the document + */ + // only package private for testing! + static String parse(final byte content[], final Metadata metadata, final int limit) throws TikaException, IOException { + // check that its not unprivileged code like a script + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + try { + return AccessController.doPrivileged(new PrivilegedExceptionAction() { + @Override + public String run() throws TikaException, IOException { + return TIKA_INSTANCE.parseToString(new ByteArrayInputStream(content), metadata, limit); + } + }, RESTRICTED_CONTEXT); + } catch (PrivilegedActionException e) { + // checked exception from tika: unbox it + Throwable cause = e.getCause(); + if (cause instanceof TikaException) { + throw (TikaException) cause; + } else if (cause instanceof IOException) { + throw (IOException) cause; + } else { + throw new AssertionError(cause); + } + } + } + + // apply additional containment for parsers, this is intersected with the current permissions + // its hairy, but worth it so we don't have some XML flaw reading random crap from the FS + private static final AccessControlContext RESTRICTED_CONTEXT = new AccessControlContext( + new ProtectionDomain[] { + new ProtectionDomain(null, getRestrictedPermissions()) + } + ); + + // compute some minimal permissions for parsers. they only get r/w access to the java temp directory, + // the ability to load some resources from JARs, and read sysprops + static PermissionCollection getRestrictedPermissions() { + Permissions perms = new Permissions(); + // property/env access needed for parsing + perms.add(new PropertyPermission("*", "read")); + perms.add(new RuntimePermission("getenv.TIKA_CONFIG")); + + // add permissions for resource access: + // classpath + addReadPermissions(perms, JarHell.parseClassPath()); + // plugin jars + if (TikaImpl.class.getClassLoader() instanceof URLClassLoader) { + addReadPermissions(perms, ((URLClassLoader)TikaImpl.class.getClassLoader()).getURLs()); + } + // jvm's java.io.tmpdir (needs read/write) + perms.add(new FilePermission(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "-", + "read,readlink,write,delete")); + // current hacks needed for POI/PDFbox issues: + perms.add(new SecurityPermission("putProviderProperty.BC")); + perms.add(new SecurityPermission("insertProvider")); + perms.add(new ReflectPermission("suppressAccessChecks")); + perms.setReadOnly(); + return perms; + } + + // add resources to (what is typically) a jar, but might not be (e.g. in tests/IDE) + @SuppressForbidden(reason = "adds access to jar resources") + static void addReadPermissions(Permissions perms, URL resources[]) { + try { + for (URL url : resources) { + Path path = PathUtils.get(url.toURI()); + // resource itself + perms.add(new FilePermission(path.toString(), "read,readlink")); + // classes underneath + perms.add(new FilePermission(path.toString() + System.getProperty("file.separator") + "-", "read,readlink")); + } + } catch (URISyntaxException bogus) { + throw new RuntimeException(bogus); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisSettingsRequired.java b/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy similarity index 57% rename from core/src/main/java/org/elasticsearch/index/analysis/AnalysisSettingsRequired.java rename to plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy index 847752c5140..e23e9f4d0cf 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisSettingsRequired.java +++ b/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy @@ -16,17 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.analysis; -import java.lang.annotation.*; +// NOTE: when modifying this file, look at restrictions in TikaImpl too +grant { + // needed to apply additional sandboxing to tika parsing + permission java.security.SecurityPermission "createAccessControlContext"; -/** - * A marker annotation on {@link CharFilterFactory}, {@link AnalyzerProvider}, {@link TokenFilterFactory}, - * or {@link TokenizerFactory} which will cause the provider/factory to only be created when explicit settings - * are provided. - */ -@Target({ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface AnalysisSettingsRequired { -} + // TODO: fix PDFBox not to actually install bouncy castle like this + permission java.security.SecurityPermission "putProviderProperty.BC"; + permission java.security.SecurityPermission "insertProvider"; + // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597 + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java new file mode 100644 index 00000000000..9b7d8afe381 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +public class AttachmentUnitTestCase extends ESTestCase { + + protected Settings testSettings; + + protected static IndicesModule getIndicesModuleWithRegisteredAttachmentMapper() { + IndicesModule indicesModule = new IndicesModule(); + indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + return indicesModule; + } + + @Before + public void createSettings() throws Exception { + testSettings = Settings.builder() + .put("path.home", createTempDir()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id) + .build(); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java new file mode 100644 index 00000000000..f93785ed14a --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.junit.Before; + +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +public class DateAttachmentMapperTests extends AttachmentUnitTestCase { + + private DocumentMapperParser mapperParser; + + @Before + public void setupMapperParser() throws Exception { + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + } + + public void testSimpleMappings() throws Exception { + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + + // Our mapping should be kept as a String + assertThat(docMapper.mappers().getMapper("file.date"), instanceOf(StringFieldMapper.class)); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java new file mode 100644 index 00000000000..10e82e24c84 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18 + * Note that we have converted /org/elasticsearch/index/mapper/xcontent/testContentLength.txt + * to a /org/elasticsearch/index/mapper/xcontent/encrypted.pdf with password `12345678`. + */ +public class EncryptedDocMapperTests extends AttachmentUnitTestCase { + + public void testMultipleDocsEncryptedLast() throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); + byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); + + BytesReference json = jsonBuilder() + .startObject() + .field("file1", html) + .field("file2", pdf) + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get(docMapper.mappers().getMapper("file1.content").fieldType().names().indexName()), containsString("World")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.title").fieldType().names().indexName()), equalTo("Hello")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.author").fieldType().names().indexName()), equalTo("kimchy")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.keywords").fieldType().names().indexName()), equalTo("elasticsearch,cool,bonsai")); + assertThat(doc.get(docMapper.mappers().getMapper("file1.content_type").fieldType().names().indexName()), startsWith("text/html;")); + assertThat(doc.getField(docMapper.mappers().getMapper("file1.content_length").fieldType().names().indexName()).numericValue().longValue(), greaterThan(0L)); + + assertThat(doc.get(docMapper.mappers().getMapper("file2").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.title").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.author").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.keywords").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file2.content_type").fieldType().names().indexName()), nullValue()); + assertThat(doc.getField(docMapper.mappers().getMapper("file2.content_length").fieldType().names().indexName()), nullValue()); + } + + public void testMultipleDocsEncryptedFirst() throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); + byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); + + BytesReference json = jsonBuilder() + .startObject() + .field("file1", pdf) + .field("file2", html) + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get(docMapper.mappers().getMapper("file1").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.title").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.author").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.keywords").fieldType().names().indexName()), nullValue()); + assertThat(doc.get(docMapper.mappers().getMapper("file1.content_type").fieldType().names().indexName()), nullValue()); + assertThat(doc.getField(docMapper.mappers().getMapper("file1.content_length").fieldType().names().indexName()), nullValue()); + + assertThat(doc.get(docMapper.mappers().getMapper("file2.content").fieldType().names().indexName()), containsString("World")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.title").fieldType().names().indexName()), equalTo("Hello")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.author").fieldType().names().indexName()), equalTo("kimchy")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.keywords").fieldType().names().indexName()), equalTo("elasticsearch,cool,bonsai")); + assertThat(doc.get(docMapper.mappers().getMapper("file2.content_type").fieldType().names().indexName()), startsWith("text/html;")); + assertThat(doc.getField(docMapper.mappers().getMapper("file2.content_length").fieldType().names().indexName()).numericValue().longValue(), greaterThan(0L)); + } + + public void testMultipleDocsEncryptedNotIgnoringErrors() throws IOException { + try { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), + getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); + byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); + + BytesReference json = jsonBuilder() + .startObject() + .field("file1", pdf) + .field("file2", html) + .endObject().bytes(); + + docMapper.parse("person", "person", "1", json); + fail("Expected doc parsing exception"); + } catch (MapperParsingException e) { + if (e.getMessage() == null || e.getMessage().contains("is encrypted") == false) { + // wrong exception + throw e; + } + } + } + +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java new file mode 100644 index 00000000000..868ecb3ae55 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java @@ -0,0 +1,126 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.junit.Before; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +public class LanguageDetectionAttachmentMapperTests extends AttachmentUnitTestCase { + + private DocumentMapper docMapper; + + @Before + public void setupMapperParser() throws IOException { + setupMapperParser(true); + } + + public void setupMapperParser(boolean langDetect) throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.settingsBuilder().put("index.mapping.attachment.detect_language", langDetect).build(), + getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json"); + docMapper = mapperParser.parse(mapping); + + assertThat(docMapper.mappers().getMapper("file.language"), instanceOf(StringFieldMapper.class)); + } + + private void testLanguage(String filename, String expected, String... forcedLanguage) throws IOException { + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename); + + XContentBuilder xcb = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", filename) + .field("_content", html); + + if (forcedLanguage.length > 0) { + xcb.field("_language", forcedLanguage[0]); + } + + xcb.endObject().endObject(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", xcb.bytes()).rootDoc(); + + // Our mapping should be kept as a String + assertThat(doc.get(docMapper.mappers().getMapper("file.language").fieldType().names().indexName()), equalTo(expected)); + } + + public void testFrDetection() throws Exception { + testLanguage("text-in-french.txt", "fr"); + } + + public void testEnDetection() throws Exception { + testLanguage("text-in-english.txt", "en"); + } + + public void testFrForced() throws Exception { + testLanguage("text-in-english.txt", "fr", "fr"); + } + + /** + * This test gives strange results! detection of ":-)" gives "lt" as a result + */ + public void testNoLanguage() throws Exception { + testLanguage("text-in-nolang.txt", "lt"); + } + + public void testLangDetectDisabled() throws Exception { + // We replace the mapper with another one which have index.mapping.attachment.detect_language = false + setupMapperParser(false); + testLanguage("text-in-english.txt", null); + } + + public void testLangDetectDocumentEnabled() throws Exception { + // We replace the mapper with another one which have index.mapping.attachment.detect_language = false + setupMapperParser(false); + + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt"); + + XContentBuilder xcb = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", "text-in-english.txt") + .field("_content", html) + .field("_detect_language", true) + .endObject().endObject(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", xcb.bytes()).rootDoc(); + + // Our mapping should be kept as a String + assertThat(doc.get(docMapper.mappers().getMapper("file.language").fieldType().names().indexName()), equalTo("en")); + } +} diff --git a/core/src/test/java/org/elasticsearch/test/rest/Rest2IT.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java similarity index 61% rename from core/src/test/java/org/elasticsearch/test/rest/Rest2IT.java rename to plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java index a2fb5ad9226..bdbafea710a 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/Rest2IT.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java @@ -17,22 +17,35 @@ * under the License. */ -package org.elasticsearch.test.rest; +package org.elasticsearch.mapper.attachments; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -/** Rest API tests subset 2 */ -public class Rest2IT extends ESRestTestCase { - public Rest2IT(@Name("yaml") RestTestCandidate testCandidate) { +public class MapperAttachmentsRestIT extends ESRestTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugin.types", MapperAttachmentsPlugin.class.getName()) + .build(); + } + + public MapperAttachmentsRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } + @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(2, 8); + return createParameters(0, 1); } } + diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java new file mode 100644 index 00000000000..acf0163acd9 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/38 + */ +public class MetadataMapperTests extends AttachmentUnitTestCase { + + protected void checkMeta(String filename, Settings otherSettings, Long expectedDate, Long expectedLength) throws IOException { + Settings settings = Settings.builder() + .put(this.testSettings) + .put(otherSettings) + .build(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename); + + BytesReference json = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", filename) + .field("_content", html) + .endObject() + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("World")); + assertThat(doc.get(docMapper.mappers().getMapper("file.name").fieldType().names().indexName()), equalTo(filename)); + if (expectedDate == null) { + assertThat(doc.getField(docMapper.mappers().getMapper("file.date").fieldType().names().indexName()), nullValue()); + } else { + assertThat(doc.getField(docMapper.mappers().getMapper("file.date").fieldType().names().indexName()).numericValue().longValue(), is(expectedDate)); + } + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("Hello")); + assertThat(doc.get(docMapper.mappers().getMapper("file.author").fieldType().names().indexName()), equalTo("kimchy")); + assertThat(doc.get(docMapper.mappers().getMapper("file.keywords").fieldType().names().indexName()), equalTo("elasticsearch,cool,bonsai")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("text/html;")); + if (expectedLength == null) { + assertNull(doc.getField(docMapper.mappers().getMapper("file.content_length").fieldType().names().indexName()).numericValue().longValue()); + } else { + assertThat(doc.getField(docMapper.mappers().getMapper("file.content_length").fieldType().names().indexName()).numericValue().longValue(), greaterThan(0L)); + } + } + + public void testIgnoreWithoutDate() throws Exception { + checkMeta("htmlWithoutDateMeta.html", Settings.builder().build(), null, 300L); + } + + public void testIgnoreWithEmptyDate() throws Exception { + checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().build(), null, 334L); + } + + public void testIgnoreWithCorrectDate() throws Exception { + checkMeta("htmlWithValidDateMeta.html", Settings.builder().build(), 1354233600000L, 344L); + } + + public void testWithoutDate() throws Exception { + checkMeta("htmlWithoutDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, 300L); + } + + public void testWithEmptyDate() throws Exception { + try { + checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, null); + } catch (MapperParsingException expected) { + assertTrue(expected.getMessage().contains("failed to parse")); + } + } + + public void testWithCorrectDate() throws Exception { + checkMeta("htmlWithValidDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), 1354233600000L, 344L); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java new file mode 100644 index 00000000000..40593ddb1bb --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java @@ -0,0 +1,151 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.common.Base64; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.core.DateFieldMapper; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; + +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { + + private DocumentMapperParser mapperParser; + private ThreadPool threadPool; + + @Before + public void setupMapperParser() throws Exception { + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + + } + + @After + public void cleanup() throws InterruptedException { + terminate(threadPool); + } + + public void testSimpleMappings() throws Exception { + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + + + assertThat(docMapper.mappers().getMapper("file.content"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.content.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.date"), instanceOf(DateFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.date.string"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.title"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.title.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.name"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.name.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.author"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.author.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.keywords"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.keywords.suggest"), instanceOf(StringFieldMapper.class)); + + assertThat(docMapper.mappers().getMapper("file.content_type"), instanceOf(StringFieldMapper.class)); + assertThat(docMapper.mappers().getMapper("file.content_type.suggest"), instanceOf(StringFieldMapper.class)); + } + + public void testExternalValues() throws Exception { + String originalText = "This is an elasticsearch mapper attachment test."; + String forcedName = "dummyname.txt"; + + String bytes = Base64.encodeBytes(originalText.getBytes(StandardCharsets.ISO_8859_1)); + threadPool = new ThreadPool("testing-only"); + + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); + + DocumentMapper documentMapper = mapperService.documentMapperParser().parse(mapping); + + ParsedDocument doc = documentMapper.parse("person", "person", "1", XContentFactory.jsonBuilder() + .startObject() + .field("file", bytes) + .endObject() + .bytes()); + + assertThat(doc.rootDoc().getField("file.content"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content").stringValue(), is(originalText + "\n")); + + assertThat(doc.rootDoc().getField("file.content_type"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type").stringValue(), startsWith("text/plain;")); + assertThat(doc.rootDoc().getField("file.content_type.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type.suggest").stringValue(), startsWith("text/plain;")); + assertThat(doc.rootDoc().getField("file.content_length"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_length").numericValue().intValue(), is(originalText.length())); + + assertThat(doc.rootDoc().getField("file.content.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content.suggest").stringValue(), is(originalText + "\n")); + + // Let's force some values + doc = documentMapper.parse("person", "person", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("file") + .field("_content", bytes) + .field("_name", forcedName) + .endObject() + .endObject() + .bytes()); + + assertThat(doc.rootDoc().getField("file.content"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content").stringValue(), is(originalText + "\n")); + + assertThat(doc.rootDoc().getField("file.content_type"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type").stringValue(), startsWith("text/plain;")); + assertThat(doc.rootDoc().getField("file.content_type.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_type.suggest").stringValue(), startsWith("text/plain;")); + assertThat(doc.rootDoc().getField("file.content_length"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content_length").numericValue().intValue(), is(originalText.length())); + + assertThat(doc.rootDoc().getField("file.content.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.content.suggest").stringValue(), is(originalText + "\n")); + + assertThat(doc.rootDoc().getField("file.name"), notNullValue()); + assertThat(doc.rootDoc().getField("file.name").stringValue(), is(forcedName)); + // In mapping we have default store:false + assertThat(doc.rootDoc().getField("file.name").fieldType().stored(), is(false)); + assertThat(doc.rootDoc().getField("file.name.suggest"), notNullValue()); + assertThat(doc.rootDoc().getField("file.name.suggest").stringValue(), is(forcedName)); + // In mapping we set store:true for suggest subfield + assertThat(doc.rootDoc().getField("file.name.suggest").fieldType().stored(), is(true)); + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java new file mode 100644 index 00000000000..01e87dc1430 --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java @@ -0,0 +1,142 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParseContext; +import org.junit.Test; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.*; + +/** + * + */ +public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { + + public void testSimpleMappings() throws Exception { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); + + BytesReference json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + + // re-parse it + String builtMapping = docMapper.mappingSource().string(); + docMapper = mapperParser.parse(builtMapping); + + json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + + doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + } + + public void testContentBackcompat() throws Exception { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(), + getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); + + BytesReference json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + assertThat(doc.get("file"), containsString("This document tests the ability of Apache Tika to extract content")); + } + + /** + * test for https://github.com/elastic/elasticsearch-mapper-attachments/issues/179 + */ + public void testSimpleMappingsWithAllFields() throws Exception { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); + + BytesReference json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + + // re-parse it + String builtMapping = docMapper.mappingSource().string(); + docMapper = mapperParser.parse(builtMapping); + + json = jsonBuilder().startObject().field("file", html).endObject().bytes(); + + doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + assertThat(doc.get(docMapper.mappers().getMapper("file.content_type").fieldType().names().indexName()), startsWith("application/xhtml+xml")); + assertThat(doc.get(docMapper.mappers().getMapper("file.title").fieldType().names().indexName()), equalTo("XHTML test document")); + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), containsString("This document tests the ability of Apache Tika to extract content")); + } + + /** + * See issue https://github.com/elastic/elasticsearch-mapper-attachments/issues/169 + * Mapping should not contain field names with dot. + */ + public void testMapperErrorWithDotTwoLevels169() throws Exception { + XContentBuilder mappingBuilder = jsonBuilder(); + mappingBuilder.startObject() + .startObject("mail") + .startObject("properties") + .startObject("attachments") + .startObject("properties") + .startObject("innerfield") + .field("type", "attachment") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + byte[] mapping = mappingBuilder.bytes().toBytes(); + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()); + DocumentMapper docMapper = mapperService.parse("mail", new CompressedXContent(mapping), true); + // this should not throw an exception + mapperService.parse("mail", new CompressedXContent(docMapper.mapping().toString()), true); + // the mapping may not contain a field name with a dot + assertFalse(docMapper.mapping().toString().contains(".")); + } + +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java new file mode 100644 index 00000000000..fcd430d0fbc --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.apache.commons.cli.CommandLine; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.cli.CliTool; +import org.elasticsearch.common.cli.CliToolConfig; +import org.elasticsearch.common.cli.Terminal; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; + +import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd; +import static org.elasticsearch.common.cli.CliToolConfig.Builder.option; +import static org.elasticsearch.common.io.Streams.copy; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.mapper.attachments.AttachmentUnitTestCase.getIndicesModuleWithRegisteredAttachmentMapper; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; + +/** + * This class provides a simple main class which can be used to test what is extracted from a given binary file. + * You can run it using + * -u file://URL/TO/YOUR/DOC + * --size set extracted size (default to mapper attachment size) + * BASE64 encoded binary + * + * Example: + * StandaloneRunner BASE64Text + * StandaloneRunner -u /tmp/mydoc.pdf + * StandaloneRunner -u /tmp/mydoc.pdf --size 1000000 + */ +@SuppressForbidden(reason = "commandline tool") +public class StandaloneRunner extends CliTool { + + private static final CliToolConfig CONFIG = CliToolConfig.config("tika", StandaloneRunner.class) + .cmds(TikaRunner.CMD) + .build(); + + static { + System.setProperty("es.path.home", "/tmp"); + } + + static class TikaRunner extends Command { + private static final String NAME = "tika"; + private final String url; + private final Integer size; + private final String base64text; + private final DocumentMapper docMapper; + + private static final CliToolConfig.Cmd CMD = cmd(NAME, TikaRunner.class) + .options(option("u", "url").required(false).hasArg(false)) + .options(option("t", "size").required(false).hasArg(false)) + .build(); + + protected TikaRunner(Terminal terminal, String url, Integer size, String base64text) throws IOException { + super(terminal); + this.size = size; + this.url = url; + this.base64text = base64text; + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); // use CWD b/c it won't be used + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json"); + docMapper = mapperParser.parse(mapping); + } + + @Override + public ExitStatus execute(Settings settings, Environment env) throws Exception { + XContentBuilder builder = jsonBuilder().startObject().field("file").startObject(); + + if (base64text != null) { + // If base64 is provided + builder.field("_content", base64text); + } else { + // A file is provided + byte[] bytes = copyToBytes(PathUtils.get(url)); + builder.field("_content", bytes); + } + + if (size >= 0) { + builder.field("_indexed_chars", size); + } + + BytesReference json = builder.endObject().endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + + terminal.println("## Extracted text"); + terminal.println("--------------------- BEGIN -----------------------"); + terminal.println("%s", doc.get("file.content")); + terminal.println("---------------------- END ------------------------"); + terminal.println("## Metadata"); + printMetadataContent(doc, AttachmentMapper.FieldNames.AUTHOR); + printMetadataContent(doc, AttachmentMapper.FieldNames.CONTENT_LENGTH); + printMetadataContent(doc, AttachmentMapper.FieldNames.CONTENT_TYPE); + printMetadataContent(doc, AttachmentMapper.FieldNames.DATE); + printMetadataContent(doc, AttachmentMapper.FieldNames.KEYWORDS); + printMetadataContent(doc, AttachmentMapper.FieldNames.LANGUAGE); + printMetadataContent(doc, AttachmentMapper.FieldNames.NAME); + printMetadataContent(doc, AttachmentMapper.FieldNames.TITLE); + + return ExitStatus.OK; + } + + private void printMetadataContent(ParseContext.Document doc, String field) { + terminal.println("- %s: %s", field, doc.get(docMapper.mappers().getMapper("file." + field).fieldType().names().indexName())); + } + + public static byte[] copyToBytes(Path path) throws IOException { + try (InputStream is = Files.newInputStream(path)) { + if (is == null) { + throw new FileNotFoundException("Resource [" + path + "] not found in classpath"); + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + copy(is, out); + return out.bytes().toBytes(); + } + } + } + + public static Command parse(Terminal terminal, CommandLine cli) throws IOException { + String url = cli.getOptionValue("u"); + String base64text = null; + String sSize = cli.getOptionValue("size"); + Integer size = sSize != null ? Integer.parseInt(sSize) : -1; + if (url == null && cli.getArgs().length == 0) { + return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided (type -h for help)"); + } + if (url == null) { + if (cli.getArgs().length == 0) { + return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided (type -h for help)"); + } + base64text = cli.getArgs()[0]; + } else { + if (cli.getArgs().length == 1) { + return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided. Not both. (type -h for help)"); + } + } + return new TikaRunner(terminal, url, size, base64text); + } + } + + public StandaloneRunner() { + super(CONFIG); + } + + + public static void main(String[] args) { + StandaloneRunner pluginManager = new StandaloneRunner(); + pluginManager.execute(args); + } + + @Override + protected Command parse(String cmdName, CommandLine cli) throws Exception { + switch (cmdName.toLowerCase(Locale.ROOT)) { + case TikaRunner.NAME: return TikaRunner.parse(terminal, cli); + default: + assert false : "can't get here as cmd name is validated before this method is called"; + return exitCmd(ExitStatus.CODE_ERROR); + } + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java new file mode 100644 index 00000000000..a5e3ec9c17c --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java @@ -0,0 +1,66 @@ +package org.elasticsearch.mapper.attachments; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; +import org.apache.lucene.util.TestUtil; +import org.apache.tika.metadata.Metadata; + +import org.elasticsearch.test.ESTestCase; + +/** + * Evil test-coverage cheat, we parse a bunch of docs from tika + * so that we have a nice grab-bag variety, and assert some content + * comes back and no exception. + */ +@SuppressFileSystems("ExtrasFS") // don't try to parse extraN +public class TikaDocTests extends ESTestCase { + + /** some test files from tika test suite, zipped up */ + static final String TIKA_FILES = "/org/elasticsearch/index/mapper/attachment/test/tika-files.zip"; + + public void testFiles() throws Exception { + Path tmp = createTempDir(); + TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES), tmp); + + try (DirectoryStream stream = Files.newDirectoryStream(tmp)) { + for (Path doc : stream) { + logger.debug("parsing: {}", doc); + assertParseable(doc); + } + } + } + + void assertParseable(Path fileName) throws Exception { + try { + byte bytes[] = Files.readAllBytes(fileName); + String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1); + assertNotNull(parsedContent); + assertFalse(parsedContent.isEmpty()); + logger.debug("extracted content: {}", parsedContent); + } catch (Throwable e) { + throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e); + } + } +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java new file mode 100644 index 00000000000..fc17d59603f --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java @@ -0,0 +1,11 @@ +package org.elasticsearch.mapper.attachments; + +import org.elasticsearch.test.ESTestCase; + +public class TikaImplTests extends ESTestCase { + + public void testTikaLoads() throws Exception { + Class.forName("org.elasticsearch.mapper.attachments.TikaImpl"); + } + +} diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java new file mode 100644 index 00000000000..5341e038cff --- /dev/null +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java @@ -0,0 +1,168 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.mapper.attachments; + +import org.apache.tika.io.IOUtils; +import org.apache.tika.metadata.Metadata; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.mapper.attachments.AttachmentMapper.FieldNames.*; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.not; + +/** + * Test for different documents + */ +public class VariousDocTests extends AttachmentUnitTestCase { + + protected DocumentMapper docMapper; + + @Before + public void createMapper() throws IOException { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); + + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json"); + docMapper = mapperParser.parse(mapping); + } + + /** + * Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/104 + */ + public void testWordDocxDocument104() throws Exception { + assertParseable("issue-104.docx"); + testMapper("issue-104.docx", false); + } + + /** + * Test for encrypted PDF + */ + public void testEncryptedPDFDocument() throws Exception { + assertException("encrypted.pdf", "is encrypted"); + testMapper("encrypted.pdf", true); + } + + /** + * Test for HTML + */ + public void testHtmlDocument() throws Exception { + assertParseable("htmlWithEmptyDateMeta.html"); + testMapper("htmlWithEmptyDateMeta.html", false); + } + + /** + * Test for XHTML + */ + public void testXHtmlDocument() throws Exception { + assertParseable("testXHTML.html"); + testMapper("testXHTML.html", false); + } + + /** + * Test for TXT + */ + public void testTxtDocument() throws Exception { + assertParseable("text-in-english.txt"); + testMapper("text-in-english.txt", false); + } + + /** + * Test for .epub + */ + public void testEpubDocument() throws Exception { + assertParseable("testEPUB.epub"); + testMapper("testEPUB.epub", false); + } + + /** + * Test for ASCIIDOC + * Not yet supported by Tika: https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/29 + */ + public void testAsciidocDocument() throws Exception { + assertParseable("asciidoc.asciidoc"); + testMapper("asciidoc.asciidoc", false); + } + + void assertException(String filename, String expectedMessage) throws Exception { + try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) { + byte bytes[] = IOUtils.toByteArray(is); + TikaImpl.parse(bytes, new Metadata(), -1); + fail("expected exception"); + } catch (Exception e) { + if (e.getMessage() != null && e.getMessage().contains(expectedMessage)) { + // ok + } else { + // unexpected + throw e; + } + } + } + + protected void assertParseable(String filename) throws Exception { + try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) { + byte bytes[] = IOUtils.toByteArray(is); + String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1); + assertThat(parsedContent, not(isEmptyOrNullString())); + logger.debug("extracted content: {}", parsedContent); + } + } + + protected void testMapper(String filename, boolean errorExpected) throws IOException { + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename); + + BytesReference json = jsonBuilder() + .startObject() + .startObject("file") + .field("_name", filename) + .field("_content", html) + .endObject() + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); + if (!errorExpected) { + assertThat(doc.get(docMapper.mappers().getMapper("file.content").fieldType().names().indexName()), not(isEmptyOrNullString())); + logger.debug("-> extracted content: {}", doc.get(docMapper.mappers().getMapper("file").fieldType().names().indexName())); + logger.debug("-> extracted metadata:"); + printMetadataContent(doc, AUTHOR); + printMetadataContent(doc, CONTENT_LENGTH); + printMetadataContent(doc, CONTENT_TYPE); + printMetadataContent(doc, DATE); + printMetadataContent(doc, KEYWORDS); + printMetadataContent(doc, LANGUAGE); + printMetadataContent(doc, NAME); + printMetadataContent(doc, TITLE); + } + } + + private void printMetadataContent(ParseContext.Document doc, String field) { + logger.debug("- [{}]: [{}]", field, doc.get(docMapper.mappers().getMapper("file." + field).fieldType().names().indexName())); + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/asciidoc.asciidoc b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/asciidoc.asciidoc new file mode 100644 index 00000000000..dc06d4e83dd --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/asciidoc.asciidoc @@ -0,0 +1,5 @@ +[[tika-asciidoc]] += AsciiDoc test + +Here is a test of the asciidoc format. + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf new file mode 100644 index 00000000000..569a904a315 Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithEmptyDateMeta.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithEmptyDateMeta.html new file mode 100644 index 00000000000..f151208e384 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithEmptyDateMeta.html @@ -0,0 +1,11 @@ + + + + Hello + + + + +World + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html new file mode 100644 index 00000000000..79b5a6234ec --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html @@ -0,0 +1,11 @@ + + + + Hello + + + + +World + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithoutDateMeta.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithoutDateMeta.html new file mode 100644 index 00000000000..3322fa3a734 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithoutDateMeta.html @@ -0,0 +1,10 @@ + + + + Hello + + + +World + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-104.docx b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-104.docx new file mode 100644 index 00000000000..f126e20b32e Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/issue-104.docx differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testContentLength.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testContentLength.txt new file mode 100644 index 00000000000..d392c2d0979 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testContentLength.txt @@ -0,0 +1,9 @@ +Begin + +BeforeLimit AfterLimit + +Broadway + +Nearing the end + +End \ No newline at end of file diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testEPUB.epub b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testEPUB.epub new file mode 100644 index 00000000000..a6fc2e634d5 Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testEPUB.epub differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html new file mode 100644 index 00000000000..f5564f025d2 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html @@ -0,0 +1,29 @@ + + + + XHTML test document + + + + +

    + This document tests the ability of Apache Tika to extract content + from an XHTML document. +

    + + diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt new file mode 100644 index 00000000000..08280926034 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt @@ -0,0 +1 @@ +"God Save the Queen" (alternatively "God Save the King" \ No newline at end of file diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-french.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-french.txt new file mode 100644 index 00000000000..e4619fb1b88 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-french.txt @@ -0,0 +1 @@ +Allons enfants de la Patrie Le jour de gloire est arrivé. Contre nous de la tyrannie \ No newline at end of file diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-nolang.txt b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-nolang.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip new file mode 100644 index 00000000000..10f5d507677 Binary files /dev/null and b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/tika-files.zip differ diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json new file mode 100644 index 00000000000..c4c90ce75e9 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json @@ -0,0 +1,12 @@ +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "date": { "type": "string" } + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json new file mode 100644 index 00000000000..7dc796c2b17 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json @@ -0,0 +1,12 @@ +{ + "person":{ + "properties":{ + "file1":{ + "type":"attachment" + }, + "file2":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json new file mode 100644 index 00000000000..02176c7ca0d --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json @@ -0,0 +1,12 @@ +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "language": { "type": "string" } + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json new file mode 100644 index 00000000000..314c70db2aa --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json @@ -0,0 +1,56 @@ +{ + "person": { + "properties": { + "file": { + "type": "attachment", + "fields": { + "content": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "date": { + "type": "date", + "fields": { + "string": { "type": "string" } + } + }, + "title": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "name": { + "type": "string", + "fields": { + "suggest": { + "type": "string", + "store": true + } + } + }, + "author": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "keywords": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + }, + "content_type": { + "type": "string", + "fields": { + "suggest": { "type": "string" } + } + } + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json new file mode 100644 index 00000000000..ea83b98ceec --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json @@ -0,0 +1,19 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment", + "fields" : { + "content" : {"store" : "yes"}, + "title" : {"store" : "yes"}, + "date" : {"store" : "yes"}, + "author" : {"analyzer" : "standard"}, + "keywords" : {"store" : "yes"}, + "content_type" : {"store" : "yes"}, + "content_length" : {"store" : "yes"}, + "language" : {"store" : "yes"} + } + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json new file mode 100644 index 00000000000..c8680cf0644 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json @@ -0,0 +1,9 @@ +{ + "person":{ + "properties":{ + "file":{ + "type":"attachment" + } + } + } +} diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/00_basic.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/00_basic.yaml new file mode 100644 index 00000000000..819478d7d56 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/00_basic.yaml @@ -0,0 +1,14 @@ +# Integration tests for plugin: check name is correct +# +"Mapper attachments loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.plugins.0.name: mapper-attachments } + - match: { nodes.$master.plugins.0.jvm: true } diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/10_index.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/10_index.yaml new file mode 100644 index 00000000000..8b2c4b0ea18 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/10_index.yaml @@ -0,0 +1,158 @@ +# Integration tests for Mapper Attachments plugin +# + +--- +# https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/23 +"Index empty attachment": + + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + file: + type: attachment + - do: + cluster.health: + wait_for_status: yellow + + - do: + catch: /(.)*mapper_parsing_exception.+No content is provided\.(.)*/ + index: + index: test + type: doc + id: 1 + body: + file: { } + +--- +# https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18 +# Encoded content with https://www.base64encode.org/ +# File1 +# +# +# +# Hello +# +# +# +# +#World +# +# File2 is an encrypted PDF with a password + +"Multiple Attachments With Encrypted Doc Ignore Failures": + + - do: + indices.create: + index: test + body: + settings: + index.mapping.attachment.ignore_errors: true + mappings: + doc: + properties: + file1: + type: attachment + file2: + type: attachment + - do: + cluster.health: + wait_for_status: yellow + + - do: + index: + index: test + type: doc + id: 1 + body: + file1: "PCFET0NUWVBFIEhUTUwgUFVCTElDICItLy9XM0MvL0RURCBIVE1MIDQuMDEgVHJhbnNpdGlvbmFsLy9FTiINCiAgICAgICAgImh0dHA6Ly93d3cudzMub3JnL1RSL2h0bWw0L2xvb3NlLmR0ZCI+DQo8aHRtbCBsYW5nPSJmciI+DQo8aGVhZD4NCiAgICA8dGl0bGU+SGVsbG88L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9ImRhdGUiIGNvbnRlbnQ9IjIwMTItMTEtMzAiPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0ia2ltY2h5Ij4NCiAgICA8bWV0YSBuYW1lPSJLZXl3b3JkcyIgY29udGVudD0iZWxhc3RpY3NlYXJjaCxjb29sLGJvbnNhaSI+DQo8L2hlYWQ+DQo8Ym9keT5Xb3JsZDwvYm9keT4NCjwvaHRtbD4NCg==" + file2: "JVBERi0xLjQNCiXDpMO8w7bDnw0KMiAwIG9iag0KPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4+DQpzdHJlYW0NCjB+az8/Pz8/bDU/Nyw+Pz/Ln1hqZT8uTD9LMS0/Jj9bRDM/SFY/Pz92P2B2P1dxRE5BP2k8Wk0/P01RP3c/PytUP2YmPz9OSD94WT8/RD8/P31AID9JZ9KeYT91Pz8/RD8/PURPP0g/P28/bz8/P3k/UVk/dD8/Pzg/P2tjP3lmPz8/Pz9uP01mfEV6dVwmP095P2osPz14Xyk/P0swbQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMyAwIG9iag0KMTg2DQplbmRvYmoNCg0KNCAwIG9iag0KPDwvVHlwZS9YT2JqZWN0DQovU3VidHlwZS9Gb3JtDQovQkJveFsgLTkgNDIwIDYwNCA0MjAuMSBdDQovR3JvdXA8PC9TL1RyYW5zcGFyZW5jeS9DUy9EZXZpY2VSR0IvSyB0cnVlPj4NCi9MZW5ndGggOA0KL0ZpbHRlci9GbGF0ZURlY29kZQ0KPj4NCnN0cmVhbQ0KPz8gPw0K24kNCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjUgMCBvYmoNCjw8L0NBIDAuNQ0KICAgL2NhIDAuNQ0KPj4NCmVuZG9iag0KDQo3IDAgb2JqDQo8PC9MZW5ndGggOCAwIFIvRmlsdGVyL0ZsYXRlRGVjb2RlL0xlbmd0aDEgMTk5MjA+Pg0Kc3RyZWFtDQo/Rz/Lij9lP1E/UT8/MCs/M0I/N00/P00/DQpfdz87ND8/TD8/QT8/Nz8/wrU/Pz9zPzlPUyQ8XT/Vsj9DP9yOT1ljPz8/Uj8/ITY/Pz9Jaz8/dT8/Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB7Pz8/Pz8/P3k/dj8/ICl0Ij9sNz8/Pz8/bWs/P9uuPz9zVSM3Pz8zP1A/P3U/VT8sP1wpPz96P0o/PT8/ID8/OkA/Ii52w6YyP+WHmj8/Pz8/dV0/P2U/KD8/P08/P3FBP25Yc3w4P3U/QHs/Pyx5WD9OY9iEYcu3Pz8/P2JfSz8mPz8/Pz8/QTI/Kig/Pz95Pz8/P20/P04/P2w/DQo/P8mxWD8/Pz8/P1Yva3l0bEs/eVxrP3c/TDZRPz9DIT8/P1s/LD8/P2Y/Oj9mPyAgPy5vP2E/DQo/ej/Qsj8/Pz8/TDZRV2k/Pz8oPz86MT8/bXd4P3IkPz9SWT8/Kz9UIcOBLD8tPz8/Pz9LP2DbnT8/P3w9Pj8/P28/Pz9UYUY/fCxOPT8/P04/cj9kP0I/DQpUUD8/zZs/WT/Zk1YmPz8/XcmQYCI/44WwPz8tP2t8Mis/c1A6Pz9NP08/ez8/T1Y/1KY/P30/Pz8/Pz8/Uj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzDQo/dz9qPz9QP013Pz9PP3E/fD/Rqz8NCiAgICAgICAgICAgICAgICAgICAgPw0KICAgICAgICAgICAgICAgICAgICAgfEE0P1RoPz8/dmsqPz9tPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/Pz86Kg0KP966Qk9bPzg/LD8/QT8/P1g/ICAgICAgID8/Wz8/Pz9aP3t5Q3I/fjt8TT8/Yz9Cw7gxRz8/Pz8/1qE/P34/P29DPz9GP3E4P2U/Pz9xPz9rWD9uP1Q/CT8/aduZ66aaPz8oCWc/Sj8/eD8/1IsNCj9MMD8/N3N4P3R+Pz8/JD8/Pz8/Kj8/P1Y/P0NWOj8+Pz9hPz9rP3E/cz8/P9yTWik/Pz/Gjz9zPz/Mvm06YNaXRc6tZT8/dNuePz89SzAoP2o/fT8/ej8/NEA/YD8/Nz9PW1k/LD8vVzA/P2I/PzFRP0I8Pz8NCj8/Rj8vPz8zP8WLPzE/Pz/hp6k/Pz/aq0trIF4/P1lUPz8/Iy5uWT8/RGc/P1g/Pz8/xJs/Pz8/IW0/Pz9GDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/SdyZP3fPsk8/xK1PP3FSP08zPz9cPzJ4Pz5rL0ZKYT8nU3VZCTR1fj81aj9hPz9p34XwuLOLSj/Ogz8/ez8gPz91ZT/Ej2o/P0k/Pz8/ST9lLj8/Tz/EhD9VRXs/OUE/P2N0P3pOXz8/cz8/Pz9oUD8/Pz8/V00/P1soPz8/Nz88NFo/Pz96Pyg/Pz82Pz/GqkI/QD84Uj9RP9uoMnk/LiFBPw0KPzQ/QSo3P3w/1YI/X3w/ID/atT/pm4g/Rj8/Nj9CPj8/PyQ/P2A/Pz9QQFw/Pz8/az/cgj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/TD8/MD8/Pz96P1w/Pz8/Pz8/Vz9JUiR+Pz9nP3stbz/Sp1I/aHE/P1drP2ljPzQiPz9CND8kID9YPz9uPz9m7JCocj91Lj8/MiE/ND9IPz8/PzM/REg/Pz9dP9ioPyc/Ync/PyM/P0Y4Py0/04g/yKY/dz9XQz8/Pz8oej8NClw/Vc6gPz9xPy8/Pz8wDQogICAgICAgICAgICAgPzFjVz8oPyY1fT8/XFY/TjM/P18/Pz8/QCE/P34/PzZRUz8/Pz8/Pz8/R3RyMj8/CVo/Tj8/Mw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9cPyo/Uj8/Pz8oPz8/Pz8xPyU/Wlw/PzQ/dT87P3o/P1c/KM6PPz/XmT8/Pz85PD8zPz9+Pz8xDQo/P9Wlbz8/YduHLj90djwmPz8tdz9vaD/Trz86Pz8yPz8pJz8/MT8nPwl8PyNT07U6Pz8/NtyHPzI/QT8qPzQxTT8/aj8/Yj8/P2pBWlQ/PzZJNT8/PzrNvT8/ID8/P2M/P1A/DQogICAgICAgICAgICAgICA/Pz82L3vMo1NdLT8lPz8vPz8/P2MxOz9qIyQjwoFkPwk/Pz9kbT8/LTc/dj8/Pz95P35uP3RNP3A35oeRIXw/Mj8/ND8/NEY/Ij/KjDc/cD/Wj30uP1M/IygNCiAgICAgICAgICAgICAgICAgICAgICBOPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgID08Pz94MD8/IT9KZT8/P20/Pz/hjLE/SD9uP1A/Sig/cVR6CT8/Tz8/P0/TvDk/1aw/Pz8oZj9LP0dAPz8/ez8/IWlCP15fMnY/VFYyOD9IPy4/WD8/ez8/Pz9DP0Y8adGF36w3LD8/cj/Rhj8/X1M6bT8/SD8/X1I/M2gJP2c/bFU2P3c/Pz8/VT8/Pz8/dj8/Pz8/P1h5IUM/biQ/Pz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9yPz8/P0rprrsnPz8/KykqTDlTPz8/Pz8/Py8/Qz8/Pz8/WG1lPz8/P3g/P1w/NSUiW34nPz9Ecj8/JX0/SHVfR0k/QUFPP9WzPz8/az8/Lz96QD8/YU0/KD9SYj8/Pz8/PyclRQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBZPT8/xKo/Pz9qUnk/UFlLPz8/P3t3XX0/UD/Zgz8/Pz8oPyE/Pz8/MzE/Pw0KICAgICAgICAgdD8/Pz9UPz8/dj94PT8/Pz8/Pz9SPzM/Pz8tKj9uSHk/PzDNkz9yUT8/3ZVaP24/Pz9hP3c/aj/Qgz8gPz9RPy8hdT9mXWclP1g/dHs/PWlgP3o/Qz9tXT9TZD0/cD8lP2g/DQogICAgICAgICAgICAgICAgICAgICAgICA/XD8/Pz8/ZT8/P2LZqmt+Pz96N96KP3Q/Ijk/Pz8/KjM/P3k/U2g/PyBnPz8/SD9rP1k/Pz8/R1A0Pz96P3Q/YENRP+eAjz8/Xj9RP3U/P05KP2NbND9xP9GVP3k/Uj8/Jz9vcj8hPyU/P2BMPz9iYz9RPz8/P3E/cD9kP1Q/Oz8/Pz93Pz8/Pz9dQkouP8+kP3NtP1UNCj92PzlvPz89KHs/PT8/cD9manc/Pz8/Tz8/Pz8/Pz9ZPz8/dnA/TD9+Pz9c1os/Pz8/aX4/JD8/Pz8+Pz8/CT/Vr0k/dz8/a3w/JlYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP35SP3U/fCY/QmE/P1ZWPz8/Pz9HKz9FJD9rSj85Pj91d09IP0Q/Pz8/Pz9hPy04Pz8/Pz9wPzE/XkQ3P3N6P1s/K1JTWj0/Pzk/Tj9Qaj9HP3s/Pz9NNA0Kej8/P2h9Pyh6Pzg/OD8/Pz/ejC4/Jz8/Jz9RUj8/QD8/JD8/JFgNCj8/Pw0KICAgPz8/PD8/WEk/P2g/cz9wPD90Pz8/PzRhP3pIRS0/e1U/Pyk/P9SrVnRYJGY/Pz9qPyIvPz9tTH0/Zdy5I1I/Q8KfPz8/Pz9CMT8/ZD9lbz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB1Kz50PtSDdEs/UT8/Pz9tXD8/Pz9PyZExP2FOPj8vNHBuVQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7Pj8qZj88P21aST8/Yj8xQD9oPz8hdT9eP3zMrj8sK21hPz8wMCc/ej9bPz8/XVw/YFpNUCI/aUE/Pz8/7IKRNGpFMz91P1RyPz9wP1BSPz8/Tj9lJH5nPz9wPz8/TkduPz8/U30/cT9SPz9Laz8/L34/P9q2P0kzPz/dpF86Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQj9cP04/dD5oaT8/Oz8/amssZz/LtD8/ej/FuiE/P3dmPz8/Pz9Ze3s/Pz8/Lj9wPm8oeG5626hFbX4zQj8/P28uPz8/Pzs/Pz8/CT97Mz8/Sz8iJi9VIj95Pz8/VT8/Pz8NCiAgICAgICAgICAgICAgPz9kTT8/Pz9VKz8/Lj9zP3dTzbtAPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/MSY/Pz9oP0g/Wz9KNz9mPz8/xZk/Mz94MjQ/OGR3Zz8/Jj8/PzB5Sj9DPz8lO8qNbj9vcj9NbT8Jw5BoPz8/TWEsPyE/Pz9sDQp5Kj94Pzc/aloiPz88Xz8/ej9APz8jPz9IPz0/1Lc/UD9XOVs/XD8/Pz8/P0huP3E9RD/Urz8/Pz8/NS1bPz8/bD8/P3tfZj8/PD90NdauPz/cvD8/P0o/XT9IfD8/Ij8kPz8/P24/Pz8nPz8/PzchLHZRPz/Mij8/PA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICo/XnFhP2o/XT8/RDzXlz8/xIQ/P1lGPz8/Rd6eITA/U1Y/P20/Pz9wXj8/MD9ePz83P9KIP2A/Pz81Zj8/Pz85TT8/Wz8/P24/ej8vPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgMD8/PHJIPz8/P34/PwlTLTA0fHU/P1pMPz8sPz8/zpc3TC0/x6A/PyEyPz8/Pz8/Ij96xa8/Sj9ddz8/Pz8/P3ZjVT8/PyA/cD8/PzpY7ZWOOlc/bD8jQGo/Py4/Pz8/Pz9ePz8/Pz8/Pz9kTjNdRy1mT3t0Pz0/ajkgSDQ/VnI/Pz8/Pz0/P2gnJD98P0lHPz91PztzNT8/Py/KsD98Pz8/Nj9wPz8/UlY/Vz9nPyw/P1p7NT96cyFvPyB0Kz8pP1dFPw0KPz8/Kj8/J8meP2E/Pz8/Pz9AXyFjPz8/PyZkx45lIj9FPz84Pz9vZT8/Pz8/YD8/P28/Pz86PykyPz8rPz9xPz8/Py0/LUhwPz/csG4/P3ghPz8wP0J5P8qjZk8/dEo/Pz9OUj82P2JWLHbKujM/DQogICAgICAgICAgICAgICAgICAgICA/Lm5AYD9Wczs/P0I/P2k/Pz8oP0c/NT8/P18/P1lyP0E/Pz8/Pyk/SyY/2a0/Pz87OD8/Pz8/P3U/Pz9QP242Iio217c/P2I/YklLP2YmPz8/Mig/Kz8/ZlwxP3pdPyU/WMugP0gvPz8/P0U/Pz9AUMyjP1c/Pzc/RT8/UG4/Pz8/Pz84K08/Pz9mej8qJj8/0p8/4oKsP04/TykkPz8/e2Q/aD887428Pz/Ut1plVD8/Pz8/Pz8nPz8/Ij8/P1MNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9OPz8/Py16OT8/1bEzPyc/ZDkxPz8/P9KrPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9vYz8/RDLVkExCPz9BPyE/P2E/yLk/eD9oPz9HP0jMnz8/Pz9RPz8/Pz8/Mj960rM/fT8kJz8/Pz8/P0o/Pz80Lz8/Pz9AVT8/Pz9kPz8/Pz8/Xnw/bj8/QdeXPz9TPzw/Pz96Z3l6P195Pz93PzE/NT8/Pz8/Pzk/w5ZzPz8uK28/Pz9ZSz8jPz8/1Jg/Pz9nP1lPPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlPz8yPz92P9anPT8/Pz8/PCU/PD9mP1lxPzk/NDtbzoQ/XD9XPz8/PzHanD8/OnTKhT8wPz8/P1w/Pz9uP0w/P3Q/ZT8/Pz8/Py10Pz8uPz8/P2QpYT8/Pzw/Pz9vdD9dDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7P0JRJlTegT9nZjk/I3bNh0A/P0JGOD9qZj9uPyVLP1R9eH5VPz8/Pz8tNSE/Yz8/cSs/Zj90NGZmbUEpPyM/PyI/dj8tWz4/Pz9+PNSuPz8/P3INCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9mPz8/Pz9+Pz8/Pz8/Jz8/Py4/Pz9NID8/Pz9idgk/P1I/Pyo/xIg/dl8/Pz/Vtk4/P2I/e2s/PzA/P0JVP2E/ID8rPz8hQVE/NT18P0FgIj8/fG4/TD8/Pz9NeS4oPz8/P3IqPz8/JUkiPz89LV4/TT8/Iz8/Pz8/P34/ME3Ht8mpPz8/RT8/cj8/P2Q/UC5LPz8/bCI/bFBdPz8kP0M/Pz9KCX0NCk4/Pz8/Pzk/bEc/Pz9szII/aVU3ICY/RChsPz9IPz8/bT8/P1cjP8yoPyvbmUo/eTo3P0A/fFd9P2E/Pz90eWg/P1o/Pz8/YT8/ej8pPz8/Pz9tPzQlPzc/0rVa2IU/Sz8/WD9+06c/Pz8/Pz8/Pz/RtT8/PzUzKjnYrz/XiQ0KRCo/NyNrP3cpP8mrUj8NCj8/Pz8/byc/dyd4Pz87eD8/NT8/Pz8vPz/Yo0FEMj8/UD8/Pz8/PyNMWj8uPz9Qw6k/MjQ/Pz9eOj/Sqj8/P1Y/240/Py15Zz8/bz8/PyQ/Pw0KLz98U1w/OVQ/Pz85Xz8/Pz9aPz8/yLEhcD90VT8/Rj8hy7RFPz9YP2NJVT8leD8/PzM/P1lYOnc/MD8/P3c/Vz8yDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/06M/Pyc/J2A/PyR7Pz8/P2kwP0JKP18/TGQ/eVk/Pz8/Pz8JP1s/Ud6GeChc1pg2Pz8iP1AlPyo/Pz8tPz9oPz9uPz8/Pj8/Pz8/Pz81LjM/Pzk/Pz89Pz9PPz9GJT9DKOODrj8/bT8/RU4/VXVQJWNvPz86P0p507k/Pzg/P1U/Pz95LFpVPz8/P2A/Oz8/Pz/LnV4jd2I6ac2wP2JRPwk/bj8/26k/WD8wPz8jaT9vPy4/QT8/NkIhPz/fjyU/wpE/cz8/P27dtT95OD8sP2prP3w/P3s/Vj88P9a3P9+KPy4/P2BAdlbSgj8/Imo/P0c/UD9iP250P9aDPz8/PyM/Jkg/Py4/Rz9oRD8/xqgiPz8lPyU/Pz8/RzM/Pz97P3c3Pz/dk0cnPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP3c/Pz8/P0liYD9ZPzo/JlZtcj8/c2lYPz8/P24/P18/Pz80ank/P0tFKj/OmFJ4P3g/OyMyM2QkNT8/cCsyIj9cPzw9JMmnP1lHPz8/X00lPz9lQnxKTEJp1oI/Nj9D1Z0/VHZWXz9yPz8/cz/apEw/w7JcWz8/Pz98bWw/CT8/P2M/PzM/Pz8/Pz9tPyciPz4/e3bUmQ0KICAgICAgICAgICAgICAgP0w/P0p1Mj9iP2A/aT9mPzw/P2BfPz9eP2FVPz/Yqj8hP0s/P1dgPz9AP0QJP08/fj8/UH4/Pz9nVD8/Pz9uZT9vP+eShD8/Pz4/P3skPz90dmo/cz8jNz9LPyY2P9alP2o/P92OP3g/Jm9jPy8/cj8/Pz8lPz8/P3A+Rz8lP0gqJj8/Pz80PyFRQz8/P8+oPz9kRT8/Pz5RPyE/VFNBSlw/YnQ/P1wgP3E4PyE/P9SMPz8/ZD8/Kg0KPz8obD9ZPyo/P3VTP11GKT8/Pys/Pz8/Pz8/34rXsnFJWz93Pz9aRT9JMzhXTlk/P3hzPz8/xJR8Pz9dSVU4Ljc/Py8/Pz8/Pz8/P2k/0Ig/PwlZeHdoPz8/UT9iWdCEP3k/Pz9IfT9PPz8/TtqBPz/Xu2A/Pz8/ID8/P1Y/DQo/P0lMRj8/Rzo7I0Y2Jj92Kj9HVzw/P0FIPz9vMD8/MmE/Pz8/P1MpVD8/Pz98S1U/Pz9UCTc/ej8/Pz/CtnVJPz9OSnI/Pz/crD8iPT8/Qj9xPz9SLj/voq0NCiAgICAgICAgICBmPz9xPzE/MT8JPz8/Pz8+JHw/Lj9qdT9zYHA/cVM/1ZEJP24/ZT8/Pz9Odjo/y7U/Pz8/LD8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA9XD8/JT8/Pz9YP8uEP1E/Pz/HjT8/V3TVkD8/dz8/fD8/Pz8tOGY0Pz/euj98Pz97aT9XQSk/Pz8/PysnPz8nP0A6TdeBPz85UW91Pyc/UzVlRD9UNz8/P0E/Pz9tZHY/PyI/cD9yT2hGPyRJP9+rMj8lWWA/Pz8/Pz9hdXk/zZcqVD9HPj8/Pz8/XT9kMD9ZND44XT8/ZT8/Q1g/Pz8/SD/YkVk/1JQ/PzM/Oz98Pz92TT8/ej8/P+mqmz8/P3U/Pz/csz9BND8/Pz8/aD8/ZT/Rm8iJPz8/Pz9qPz8/PyhXZD8/P2lCUVA/IEQ/Pz/nlrc/Pz/Viz8/P0NcNT8vQXY/ZnA/Pz9vQD8/Pz8/P29EJD8/Pzc/Pz9YdT8rDQogICAgICAgICAgICAgIO2GsVg/UT87XFE/Pz8/Wz8/2Kc/2Jc/dw0Kdj94Uz8/YD8lP28/Pz80Pz9aRD8yP2ZjPz8uP0R6Pz8/VW8/Xj4/Jj8/dT8/SD/aoz8/Jg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfj8/Pz8/PXo/Z0dUPz9iPz8/czwnPz9tP2t4P1I/I3k/Nj8/Pz8/RlA0PD8/QT9oPz8sPz97MT8/RGdFcEY/Pz8JPz91Jj8/P2523r02Pz8/P2w3P1VPdD8qP2U/Pz9XcT8/Xj8/K1d6Pz8xP20/woEoXFpCP3Y9dC9NKD8/JD9fPzU/P0x+Pz8/Pz8tRFxSVj993JI/PzQ/Pw0KPz8NCj8hPz8/dyjJgz8/Pz86Nz8ye0c/Pyg/Zj8/LE0/Pz8/MD9yaktHWNyEMFU/fT8/P9OLPzM/bD9wP3cNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/PWU2Pz8/MmZSPyIraA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgS1A/Jz8/XSYoPz8/dj9AbD8/P3J4P3ZoUnJjP1QlPz9F3rxrPz8rPz8/Nj9SaT46Zz8/Pz8zPz8/Pz8/X1Y/P2ZYPz8/Pz9sP++sryU/Pz8kMT8NCiIyPz8/NUdKPz9PNj8/PzE/T3c/Pz8NCmhRSD8/KWU0Pz8/PVw/TD8vPyFBPz0/Xj8kPz9lPz8/Pz9yPmthP08/ajM/KT8/Pz8/ZUE/27g/Pz8hPy8/XW8/Yj9HP0Mjbms/75qFPzFMYj9YPz9oPz8/P8KHPz8/OCY/cT9xIQ0KP28/P1k5KCA/SEA/IVw/ciA/Rz8/Pz8rDQogICAgICAgICAgICAgICAgICAgIC4w34MrPz8/LT8zJGk/Pz9EP0E/PzJoPz92P2A/LT97PycyP1hHcj8/P0ghdXF5XmNkP2M/Pz8/PypINtqDMz8zPz8/ST8NCiAgICAgICAgwp0/PycvTSA/UT8/Sz8/Pz8qDQogICAgICAgICAgICAgICAgICAgICAgICAgPw0KOD9sP8WiPz8/P3E8y595Pz8oPz94Pz8/cj9sPz8NCj8/U0E/fHo5P2cvOE1mP2smPz8qOj9Xaz8/Pz8/Qj8/Tj8/Pz/bh14/SD9cMHI/P1w/Pz8/x6A/Pz8/IVI/PzZfPwkkPzNZWSdtPz9SP2BiP2U/P2c6P3VkP18/Pz85fj8/Pyo/aEjfoT8/PzU/P0FPTz80Pz9NcD91PzJoPz9eP3tiYVQ/Pz8/Uz8iYj8/Pz9rP2w/Pz8/Pz9zcD9geU8/Lio/cz8wPz9xPzc/Nd+FRmw/YG0/Pzw/zYU/TWk/Pz8/VD8/Rz8/Pz8/JXZHfT81zac/Pz8/eD8/fUAlPz8/Z1N2bN2NPyMuPz8gP2xpZD8/NFQ/Kz8/Pz8/Pz8/P1Y/Lj8/P1o/PwkhPz8/cz8/RjYNClkNCiA/MT8/fQ0KP20/YXE/L0A/WDtHP2E/Nm0/cD9VZnFaP0k/TD8nUz8/PyU/Pzk/Pz83P05OPz9wP0EkbUo/WD8/PzLNqT8/bD97Kj8/R8yrPz8/yJ4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPz8/SD8/SD9xUT8/P0NrPHk/aj8/Ow0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUTk/cz8/Pz96IV3VuW1hTD8/MT9rPz/Isj8/Pz94PzE/RD99PzU/PyA/Pz8/Pz8/P111Pz8/IU8/UD84TT8/Mj9QPz8/OShDPyE/W3BKPz/JknM/ZT88P1Y/P3Y/RT82Jz8/VT8kP0Y/Vj9QOz8/aj9+Pz8/J1Y/dWQ/b3E/Pz9iPz8/cnE/Pz8/Jz9OP9CTOHvSpT8wP1Fc3oJPbT8nPz1PP2ojPz8nPz8/Pz8tKuSrpj8/OT8lWD8/Pz8/Pz8/Pz9rU+OYnSQkP1E/xrw/PyHMiD9RemY/P9GYPz8/S1E/SjE/Pz8/P3I/bz/WhD9uS3k/NX4/P1nelD8/TT92Pz8/eD8/YTw/P10/Uz8wP31hP3U/Ol1+SzM/cmh5PypWPz9haj92MlQ/Wz8/Pz8/P00/Uz8/Pz8/dEQ2P0trdz8/Pz/fhHINCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFAiPz8/P2s/P2I/UXs/Pz8/Pz8/Wj8/DQpwPz8wPzc/Pz8/PzVRP0LTjj86aj8/Pik/cj8/PzkNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPyA/Pz/Ltj94P1Rfej98Tj8/P2tVSz9pP3M/bkc/anA/Pz8hP0A3fT8/Pz9uTz8/Pz8/djomPzNnTnY/TD8semQ/P0U/az8md3k/P9mKPz8/P3w/Pz8/Pz8/Pz83Pz8/Pw0KP3M/Pz8/P1U/Pz8/P1Y/P1R3K1zSpio/P3rRkD8rPzM/P0AzWD8/Pyg/dFM/P1I/cz8/Lz9J0Yc/Zj9qU9uUPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbj8/ez8/aD8vPz/Iqgk/Pz8rJkY/RNKIPz9NP0Q7P144Pz9UUD8/Pz8rPz8/Pz98Mj9TPz97Pz8yRT8/KT8/XnNrY3g/Zj8/VT9FQl1dOj9QP3BmPz99P/C5jLFxJT9ucz9fRls/cUbSk1t5MFk/P2dHTz8/PylrWmk/fU5dPz8/SD8/ZWopPz86zJk/Jkk/Pz85P2k/aSE/Xz9wb0c/MnI/Klk/dT8/JMiM17lZUseJPz8/VD9PPz8/JiU+P9OEPzDUkDAoME8/Yz9CCT8/P3M/P0ddPzc8RsWH7qKKOXRjetyWPz9dPz/jvIg/Tgk/Pz8/PyU/P0jOvj8/Pz8/PytDUlE/bD9JRkk/JWd9e25iPzg/IT9QPz8/Pz93Pz8/2L4hK9Ccbj8/Pyk/Pz8/dT8mP0XGrD8/MD9vd1Q/fT/GtEQ/XT9RP1s/fChiPz9YP2M/P3ZXLT8/Uj8+MHg/Zz8/xpBKPy8xPyM/dz9RPz9OP0s6Pz8/NT9+Pz8/P2RHbz8JxIc/PyQ/P08/Uic7ND8/QU1iPz8/J0FfOz8/Sz9eJj8qP0ZpIj9cPz95Vj87SyRbPz8/Pj8/L1E/RS0/P1c/dTs8UDN8OVk/Qz9yPz86PzE/OjskKt2EP9GuPz8/P1/ZkGM/Pz8/Mz8/Z3I/Pz9APz86Pz8xPz8/DQo/IT9QX9yBeCNTQ2A/PUBVP0lyaj8/P8eSPzgtej8/Pzs/P2INCj8kKz8/c2g/Pz9lLiPdhA0KLiUzPz9QP8e9fGdnP0o/Py1gP3pdPz8/eilofUcvPz8/Vz8gP3E/Rz8nOj8/DQogICAgICAgICAgICAgICAgICAgICAgICYxPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9kcT8/cj9wYz/GgW5SPz8xTQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/PzdyPD8lVz9UPz8/Pz9sPykgPy/KmFd2Pz8/Nj8/3Lk/P1x1P3g/Oyc/Pz8/IH5fyJPdsz8yRMafND8/P0A/P2w9Wj8/Pz8vOnY/PzI/Pz94Pz8/2oxAPz9bbj8/Ij8/Pyh+LSk/eCNmPz9cPz8/cD8nfj8/d3YjMz/JsT8/KD8/ZD8lDQo/P0s/1pRuPz8/P0o/Oj1yPz8/PyY/P1F6Pz8zPzk/ez8/Pz8/KlA/Pz9gbT8/MT8NCj8/Pz9sPj9MPz8xP3M/P1c/P18/fWNTP01wP04vPGs/aFZ1Sj9aUj8/Pz8rPz8/P08/eN2yeEg/Rz8/QD9aLj90P3R8Lz80P09YOT8/JUg/P9ytPz/Unz8/Pz8/fj9cPz8/VHk/Pz/WqWU/Pz8/Tz8/aeKAmD8/UD8/dj9hPz8/P+ajsT9sLWxSPz9cPz8/Kj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHLCikk/P9SwPz86Rj9DPz8/azs/PzR70po/UT98Pz8/RA0KICAgICAgID8/PzA/PyVVP0U/Pz8qMj8/Pzk0Pz8/Lz8/U3A/Pz9IJD/gvIg/QFtxPz8jRD8/Oj8NCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjggMCBvYmoNCjExODcyDQplbmRvYmoNCg0KOSAwIG9iag0KPDwvVHlwZS9Gb250RGVzY3JpcHRvci9Gb250TmFtZS9CQUFBQUErQ291cmllck5ld1BTTVQNCi9GbGFncyA1DQovRm9udEJCb3hbLTEyMSAtNjc5IDYyMiAxMDIxXS9JdGFsaWNBbmdsZSAwDQovQXNjZW50IDgzMg0KL0Rlc2NlbnQgLTMwMA0KL0NhcEhlaWdodCAxMDIwDQovU3RlbVYgODANCi9Gb250RmlsZTIgNyAwIFINCj4+DQplbmRvYmoNCg0KMTAgMCBvYmoNCjw8L0xlbmd0aCAzMTQvRmlsdGVyL0ZsYXRlRGVjb2RlPj4NCnN0cmVhbQ0KMT8/RT8ycSE/NT8/zpw/yKI/Pz8gUT8/P0x4fXo/Mz8/Pz9jP0U/R20rNlQ/Iz8/Pz8/VT8/P2c/S9+oP1vMvU0/O8SwPz/PqWw/P2c/QD83P3pVPnw/Pz9wPzs/PyhyLDY/P3ZMP3JqPz8/PzU/fj4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgP1XNuT95JT8/d1snP34NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERSPz/mlaA/P3wzPz8/Pz9F3J8m0oA/Ik4zPz8/Sc6FP2o/Lz8/eT90Pz8/Pw0K56SMbVA/PzLZhj8/IEF2Pz8/1o0/Pyk/Pz8/Pz8/Vl0/PT9DP0g/bm12Pz8/DQp0Pz8iPyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBMJ14/Pw0KICAgICB5Pz8/Xz8/Yj9+Pz9OPy0zP0o/WGloND4/P00/Pz8/CQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMTEgMCBvYmoNCjw8L1R5cGUvRm9udC9TdWJ0eXBlL1RydWVUeXBlL0Jhc2VGb250L0JBQUFBQStDb3VyaWVyTmV3UFNNVA0KL0ZpcnN0Q2hhciAwDQovTGFzdENoYXIgMjANCi9XaWR0aHNbNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwDQo2MDAgNjAwIDYwMCA2MDAgNjAwIF0NCi9Gb250RGVzY3JpcHRvciA5IDAgUg0KL1RvVW5pY29kZSAxMCAwIFINCj4+DQplbmRvYmoNCg0KMTIgMCBvYmoNCjw8L0YxIDExIDAgUg0KPj4NCmVuZG9iag0KDQoxMyAwIG9iag0KPDwvRm9udCAxMiAwIFINCi9YT2JqZWN0PDwvVHI0IDQgMCBSPj4NCi9FeHRHU3RhdGU8PC9FR1M1IDUgMCBSPj4NCi9Qcm9jU2V0Wy9QREYvVGV4dC9JbWFnZUMvSW1hZ2VJL0ltYWdlQl0NCj4+DQplbmRvYmoNCg0KMSAwIG9iag0KPDwvVHlwZS9QYWdlL1BhcmVudCA2IDAgUi9SZXNvdXJjZXMgMTMgMCBSL01lZGlhQm94WzAgMCA1OTUgODQyXS9Hcm91cDw8L1MvVHJhbnNwYXJlbmN5L0NTL0RldmljZVJHQi9JIHRydWU+Pi9Db250ZW50cyAyIDAgUj4+DQplbmRvYmoNCg0KNiAwIG9iag0KPDwvVHlwZS9QYWdlcw0KL1Jlc291cmNlcyAxMyAwIFINCi9NZWRpYUJveFsgMCAwIDU5NSA4NDIgXQ0KL0tpZHNbIDEgMCBSIF0NCi9Db3VudCAxPj4NCmVuZG9iag0KDQoxNCAwIG9iag0KPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDYgMCBSDQovT3BlbkFjdGlvblsxIDAgUiAvWFlaIG51bGwgbnVsbCAwXQ0KL0xhbmcoSj9Abj8pDQo+Pg0KZW5kb2JqDQoNCjE1IDAgb2JqDQo8PC9DcmVhdG9yPEQ3OUM1NUQ1OTdDMDUxMjJCQkE3N0M2NDIyRjg+DQovUHJvZHVjZXI8RDc5QzU1Q0U5N0RCNTEyOUJCQTE3QzY0MjJDNTY2M0VDQ0I1QzM0OTdENURDQjAwQUJEM0IyMkE4NkMxNDlEMz4NCi9DcmVhdGlvbkRhdGUobVlnPz8/YXM/P005P1ZrPz8/TT8pPj4NCmVuZG9iag0KDQoxNiAwIG9iag0KPDwvRmlsdGVyL1N0YW5kYXJkL1YgMi9MZW5ndGggMTI4L1IgMy9PKD98QTptP3AhUz8zP0fRhT8/SD8/P86nVzs/VD9cYikvVSg/PzxbP05WYz/Qvz8/fT8pL1AgLTEwMjg+Pg0KZW5kb2JqDQoNCnhyZWYNCjAgMTcNCjAwMDAwMDAwMDAgNjU1MzUgZiANCjAwMDAwMTM0NzAgMDAwMDAgbiANCjAwMDAwMDAwMTkgMDAwMDAgbiANCjAwMDAwMDAyNzYgMDAwMDAgbiANCjAwMDAwMDAyOTYgMDAwMDAgbiANCjAwMDAwMDA0NzMgMDAwMDAgbiANCjAwMDAwMTM2MTMgMDAwMDAgbiANCjAwMDAwMDA1MTMgMDAwMDAgbiANCjAwMDAwMTI0NzAgMDAwMDAgbiANCjAwMDAwMTI0OTIgMDAwMDAgbiANCjAwMDAwMTI2ODcgMDAwMDAgbiANCjAwMDAwMTMwNzEgMDAwMDAgbiANCjAwMDAwMTMzMTIgMDAwMDAgbiANCjAwMDAwMTMzNDUgMDAwMDAgbiANCjAwMDAwMTM3MTIgMDAwMDAgbiANCjAwMDAwMTM4MDkgMDAwMDAgbiANCjAwMDAwMTM5ODQgMDAwMDAgbiANCnRyYWlsZXINCjw8L1NpemUgMTcvUm9vdCAxNCAwIFINCi9FbmNyeXB0IDE2IDAgUg0KL0luZm8gMTUgMCBSDQovSUQgWyA8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+DQo8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+IF0NCi9Eb2NDaGVja3N1bSAvNEE4NTkwMDE4QURDRjVCQjRCQTQ1NDFDQUFDQjFBNEUNCj4+DQpzdGFydHhyZWYNCjE0MTIyDQolJUVPRg==" + hello: "world" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + hello: "world" + + - match: { hits.total: 1 } + + - do: + search: + index: test + body: + query: + match: + file1.author: "kimchy" + + - match: { hits.total: 1 } + +#--- +# This test has been disabled as it tries to decode an encoded PDF using BouncyCastle lib +# Elasticsearch security manager does not allow permission java.security.SecurityPermission "insertProvider.BC"; +# See https://github.com/elastic/elasticsearch/pull/13077 +# See https://github.com/elastic/elasticsearch-mapper-attachments/pull/150#issuecomment-134247110 +# +# https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18 +# Encoded content with https://www.base64encode.org/ +# File1 +# +# +# +# Hello +# +# +# +# +#World +# +# File2 is an encrypted PDF with a password + +#"Multiple Attachments With Encrypted Doc Should Fail": +# +# - do: +# indices.create: +# index: test +# body: +# settings: +# index.mapping.attachment.ignore_errors: false +# mappings: +# doc: +# properties: +# file1: +# type: attachment +# file2: +# type: attachment +# - do: +# cluster.health: +# wait_for_status: yellow +# +# - do: +# catch: /(.)*mapper_parsing_exception(.)*The supplied password does not match either the owner or user password in the document\.(.)*/ +# index: +# index: test +# type: doc +# id: 1 +# body: +# file1: "PCFET0NUWVBFIEhUTUwgUFVCTElDICItLy9XM0MvL0RURCBIVE1MIDQuMDEgVHJhbnNpdGlvbmFsLy9FTiINCiAgICAgICAgImh0dHA6Ly93d3cudzMub3JnL1RSL2h0bWw0L2xvb3NlLmR0ZCI+DQo8aHRtbCBsYW5nPSJmciI+DQo8aGVhZD4NCiAgICA8dGl0bGU+SGVsbG88L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9ImRhdGUiIGNvbnRlbnQ9IjIwMTItMTEtMzAiPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0ia2ltY2h5Ij4NCiAgICA8bWV0YSBuYW1lPSJLZXl3b3JkcyIgY29udGVudD0iZWxhc3RpY3NlYXJjaCxjb29sLGJvbnNhaSI+DQo8L2hlYWQ+DQo8Ym9keT5Xb3JsZDwvYm9keT4NCjwvaHRtbD4NCg==" +# file2: "JVBERi0xLjQNCiXDpMO8w7bDnw0KMiAwIG9iag0KPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4+DQpzdHJlYW0NCjB+az8/Pz8/bDU/Nyw+Pz/Ln1hqZT8uTD9LMS0/Jj9bRDM/SFY/Pz92P2B2P1dxRE5BP2k8Wk0/P01RP3c/PytUP2YmPz9OSD94WT8/RD8/P31AID9JZ9KeYT91Pz8/RD8/PURPP0g/P28/bz8/P3k/UVk/dD8/Pzg/P2tjP3lmPz8/Pz9uP01mfEV6dVwmP095P2osPz14Xyk/P0swbQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMyAwIG9iag0KMTg2DQplbmRvYmoNCg0KNCAwIG9iag0KPDwvVHlwZS9YT2JqZWN0DQovU3VidHlwZS9Gb3JtDQovQkJveFsgLTkgNDIwIDYwNCA0MjAuMSBdDQovR3JvdXA8PC9TL1RyYW5zcGFyZW5jeS9DUy9EZXZpY2VSR0IvSyB0cnVlPj4NCi9MZW5ndGggOA0KL0ZpbHRlci9GbGF0ZURlY29kZQ0KPj4NCnN0cmVhbQ0KPz8gPw0K24kNCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjUgMCBvYmoNCjw8L0NBIDAuNQ0KICAgL2NhIDAuNQ0KPj4NCmVuZG9iag0KDQo3IDAgb2JqDQo8PC9MZW5ndGggOCAwIFIvRmlsdGVyL0ZsYXRlRGVjb2RlL0xlbmd0aDEgMTk5MjA+Pg0Kc3RyZWFtDQo/Rz/Lij9lP1E/UT8/MCs/M0I/N00/P00/DQpfdz87ND8/TD8/QT8/Nz8/wrU/Pz9zPzlPUyQ8XT/Vsj9DP9yOT1ljPz8/Uj8/ITY/Pz9Jaz8/dT8/Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB7Pz8/Pz8/P3k/dj8/ICl0Ij9sNz8/Pz8/bWs/P9uuPz9zVSM3Pz8zP1A/P3U/VT8sP1wpPz96P0o/PT8/ID8/OkA/Ii52w6YyP+WHmj8/Pz8/dV0/P2U/KD8/P08/P3FBP25Yc3w4P3U/QHs/Pyx5WD9OY9iEYcu3Pz8/P2JfSz8mPz8/Pz8/QTI/Kig/Pz95Pz8/P20/P04/P2w/DQo/P8mxWD8/Pz8/P1Yva3l0bEs/eVxrP3c/TDZRPz9DIT8/P1s/LD8/P2Y/Oj9mPyAgPy5vP2E/DQo/ej/Qsj8/Pz8/TDZRV2k/Pz8oPz86MT8/bXd4P3IkPz9SWT8/Kz9UIcOBLD8tPz8/Pz9LP2DbnT8/P3w9Pj8/P28/Pz9UYUY/fCxOPT8/P04/cj9kP0I/DQpUUD8/zZs/WT/Zk1YmPz8/XcmQYCI/44WwPz8tP2t8Mis/c1A6Pz9NP08/ez8/T1Y/1KY/P30/Pz8/Pz8/Uj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzDQo/dz9qPz9QP013Pz9PP3E/fD/Rqz8NCiAgICAgICAgICAgICAgICAgICAgPw0KICAgICAgICAgICAgICAgICAgICAgfEE0P1RoPz8/dmsqPz9tPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/Pz86Kg0KP966Qk9bPzg/LD8/QT8/P1g/ICAgICAgID8/Wz8/Pz9aP3t5Q3I/fjt8TT8/Yz9Cw7gxRz8/Pz8/1qE/P34/P29DPz9GP3E4P2U/Pz9xPz9rWD9uP1Q/CT8/aduZ66aaPz8oCWc/Sj8/eD8/1IsNCj9MMD8/N3N4P3R+Pz8/JD8/Pz8/Kj8/P1Y/P0NWOj8+Pz9hPz9rP3E/cz8/P9yTWik/Pz/Gjz9zPz/Mvm06YNaXRc6tZT8/dNuePz89SzAoP2o/fT8/ej8/NEA/YD8/Nz9PW1k/LD8vVzA/P2I/PzFRP0I8Pz8NCj8/Rj8vPz8zP8WLPzE/Pz/hp6k/Pz/aq0trIF4/P1lUPz8/Iy5uWT8/RGc/P1g/Pz8/xJs/Pz8/IW0/Pz9GDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/SdyZP3fPsk8/xK1PP3FSP08zPz9cPzJ4Pz5rL0ZKYT8nU3VZCTR1fj81aj9hPz9p34XwuLOLSj/Ogz8/ez8gPz91ZT/Ej2o/P0k/Pz8/ST9lLj8/Tz/EhD9VRXs/OUE/P2N0P3pOXz8/cz8/Pz9oUD8/Pz8/V00/P1soPz8/Nz88NFo/Pz96Pyg/Pz82Pz/GqkI/QD84Uj9RP9uoMnk/LiFBPw0KPzQ/QSo3P3w/1YI/X3w/ID/atT/pm4g/Rj8/Nj9CPj8/PyQ/P2A/Pz9QQFw/Pz8/az/cgj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/TD8/MD8/Pz96P1w/Pz8/Pz8/Vz9JUiR+Pz9nP3stbz/Sp1I/aHE/P1drP2ljPzQiPz9CND8kID9YPz9uPz9m7JCocj91Lj8/MiE/ND9IPz8/PzM/REg/Pz9dP9ioPyc/Ync/PyM/P0Y4Py0/04g/yKY/dz9XQz8/Pz8oej8NClw/Vc6gPz9xPy8/Pz8wDQogICAgICAgICAgICAgPzFjVz8oPyY1fT8/XFY/TjM/P18/Pz8/QCE/P34/PzZRUz8/Pz8/Pz8/R3RyMj8/CVo/Tj8/Mw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9cPyo/Uj8/Pz8oPz8/Pz8xPyU/Wlw/PzQ/dT87P3o/P1c/KM6PPz/XmT8/Pz85PD8zPz9+Pz8xDQo/P9Wlbz8/YduHLj90djwmPz8tdz9vaD/Trz86Pz8yPz8pJz8/MT8nPwl8PyNT07U6Pz8/NtyHPzI/QT8qPzQxTT8/aj8/Yj8/P2pBWlQ/PzZJNT8/PzrNvT8/ID8/P2M/P1A/DQogICAgICAgICAgICAgICA/Pz82L3vMo1NdLT8lPz8vPz8/P2MxOz9qIyQjwoFkPwk/Pz9kbT8/LTc/dj8/Pz95P35uP3RNP3A35oeRIXw/Mj8/ND8/NEY/Ij/KjDc/cD/Wj30uP1M/IygNCiAgICAgICAgICAgICAgICAgICAgICBOPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgID08Pz94MD8/IT9KZT8/P20/Pz/hjLE/SD9uP1A/Sig/cVR6CT8/Tz8/P0/TvDk/1aw/Pz8oZj9LP0dAPz8/ez8/IWlCP15fMnY/VFYyOD9IPy4/WD8/ez8/Pz9DP0Y8adGF36w3LD8/cj/Rhj8/X1M6bT8/SD8/X1I/M2gJP2c/bFU2P3c/Pz8/VT8/Pz8/dj8/Pz8/P1h5IUM/biQ/Pz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9yPz8/P0rprrsnPz8/KykqTDlTPz8/Pz8/Py8/Qz8/Pz8/WG1lPz8/P3g/P1w/NSUiW34nPz9Ecj8/JX0/SHVfR0k/QUFPP9WzPz8/az8/Lz96QD8/YU0/KD9SYj8/Pz8/PyclRQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBZPT8/xKo/Pz9qUnk/UFlLPz8/P3t3XX0/UD/Zgz8/Pz8oPyE/Pz8/MzE/Pw0KICAgICAgICAgdD8/Pz9UPz8/dj94PT8/Pz8/Pz9SPzM/Pz8tKj9uSHk/PzDNkz9yUT8/3ZVaP24/Pz9hP3c/aj/Qgz8gPz9RPy8hdT9mXWclP1g/dHs/PWlgP3o/Qz9tXT9TZD0/cD8lP2g/DQogICAgICAgICAgICAgICAgICAgICAgICA/XD8/Pz8/ZT8/P2LZqmt+Pz96N96KP3Q/Ijk/Pz8/KjM/P3k/U2g/PyBnPz8/SD9rP1k/Pz8/R1A0Pz96P3Q/YENRP+eAjz8/Xj9RP3U/P05KP2NbND9xP9GVP3k/Uj8/Jz9vcj8hPyU/P2BMPz9iYz9RPz8/P3E/cD9kP1Q/Oz8/Pz93Pz8/Pz9dQkouP8+kP3NtP1UNCj92PzlvPz89KHs/PT8/cD9manc/Pz8/Tz8/Pz8/Pz9ZPz8/dnA/TD9+Pz9c1os/Pz8/aX4/JD8/Pz8+Pz8/CT/Vr0k/dz8/a3w/JlYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP35SP3U/fCY/QmE/P1ZWPz8/Pz9HKz9FJD9rSj85Pj91d09IP0Q/Pz8/Pz9hPy04Pz8/Pz9wPzE/XkQ3P3N6P1s/K1JTWj0/Pzk/Tj9Qaj9HP3s/Pz9NNA0Kej8/P2h9Pyh6Pzg/OD8/Pz/ejC4/Jz8/Jz9RUj8/QD8/JD8/JFgNCj8/Pw0KICAgPz8/PD8/WEk/P2g/cz9wPD90Pz8/PzRhP3pIRS0/e1U/Pyk/P9SrVnRYJGY/Pz9qPyIvPz9tTH0/Zdy5I1I/Q8KfPz8/Pz9CMT8/ZD9lbz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB1Kz50PtSDdEs/UT8/Pz9tXD8/Pz9PyZExP2FOPj8vNHBuVQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7Pj8qZj88P21aST8/Yj8xQD9oPz8hdT9eP3zMrj8sK21hPz8wMCc/ej9bPz8/XVw/YFpNUCI/aUE/Pz8/7IKRNGpFMz91P1RyPz9wP1BSPz8/Tj9lJH5nPz9wPz8/TkduPz8/U30/cT9SPz9Laz8/L34/P9q2P0kzPz/dpF86Pw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQj9cP04/dD5oaT8/Oz8/amssZz/LtD8/ej/FuiE/P3dmPz8/Pz9Ze3s/Pz8/Lj9wPm8oeG5626hFbX4zQj8/P28uPz8/Pzs/Pz8/CT97Mz8/Sz8iJi9VIj95Pz8/VT8/Pz8NCiAgICAgICAgICAgICAgPz9kTT8/Pz9VKz8/Lj9zP3dTzbtAPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/MSY/Pz9oP0g/Wz9KNz9mPz8/xZk/Mz94MjQ/OGR3Zz8/Jj8/PzB5Sj9DPz8lO8qNbj9vcj9NbT8Jw5BoPz8/TWEsPyE/Pz9sDQp5Kj94Pzc/aloiPz88Xz8/ej9APz8jPz9IPz0/1Lc/UD9XOVs/XD8/Pz8/P0huP3E9RD/Urz8/Pz8/NS1bPz8/bD8/P3tfZj8/PD90NdauPz/cvD8/P0o/XT9IfD8/Ij8kPz8/P24/Pz8nPz8/PzchLHZRPz/Mij8/PA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICo/XnFhP2o/XT8/RDzXlz8/xIQ/P1lGPz8/Rd6eITA/U1Y/P20/Pz9wXj8/MD9ePz83P9KIP2A/Pz81Zj8/Pz85TT8/Wz8/P24/ej8vPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgMD8/PHJIPz8/P34/PwlTLTA0fHU/P1pMPz8sPz8/zpc3TC0/x6A/PyEyPz8/Pz8/Ij96xa8/Sj9ddz8/Pz8/P3ZjVT8/PyA/cD8/PzpY7ZWOOlc/bD8jQGo/Py4/Pz8/Pz9ePz8/Pz8/Pz9kTjNdRy1mT3t0Pz0/ajkgSDQ/VnI/Pz8/Pz0/P2gnJD98P0lHPz91PztzNT8/Py/KsD98Pz8/Nj9wPz8/UlY/Vz9nPyw/P1p7NT96cyFvPyB0Kz8pP1dFPw0KPz8/Kj8/J8meP2E/Pz8/Pz9AXyFjPz8/PyZkx45lIj9FPz84Pz9vZT8/Pz8/YD8/P28/Pz86PykyPz8rPz9xPz8/Py0/LUhwPz/csG4/P3ghPz8wP0J5P8qjZk8/dEo/Pz9OUj82P2JWLHbKujM/DQogICAgICAgICAgICAgICAgICAgICA/Lm5AYD9Wczs/P0I/P2k/Pz8oP0c/NT8/P18/P1lyP0E/Pz8/Pyk/SyY/2a0/Pz87OD8/Pz8/P3U/Pz9QP242Iio217c/P2I/YklLP2YmPz8/Mig/Kz8/ZlwxP3pdPyU/WMugP0gvPz8/P0U/Pz9AUMyjP1c/Pzc/RT8/UG4/Pz8/Pz84K08/Pz9mej8qJj8/0p8/4oKsP04/TykkPz8/e2Q/aD887428Pz/Ut1plVD8/Pz8/Pz8nPz8/Ij8/P1MNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9OPz8/Py16OT8/1bEzPyc/ZDkxPz8/P9KrPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9vYz8/RDLVkExCPz9BPyE/P2E/yLk/eD9oPz9HP0jMnz8/Pz9RPz8/Pz8/Mj960rM/fT8kJz8/Pz8/P0o/Pz80Lz8/Pz9AVT8/Pz9kPz8/Pz8/Xnw/bj8/QdeXPz9TPzw/Pz96Z3l6P195Pz93PzE/NT8/Pz8/Pzk/w5ZzPz8uK28/Pz9ZSz8jPz8/1Jg/Pz9nP1lPPz8/DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlPz8yPz92P9anPT8/Pz8/PCU/PD9mP1lxPzk/NDtbzoQ/XD9XPz8/PzHanD8/OnTKhT8wPz8/P1w/Pz9uP0w/P3Q/ZT8/Pz8/Py10Pz8uPz8/P2QpYT8/Pzw/Pz9vdD9dDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA7P0JRJlTegT9nZjk/I3bNh0A/P0JGOD9qZj9uPyVLP1R9eH5VPz8/Pz8tNSE/Yz8/cSs/Zj90NGZmbUEpPyM/PyI/dj8tWz4/Pz9+PNSuPz8/P3INCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgID9mPz8/Pz9+Pz8/Pz8/Jz8/Py4/Pz9NID8/Pz9idgk/P1I/Pyo/xIg/dl8/Pz/Vtk4/P2I/e2s/PzA/P0JVP2E/ID8rPz8hQVE/NT18P0FgIj8/fG4/TD8/Pz9NeS4oPz8/P3IqPz8/JUkiPz89LV4/TT8/Iz8/Pz8/P34/ME3Ht8mpPz8/RT8/cj8/P2Q/UC5LPz8/bCI/bFBdPz8kP0M/Pz9KCX0NCk4/Pz8/Pzk/bEc/Pz9szII/aVU3ICY/RChsPz9IPz8/bT8/P1cjP8yoPyvbmUo/eTo3P0A/fFd9P2E/Pz90eWg/P1o/Pz8/YT8/ej8pPz8/Pz9tPzQlPzc/0rVa2IU/Sz8/WD9+06c/Pz8/Pz8/Pz/RtT8/PzUzKjnYrz/XiQ0KRCo/NyNrP3cpP8mrUj8NCj8/Pz8/byc/dyd4Pz87eD8/NT8/Pz8vPz/Yo0FEMj8/UD8/Pz8/PyNMWj8uPz9Qw6k/MjQ/Pz9eOj/Sqj8/P1Y/240/Py15Zz8/bz8/PyQ/Pw0KLz98U1w/OVQ/Pz85Xz8/Pz9aPz8/yLEhcD90VT8/Rj8hy7RFPz9YP2NJVT8leD8/PzM/P1lYOnc/MD8/P3c/Vz8yDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/06M/Pyc/J2A/PyR7Pz8/P2kwP0JKP18/TGQ/eVk/Pz8/Pz8JP1s/Ud6GeChc1pg2Pz8iP1AlPyo/Pz8tPz9oPz9uPz8/Pj8/Pz8/Pz81LjM/Pzk/Pz89Pz9PPz9GJT9DKOODrj8/bT8/RU4/VXVQJWNvPz86P0p507k/Pzg/P1U/Pz95LFpVPz8/P2A/Oz8/Pz/LnV4jd2I6ac2wP2JRPwk/bj8/26k/WD8wPz8jaT9vPy4/QT8/NkIhPz/fjyU/wpE/cz8/P27dtT95OD8sP2prP3w/P3s/Vj88P9a3P9+KPy4/P2BAdlbSgj8/Imo/P0c/UD9iP250P9aDPz8/PyM/Jkg/Py4/Rz9oRD8/xqgiPz8lPyU/Pz8/RzM/Pz97P3c3Pz/dk0cnPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgP3c/Pz8/P0liYD9ZPzo/JlZtcj8/c2lYPz8/P24/P18/Pz80ank/P0tFKj/OmFJ4P3g/OyMyM2QkNT8/cCsyIj9cPzw9JMmnP1lHPz8/X00lPz9lQnxKTEJp1oI/Nj9D1Z0/VHZWXz9yPz8/cz/apEw/w7JcWz8/Pz98bWw/CT8/P2M/PzM/Pz8/Pz9tPyciPz4/e3bUmQ0KICAgICAgICAgICAgICAgP0w/P0p1Mj9iP2A/aT9mPzw/P2BfPz9eP2FVPz/Yqj8hP0s/P1dgPz9AP0QJP08/fj8/UH4/Pz9nVD8/Pz9uZT9vP+eShD8/Pz4/P3skPz90dmo/cz8jNz9LPyY2P9alP2o/P92OP3g/Jm9jPy8/cj8/Pz8lPz8/P3A+Rz8lP0gqJj8/Pz80PyFRQz8/P8+oPz9kRT8/Pz5RPyE/VFNBSlw/YnQ/P1wgP3E4PyE/P9SMPz8/ZD8/Kg0KPz8obD9ZPyo/P3VTP11GKT8/Pys/Pz8/Pz8/34rXsnFJWz93Pz9aRT9JMzhXTlk/P3hzPz8/xJR8Pz9dSVU4Ljc/Py8/Pz8/Pz8/P2k/0Ig/PwlZeHdoPz8/UT9iWdCEP3k/Pz9IfT9PPz8/TtqBPz/Xu2A/Pz8/ID8/P1Y/DQo/P0lMRj8/Rzo7I0Y2Jj92Kj9HVzw/P0FIPz9vMD8/MmE/Pz8/P1MpVD8/Pz98S1U/Pz9UCTc/ej8/Pz/CtnVJPz9OSnI/Pz/crD8iPT8/Qj9xPz9SLj/voq0NCiAgICAgICAgICBmPz9xPzE/MT8JPz8/Pz8+JHw/Lj9qdT9zYHA/cVM/1ZEJP24/ZT8/Pz9Odjo/y7U/Pz8/LD8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA9XD8/JT8/Pz9YP8uEP1E/Pz/HjT8/V3TVkD8/dz8/fD8/Pz8tOGY0Pz/euj98Pz97aT9XQSk/Pz8/PysnPz8nP0A6TdeBPz85UW91Pyc/UzVlRD9UNz8/P0E/Pz9tZHY/PyI/cD9yT2hGPyRJP9+rMj8lWWA/Pz8/Pz9hdXk/zZcqVD9HPj8/Pz8/XT9kMD9ZND44XT8/ZT8/Q1g/Pz8/SD/YkVk/1JQ/PzM/Oz98Pz92TT8/ej8/P+mqmz8/P3U/Pz/csz9BND8/Pz8/aD8/ZT/Rm8iJPz8/Pz9qPz8/PyhXZD8/P2lCUVA/IEQ/Pz/nlrc/Pz/Viz8/P0NcNT8vQXY/ZnA/Pz9vQD8/Pz8/P29EJD8/Pzc/Pz9YdT8rDQogICAgICAgICAgICAgIO2GsVg/UT87XFE/Pz8/Wz8/2Kc/2Jc/dw0Kdj94Uz8/YD8lP28/Pz80Pz9aRD8yP2ZjPz8uP0R6Pz8/VW8/Xj4/Jj8/dT8/SD/aoz8/Jg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfj8/Pz8/PXo/Z0dUPz9iPz8/czwnPz9tP2t4P1I/I3k/Nj8/Pz8/RlA0PD8/QT9oPz8sPz97MT8/RGdFcEY/Pz8JPz91Jj8/P2523r02Pz8/P2w3P1VPdD8qP2U/Pz9XcT8/Xj8/K1d6Pz8xP20/woEoXFpCP3Y9dC9NKD8/JD9fPzU/P0x+Pz8/Pz8tRFxSVj993JI/PzQ/Pw0KPz8NCj8hPz8/dyjJgz8/Pz86Nz8ye0c/Pyg/Zj8/LE0/Pz8/MD9yaktHWNyEMFU/fT8/P9OLPzM/bD9wP3cNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/PWU2Pz8/MmZSPyIraA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgS1A/Jz8/XSYoPz8/dj9AbD8/P3J4P3ZoUnJjP1QlPz9F3rxrPz8rPz8/Nj9SaT46Zz8/Pz8zPz8/Pz8/X1Y/P2ZYPz8/Pz9sP++sryU/Pz8kMT8NCiIyPz8/NUdKPz9PNj8/PzE/T3c/Pz8NCmhRSD8/KWU0Pz8/PVw/TD8vPyFBPz0/Xj8kPz9lPz8/Pz9yPmthP08/ajM/KT8/Pz8/ZUE/27g/Pz8hPy8/XW8/Yj9HP0Mjbms/75qFPzFMYj9YPz9oPz8/P8KHPz8/OCY/cT9xIQ0KP28/P1k5KCA/SEA/IVw/ciA/Rz8/Pz8rDQogICAgICAgICAgICAgICAgICAgIC4w34MrPz8/LT8zJGk/Pz9EP0E/PzJoPz92P2A/LT97PycyP1hHcj8/P0ghdXF5XmNkP2M/Pz8/PypINtqDMz8zPz8/ST8NCiAgICAgICAgwp0/PycvTSA/UT8/Sz8/Pz8qDQogICAgICAgICAgICAgICAgICAgICAgICAgPw0KOD9sP8WiPz8/P3E8y595Pz8oPz94Pz8/cj9sPz8NCj8/U0E/fHo5P2cvOE1mP2smPz8qOj9Xaz8/Pz8/Qj8/Tj8/Pz/bh14/SD9cMHI/P1w/Pz8/x6A/Pz8/IVI/PzZfPwkkPzNZWSdtPz9SP2BiP2U/P2c6P3VkP18/Pz85fj8/Pyo/aEjfoT8/PzU/P0FPTz80Pz9NcD91PzJoPz9eP3tiYVQ/Pz8/Uz8iYj8/Pz9rP2w/Pz8/Pz9zcD9geU8/Lio/cz8wPz9xPzc/Nd+FRmw/YG0/Pzw/zYU/TWk/Pz8/VD8/Rz8/Pz8/JXZHfT81zac/Pz8/eD8/fUAlPz8/Z1N2bN2NPyMuPz8gP2xpZD8/NFQ/Kz8/Pz8/Pz8/P1Y/Lj8/P1o/PwkhPz8/cz8/RjYNClkNCiA/MT8/fQ0KP20/YXE/L0A/WDtHP2E/Nm0/cD9VZnFaP0k/TD8nUz8/PyU/Pzk/Pz83P05OPz9wP0EkbUo/WD8/PzLNqT8/bD97Kj8/R8yrPz8/yJ4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPz8/SD8/SD9xUT8/P0NrPHk/aj8/Ow0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUTk/cz8/Pz96IV3VuW1hTD8/MT9rPz/Isj8/Pz94PzE/RD99PzU/PyA/Pz8/Pz8/P111Pz8/IU8/UD84TT8/Mj9QPz8/OShDPyE/W3BKPz/JknM/ZT88P1Y/P3Y/RT82Jz8/VT8kP0Y/Vj9QOz8/aj9+Pz8/J1Y/dWQ/b3E/Pz9iPz8/cnE/Pz8/Jz9OP9CTOHvSpT8wP1Fc3oJPbT8nPz1PP2ojPz8nPz8/Pz8tKuSrpj8/OT8lWD8/Pz8/Pz8/Pz9rU+OYnSQkP1E/xrw/PyHMiD9RemY/P9GYPz8/S1E/SjE/Pz8/P3I/bz/WhD9uS3k/NX4/P1nelD8/TT92Pz8/eD8/YTw/P10/Uz8wP31hP3U/Ol1+SzM/cmh5PypWPz9haj92MlQ/Wz8/Pz8/P00/Uz8/Pz8/dEQ2P0trdz8/Pz/fhHINCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFAiPz8/P2s/P2I/UXs/Pz8/Pz8/Wj8/DQpwPz8wPzc/Pz8/PzVRP0LTjj86aj8/Pik/cj8/PzkNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPyA/Pz/Ltj94P1Rfej98Tj8/P2tVSz9pP3M/bkc/anA/Pz8hP0A3fT8/Pz9uTz8/Pz8/djomPzNnTnY/TD8semQ/P0U/az8md3k/P9mKPz8/P3w/Pz8/Pz8/Pz83Pz8/Pw0KP3M/Pz8/P1U/Pz8/P1Y/P1R3K1zSpio/P3rRkD8rPzM/P0AzWD8/Pyg/dFM/P1I/cz8/Lz9J0Yc/Zj9qU9uUPw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbj8/ez8/aD8vPz/Iqgk/Pz8rJkY/RNKIPz9NP0Q7P144Pz9UUD8/Pz8rPz8/Pz98Mj9TPz97Pz8yRT8/KT8/XnNrY3g/Zj8/VT9FQl1dOj9QP3BmPz99P/C5jLFxJT9ucz9fRls/cUbSk1t5MFk/P2dHTz8/PylrWmk/fU5dPz8/SD8/ZWopPz86zJk/Jkk/Pz85P2k/aSE/Xz9wb0c/MnI/Klk/dT8/JMiM17lZUseJPz8/VD9PPz8/JiU+P9OEPzDUkDAoME8/Yz9CCT8/P3M/P0ddPzc8RsWH7qKKOXRjetyWPz9dPz/jvIg/Tgk/Pz8/PyU/P0jOvj8/Pz8/PytDUlE/bD9JRkk/JWd9e25iPzg/IT9QPz8/Pz93Pz8/2L4hK9Ccbj8/Pyk/Pz8/dT8mP0XGrD8/MD9vd1Q/fT/GtEQ/XT9RP1s/fChiPz9YP2M/P3ZXLT8/Uj8+MHg/Zz8/xpBKPy8xPyM/dz9RPz9OP0s6Pz8/NT9+Pz8/P2RHbz8JxIc/PyQ/P08/Uic7ND8/QU1iPz8/J0FfOz8/Sz9eJj8qP0ZpIj9cPz95Vj87SyRbPz8/Pj8/L1E/RS0/P1c/dTs8UDN8OVk/Qz9yPz86PzE/OjskKt2EP9GuPz8/P1/ZkGM/Pz8/Mz8/Z3I/Pz9APz86Pz8xPz8/DQo/IT9QX9yBeCNTQ2A/PUBVP0lyaj8/P8eSPzgtej8/Pzs/P2INCj8kKz8/c2g/Pz9lLiPdhA0KLiUzPz9QP8e9fGdnP0o/Py1gP3pdPz8/eilofUcvPz8/Vz8gP3E/Rz8nOj8/DQogICAgICAgICAgICAgICAgICAgICAgICYxPz8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgID9kcT8/cj9wYz/GgW5SPz8xTQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICA/Pz8/PzdyPD8lVz9UPz8/Pz9sPykgPy/KmFd2Pz8/Nj8/3Lk/P1x1P3g/Oyc/Pz8/IH5fyJPdsz8yRMafND8/P0A/P2w9Wj8/Pz8vOnY/PzI/Pz94Pz8/2oxAPz9bbj8/Ij8/Pyh+LSk/eCNmPz9cPz8/cD8nfj8/d3YjMz/JsT8/KD8/ZD8lDQo/P0s/1pRuPz8/P0o/Oj1yPz8/PyY/P1F6Pz8zPzk/ez8/Pz8/KlA/Pz9gbT8/MT8NCj8/Pz9sPj9MPz8xP3M/P1c/P18/fWNTP01wP04vPGs/aFZ1Sj9aUj8/Pz8rPz8/P08/eN2yeEg/Rz8/QD9aLj90P3R8Lz80P09YOT8/JUg/P9ytPz/Unz8/Pz8/fj9cPz8/VHk/Pz/WqWU/Pz8/Tz8/aeKAmD8/UD8/dj9hPz8/P+ajsT9sLWxSPz9cPz8/Kj8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHLCikk/P9SwPz86Rj9DPz8/azs/PzR70po/UT98Pz8/RA0KICAgICAgID8/PzA/PyVVP0U/Pz8qMj8/Pzk0Pz8/Lz8/U3A/Pz9IJD/gvIg/QFtxPz8jRD8/Oj8NCmVuZHN0cmVhbQ0KZW5kb2JqDQoNCjggMCBvYmoNCjExODcyDQplbmRvYmoNCg0KOSAwIG9iag0KPDwvVHlwZS9Gb250RGVzY3JpcHRvci9Gb250TmFtZS9CQUFBQUErQ291cmllck5ld1BTTVQNCi9GbGFncyA1DQovRm9udEJCb3hbLTEyMSAtNjc5IDYyMiAxMDIxXS9JdGFsaWNBbmdsZSAwDQovQXNjZW50IDgzMg0KL0Rlc2NlbnQgLTMwMA0KL0NhcEhlaWdodCAxMDIwDQovU3RlbVYgODANCi9Gb250RmlsZTIgNyAwIFINCj4+DQplbmRvYmoNCg0KMTAgMCBvYmoNCjw8L0xlbmd0aCAzMTQvRmlsdGVyL0ZsYXRlRGVjb2RlPj4NCnN0cmVhbQ0KMT8/RT8ycSE/NT8/zpw/yKI/Pz8gUT8/P0x4fXo/Mz8/Pz9jP0U/R20rNlQ/Iz8/Pz8/VT8/P2c/S9+oP1vMvU0/O8SwPz/PqWw/P2c/QD83P3pVPnw/Pz9wPzs/PyhyLDY/P3ZMP3JqPz8/PzU/fj4NCiAgICAgICAgICAgICAgICAgICAgICAgICAgP1XNuT95JT8/d1snP34NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERSPz/mlaA/P3wzPz8/Pz9F3J8m0oA/Ik4zPz8/Sc6FP2o/Lz8/eT90Pz8/Pw0K56SMbVA/PzLZhj8/IEF2Pz8/1o0/Pyk/Pz8/Pz8/Vl0/PT9DP0g/bm12Pz8/DQp0Pz8iPyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBMJ14/Pw0KICAgICB5Pz8/Xz8/Yj9+Pz9OPy0zP0o/WGloND4/P00/Pz8/CQ0KZW5kc3RyZWFtDQplbmRvYmoNCg0KMTEgMCBvYmoNCjw8L1R5cGUvRm9udC9TdWJ0eXBlL1RydWVUeXBlL0Jhc2VGb250L0JBQUFBQStDb3VyaWVyTmV3UFNNVA0KL0ZpcnN0Q2hhciAwDQovTGFzdENoYXIgMjANCi9XaWR0aHNbNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwDQo2MDAgNjAwIDYwMCA2MDAgNjAwIF0NCi9Gb250RGVzY3JpcHRvciA5IDAgUg0KL1RvVW5pY29kZSAxMCAwIFINCj4+DQplbmRvYmoNCg0KMTIgMCBvYmoNCjw8L0YxIDExIDAgUg0KPj4NCmVuZG9iag0KDQoxMyAwIG9iag0KPDwvRm9udCAxMiAwIFINCi9YT2JqZWN0PDwvVHI0IDQgMCBSPj4NCi9FeHRHU3RhdGU8PC9FR1M1IDUgMCBSPj4NCi9Qcm9jU2V0Wy9QREYvVGV4dC9JbWFnZUMvSW1hZ2VJL0ltYWdlQl0NCj4+DQplbmRvYmoNCg0KMSAwIG9iag0KPDwvVHlwZS9QYWdlL1BhcmVudCA2IDAgUi9SZXNvdXJjZXMgMTMgMCBSL01lZGlhQm94WzAgMCA1OTUgODQyXS9Hcm91cDw8L1MvVHJhbnNwYXJlbmN5L0NTL0RldmljZVJHQi9JIHRydWU+Pi9Db250ZW50cyAyIDAgUj4+DQplbmRvYmoNCg0KNiAwIG9iag0KPDwvVHlwZS9QYWdlcw0KL1Jlc291cmNlcyAxMyAwIFINCi9NZWRpYUJveFsgMCAwIDU5NSA4NDIgXQ0KL0tpZHNbIDEgMCBSIF0NCi9Db3VudCAxPj4NCmVuZG9iag0KDQoxNCAwIG9iag0KPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDYgMCBSDQovT3BlbkFjdGlvblsxIDAgUiAvWFlaIG51bGwgbnVsbCAwXQ0KL0xhbmcoSj9Abj8pDQo+Pg0KZW5kb2JqDQoNCjE1IDAgb2JqDQo8PC9DcmVhdG9yPEQ3OUM1NUQ1OTdDMDUxMjJCQkE3N0M2NDIyRjg+DQovUHJvZHVjZXI8RDc5QzU1Q0U5N0RCNTEyOUJCQTE3QzY0MjJDNTY2M0VDQ0I1QzM0OTdENURDQjAwQUJEM0IyMkE4NkMxNDlEMz4NCi9DcmVhdGlvbkRhdGUobVlnPz8/YXM/P005P1ZrPz8/TT8pPj4NCmVuZG9iag0KDQoxNiAwIG9iag0KPDwvRmlsdGVyL1N0YW5kYXJkL1YgMi9MZW5ndGggMTI4L1IgMy9PKD98QTptP3AhUz8zP0fRhT8/SD8/P86nVzs/VD9cYikvVSg/PzxbP05WYz/Qvz8/fT8pL1AgLTEwMjg+Pg0KZW5kb2JqDQoNCnhyZWYNCjAgMTcNCjAwMDAwMDAwMDAgNjU1MzUgZiANCjAwMDAwMTM0NzAgMDAwMDAgbiANCjAwMDAwMDAwMTkgMDAwMDAgbiANCjAwMDAwMDAyNzYgMDAwMDAgbiANCjAwMDAwMDAyOTYgMDAwMDAgbiANCjAwMDAwMDA0NzMgMDAwMDAgbiANCjAwMDAwMTM2MTMgMDAwMDAgbiANCjAwMDAwMDA1MTMgMDAwMDAgbiANCjAwMDAwMTI0NzAgMDAwMDAgbiANCjAwMDAwMTI0OTIgMDAwMDAgbiANCjAwMDAwMTI2ODcgMDAwMDAgbiANCjAwMDAwMTMwNzEgMDAwMDAgbiANCjAwMDAwMTMzMTIgMDAwMDAgbiANCjAwMDAwMTMzNDUgMDAwMDAgbiANCjAwMDAwMTM3MTIgMDAwMDAgbiANCjAwMDAwMTM4MDkgMDAwMDAgbiANCjAwMDAwMTM5ODQgMDAwMDAgbiANCnRyYWlsZXINCjw8L1NpemUgMTcvUm9vdCAxNCAwIFINCi9FbmNyeXB0IDE2IDAgUg0KL0luZm8gMTUgMCBSDQovSUQgWyA8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+DQo8NTJBOUE5NEE2MzExODQ3QTk2NkI1NjIxRDc3QTRERDM+IF0NCi9Eb2NDaGVja3N1bSAvNEE4NTkwMDE4QURDRjVCQjRCQTQ1NDFDQUFDQjFBNEUNCj4+DQpzdGFydHhyZWYNCjE0MTIyDQolJUVPRg==" +# hello: "world" +# +# - do: +# indices.refresh: {} +# +# - do: +# search: +# index: test +# +# - match: { hits.total: 0 } diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/20_search.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/20_search.yaml new file mode 100644 index 00000000000..95d9cef2cbf --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/20_search.yaml @@ -0,0 +1,114 @@ +# Integration tests for Mapper Attachments plugin +# + +setup: + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + file: + type: attachment + - do: + cluster.health: + wait_for_status: yellow + +--- +# Encoded content with https://www.base64encode.org/ +# +# +# XHTML test document +# +# +# +# +#

    +# This document tests the ability of Apache Tika to extract content +# from an XHTML document. +#

    +# +# + +"Mapper Attachment Simple": + + - do: + index: + index: test + type: doc + id: 1 + body: + file: "PGh0bWwgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGh0bWwiPg0KPGhlYWQ+DQogICAgPHRpdGxlPlhIVE1MIHRlc3QgZG9jdW1lbnQ8L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0iVGlrYSBEZXZlbG9wZXJzIi8+DQogICAgPG1ldGEgaHR0cC1lcXVpdj0icmVmcmVzaCIgY29udGVudD0iNSIvPg0KPC9oZWFkPg0KPGJvZHk+DQo8cD4NCiAgICBUaGlzIGRvY3VtZW50IHRlc3RzIHRoZSBhYmlsaXR5IG9mIEFwYWNoZSBUaWthIHRvIGV4dHJhY3QgY29udGVudA0KICAgIGZyb20gYW4gPGEgaHJlZj0iaHR0cDovL3d3dy53My5vcmcvVFIveGh0bWwxLyI+WEhUTUwgZG9jdW1lbnQ8L2E+Lg0KPC9wPg0KPC9ib2R5Pg0KPC9odG1sPg==" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + file.title: "test document" + + - match: { hits.total: 1 } + +--- +# Encoded content with https://www.base64encode.org/ +#Begin +# +#BeforeLimit AfterLimit +# +#Broadway +# +#Nearing the end +# +#End + +"Mapper Attachment ContentLength Limit": + + - do: + index: + index: test + type: doc + id: "withlimit" + body: + file: + _indexed_chars: 20 + _content: "QmVnaW4NCg0KQmVmb3JlTGltaXQgQWZ0ZXJMaW1pdA0KDQpCcm9hZHdheQ0KDQpOZWFyaW5nIHRoZSBlbmQNCg0KRW5k" + + - do: + index: + index: test + type: doc + id: "nolimit" + body: + file: + _indexed_chars: -1 + _content: "QmVnaW4NCg0KQmVmb3JlTGltaXQgQWZ0ZXJMaW1pdA0KDQpCcm9hZHdheQ0KDQpOZWFyaW5nIHRoZSBlbmQNCg0KRW5k" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + file.content: "BeforeLimit" + + - match: { hits.total: 2 } + + - do: + search: + index: test + body: + query: + match: + file.content: "AfterLimit" + + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "nolimit" } + diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml new file mode 100644 index 00000000000..170a8bf7382 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml @@ -0,0 +1,61 @@ +# Integration tests for Mapper Attachments plugin +# + +--- +# Encoded content with https://www.base64encode.org/ +# +# +# XHTML test document +# +# +# +# +#

    +# This document tests the ability of Apache Tika to extract content +# from an XHTML document. +#

    +# +# +"ContentType and Name": + + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + "file": + "type": "attachment" + "fields": + "content_type": + "store": "yes" + "name": + "store": "yes" + - do: + cluster.health: + wait_for_status: yellow + + - do: + index: + index: test + type: doc + id: 1 + body: + file: + _content: "QmVnaW4NCg0KQmVmb3JlTGltaXQgQWZ0ZXJMaW1pdA0KDQpCcm9hZHdheQ0KDQpOZWFyaW5nIHRoZSBlbmQNCg0KRW5k" + _content_type: "text/my-dummy-content-type" + _name: "my-dummy-name-txt" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + fields: [file.content_type,file.name] + + - match: { hits.total: 1 } + - match: { hits.hits.0.fields: { file.content_type: ["text/my-dummy-content-type"], file.name: ["my-dummy-name-txt"] }} + diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml new file mode 100644 index 00000000000..286dae8b976 --- /dev/null +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml @@ -0,0 +1,67 @@ +# Integration tests for Mapper Attachments plugin +# + +setup: + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + "file": + "type": "attachment" + "fields": + "content" : + "type": "string" + "store" : "yes" + "term_vector": "with_positions_offsets" + + - do: + cluster.health: + wait_for_status: yellow + +--- +# Encoded content with https://www.base64encode.org/ +# +# +# XHTML test document +# +# +# +# +#

    +# This document tests the ability of Apache Tika to extract content +# from an XHTML document. +#

    +# +# + +"Highlight content": + + - do: + index: + index: test + type: doc + id: 1 + body: + file: "PGh0bWwgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGh0bWwiPg0KPGhlYWQ+DQogICAgPHRpdGxlPlhIVE1MIHRlc3QgZG9jdW1lbnQ8L3RpdGxlPg0KICAgIDxtZXRhIG5hbWU9IkF1dGhvciIgY29udGVudD0iVGlrYSBEZXZlbG9wZXJzIi8+DQogICAgPG1ldGEgaHR0cC1lcXVpdj0icmVmcmVzaCIgY29udGVudD0iNSIvPg0KPC9oZWFkPg0KPGJvZHk+DQo8cD4NCiAgICBUaGlzIGRvY3VtZW50IHRlc3RzIHRoZSBhYmlsaXR5IG9mIEFwYWNoZSBUaWthIHRvIGV4dHJhY3QgY29udGVudA0KICAgIGZyb20gYW4gPGEgaHJlZj0iaHR0cDovL3d3dy53My5vcmcvVFIveGh0bWwxLyI+WEhUTUwgZG9jdW1lbnQ8L2E+Lg0KPC9wPg0KPC9ib2R5Pg0KPC9odG1sPg==" + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + file.content: "apache tika" + fields: [] + highlight: + fields: + file.content: {} + + - match: { hits.total: 1 } + - match: { hits.hits.0.highlight: { file.content : [ "\n\n This document tests the ability of Apache Tika to extract content\n from an XHTML document.\n" ] }} + diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle new file mode 100644 index 00000000000..ca93c118487 --- /dev/null +++ b/plugins/mapper-murmur3/build.gradle @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.' + classname 'org.elasticsearch.plugin.mapper.MapperMurmur3Plugin' +} + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" diff --git a/plugins/mapper-murmur3/licenses/no_deps.txt b/plugins/mapper-murmur3/licenses/no_deps.txt deleted file mode 100644 index 8cce254d037..00000000000 --- a/plugins/mapper-murmur3/licenses/no_deps.txt +++ /dev/null @@ -1 +0,0 @@ -This plugin has no third party dependencies diff --git a/plugins/mapper-murmur3/pom.xml b/plugins/mapper-murmur3/pom.xml deleted file mode 100644 index 2e78be25c09..00000000000 --- a/plugins/mapper-murmur3/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - mapper-murmur3 - Plugin: Mapper: Murmur3 - The Mapper Murmur3 plugin allows to compute hashes of a field's values at index-time and to store them in the index. - - - org.elasticsearch.plugin.mapper.MapperMurmur3Plugin - mapper_murmur3 - false - -Xlint:-rawtypes - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/RegisterMurmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/RegisterMurmur3FieldMapper.java deleted file mode 100644 index 5a6a71222c0..00000000000 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/RegisterMurmur3FieldMapper.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.murmur3; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.mapper.MapperService; - -public class RegisterMurmur3FieldMapper extends AbstractIndexComponent { - - @Inject - public RegisterMurmur3FieldMapper(Index index, Settings indexSettings, MapperService mapperService) { - super(index, indexSettings); - mapperService.documentMapperParser().putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); - } - -} diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java index 429db165b75..19b2f0fb9c6 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java @@ -19,13 +19,10 @@ package org.elasticsearch.plugin.mapper; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.murmur3.Murmur3FieldMapper; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - public class MapperMurmur3Plugin extends Plugin { @Override @@ -38,9 +35,8 @@ public class MapperMurmur3Plugin extends Plugin { return "A mapper that allows to precompute murmur3 hashes of values at index-time and store them in the index"; } - @Override - public Collection indexModules(Settings settings) { - return Collections.singletonList(new MapperMurmur3IndexModule()); + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMapper(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 676a5c4c1cb..da65210f6d9 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -31,21 +31,27 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; import java.util.Arrays; +import java.util.Collections; public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { + MapperRegistry mapperRegistry; IndexService indexService; DocumentMapperParser parser; @Before public void before() { indexService = createIndex("test"); - parser = indexService.mapperService().documentMapperParser(); - parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + mapperRegistry = new MapperRegistry( + Collections.singletonMap(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()), + Collections.emptyMap()); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); } public void testDefaults() throws Exception { @@ -120,8 +126,8 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { public void testDocValuesSettingBackcompat() throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); - parser = indexService.mapperService().documentMapperParser(); - parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") @@ -136,8 +142,8 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { public void testIndexSettingBackcompat() throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); - parser = indexService.mapperService().documentMapperParser(); - parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java new file mode 100644 index 00000000000..b3ad01bae49 --- /dev/null +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.murmur3; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugin.mapper.MapperMurmur3Plugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.ExecutionException; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +@LuceneTestCase.SuppressFileSystems("ExtrasFS") +public class Murmur3FieldMapperUpgradeTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MapperMurmur3Plugin.class); + } + + public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException { + final String indexName = "index-mapper-murmur3-2.0.0"; + InternalTestCluster.Async master = internalCluster().startNodeAsync(); + Path unzipDir = createTempDir(); + Path unzipDataDir = unzipDir.resolve("data"); + Path backwardsIndex = getBwcIndicesPath().resolve(indexName + ".zip"); + try (InputStream stream = Files.newInputStream(backwardsIndex)) { + TestUtil.unzip(stream, unzipDir); + } + assertTrue(Files.exists(unzipDataDir)); + + Path dataPath = createTempDir(); + Settings settings = Settings.builder() + .put("path.data", dataPath) + .build(); + final String node = internalCluster().startDataOnlyNode(settings); // workaround for dangling index loading issue when node is master + Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, node).nodeDataPaths(); + assertEquals(1, nodePaths.length); + dataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER); + assertFalse(Files.exists(dataPath)); + Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices"); + Files.move(src, dataPath); + + master.get(); + // force reloading dangling indices with a cluster state republish + client().admin().cluster().prepareReroute().get(); + ensureGreen(indexName); + final SearchResponse countResponse = client().prepareSearch(indexName).setSize(0).get(); + ElasticsearchAssertions.assertHitCount(countResponse, 3L); + + final SearchResponse cardinalityResponse = client().prepareSearch(indexName).addAggregation( + AggregationBuilders.cardinality("card").field("foo.hash")).get(); + Cardinality cardinality = cardinalityResponse.getAggregations().get("card"); + assertEquals(3L, cardinality.getValue()); + } +} diff --git a/plugins/mapper-murmur3/src/test/resources/indices/bwc/index-mapper-murmur3-2.0.0.zip b/plugins/mapper-murmur3/src/test/resources/indices/bwc/index-mapper-murmur3-2.0.0.zip new file mode 100644 index 00000000000..0b69aac180b Binary files /dev/null and b/plugins/mapper-murmur3/src/test/resources/indices/bwc/index-mapper-murmur3-2.0.0.zip differ diff --git a/plugins/mapper-size/build.gradle b/plugins/mapper-size/build.gradle new file mode 100644 index 00000000000..7af65d19ef3 --- /dev/null +++ b/plugins/mapper-size/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Mapper Size plugin allows document to record their uncompressed size at index time.' + classname 'org.elasticsearch.plugin.mapper.MapperSizePlugin' +} + diff --git a/plugins/mapper-size/licenses/no_deps.txt b/plugins/mapper-size/licenses/no_deps.txt deleted file mode 100644 index 8cce254d037..00000000000 --- a/plugins/mapper-size/licenses/no_deps.txt +++ /dev/null @@ -1 +0,0 @@ -This plugin has no third party dependencies diff --git a/plugins/mapper-size/pom.xml b/plugins/mapper-size/pom.xml deleted file mode 100644 index 3e148cdfc28..00000000000 --- a/plugins/mapper-size/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - mapper-size - Plugin: Mapper: Size - The Mapper Size plugin allows document to record their uncompressed size at index time. - - - org.elasticsearch.plugin.mapper.MapperSizePlugin - mapper_size - false - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/RegisterSizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/RegisterSizeFieldMapper.java deleted file mode 100644 index e71018fb159..00000000000 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/RegisterSizeFieldMapper.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.size; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.mapper.MapperService; - -public class RegisterSizeFieldMapper extends AbstractIndexComponent { - - @Inject - public RegisterSizeFieldMapper(Index index, Settings indexSettings, MapperService mapperService) { - super(index, indexSettings); - mapperService.documentMapperParser().putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); - } - -} diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 6c80bcd1c3e..aaf46553a75 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.analysis.NumericIntegerAnalyzer; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; @@ -85,9 +84,9 @@ public class SizeFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -103,10 +102,19 @@ public class SizeFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new SizeFieldMapper(indexSettings, fieldType); + } } private EnabledAttributeMapper enabledState; + private SizeFieldMapper(Settings indexSettings, MappedFieldType mappedFieldType) { + this(Defaults.ENABLED_STATE, mappedFieldType == null ? Defaults.SIZE_FIELD_TYPE : mappedFieldType, indexSettings); + } + private SizeFieldMapper(EnabledAttributeMapper enabled, MappedFieldType fieldType, Settings indexSettings) { super(NAME, fieldType, Defaults.SIZE_FIELD_TYPE, indexSettings); this.enabledState = enabled; @@ -169,7 +177,7 @@ public class SizeFieldMapper extends MetadataFieldMapper { } @Override - public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + public void merge(Mapper mergeWith, MergeResult mergeResult) { SizeFieldMapper sizeFieldMapperMergeWith = (SizeFieldMapper) mergeWith; if (!mergeResult.simulate()) { if (sizeFieldMapperMergeWith.enabledState != enabledState && !sizeFieldMapperMergeWith.enabledState.unset()) { diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizeIndexModule.java b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizeIndexModule.java deleted file mode 100644 index df9585b6e97..00000000000 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizeIndexModule.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugin.mapper; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.mapper.size.RegisterSizeFieldMapper; - -public class MapperSizeIndexModule extends AbstractModule { - - @Override - protected void configure() { - bind(RegisterSizeFieldMapper.class).asEagerSingleton(); - } - -} diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java index df95a4e9e67..a4c34e97cd8 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java @@ -19,13 +19,10 @@ package org.elasticsearch.plugin.mapper; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.size.SizeFieldMapper; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - public class MapperSizePlugin extends Plugin { @Override @@ -38,9 +35,7 @@ public class MapperSizePlugin extends Plugin { return "A mapper that allows document to record their uncompressed size"; } - @Override - public Collection indexModules(Settings indexSettings) { - return Collections.singletonList(new MapperSizeIndexModule()); + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); } - } diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java new file mode 100644 index 00000000000..4529111c16e --- /dev/null +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.size; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugin.mapper.MapperSizePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +@LuceneTestCase.SuppressFileSystems("ExtrasFS") +public class SizeFieldMapperUpgradeTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MapperSizePlugin.class); + } + + public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException { + final String indexName = "index-mapper-size-2.0.0"; + InternalTestCluster.Async master = internalCluster().startNodeAsync(); + Path unzipDir = createTempDir(); + Path unzipDataDir = unzipDir.resolve("data"); + Path backwardsIndex = getBwcIndicesPath().resolve(indexName + ".zip"); + try (InputStream stream = Files.newInputStream(backwardsIndex)) { + TestUtil.unzip(stream, unzipDir); + } + assertTrue(Files.exists(unzipDataDir)); + + Path dataPath = createTempDir(); + Settings settings = Settings.builder() + .put("path.data", dataPath) + .build(); + final String node = internalCluster().startDataOnlyNode(settings); // workaround for dangling index loading issue when node is master + Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, node).nodeDataPaths(); + assertEquals(1, nodePaths.length); + dataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER); + assertFalse(Files.exists(dataPath)); + Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices"); + Files.move(src, dataPath); + master.get(); + // force reloading dangling indices with a cluster state republish + client().admin().cluster().prepareReroute().get(); + ensureGreen(indexName); + final SearchResponse countResponse = client().prepareSearch(indexName).setSize(0).get(); + ElasticsearchAssertions.assertHitCount(countResponse, 3L); + + final SearchResponse sizeResponse = client().prepareSearch(indexName) + .addField("_source") + .addField("_size") + .get(); + ElasticsearchAssertions.assertHitCount(sizeResponse, 3L); + for (SearchHit hit : sizeResponse.getHits().getHits()) { + String source = hit.getSourceAsString(); + assertNotNull(source); + Map fields = hit.getFields(); + assertTrue(fields.containsKey("_size")); + Number size = fields.get("_size").getValue(); + assertNotNull(size); + assertEquals(source.length(), size.longValue()); + } + } + +} diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index df2f060d2fd..e07b76bfc92 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -19,27 +19,47 @@ package org.elasticsearch.index.mapper.size; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +import java.util.Collections; + import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; - -import static org.hamcrest.Matchers.*; +import org.junit.Before; public class SizeMappingTests extends ESSingleNodeTestCase { + MapperRegistry mapperRegistry; + IndexService indexService; + DocumentMapperParser parser; + + @Before + public void before() { + indexService = createIndex("test"); + mapperRegistry = new MapperRegistry( + Collections.emptyMap(), + Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + } + public void testSizeEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() @@ -59,8 +79,12 @@ public class SizeMappingTests extends ESSingleNodeTestCase { .endObject().endObject().string(); Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - DocumentMapperParser parser = createIndex("test", indexSettings).mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); + indexService = createIndex("test2", indexSettings); + mapperRegistry = new MapperRegistry( + Collections.emptyMap(), + Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() @@ -78,8 +102,6 @@ public class SizeMappingTests extends ESSingleNodeTestCase { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", false).endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() @@ -95,7 +117,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase { public void testSizeNotSet() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() .startObject() @@ -111,8 +133,6 @@ public class SizeMappingTests extends ESSingleNodeTestCase { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); DocumentMapper enabledMapper = parser.parse(enabledMapping); String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -121,6 +141,6 @@ public class SizeMappingTests extends ESSingleNodeTestCase { DocumentMapper disabledMapper = parser.parse(disabledMapping); enabledMapper.merge(disabledMapper.mapping(), false, false); - assertThat(enabledMapper.rootMapper(SizeFieldMapper.class).enabled(), is(false)); + assertThat(enabledMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(false)); } } \ No newline at end of file diff --git a/plugins/mapper-size/src/test/resources/indices/bwc/index-mapper-size-2.0.0.zip b/plugins/mapper-size/src/test/resources/indices/bwc/index-mapper-size-2.0.0.zip new file mode 100644 index 00000000000..0a74f835c3e Binary files /dev/null and b/plugins/mapper-size/src/test/resources/indices/bwc/index-mapper-size-2.0.0.zip differ diff --git a/plugins/pom.xml b/plugins/pom.xml deleted file mode 100644 index 7109ba31a4d..00000000000 --- a/plugins/pom.xml +++ /dev/null @@ -1,437 +0,0 @@ - - - - 4.0.0 - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - pom - Plugin: Parent POM - 2009 - A parent project for Elasticsearch plugins - - - org.elasticsearch - parent - 3.0.0-SNAPSHOT - - - - ${elasticsearch.tools.directory}/plugin-metadata/plugin-assembly.xml - false - ${project.artifactId} - true - true - false - 1.10.19 - - - - - - org.hamcrest - hamcrest-all - test - - - org.apache.lucene - lucene-test-framework - test - - - org.elasticsearch - elasticsearch - test-jar - test - - - - - org.elasticsearch - elasticsearch - provided - - - org.apache.lucene - lucene-core - provided - - - org.apache.lucene - lucene-backward-codecs - provided - - - org.apache.lucene - lucene-analyzers-common - provided - - - org.apache.lucene - lucene-queries - provided - - - org.apache.lucene - lucene-memory - provided - - - org.apache.lucene - lucene-highlighter - provided - - - org.apache.lucene - lucene-queryparser - provided - - - org.apache.lucene - lucene-suggest - provided - - - org.apache.lucene - lucene-join - provided - - - org.apache.lucene - lucene-spatial - provided - - - com.spatial4j - spatial4j - provided - - - com.vividsolutions - jts - provided - - - com.github.spullara.mustache.java - compiler - provided - - - com.carrotsearch - hppc - provided - - - joda-time - joda-time - provided - - - org.joda - joda-convert - provided - - - com.fasterxml.jackson.core - jackson-core - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - provided - - - io.netty - netty - provided - - - com.ning - compress-lzf - provided - - - com.tdunning - t-digest - provided - - - commons-cli - commons-cli - provided - - - log4j - log4j - provided - - - log4j - apache-log4j-extras - provided - - - org.slf4j - slf4j-api - provided - - - net.java.dev.jna - jna - provided - - - - - - org.apache.httpcomponents - httpclient - test - - - - - - - - src/test/resources - - **/* - - - - - ${elasticsearch.tools.directory}/rest-api-spec - rest-api-spec - - - api/info.json - api/cluster.health.json - api/cluster.state.json - - api/index.json - api/get.json - api/get_script.json - api/put_script.json - api/delete_script.json - api/update.json - api/search.json - api/indices.analyze.json - api/indices.create.json - api/indices.refresh.json - api/nodes.info.json - api/count.json - - api/snapshot.create_repository.json - api/snapshot.get_repository.json - - - - - ${elasticsearch.tools.directory}/shared-test-resources - false - - - - ${basedir}/target/metadata-test-resources - false - - - - - - - maven-resources-plugin - - - copy-resources - - generate-resources - - copy-resources - - - ${basedir}/target/metadata-test-resources - - - src/main/plugin-metadata - false - - - ${elasticsearch.tools.directory}/plugin-metadata - true - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - integ-setup - pre-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - integ-teardown - post-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - ${elasticsearch.assembly.appendId} - ${project.build.directory}/releases/ - - ${elasticsearch.assembly.descriptor} - - - - - package - - single - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - - - enforce-plugin-classname - - enforce - - - - - elasticsearch.plugin.classname - elasticsearch.plugin.name - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - integ-setup-dependencies - pre-integration-test - - copy - - - ${skip.integ.tests} - - - org.elasticsearch.distribution.zip - elasticsearch - ${elasticsearch.version} - zip - true - - - true - ${integ.deps} - - - - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - - - integ-tests - - - 1 - - - localhost:${integ.transport.port} - - - - - - - - - - - analysis-icu - analysis-kuromoji - analysis-phonetic - analysis-smartcn - analysis-stempel - delete-by-query - discovery-azure - discovery-ec2 - discovery-gce - discovery-multicast - lang-expression - lang-groovy - lang-javascript - lang-python - mapper-murmur3 - mapper-size - repository-azure - repository-s3 - store-smb - - - jvm-example - site-example - - diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle new file mode 100644 index 00000000000..f0c21bb3828 --- /dev/null +++ b/plugins/repository-azure/build.gradle @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Azure Repository plugin adds support for Azure storage repositories.' + classname 'org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin' +} + +dependencies { + compile 'com.microsoft.azure:azure-storage:2.0.0' + compile 'org.apache.commons:commons-lang3:3.3.2' +} + +dependencyLicenses { + mapping from: /azure-.*/, to: 'azure' + mapping from: /jackson-.*/, to: 'jackson' + mapping from: /jersey-.*/, to: 'jersey' + mapping from: /jaxb-.*/, to: 'jaxb' + mapping from: /stax-.*/, to: 'stax' +} + +compileJava.options.compilerArgs << '-Xlint:-deprecation,-serial' +compileTestJava.options.compilerArgs << '-Xlint:-deprecation' + diff --git a/plugins/repository-azure/pom.xml b/plugins/repository-azure/pom.xml deleted file mode 100644 index a6aff7f9cf7..00000000000 --- a/plugins/repository-azure/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - repository-azure - Plugin: Repository: Azure - The Azure Repository plugin adds support for Azure storage repositories. - - - org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin - 1 - repository_azure - false - -Xlint:-deprecation,-serial - - - - - - com.microsoft.azure - azure-storage - 2.0.0 - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java index e89f523b4ce..aa78ed4d835 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java @@ -19,12 +19,9 @@ package org.elasticsearch.cloud.azure; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cloud.azure.storage.AzureStorageService; -import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl; import org.elasticsearch.cloud.azure.storage.AzureStorageSettingsFilter; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; @@ -43,18 +40,12 @@ import org.elasticsearch.common.settings.Settings; */ public class AzureRepositoryModule extends AbstractModule { protected final ESLogger logger; - private Settings settings; // pkg private so it is settable by tests static Class storageServiceImpl = AzureStorageServiceImpl.class; - public static Class getStorageServiceImpl() { - return storageServiceImpl; - } - @Inject public AzureRepositoryModule(Settings settings) { - this.settings = settings; this.logger = Loggers.getLogger(getClass(), settings); } @@ -64,35 +55,7 @@ public class AzureRepositoryModule extends AbstractModule { bind(AzureStorageSettingsFilter.class).asEagerSingleton(); // If we have settings for azure repository, let's start the azure storage service - if (isSnapshotReady(settings, logger)) { - logger.debug("starting azure repository service"); - bind(AzureStorageService.class).to(storageServiceImpl).asEagerSingleton(); - } + logger.debug("starting azure repository service"); + bind(AzureStorageService.class).to(storageServiceImpl).asEagerSingleton(); } - - /** - * Check if we have repository azure settings available - * @return true if we can use snapshot and restore - */ - public static boolean isSnapshotReady(Settings settings, ESLogger logger) { - if (isPropertyMissing(settings, Storage.ACCOUNT) || - isPropertyMissing(settings, Storage.KEY)) { - logger.debug("azure repository is not set using [{}] and [{}] properties", - Storage.ACCOUNT, - Storage.KEY); - return false; - } - - logger.trace("all required properties for azure repository are set!"); - - return true; - } - - public static boolean isPropertyMissing(Settings settings, String name) throws ElasticsearchException { - if (!Strings.hasText(settings.get(name))) { - return true; - } - return false; - } - } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java index a7c980c8159..cf25e5d8b77 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java @@ -23,7 +23,9 @@ import com.microsoft.azure.storage.StorageException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.blobstore.support.AbstractLegacyBlobContainer; +import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.repositories.RepositoryException; @@ -39,7 +41,7 @@ import java.util.Map; /** * */ -public class AzureBlobContainer extends AbstractLegacyBlobContainer { +public class AzureBlobContainer extends AbstractBlobContainer { protected final ESLogger logger = Loggers.getLogger(AzureBlobContainer.class); protected final AzureBlobStore blobStore; @@ -61,7 +63,7 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { @Override public boolean blobExists(String blobName) { try { - return blobStore.client().blobExists(blobStore.container(), buildKey(blobName)); + return blobStore.blobExists(blobStore.container(), buildKey(blobName)); } catch (URISyntaxException | StorageException e) { logger.warn("can not access [{}] in container {{}}: {}", blobName, blobStore.container(), e.getMessage()); } @@ -69,9 +71,9 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { } @Override - public InputStream openInput(String blobName) throws IOException { + public InputStream readBlob(String blobName) throws IOException { try { - return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName)); + return blobStore.getInputStream(blobStore.container(), buildKey(blobName)); } catch (StorageException e) { if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new FileNotFoundException(e.getMessage()); @@ -83,9 +85,22 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { } @Override - public OutputStream createOutput(String blobName) throws IOException { + public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + Streams.copy(inputStream, stream); + } + } + + @Override + public void writeBlob(String blobName, BytesReference bytes) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + bytes.writeTo(stream); + } + } + + private OutputStream createOutput(String blobName) throws IOException { try { - return new AzureOutputStream(blobStore.client().getOutputStream(blobStore.container(), buildKey(blobName))); + return new AzureOutputStream(blobStore.getOutputStream(blobStore.container(), buildKey(blobName))); } catch (StorageException e) { if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new FileNotFoundException(e.getMessage()); @@ -101,7 +116,7 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { @Override public void deleteBlob(String blobName) throws IOException { try { - blobStore.client().deleteBlob(blobStore.container(), buildKey(blobName)); + blobStore.deleteBlob(blobStore.container(), buildKey(blobName)); } catch (URISyntaxException | StorageException e) { logger.warn("can not access [{}] in container {{}}: {}", blobName, blobStore.container(), e.getMessage()); throw new IOException(e); @@ -112,7 +127,7 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { public Map listBlobsByPrefix(@Nullable String prefix) throws IOException { try { - return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix); + return blobStore.listBlobsByPrefix(blobStore.container(), keyPath, prefix); } catch (URISyntaxException | StorageException e) { logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage()); throw new IOException(e); @@ -127,7 +142,7 @@ public class AzureBlobContainer extends AbstractLegacyBlobContainer { logger.debug("moving blob [{}] to [{}] in container {{}}", source, target, blobStore.container()); - blobStore.client().moveBlob(blobStore.container(), source, target); + blobStore.moveBlob(blobStore.container(), source, target); } catch (URISyntaxException e) { logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore.container(), e.getMessage()); throw new IOException(e); diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java index 6edcae73e07..99a505c5666 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java @@ -19,9 +19,11 @@ package org.elasticsearch.cloud.azure.blobstore; +import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.component.AbstractComponent; @@ -30,18 +32,22 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositorySettings; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URISyntaxException; +import java.util.Locale; +import java.util.Map; import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.CONTAINER; import static org.elasticsearch.repositories.azure.AzureRepository.CONTAINER_DEFAULT; +import static org.elasticsearch.repositories.azure.AzureRepository.Repository; -/** - * - */ public class AzureBlobStore extends AbstractComponent implements BlobStore { private final AzureStorageService client; + private final String accountName; + private final LocationMode locMode; private final String container; private final String repositoryName; @@ -49,9 +55,19 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { public AzureBlobStore(RepositoryName name, Settings settings, RepositorySettings repositorySettings, AzureStorageService client) throws URISyntaxException, StorageException { super(settings); - this.client = client; + this.client = client.start(); this.container = repositorySettings.settings().get("container", settings.get(CONTAINER, CONTAINER_DEFAULT)); this.repositoryName = name.getName(); + + // NOTE: null account means to use the first one specified in config + this.accountName = repositorySettings.settings().get(Repository.ACCOUNT, null); + + String modeStr = repositorySettings.settings().get(Repository.LOCATION_MODE, null); + if (modeStr == null) { + this.locMode = LocationMode.PRIMARY_ONLY; + } else { + this.locMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT)); + } } @Override @@ -59,10 +75,6 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { return container; } - public AzureStorageService client() { - return client; - } - public String container() { return container; } @@ -80,7 +92,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { } try { - client.deleteFiles(container, keyPath); + this.client.deleteFiles(this.accountName, this.locMode, container, keyPath); } catch (URISyntaxException | StorageException e) { logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage()); } @@ -89,4 +101,54 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { @Override public void close() { } + + public boolean doesContainerExist(String container) + { + return this.client.doesContainerExist(this.accountName, this.locMode, container); + } + + public void removeContainer(String container) throws URISyntaxException, StorageException + { + this.client.removeContainer(this.accountName, this.locMode, container); + } + + public void createContainer(String container) throws URISyntaxException, StorageException + { + this.client.createContainer(this.accountName, this.locMode, container); + } + + public void deleteFiles(String container, String path) throws URISyntaxException, StorageException + { + this.client.deleteFiles(this.accountName, this.locMode, container, path); + } + + public boolean blobExists(String container, String blob) throws URISyntaxException, StorageException + { + return this.client.blobExists(this.accountName, this.locMode, container, blob); + } + + public void deleteBlob(String container, String blob) throws URISyntaxException, StorageException + { + this.client.deleteBlob(this.accountName, this.locMode, container, blob); + } + + public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException + { + return this.client.getInputStream(this.accountName, this.locMode, container, blob); + } + + public OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException + { + return this.client.getOutputStream(this.accountName, this.locMode, container, blob); + } + + public Map listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException + { + return this.client.listBlobsByPrefix(this.accountName, this.locMode, container, keyPath, prefix); + } + + public void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException + { + this.client.moveBlob(this.accountName, this.locMode, container, sourceBlob, targetBlob); + } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java index c9b48aea052..5a551f54de3 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java @@ -20,6 +20,7 @@ package org.elasticsearch.cloud.azure.storage; import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.LocationMode; import org.elasticsearch.common.blobstore.BlobMetaData; import java.io.InputStream; @@ -32,9 +33,12 @@ import java.util.Map; * @see AzureStorageServiceImpl for Azure REST API implementation */ public interface AzureStorageService { - static public final class Storage { - public static final String API_IMPLEMENTATION = "cloud.azure.storage.api.impl"; + + final class Storage { + public static final String PREFIX = "cloud.azure.storage."; + @Deprecated public static final String ACCOUNT = "cloud.azure.storage.account"; + @Deprecated public static final String KEY = "cloud.azure.storage.key"; public static final String CONTAINER = "repositories.azure.container"; public static final String BASE_PATH = "repositories.azure.base_path"; @@ -42,23 +46,25 @@ public interface AzureStorageService { public static final String COMPRESS = "repositories.azure.compress"; } - boolean doesContainerExist(String container); + boolean doesContainerExist(String account, LocationMode mode, String container); - void removeContainer(String container) throws URISyntaxException, StorageException; + void removeContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException; - void createContainer(String container) throws URISyntaxException, StorageException; + void createContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException; - void deleteFiles(String container, String path) throws URISyntaxException, StorageException; + void deleteFiles(String account, LocationMode mode, String container, String path) throws URISyntaxException, StorageException; - boolean blobExists(String container, String blob) throws URISyntaxException, StorageException; + boolean blobExists(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException; - void deleteBlob(String container, String blob) throws URISyntaxException, StorageException; + void deleteBlob(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException; - InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException; + InputStream getInputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException; - OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException; + OutputStream getOutputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException; - Map listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException; + Map listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException; - void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException; + void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException; + + AzureStorageService start(); } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java index f38b23250cd..56e75d7386c 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java @@ -20,12 +20,14 @@ package org.elasticsearch.cloud.azure.storage; import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.*; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -35,54 +37,91 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; +import java.util.Hashtable; import java.util.Map; -import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.*; - -/** - * - */ public class AzureStorageServiceImpl extends AbstractLifecycleComponent implements AzureStorageService { - private final String account; - private final String key; - private final String blob; - - private CloudBlobClient client; + final AzureStorageSettings primaryStorageSettings; + final Map secondariesStorageSettings; + final Map clients; + @Inject public AzureStorageServiceImpl(Settings settings) { super(settings); - // We try to load storage API settings from `cloud.azure.` - account = settings.get(ACCOUNT); - key = settings.get(KEY); - blob = "https://" + account + ".blob.core.windows.net/"; - try { - if (account != null) { - logger.trace("creating new Azure storage client using account [{}], key [{}], blob [{}]", account, key, blob); + Tuple> storageSettings = AzureStorageSettings.parse(settings); + this.primaryStorageSettings = storageSettings.v1(); + this.secondariesStorageSettings = storageSettings.v2(); - String storageConnectionString = - "DefaultEndpointsProtocol=https;" - + "AccountName="+ account +";" - + "AccountKey=" + key; - - // Retrieve storage account from connection-string. - CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); - - // Create the blob client. - client = storageAccount.createCloudBlobClient(); - } - } catch (Exception e) { - // Can not start Azure Storage Client - logger.error("can not start azure storage client: {}", e.getMessage()); - } + this.clients = new Hashtable<>(); } - @Override - public boolean doesContainerExist(String container) { + void createClient(AzureStorageSettings azureStorageSettings) { try { + logger.trace("creating new Azure storage client using account [{}], key [{}]", + azureStorageSettings.getAccount(), azureStorageSettings.getKey()); + + String storageConnectionString = + "DefaultEndpointsProtocol=https;" + + "AccountName="+ azureStorageSettings.getAccount() +";" + + "AccountKey=" + azureStorageSettings.getKey(); + + // Retrieve storage account from connection-string. + CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); + + // Create the blob client. + CloudBlobClient client = storageAccount.createCloudBlobClient(); + + // Register the client + this.clients.put(azureStorageSettings.getAccount(), client); + } catch (Exception e) { + logger.error("can not create azure storage client: {}", e.getMessage()); + } + } + + CloudBlobClient getSelectedClient(String account, LocationMode mode) { + logger.trace("selecting a client for account [{}], mode [{}]", account, mode.name()); + AzureStorageSettings azureStorageSettings = null; + + if (this.primaryStorageSettings == null || this.secondariesStorageSettings.isEmpty()) { + throw new IllegalArgumentException("No azure storage can be found. Check your elasticsearch.yml."); + } + + if (account != null) { + azureStorageSettings = this.secondariesStorageSettings.get(account); + } + + // if account is not secondary, it's the primary + if (azureStorageSettings == null) { + if (account == null || primaryStorageSettings.getName() == null || account.equals(primaryStorageSettings.getName())) { + azureStorageSettings = primaryStorageSettings; + } + } + + if (azureStorageSettings == null) { + // We did not get an account. That's bad. + throw new IllegalArgumentException("Can not find azure account [" + account + "]. Check your elasticsearch.yml."); + } + + CloudBlobClient client = this.clients.get(azureStorageSettings.getAccount()); + + if (client == null) { + throw new IllegalArgumentException("Can not find an azure client for account [" + account + "]"); + } + + // NOTE: for now, just set the location mode in case it is different; + // only one mode per storage account can be active at a time + client.getDefaultRequestOptions().setLocationMode(mode); + return client; + } + + @Override + public boolean doesContainerExist(String account, LocationMode mode, String container) { + try { + CloudBlobClient client = this.getSelectedClient(account, mode); CloudBlobContainer blob_container = client.getContainerReference(container); return blob_container.exists(); } catch (Exception e) { @@ -92,7 +131,8 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException { + public Map listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException { + // NOTE: this should be here: if (prefix == null) prefix = ""; + // however, this is really inefficient since deleteBlobsByPrefix enumerates everything and + // then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix! + logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix); MapBuilder blobsBuilder = MapBuilder.newMapBuilder(); + CloudBlobClient client = this.getSelectedClient(account, mode); CloudBlobContainer blobContainer = client.getContainerReference(container); if (blobContainer.exists()) { for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath + (prefix == null ? "" : prefix))) { @@ -200,8 +251,10 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent azureStorageSettingsEntry : secondariesStorageSettings.entrySet()) { + logger.debug("registering secondary client for account [{}]", azureStorageSettingsEntry.getKey()); + createClient(azureStorageSettingsEntry.getValue()); + } } @Override protected void doStop() throws ElasticsearchException { logger.debug("stopping azure storage client instance"); + // We should stop all clients but it does sound like CloudBlobClient has + // any shutdown method... } @Override diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java new file mode 100644 index 00000000000..7fd0312df29 --- /dev/null +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java @@ -0,0 +1,122 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.azure.storage; + +import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; + +import java.util.HashMap; +import java.util.Map; + +public class AzureStorageSettings { + private static ESLogger logger = ESLoggerFactory.getLogger(AzureStorageSettings.class.getName()); + + private String name; + private String account; + private String key; + + public AzureStorageSettings(String name, String account, String key) { + this.name = name; + this.account = account; + this.key = key; + } + + public String getName() { + return name; + } + + public String getKey() { + return key; + } + + public String getAccount() { + return account; + } + + @Override + public String toString() { + final StringBuffer sb = new StringBuffer("AzureStorageSettings{"); + sb.append("name='").append(name).append('\''); + sb.append(", account='").append(account).append('\''); + sb.append(", key='").append(key).append('\''); + sb.append('}'); + return sb.toString(); + } + + /** + * Parses settings and read all settings available under cloud.azure.storage.* + * @param settings settings to parse + * @return A tuple with v1 = primary storage and v2 = secondary storage + */ + public static Tuple> parse(Settings settings) { + AzureStorageSettings primaryStorage = null; + Map secondaryStorage = new HashMap<>(); + + // We check for deprecated settings + String account = settings.get(Storage.ACCOUNT); + String key = settings.get(Storage.KEY); + if (account != null) { + logger.warn("[{}] and [{}] have been deprecated. Use now [{}xxx.account] and [{}xxx.key] where xxx is any name", + Storage.ACCOUNT, Storage.KEY, Storage.PREFIX, Storage.PREFIX); + primaryStorage = new AzureStorageSettings(null, account, key); + } else { + Settings storageSettings = settings.getByPrefix(Storage.PREFIX); + if (storageSettings != null) { + Map asMap = storageSettings.getAsStructuredMap(); + for (Map.Entry storage : asMap.entrySet()) { + if (storage.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) storage.getValue(); + AzureStorageSettings current = new AzureStorageSettings(storage.getKey(), map.get("account"), map.get("key")); + boolean activeByDefault = Boolean.parseBoolean(map.getOrDefault("default", "false")); + if (activeByDefault) { + if (primaryStorage == null) { + primaryStorage = current; + } else { + logger.warn("default storage settings has already been defined. You can not define it to [{}]", storage.getKey()); + secondaryStorage.put(storage.getKey(), current); + } + } else { + secondaryStorage.put(storage.getKey(), current); + } + } + } + // If we did not set any default storage, we should complain and define it + if (primaryStorage == null && secondaryStorage.isEmpty() == false) { + Map.Entry fallback = secondaryStorage.entrySet().iterator().next(); + // We only warn if the number of secondary storage if > to 1 + // If the user defined only one storage account, that's fine. We know it's the default one. + if (secondaryStorage.size() > 1) { + logger.warn("no default storage settings has been defined. " + + "Add \"default\": true to the settings you want to activate by default. " + + "Forcing default to [{}].", fallback.getKey()); + } + primaryStorage = fallback.getValue(); + secondaryStorage.remove(fallback.getKey()); + } + } + } + + return Tuple.tuple(primaryStorage, secondaryStorage); + } +} diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java index da3aa8c15d1..c061d262f0b 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilter.java @@ -24,15 +24,12 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.*; - public class AzureStorageSettingsFilter extends AbstractComponent { @Inject public AzureStorageSettingsFilter(Settings settings, SettingsFilter settingsFilter) { super(settings); // Cloud storage API settings needed to be hidden - settingsFilter.addFilter(ACCOUNT); - settingsFilter.addFilter(KEY); + settingsFilter.addFilter("cloud.azure.storage.*"); } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java index 6256115be06..e32c10562c7 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java @@ -32,8 +32,6 @@ import org.elasticsearch.repositories.azure.AzureRepository; import java.util.Collection; import java.util.Collections; -import static org.elasticsearch.cloud.azure.AzureRepositoryModule.isSnapshotReady; - /** * */ @@ -63,8 +61,7 @@ public class AzureRepositoryPlugin extends Plugin { } public void onModule(RepositoriesModule module) { - if (isSnapshotReady(settings, logger)) { - module.registerRepository(AzureRepository.TYPE, AzureRepository.class, BlobStoreIndexShardRepository.class); - } + logger.debug("registering repository type [{}]", AzureRepository.TYPE); + module.registerRepository(AzureRepository.TYPE, AzureRepository.class, BlobStoreIndexShardRepository.class); } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index eb145d850bd..829ccb7e95e 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -20,6 +20,7 @@ package org.elasticsearch.repositories.azure; import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.LocationMode; import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore; import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.cluster.metadata.MetaData; @@ -40,6 +41,7 @@ import org.elasticsearch.snapshots.SnapshotCreationException; import java.io.IOException; import java.net.URISyntaxException; import java.util.List; +import java.util.Locale; /** * Azure file system implementation of the BlobStoreRepository @@ -58,6 +60,8 @@ public class AzureRepository extends BlobStoreRepository { public final static String CONTAINER_DEFAULT = "elasticsearch-snapshots"; static public final class Repository { + public static final String ACCOUNT = "account"; + public static final String LOCATION_MODE = "location_mode"; public static final String CONTAINER = "container"; public static final String CHUNK_SIZE = "chunk_size"; public static final String COMPRESS = "compress"; @@ -71,6 +75,7 @@ public class AzureRepository extends BlobStoreRepository { private ByteSizeValue chunkSize; private boolean compress; + private final boolean readonly; @Inject public AzureRepository(RepositoryName name, RepositorySettings repositorySettings, @@ -92,6 +97,18 @@ public class AzureRepository extends BlobStoreRepository { this.compress = repositorySettings.settings().getAsBoolean(Repository.COMPRESS, settings.getAsBoolean(Storage.COMPRESS, false)); + String modeStr = repositorySettings.settings().get(Repository.LOCATION_MODE, null); + if (modeStr != null) { + LocationMode locationMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT)); + if (locationMode == LocationMode.SECONDARY_ONLY) { + readonly = true; + } else { + readonly = false; + } + } else { + readonly = false; + } + String basePath = repositorySettings.settings().get(Repository.BASE_PATH, null); if (Strings.hasLength(basePath)) { @@ -141,15 +158,12 @@ public class AzureRepository extends BlobStoreRepository { @Override public void initializeSnapshot(SnapshotId snapshotId, List indices, MetaData metaData) { try { - if (!blobStore.client().doesContainerExist(blobStore.container())) { + if (!blobStore.doesContainerExist(blobStore.container())) { logger.debug("container [{}] does not exist. Creating...", blobStore.container()); - blobStore.client().createContainer(blobStore.container()); + blobStore.createContainer(blobStore.container()); } super.initializeSnapshot(snapshotId, indices, metaData); - } catch (StorageException e) { - logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage()); - throw new SnapshotCreationException(snapshotId, e); - } catch (URISyntaxException e) { + } catch (StorageException | URISyntaxException e) { logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage()); throw new SnapshotCreationException(snapshotId, e); } @@ -157,18 +171,22 @@ public class AzureRepository extends BlobStoreRepository { @Override public String startVerification() { - try { - if (!blobStore.client().doesContainerExist(blobStore.container())) { - logger.debug("container [{}] does not exist. Creating...", blobStore.container()); - blobStore.client().createContainer(blobStore.container()); + if (readonly == false) { + try { + if (!blobStore.doesContainerExist(blobStore.container())) { + logger.debug("container [{}] does not exist. Creating...", blobStore.container()); + blobStore.createContainer(blobStore.container()); + } + } catch (StorageException | URISyntaxException e) { + logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage()); + throw new RepositoryVerificationException(repositoryName, "can not initialize container " + blobStore.container(), e); } - return super.startVerification(); - } catch (StorageException e) { - logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage()); - throw new RepositoryVerificationException(repositoryName, "can not initialize container " + blobStore.container(), e); - } catch (URISyntaxException e) { - logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage()); - throw new RepositoryVerificationException(repositoryName, "can not initialize container " + blobStore.container(), e); } + return super.startVerification(); + } + + @Override + public boolean readOnly() { + return readonly; } } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java index a1abded4546..8a17f83d92d 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java @@ -20,6 +20,7 @@ package org.elasticsearch.cloud.azure; import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.LocationMode; import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.cloud.azure.storage.AzureStorageServiceMock; @@ -115,6 +116,6 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container"); logger.info("--> remove blobs in container [{}]", container); AzureStorageService client = internalCluster().getInstance(AzureStorageService.class); - client.deleteFiles(container, path); + client.deleteFiles(null, LocationMode.PRIMARY_ONLY, container, path); } } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java index 8fe19232c26..90e44d97a3f 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java @@ -20,6 +20,7 @@ package org.elasticsearch.cloud.azure.storage; import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.LocationMode; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; @@ -51,46 +52,46 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent listBlobsByPrefix(String container, String keyPath, String prefix) { + public Map listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) { MapBuilder blobsBuilder = MapBuilder.newMapBuilder(); for (String blobName : blobs.keySet()) { if (startsWithIgnoreCase(blobName, prefix)) { @@ -101,7 +102,7 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent + # Clean test env + curl -XDELETE localhost:9200/foo?pretty + curl -XDELETE localhost:9200/_snapshot/my_backup1?pretty + curl -XDELETE localhost:9200/_snapshot/my_backup2?pretty + + # Create data + curl -XPUT localhost:9200/foo/bar/1?pretty -d '{ + "foo": "bar" + }' + curl -XPOST localhost:9200/foo/_refresh?pretty + curl -XGET localhost:9200/foo/_count?pretty + + # Create repository using default account + curl -XPUT localhost:9200/_snapshot/my_backup1?pretty -d '{ + "type": "azure" + }' + + # Backup + curl -XPOST "localhost:9200/_snapshot/my_backup1/snap1?pretty&wait_for_completion=true" + + # Remove data + curl -XDELETE localhost:9200/foo?pretty + + # Restore data + curl -XPOST "localhost:9200/_snapshot/my_backup1/snap1/_restore?pretty&wait_for_completion=true" + curl -XGET localhost:9200/foo/_count?pretty + + * + * If you want to define a secondary repository: + * + * 4) Set `-Dcloud.azure.storage.my_account.default=true` + * 5) Set `-Dcloud.azure.storage.my_account2.account=account_name` + * 6) Set `-Dcloud.azure.storage.my_account2.key=account_key_secondary` + * + * Then you can run REST calls like: + * 
    + # Remove data
+ curl -XDELETE localhost:9200/foo?pretty
+
+ # Create repository using account2 (secondary)
+ curl -XPUT localhost:9200/_snapshot/my_backup2?pretty -d '{
+   "type": "azure",
+   "settings": {
+     "account" : "my_account2",
+     "location_mode": "secondary_only"
+   }
+ }'
+
+ # Restore data from the secondary endpoint
+ curl -XPOST "localhost:9200/_snapshot/my_backup2/snap1/_restore?pretty&wait_for_completion=true"
+ curl -XGET localhost:9200/foo/_count?pretty
+
    + */ +public class AzureRepositoryF { + public static void main(String[] args) throws Throwable { + Settings.Builder settings = Settings.builder(); + settings.put("http.cors.enabled", "true"); + settings.put("http.cors.allow-origin", "*"); + settings.put("cluster.name", AzureRepositoryF.class.getSimpleName()); + + // Example for azure repo settings + // settings.put("cloud.azure.storage.my_account1.account", "account_name"); + // settings.put("cloud.azure.storage.my_account1.key", "account_key"); + // settings.put("cloud.azure.storage.my_account1.default", true); + // settings.put("cloud.azure.storage.my_account2.account", "account_name"); + // settings.put("cloud.azure.storage.my_account2.key", "account_key_secondary"); + + final CountDownLatch latch = new CountDownLatch(1); + final Node node = new MockNode(settings.build(), Version.CURRENT, Collections.singletonList(AzureRepositoryPlugin.class)); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + node.close(); + latch.countDown(); + } + }); + node.start(); + latch.await(); + } +} diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java new file mode 100644 index 00000000000..17c0b01850f --- /dev/null +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.azure; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; +import org.elasticsearch.cloud.azure.storage.AzureStorageSettings; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Settings; + +import java.util.Map; + +import static org.hamcrest.Matchers.*; + +public class AzureSettingsParserTest extends LuceneTestCase { + + public void testParseTwoSettingsExplicitDefault() { + Settings settings = Settings.builder() + .put("cloud.azure.storage.azure1.account", "myaccount1") + .put("cloud.azure.storage.azure1.key", "mykey1") + .put("cloud.azure.storage.azure1.default", true) + .put("cloud.azure.storage.azure2.account", "myaccount2") + .put("cloud.azure.storage.azure2.key", "mykey2") + .build(); + + Tuple> tuple = AzureStorageSettings.parse(settings); + assertThat(tuple.v1(), notNullValue()); + assertThat(tuple.v1().getAccount(), is("myaccount1")); + assertThat(tuple.v1().getKey(), is("mykey1")); + assertThat(tuple.v2().keySet(), hasSize(1)); + assertThat(tuple.v2().get("azure2"), notNullValue()); + assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2")); + assertThat(tuple.v2().get("azure2").getKey(), is("mykey2")); + } + + public void testParseUniqueSettings() { + Settings settings = Settings.builder() + .put("cloud.azure.storage.azure1.account", "myaccount1") + .put("cloud.azure.storage.azure1.key", "mykey1") + .build(); + + Tuple> tuple = AzureStorageSettings.parse(settings); + assertThat(tuple.v1(), notNullValue()); + assertThat(tuple.v1().getAccount(), is("myaccount1")); + assertThat(tuple.v1().getKey(), is("mykey1")); + assertThat(tuple.v2().keySet(), hasSize(0)); + } + + public void testDeprecatedSettings() { + Settings settings = Settings.builder() + .put(Storage.ACCOUNT, "myaccount1") + .put(Storage.KEY, "mykey1") + .build(); + + Tuple> tuple = AzureStorageSettings.parse(settings); + assertThat(tuple.v1(), notNullValue()); + assertThat(tuple.v1().getAccount(), is("myaccount1")); + assertThat(tuple.v1().getKey(), is("mykey1")); + assertThat(tuple.v2().keySet(), hasSize(0)); + } + + public void testParseTwoSettingsNoDefault() { + Settings settings = Settings.builder() + .put("cloud.azure.storage.azure1.account", "myaccount1") + .put("cloud.azure.storage.azure1.key", "mykey1") + .put("cloud.azure.storage.azure2.account", "myaccount2") + .put("cloud.azure.storage.azure2.key", "mykey2") + .build(); + + Tuple> tuple = AzureStorageSettings.parse(settings); + assertThat(tuple.v1(), notNullValue()); + assertThat(tuple.v1().getAccount(), is("myaccount1")); + assertThat(tuple.v1().getKey(), is("mykey1")); + assertThat(tuple.v2().keySet(), hasSize(1)); + assertThat(tuple.v2().get("azure2"), notNullValue()); + assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2")); + assertThat(tuple.v2().get("azure2").getKey(), is("mykey2")); + } + + public void testParseTwoSettingsTooManyDefaultSet() { + Settings settings = Settings.builder() + .put("cloud.azure.storage.azure1.account", "myaccount1") + .put("cloud.azure.storage.azure1.key", "mykey1") + .put("cloud.azure.storage.azure1.default", true) + .put("cloud.azure.storage.azure2.account", "myaccount2") + .put("cloud.azure.storage.azure2.key", "mykey2") + .put("cloud.azure.storage.azure2.default", true) + .build(); + + Tuple> tuple = AzureStorageSettings.parse(settings); + assertThat(tuple.v1(), notNullValue()); + assertThat(tuple.v1().getAccount(), is("myaccount1")); + assertThat(tuple.v1().getKey(), is("mykey1")); + assertThat(tuple.v2().keySet(), hasSize(1)); + assertThat(tuple.v2().get("azure2"), notNullValue()); + assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2")); + assertThat(tuple.v2().get("azure2").getKey(), is("mykey2")); + } + + public void testParseEmptySettings() { + Tuple> tuple = AzureStorageSettings.parse(Settings.EMPTY); + assertThat(tuple.v1(), nullValue()); + assertThat(tuple.v2().keySet(), hasSize(0)); + } +} diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceTests.java index 37a21a38a8a..995ae9256ac 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -41,12 +40,10 @@ import static org.hamcrest.Matchers.greaterThan; numClientNodes = 0, transportClientRatio = 0.0) public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositoryServiceTestCase { - public AzureSnapshotRestoreServiceTests() { super("/snapshot-test/repo-" + randomInt()); } - @Test public void testSimpleWorkflow() { Client client = client(); logger.info("--> creating azure repository with path [{}]", basePath); @@ -67,9 +64,9 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); @@ -89,9 +86,9 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); @@ -101,9 +98,9 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); // Test restore after index deletion logger.info("--> delete indices"); @@ -112,7 +109,7 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java index 670675aff52..7e4285829a8 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java @@ -22,6 +22,8 @@ package org.elasticsearch.repositories.azure; import com.carrotsearch.randomizedtesting.RandomizedTest; import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.LocationMode; + import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -44,7 +46,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.store.MockFSDirectoryService; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.net.URISyntaxException; import java.util.Locale; @@ -63,7 +64,6 @@ import static org.hamcrest.Matchers.greaterThan; numDataNodes = 1, transportClientRatio = 0.0) public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCase { - private String getRepositoryPath() { String testName = "it-".concat(Strings.toUnderscoreCase(getTestName()).replaceAll("_", "-")); return testName.contains(" ") ? Strings.split(testName, " ")[0] : testName; @@ -101,7 +101,6 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa getContainerName().concat("-2")); } - @Test public void testSimpleWorkflow() { Client client = client(); logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); @@ -123,9 +122,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); @@ -145,9 +144,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); @@ -157,9 +156,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); // Test restore after index deletion logger.info("--> delete indices"); @@ -168,7 +167,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); @@ -177,7 +176,6 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa /** * For issue #51: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/51 */ - @Test public void testMultipleSnapshots() throws URISyntaxException, StorageException { final String indexName = "test-idx-1"; final String typeName = "doc"; @@ -194,7 +192,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("indexing first document"); index(indexName, typeName, Integer.toString(1), "foo", "bar " + Integer.toString(1)); refresh(); - assertThat(client.prepareCount(indexName).get().getCount(), equalTo(1L)); + assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(1L)); logger.info("creating Azure repository with path [{}]", getRepositoryPath()); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository(repositoryName) @@ -215,7 +213,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa logger.info("indexing second document"); index(indexName, typeName, Integer.toString(2), "foo", "bar " + Integer.toString(2)); refresh(); - assertThat(client.prepareCount(indexName).get().getCount(), equalTo(2L)); + assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(2L)); logger.info("creating snapshot [{}]", snapshot2Name); CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot2Name).setWaitForCompletion(true).setIndices(indexName).get(); @@ -231,10 +229,9 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repositoryName, snapshot1Name).setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount(indexName).get().getCount(), equalTo(1L)); + assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(1L)); } - @Test public void testMultipleRepositories() { Client client = client(); logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); @@ -262,8 +259,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot 1"); CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot("test-repo1", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get(); @@ -285,7 +282,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin().cluster().prepareRestoreSnapshot("test-repo1", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet(); assertThat(restoreSnapshotResponse1.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); @@ -294,7 +291,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin().cluster().prepareRestoreSnapshot("test-repo2", "test-snap").setWaitForCompletion(true).setIndices("test-idx-2").execute().actionGet(); assertThat(restoreSnapshotResponse2.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(true)); @@ -303,7 +300,6 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa /** * For issue #26: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/26 */ - @Test public void testListBlobs_26() throws StorageException, URISyntaxException { createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); @@ -362,7 +358,6 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa /** * For issue #28: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/28 */ - @Test public void testGetDeleteNonExistingSnapshot_28() throws StorageException, URISyntaxException { ClusterAdminClient client = client().admin().cluster(); logger.info("--> creating azure repository without any path"); @@ -390,7 +385,6 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa /** * For issue #21: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/21 */ - @Test public void testForbiddenContainerName() throws Exception { checkContainerName("", false); checkContainerName("es", false); @@ -415,6 +409,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa // we can not create it yet. assertBusy(new Runnable() { + @Override public void run() { try { PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo") @@ -444,7 +439,6 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa /** * Test case for issue #23: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/23 */ - @Test public void testNonExistingRepo_23() { Client client = client(); logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); @@ -468,7 +462,6 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa /** * When a user remove a container you can not immediately create it again. */ - @Test public void testRemoveAndCreateContainer() throws Exception { final String container = getContainerName().concat("-testremove"); final AzureStorageService storageService = internalCluster().getInstance(AzureStorageService.class); @@ -477,9 +470,10 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa // so we might need some time to be able to create the container assertBusy(new Runnable() { + @Override public void run() { try { - storageService.createContainer(container); + storageService.createContainer(null, LocationMode.PRIMARY_ONLY, container); logger.debug(" -> container created..."); } catch (URISyntaxException e) { // Incorrect URL. This should never happen. @@ -491,7 +485,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa } } }, 30, TimeUnit.SECONDS); - storageService.removeContainer(container); + storageService.removeContainer(null, LocationMode.PRIMARY_ONLY, container); ClusterAdminClient client = client().admin().cluster(); logger.info("--> creating azure repository while container is being removed"); @@ -530,7 +524,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa Settings settings = readSettingsFromFile(); AzureStorageService client = new AzureStorageServiceImpl(settings); for (String container : containers) { - client.removeContainer(container); + client.removeContainer(null, LocationMode.PRIMARY_ONLY, container); } } } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle new file mode 100644 index 00000000000..8f18f67f70d --- /dev/null +++ b/plugins/repository-hdfs/build.gradle @@ -0,0 +1,203 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//apply plugin: 'nebula.provided-base' + +esplugin { + description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.' + classname 'org.elasticsearch.plugin.hadoop.hdfs.HdfsPlugin' +} + +configurations { + hadoop1 + hadoop2 +} + +versions << [ + 'hadoop1': '1.2.1', + 'hadoop2': '2.7.1' +] + +dependencies { + provided "org.elasticsearch:elasticsearch:${versions.elasticsearch}" + provided "org.apache.hadoop:hadoop-core:${versions.hadoop1}" + + // use Hadoop1 to compile and test things (a subset of Hadoop2) + testCompile "org.apache.hadoop:hadoop-core:${versions.hadoop1}" + testCompile "org.apache.hadoop:hadoop-test:${versions.hadoop1}" + // Hadoop dependencies + testCompile "commons-configuration:commons-configuration:1.6" + testCompile "commons-lang:commons-lang:${versions.commonslang}" + testCompile "commons-collections:commons-collections:3.2.2" + testCompile "commons-net:commons-net:1.4.1" + testCompile "org.mortbay.jetty:jetty:6.1.26" + testCompile "org.mortbay.jetty:jetty-util:6.1.26" + testCompile "org.mortbay.jetty:servlet-api:2.5-20081211" + testCompile "com.sun.jersey:jersey-core:1.8" + + + hadoop1("org.apache.hadoop:hadoop-core:${versions.hadoop1}") { + exclude module: "commons-cli" + exclude group: "com.sun.jersey" + exclude group: "org.mortbay.jetty" + exclude group: "tomcat" + exclude module: "commons-el" + exclude module: "hsqldb" + exclude group: "org.eclipse.jdt" + exclude module: "commons-beanutils" + exclude module: "commons-beanutils-core" + exclude module: "junit" + // provided by ES itself + exclude group: "log4j" + } + + hadoop2("org.apache.hadoop:hadoop-client:${versions.hadoop2}") { + exclude module: "commons-cli" + exclude group: "com.sun.jersey" + exclude group: "com.sun.jersey.contribs" + exclude group: "com.sun.jersey.jersey-test-framework" + exclude module: "guice" + exclude group: "org.mortbay.jetty" + exclude group: "tomcat" + exclude module: "commons-el" + exclude module: "hsqldb" + exclude group: "org.eclipse.jdt" + exclude module: "commons-beanutils" + exclude module: "commons-beanutils-core" + exclude module: "javax.servlet" + exclude module: "junit" + // provided by ES itself + exclude group: "log4j" + } + + hadoop2("org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}") { + exclude module: "guava" + exclude module: "junit" + // provided by ES itself + exclude group: "log4j" + } +} + +configurations.all { + resolutionStrategy { + force "commons-codec:commons-codec:${versions.commonscodec}" + force "commons-logging:commons-logging:${versions.commonslogging}" + force "commons-lang:commons-lang:2.6" + force "commons-httpclient:commons-httpclient:3.0.1" + force "org.codehaus.jackson:jackson-core-asl:1.8.8" + force "org.codehaus.jackson:jackson-mapper-asl:1.8.8" + force "com.google.code.findbugs:jsr305:3.0.0" + force "com.google.guava:guava:16.0.1" + force "org.slf4j:slf4j-api:1.7.10" + force "org.slf4j:slf4j-log4j12:1.7.10" + } +} + + +dependencyLicenses { + mapping from: /hadoop-core.*/, to: 'hadoop-1' + mapping from: /hadoop-.*/, to: 'hadoop-2' +} + +compileJava.options.compilerArgs << '-Xlint:-deprecation,-rawtypes' + +// main jar includes just the plugin classes +jar { + include "org/elasticsearch/plugin/hadoop/hdfs/*" +} + +// hadoop jar (which actually depend on Hadoop) +task hadoopLinkedJar(type: Jar, dependsOn:jar) { + appendix "internal" + from sourceSets.main.output.classesDir + // exclude plugin + exclude "org/elasticsearch/plugin/hadoop/hdfs/*" +} + + +bundlePlugin.dependsOn hadoopLinkedJar + +// configure 'bundle' as being w/o Hadoop deps +bundlePlugin { + into ("internal-libs") { + from hadoopLinkedJar.archivePath + } + + into ("hadoop-libs") { + from configurations.hadoop2.allArtifacts.files + from configurations.hadoop2 + } +} + + +task distZipHadoop1(type: Zip, dependsOn: [hadoopLinkedJar, jar]) { zipTask -> + from (zipTree(bundlePlugin.archivePath)) { + include "*" + include "internal-libs/**" + } + + description = "Builds archive (with Hadoop1 dependencies) suitable for download page." + classifier = "hadoop1" + + into ("hadoop-libs") { + from configurations.hadoop1.allArtifacts.files + from configurations.hadoop1 + } +} + +task distZipHadoop2(type: Zip, dependsOn: [hadoopLinkedJar, jar]) { zipTask -> + from (zipTree(bundlePlugin.archivePath)) { + include "*" + include "internal-libs/**" + } + + description = "Builds archive (with Hadoop2/YARN dependencies) suitable for download page." + classifier = "hadoop2" + + into ("hadoop-libs") { + from configurations.hadoop2.allArtifacts.files + from configurations.hadoop2 + } +} + +task distZipNoHadoop(type: Zip, dependsOn: [hadoopLinkedJar, jar]) { zipTask -> + from (zipTree(bundlePlugin.archivePath)) { + exclude "hadoop-libs/**" + } + + from sourceSets.main.output.resourcesDir + + description = "Builds archive (without any Hadoop dependencies) suitable for download page." + classifier = "lite" +} + + +artifacts { + archives bundlePlugin + 'default' bundlePlugin + archives distZipHadoop1 + archives distZipHadoop2 + archives distZipNoHadoop +} + +integTest { + cluster { + plugin(pluginProperties.extension.name, zipTree(distZipHadoop2.archivePath)) + } +} \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsPlugin.java new file mode 100644 index 00000000000..9b65f7bec2f --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsPlugin.java @@ -0,0 +1,173 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.plugin.hadoop.hdfs; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoriesModule; +import org.elasticsearch.repositories.Repository; + +// +// Note this plugin is somewhat special as Hadoop itself loads a number of libraries and thus requires a number of permissions to run even in client mode. +// This poses two problems: +// - Hadoop itself comes with tons of jars, many providing the same classes across packages. In particular Hadoop 2 provides package annotations in the same +// package across jars which trips JarHell. Thus, to allow Hadoop jars to load, the plugin uses a dedicated CL which picks them up from the hadoop-libs folder. +// - The issue though with using a different CL is that it picks up the jars from a different location / codeBase and thus it does not fall under the plugin +// permissions. In other words, the plugin permissions don't apply to the hadoop libraries. +// There are different approaches here: +// - implement a custom classloader that loads the jars but 'lies' about the codesource. It is doable but since URLClassLoader is locked down, one would +// would have to implement the whole jar opening and loading from it. Not impossible but still fairly low-level. +// Further more, even if the code has the proper credentials, it needs to use the proper Privileged blocks to use its full permissions which does not +// happen in the Hadoop code base. +// - use a different Policy. Works but the Policy is JVM wide and thus the code needs to be quite efficient - quite a bit impact to cover just some plugin +// libraries +// - use a DomainCombiner. This doesn't change the semantics (it's clear where the code is loaded from, etc..) however it gives us a scoped, fine-grained +// callback on handling the permission intersection for secured calls. Note that DC works only in the current PAC call - the moment another PA is used, +// the domain combiner is going to be ignored (unless the caller specifically uses it). Due to its scoped impact and official Java support, this approach +// was used. + +// ClassLoading info +// - package plugin.hadoop.hdfs is part of the plugin +// - all the other packages are assumed to be in the nested Hadoop CL. + +// Code +public class HdfsPlugin extends Plugin { + + @Override + public String name() { + return "repository-hdfs"; + } + + @Override + public String description() { + return "HDFS Repository Plugin"; + } + + @SuppressWarnings("unchecked") + public void onModule(RepositoriesModule repositoriesModule) { + String baseLib = Utils.detectLibFolder(); + List cp = getHadoopClassLoaderPath(baseLib); + + ClassLoader hadoopCL = URLClassLoader.newInstance(cp.toArray(new URL[cp.size()]), getClass().getClassLoader()); + + Class repository = null; + try { + repository = (Class) hadoopCL.loadClass("org.elasticsearch.repositories.hdfs.HdfsRepository"); + } catch (ClassNotFoundException cnfe) { + throw new IllegalStateException("Cannot load plugin class; is the plugin class setup correctly?", cnfe); + } + + repositoriesModule.registerRepository("hdfs", repository, BlobStoreIndexShardRepository.class); + Loggers.getLogger(HdfsPlugin.class).info("Loaded Hadoop [{}] libraries from {}", getHadoopVersion(hadoopCL), baseLib); + } + + protected List getHadoopClassLoaderPath(String baseLib) { + List cp = new ArrayList<>(); + // add plugin internal jar + discoverJars(createURI(baseLib, "internal-libs"), cp, false); + // add Hadoop jars + discoverJars(createURI(baseLib, "hadoop-libs"), cp, true); + return cp; + } + + private String getHadoopVersion(ClassLoader hadoopCL) { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + // unprivileged code such as scripts do not have SpecialPermission + sm.checkPermission(new SpecialPermission()); + } + + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public String run() { + // Hadoop 2 relies on TCCL to determine the version + ClassLoader tccl = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(hadoopCL); + return doGetHadoopVersion(hadoopCL); + } finally { + Thread.currentThread().setContextClassLoader(tccl); + } + } + }, Utils.hadoopACC()); + } + + private String doGetHadoopVersion(ClassLoader hadoopCL) { + String version = "Unknown"; + + Class clz = null; + try { + clz = hadoopCL.loadClass("org.apache.hadoop.util.VersionInfo"); + } catch (ClassNotFoundException cnfe) { + // unknown + } + if (clz != null) { + try { + Method method = clz.getMethod("getVersion"); + version = method.invoke(null).toString(); + } catch (Exception ex) { + // class has changed, ignore + } + } + + return version; + } + + private URI createURI(String base, String suffix) { + String location = base + suffix; + try { + return new URI(location); + } catch (URISyntaxException ex) { + throw new IllegalStateException(String.format(Locale.ROOT, "Cannot detect plugin folder; [%s] seems invalid", location), ex); + } + } + + @SuppressForbidden(reason = "discover nested jar") + private void discoverJars(URI libPath, List cp, boolean optional) { + try { + Path[] jars = FileSystemUtils.files(PathUtils.get(libPath), "*.jar"); + + for (Path path : jars) { + cp.add(path.toUri().toURL()); + } + } catch (IOException ex) { + if (!optional) { + throw new IllegalStateException("Cannot compute plugin classpath", ex); + } + } + } +} \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/plugin/hadoop/hdfs/Utils.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/plugin/hadoop/hdfs/Utils.java new file mode 100644 index 00000000000..101025d029e --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/plugin/hadoop/hdfs/Utils.java @@ -0,0 +1,84 @@ +package org.elasticsearch.plugin.hadoop.hdfs; + +import java.net.URL; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.DomainCombiner; +import java.security.PrivilegedAction; +import java.security.ProtectionDomain; + +import org.elasticsearch.SpecialPermission; + +public abstract class Utils { + + protected static AccessControlContext hadoopACC() { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + // unprivileged code such as scripts do not have SpecialPermission + sm.checkPermission(new SpecialPermission()); + } + + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public AccessControlContext run() { + return new AccessControlContext(AccessController.getContext(), new HadoopDomainCombiner()); + } + }); + } + + private static class HadoopDomainCombiner implements DomainCombiner { + + private static String BASE_LIB = detectLibFolder(); + + @Override + public ProtectionDomain[] combine(ProtectionDomain[] currentDomains, ProtectionDomain[] assignedDomains) { + for (ProtectionDomain pd : assignedDomains) { + if (pd.getCodeSource().getLocation().toString().startsWith(BASE_LIB)) { + return assignedDomains; + } + } + + return currentDomains; + } + } + + static String detectLibFolder() { + ClassLoader cl = Utils.class.getClassLoader(); + + // we could get the URL from the URLClassloader directly + // but that can create issues when running the tests from the IDE + // we could detect that by loading resources but that as well relies on + // the JAR URL + String classToLookFor = HdfsPlugin.class.getName().replace(".", "/").concat(".class"); + URL classURL = cl.getResource(classToLookFor); + if (classURL == null) { + throw new IllegalStateException("Cannot detect itself; something is wrong with this ClassLoader " + cl); + } + + String base = classURL.toString(); + + // extract root + // typically a JAR URL + int index = base.indexOf("!/"); + if (index > 0) { + base = base.substring(0, index); + // remove its prefix (jar:) + base = base.substring(4); + // remove the trailing jar + index = base.lastIndexOf("/"); + base = base.substring(0, index + 1); + } + // not a jar - something else, do a best effort here + else { + // remove the class searched + base = base.substring(0, base.length() - classToLookFor.length()); + } + + // append / + if (!base.endsWith("/")) { + base = base.concat("/"); + } + + return base; + } +} \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/FileSystemFactory.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/FileSystemFactory.java new file mode 100644 index 00000000000..5e7c4d3fa57 --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/FileSystemFactory.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.hdfs; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; + +interface FileSystemFactory { + + FileSystem getFileSystem() throws IOException; +} diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/FsCallback.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/FsCallback.java new file mode 100644 index 00000000000..3eda2272149 --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/FsCallback.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.hdfs; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; + +interface FsCallback { + + V doInHdfs(FileSystem fs) throws IOException; +} diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java new file mode 100644 index 00000000000..f71ca7020a8 --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -0,0 +1,173 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.hdfs; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.blobstore.BlobMetaData; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; +import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +public class HdfsBlobContainer extends AbstractBlobContainer { + + protected final HdfsBlobStore blobStore; + protected final Path path; + + public HdfsBlobContainer(BlobPath blobPath, HdfsBlobStore blobStore, Path path) { + super(blobPath); + this.blobStore = blobStore; + this.path = path; + } + + @Override + public boolean blobExists(String blobName) { + try { + return SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public Boolean doInHdfs(FileSystem fs) throws IOException { + return fs.exists(new Path(path, blobName)); + } + }); + } catch (Exception e) { + return false; + } + } + + @Override + public void deleteBlob(String blobName) throws IOException { + SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public Boolean doInHdfs(FileSystem fs) throws IOException { + return fs.delete(new Path(path, blobName), true); + } + }); + } + + @Override + public void move(String sourceBlobName, String targetBlobName) throws IOException { + boolean rename = SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public Boolean doInHdfs(FileSystem fs) throws IOException { + return fs.rename(new Path(path, sourceBlobName), new Path(path, targetBlobName)); + } + }); + + if (!rename) { + throw new IOException(String.format(Locale.ROOT, "can not move blob from [%s] to [%s]", sourceBlobName, targetBlobName)); + } + } + + @Override + public InputStream readBlob(String blobName) throws IOException { + // FSDataInputStream does buffering internally + return SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public InputStream doInHdfs(FileSystem fs) throws IOException { + return fs.open(new Path(path, blobName), blobStore.bufferSizeInBytes()); + } + }); + } + + @Override + public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { + SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public Void doInHdfs(FileSystem fs) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + Streams.copy(inputStream, stream); + } + return null; + } + }); + } + + @Override + public void writeBlob(String blobName, BytesReference bytes) throws IOException { + SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public Void doInHdfs(FileSystem fs) throws IOException { + try (OutputStream stream = createOutput(blobName)) { + bytes.writeTo(stream); + } + return null; + } + }); + } + + private OutputStream createOutput(String blobName) throws IOException { + Path file = new Path(path, blobName); + // FSDataOutputStream does buffering internally + return blobStore.fileSystemFactory().getFileSystem().create(file, true, blobStore.bufferSizeInBytes()); + } + + @Override + public Map listBlobsByPrefix(final @Nullable String blobNamePrefix) throws IOException { + FileStatus[] files = SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public FileStatus[] doInHdfs(FileSystem fs) throws IOException { + return fs.listStatus(path, new PathFilter() { + @Override + public boolean accept(Path path) { + return path.getName().startsWith(blobNamePrefix); + } + }); + } + }); + if (files == null || files.length == 0) { + return Collections.emptyMap(); + } + Map map = new LinkedHashMap(); + for (FileStatus file : files) { + map.put(file.getPath().getName(), new PlainBlobMetaData(file.getPath().getName(), file.getLen())); + } + return Collections.unmodifiableMap(map); + } + + @Override + public Map listBlobs() throws IOException { + FileStatus[] files = SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback() { + @Override + public FileStatus[] doInHdfs(FileSystem fs) throws IOException { + return fs.listStatus(path); + } + }); + if (files == null || files.length == 0) { + return Collections.emptyMap(); + } + Map map = new LinkedHashMap(); + for (FileStatus file : files) { + map.put(file.getPath().getName(), new PlainBlobMetaData(file.getPath().getName(), file.getLen())); + } + return Collections.unmodifiableMap(map); + } +} \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java new file mode 100644 index 00000000000..b75485fa7fe --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java @@ -0,0 +1,125 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.hdfs; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.concurrent.Executor; + +public class HdfsBlobStore extends AbstractComponent implements BlobStore { + + private final FileSystemFactory ffs; + private final Path rootHdfsPath; + private final ThreadPool threadPool; + private final int bufferSizeInBytes; + + public HdfsBlobStore(Settings settings, FileSystemFactory ffs, Path path, ThreadPool threadPool) throws IOException { + super(settings); + this.ffs = ffs; + this.rootHdfsPath = path; + this.threadPool = threadPool; + + this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); + + mkdirs(path); + } + + private void mkdirs(Path path) throws IOException { + SecurityUtils.execute(ffs, new FsCallback() { + @Override + public Void doInHdfs(FileSystem fs) throws IOException { + if (!fs.exists(path)) { + fs.mkdirs(path); + } + return null; + } + }); + } + + @Override + public String toString() { + return rootHdfsPath.toUri().toString(); + } + + public FileSystemFactory fileSystemFactory() { + return ffs; + } + + public Path path() { + return rootHdfsPath; + } + + public Executor executor() { + return threadPool.executor(ThreadPool.Names.SNAPSHOT); + } + + public int bufferSizeInBytes() { + return bufferSizeInBytes; + } + + @Override + public BlobContainer blobContainer(BlobPath path) { + return new HdfsBlobContainer(path, this, buildHdfsPath(path)); + } + + @Override + public void delete(BlobPath path) throws IOException { + SecurityUtils.execute(ffs, new FsCallback() { + @Override + public Void doInHdfs(FileSystem fs) throws IOException { + fs.delete(translateToHdfsPath(path), true); + return null; + } + }); + } + + private Path buildHdfsPath(BlobPath blobPath) { + final Path path = translateToHdfsPath(blobPath); + try { + mkdirs(path); + } catch (IOException ex) { + throw new ElasticsearchException("failed to create blob container", ex); + } + return path; + } + + private Path translateToHdfsPath(BlobPath blobPath) { + Path path = path(); + for (String p : blobPath) { + path = new Path(path, p); + } + return path; + } + + @Override + public void close() { + // + } +} \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java new file mode 100644 index 00000000000..11081445fd4 --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java @@ -0,0 +1,259 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.hdfs; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.file.Files; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.snapshots.IndexShardRepository; +import org.elasticsearch.repositories.RepositoryName; +import org.elasticsearch.repositories.RepositorySettings; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.threadpool.ThreadPool; + +public class HdfsRepository extends BlobStoreRepository implements FileSystemFactory { + + public final static String TYPE = "hdfs"; + + private final HdfsBlobStore blobStore; + private final BlobPath basePath; + private final ByteSizeValue chunkSize; + private final boolean compress; + private final RepositorySettings repositorySettings; + private FileSystem fs; + + @Inject + public HdfsRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, ThreadPool threadPool) throws IOException { + super(name.getName(), repositorySettings, indexShardRepository); + + this.repositorySettings = repositorySettings; + + String path = repositorySettings.settings().get("path", settings.get("path")); + if (path == null) { + throw new IllegalArgumentException("no 'path' defined for hdfs snapshot/restore"); + } + + // get configuration + fs = getFileSystem(); + Path hdfsPath = SecurityUtils.execute(fs, new FsCallback() { + @Override + public Path doInHdfs(FileSystem fs) throws IOException { + return fs.makeQualified(new Path(path)); + } + }); + this.basePath = BlobPath.cleanPath(); + + logger.debug("Using file-system [{}] for URI [{}], path [{}]", fs, fs.getUri(), hdfsPath); + blobStore = new HdfsBlobStore(settings, this, hdfsPath, threadPool); + this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", settings.getAsBytesSize("chunk_size", null)); + this.compress = repositorySettings.settings().getAsBoolean("compress", settings.getAsBoolean("compress", false)); + } + + // as the FileSystem is long-lived and might go away, make sure to check it before it's being used. + @Override + public FileSystem getFileSystem() throws IOException { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + // unprivileged code such as scripts do not have SpecialPermission + sm.checkPermission(new SpecialPermission()); + } + + try { + return AccessController.doPrivileged(new PrivilegedExceptionAction() { + @Override + public FileSystem run() throws IOException { + return doGetFileSystem(); + } + }, SecurityUtils.AccBridge.acc()); + } catch (PrivilegedActionException pae) { + Throwable th = pae.getCause(); + if (th instanceof Error) { + throw (Error) th; + } + if (th instanceof RuntimeException) { + throw (RuntimeException) th; + } + if (th instanceof IOException) { + throw (IOException) th; + } + throw new ElasticsearchException(pae); + } + } + + private FileSystem doGetFileSystem() throws IOException { + // check if the fs is still alive + // make a cheap call that triggers little to no security checks + if (fs != null) { + try { + fs.isFile(fs.getWorkingDirectory()); + } catch (IOException ex) { + if (ex.getMessage().contains("Filesystem closed")) { + fs = null; + } + else { + throw ex; + } + } + } + if (fs == null) { + Thread th = Thread.currentThread(); + ClassLoader oldCL = th.getContextClassLoader(); + try { + th.setContextClassLoader(getClass().getClassLoader()); + return initFileSystem(repositorySettings); + } catch (IOException ex) { + throw ex; + } finally { + th.setContextClassLoader(oldCL); + } + } + return fs; + } + + private FileSystem initFileSystem(RepositorySettings repositorySettings) throws IOException { + + Configuration cfg = new Configuration(repositorySettings.settings().getAsBoolean("load_defaults", settings.getAsBoolean("load_defaults", true))); + cfg.setClassLoader(this.getClass().getClassLoader()); + cfg.reloadConfiguration(); + + String confLocation = repositorySettings.settings().get("conf_location", settings.get("conf_location")); + if (Strings.hasText(confLocation)) { + for (String entry : Strings.commaDelimitedListToStringArray(confLocation)) { + addConfigLocation(cfg, entry.trim()); + } + } + + Map map = repositorySettings.settings().getByPrefix("conf.").getAsMap(); + for (Entry entry : map.entrySet()) { + cfg.set(entry.getKey(), entry.getValue()); + } + + try { + UserGroupInformation.setConfiguration(cfg); + } catch (Throwable th) { + throw new ElasticsearchGenerationException(String.format(Locale.ROOT, "Cannot initialize Hadoop"), th); + } + + String uri = repositorySettings.settings().get("uri", settings.get("uri")); + URI actualUri = (uri != null ? URI.create(uri) : FileSystem.getDefaultUri(cfg)); + String user = repositorySettings.settings().get("user", settings.get("user")); + + try { + // disable FS cache + String disableFsCache = String.format(Locale.ROOT, "fs.%s.impl.disable.cache", actualUri.getScheme()); + cfg.setBoolean(disableFsCache, true); + + return (user != null ? FileSystem.get(actualUri, cfg, user) : FileSystem.get(actualUri, cfg)); + } catch (Exception ex) { + throw new ElasticsearchGenerationException(String.format(Locale.ROOT, "Cannot create Hdfs file-system for uri [%s]", actualUri), ex); + } + } + + @SuppressForbidden(reason = "pick up Hadoop config (which can be on HDFS)") + private void addConfigLocation(Configuration cfg, String confLocation) { + URL cfgURL = null; + // it's an URL + if (!confLocation.contains(":")) { + cfgURL = cfg.getClassLoader().getResource(confLocation); + + // fall back to file + if (cfgURL == null) { + java.nio.file.Path path = PathUtils.get(confLocation); + if (!Files.isReadable(path)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, + "Cannot find classpath resource or file 'conf_location' [%s] defined for hdfs snapshot/restore", + confLocation)); + } + String pathLocation = path.toUri().toString(); + logger.debug("Adding path [{}] as file [{}]", confLocation, pathLocation); + confLocation = pathLocation; + } + else { + logger.debug("Resolving path [{}] to classpath [{}]", confLocation, cfgURL); + } + } + else { + logger.debug("Adding path [{}] as URL", confLocation); + } + + if (cfgURL == null) { + try { + cfgURL = new URL(confLocation); + } catch (MalformedURLException ex) { + throw new IllegalArgumentException(String.format(Locale.ROOT, + "Invalid 'conf_location' URL [%s] defined for hdfs snapshot/restore", confLocation), ex); + } + } + + cfg.addResource(cfgURL); + } + + @Override + protected BlobStore blobStore() { + return blobStore; + } + + @Override + protected BlobPath basePath() { + return basePath; + } + + @Override + protected boolean isCompress() { + return compress; + } + + @Override + protected ByteSizeValue chunkSize() { + return chunkSize; + } + + @Override + protected void doClose() throws ElasticsearchException { + super.doClose(); + + IOUtils.closeStream(fs); + fs = null; + } +} \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/SecurityUtils.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/SecurityUtils.java new file mode 100644 index 00000000000..6a0d4ffa818 --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/SecurityUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.hdfs; + +import java.io.IOException; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; + +import org.apache.hadoop.fs.FileSystem; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.plugin.hadoop.hdfs.Utils; + +class SecurityUtils { + + abstract static class AccBridge extends Utils { + static AccessControlContext acc() { + return Utils.hadoopACC(); + } + } + + static V execute(FileSystemFactory ffs, FsCallback callback) throws IOException { + return execute(ffs.getFileSystem(), callback); + } + + static V execute(FileSystem fs, FsCallback callback) throws IOException { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + // unprivileged code such as scripts do not have SpecialPermission + sm.checkPermission(new SpecialPermission()); + } + + try { + return AccessController.doPrivileged(new PrivilegedExceptionAction() { + @Override + public V run() throws IOException { + return callback.doInHdfs(fs); + } + }, AccBridge.acc()); + } catch (PrivilegedActionException pae) { + Throwable th = pae.getCause(); + if (th instanceof Error) { + throw (Error) th; + } + if (th instanceof RuntimeException) { + throw (RuntimeException) th; + } + if (th instanceof IOException) { + throw (IOException) th; + } + throw new ElasticsearchException(pae); + } + } +} diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/TestingFs.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/TestingFs.java new file mode 100644 index 00000000000..46cb0a263fe --- /dev/null +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/TestingFs.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.hdfs; + +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.elasticsearch.common.SuppressForbidden; + +import java.io.File; +import java.io.IOException; + +/** + * Extends LFS to improve some operations to keep the security permissions at + * bay. In particular mkdir is smarter and doesn't have to walk all the file + * hierarchy but rather only limits itself to the parent/working dir and creates + * a file only when necessary. + */ +public class TestingFs extends LocalFileSystem { + + private static class ImprovedRawLocalFileSystem extends RawLocalFileSystem { + @Override + @SuppressForbidden(reason = "the Hadoop API depends on java.io.File") + public boolean mkdirs(Path f) throws IOException { + File wd = pathToFile(getWorkingDirectory()); + File local = pathToFile(f); + if (wd.equals(local) || local.exists()) { + return true; + } + return mkdirs(f.getParent()) && local.mkdir(); + } + } + + public TestingFs() { + super(new ImprovedRawLocalFileSystem()); + // use the build path instead of the starting dir as that one has read permissions + //setWorkingDirectory(new Path(getClass().getProtectionDomain().getCodeSource().getLocation().toString())); + setWorkingDirectory(new Path(System.getProperty("java.io.tmpdir"))); + } +} diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..d26acd121e4 --- /dev/null +++ b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grant { + // used by the plugin to get the TCCL to properly initialize all of Hadoop components + permission java.lang.RuntimePermission "getClassLoader"; + + // used for DomainCombiner + permission java.security.SecurityPermission "createAccessControlContext"; + + // set TCCL used for bootstrapping Hadoop Configuration and JAAS + permission java.lang.RuntimePermission "setContextClassLoader"; + + // + // Hadoop 1 + // + + // UserGroupInformation (UGI) + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + + // UGI triggers JAAS + permission javax.security.auth.AuthPermission "getSubject"; + + // JAAS libraries are not loaded with the proper context in Hadoop, hence why the permission is needed here + permission java.lang.RuntimePermission "loadLibrary.jaas_nt"; + + // which triggers the use of the Kerberos library + permission java.lang.RuntimePermission "accessClassInPackage.sun.security.krb5"; + + // plus LoginContext + permission javax.security.auth.AuthPermission "modifyPrincipals"; + + permission javax.security.auth.AuthPermission "modifyPublicCredentials"; + + permission javax.security.auth.AuthPermission "modifyPrivateCredentials"; + + // + // Hadoop 2 + // + + // UGI (Ugi Metrics) + permission java.lang.RuntimePermission "accessDeclaredMembers"; + + // Shell initialization - reading system props + permission java.util.PropertyPermission "*", "read,write"; + + permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials \"*\"", "read"; + + // HftpFileSystem (all present FS are loaded and initialized at startup ...) + permission java.lang.RuntimePermission "setFactory"; +}; \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/resources/hadoop-libs/README.asciidoc b/plugins/repository-hdfs/src/main/resources/hadoop-libs/README.asciidoc new file mode 100644 index 00000000000..e9f85f3cdf7 --- /dev/null +++ b/plugins/repository-hdfs/src/main/resources/hadoop-libs/README.asciidoc @@ -0,0 +1 @@ +Folder containing the required Hadoop client libraries and dependencies. \ No newline at end of file diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsRepositoryRestIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsRepositoryRestIT.java new file mode 100644 index 00000000000..fd87e18cbce --- /dev/null +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsRepositoryRestIT.java @@ -0,0 +1,30 @@ +package org.elasticsearch.plugin.hadoop.hdfs; + +import java.io.IOException; +import java.util.Collection; + +import org.elasticsearch.plugin.hadoop.hdfs.HdfsPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +public class HdfsRepositoryRestIT extends ESRestTestCase { + + @Override + protected Collection> nodePlugins() { + return pluginList(HdfsPlugin.class); + } + + public HdfsRepositoryRestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESRestTestCase.createParameters(0, 1); + } +} diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsTestPlugin.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsTestPlugin.java new file mode 100644 index 00000000000..4b4e2aa05ef --- /dev/null +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsTestPlugin.java @@ -0,0 +1,15 @@ +package org.elasticsearch.plugin.hadoop.hdfs; + +import java.net.URL; +import java.util.Collections; +import java.util.List; + +import org.elasticsearch.plugin.hadoop.hdfs.HdfsPlugin; + +public class HdfsTestPlugin extends HdfsPlugin { + + @Override + protected List getHadoopClassLoaderPath(String baseLib) { + return Collections.emptyList(); + } +} diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsTests.java new file mode 100644 index 00000000000..d1b23e92538 --- /dev/null +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsTests.java @@ -0,0 +1,218 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.plugin.hadoop.hdfs; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +import java.util.Collection; + +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.hdfs.TestingFs; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.ESIntegTestCase.ThirdParty; +import org.elasticsearch.test.store.MockFSDirectoryService; +import org.junit.After; +import org.junit.Before; + +/** + * You must specify {@code -Dtests.thirdparty=true} + */ +@ThirdParty +@ClusterScope(scope = Scope.SUITE, numDataNodes = 1, transportClientRatio = 0.0) +public class HdfsTests extends ESIntegTestCase { + + @Override + public Settings indexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE, false) + .put(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE, false) + .build(); + } + + @Override + protected Settings nodeSettings(int ordinal) { + Settings.Builder settings = Settings.builder() + .put(super.nodeSettings(ordinal)) + .put("path.home", createTempDir()) + .put("path.repo", "") + .put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE, false) + .put(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE, false); + return settings.build(); + } + + @Override + protected Collection> nodePlugins() { + return pluginList(HdfsTestPlugin.class); + } + + private String path; + + @Before + public final void wipeBefore() throws Exception { + wipeRepositories(); + path = "build/data/repo-" + randomInt(); + } + + @After + public final void wipeAfter() throws Exception { + wipeRepositories(); + } + + public void testSimpleWorkflow() { + Client client = client(); + logger.info("--> creating hdfs repository with path [{}]", path); + + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("hdfs") + .setSettings(Settings.settingsBuilder() + //.put("uri", "hdfs://127.0.0.1:51227") + .put("conf.fs.es-hdfs.impl", TestingFs.class.getName()) + .put("uri", "es-hdfs://./build/") + .put("path", path) + .put("conf", "additional-cfg.xml, conf-2.xml") + .put("chunk_size", randomIntBetween(100, 1000) + "k") + .put("compress", randomBoolean()) + ).get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + createIndex("test-idx-1", "test-idx-2", "test-idx-3"); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); + index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); + index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); + } + refresh(); + assertThat(count(client, "test-idx-1"), equalTo(100L)); + assertThat(count(client, "test-idx-2"), equalTo(100L)); + assertThat(count(client, "test-idx-3"), equalTo(100L)); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + + logger.info("--> delete some data"); + for (int i = 0; i < 50; i++) { + client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); + } + for (int i = 50; i < 100; i++) { + client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get(); + } + for (int i = 0; i < 100; i += 2) { + client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); + } + refresh(); + assertThat(count(client, "test-idx-1"), equalTo(50L)); + assertThat(count(client, "test-idx-2"), equalTo(50L)); + assertThat(count(client, "test-idx-3"), equalTo(50L)); + + logger.info("--> close indices"); + client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); + + logger.info("--> restore all indices from the snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + ensureGreen(); + assertThat(count(client, "test-idx-1"), equalTo(100L)); + assertThat(count(client, "test-idx-2"), equalTo(100L)); + assertThat(count(client, "test-idx-3"), equalTo(50L)); + + // Test restore after index deletion + logger.info("--> delete indices"); + wipeIndices("test-idx-1", "test-idx-2"); + logger.info("--> restore one index after deletion"); + restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + ensureGreen(); + assertThat(count(client, "test-idx-1"), equalTo(100L)); + ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); + assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); + assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); + } + + private void wipeIndices(String... indices) { + cluster().wipeIndices(indices); + } + + // RepositoryVerificationException.class + public void testWrongPath() { + Client client = client(); + logger.info("--> creating hdfs repository with path [{}]", path); + + try { + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("hdfs") + .setSettings(Settings.settingsBuilder() + // .put("uri", "hdfs://127.0.0.1:51227/") + .put("conf.fs.es-hdfs.impl", TestingFs.class.getName()) + .put("uri", "es-hdfs:///") + .put("path", path + "a@b$c#11:22") + .put("chunk_size", randomIntBetween(100, 1000) + "k") + .put("compress", randomBoolean())) + .get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + createIndex("test-idx-1", "test-idx-2", "test-idx-3"); + ensureGreen(); + fail("Path name is invalid"); + } catch (RepositoryException re) { + // expected + } + } + + /** + * Deletes repositories, supports wildcard notation. + */ + public static void wipeRepositories(String... repositories) { + // if nothing is provided, delete all + if (repositories.length == 0) { + repositories = new String[]{"*"}; + } + for (String repository : repositories) { + try { + client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet(); + } catch (RepositoryMissingException ex) { + // ignore + } + } + } + + private long count(Client client, String index) { + return client.prepareSearch(index).setSize(0).get().getHits().totalHits(); + } +} \ No newline at end of file diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/MiniHDFSCluster.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/MiniHDFSCluster.java new file mode 100644 index 00000000000..0d700615a1a --- /dev/null +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/MiniHDFSCluster.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.hadoop.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.elasticsearch.common.SuppressForbidden; + +import java.io.File; + +public class MiniHDFSCluster { + + @SuppressForbidden(reason = "Hadoop is messy") + public static void main(String[] args) throws Exception { + FileUtil.fullyDelete(new File(System.getProperty("test.build.data", "build/test/data"), "dfs/")); + // MiniHadoopClusterManager.main(new String[] { "-nomr" }); + Configuration cfg = new Configuration(); + cfg.set(DataNode.DATA_DIR_PERMISSION_KEY, "666"); + cfg.set("dfs.replication", "0"); + MiniDFSCluster dfsCluster = new MiniDFSCluster(cfg, 1, true, null); + FileSystem fs = dfsCluster.getFileSystem(); + System.out.println(fs.getClass()); + System.out.println(fs.getUri()); + System.out.println(dfsCluster.getHftpFileSystem().getClass()); + + // dfsCluster.shutdown(); + } +} diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/UtilsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/UtilsTests.java new file mode 100644 index 00000000000..2f492eee343 --- /dev/null +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/UtilsTests.java @@ -0,0 +1,11 @@ +package org.elasticsearch.plugin.hadoop.hdfs; + +import org.elasticsearch.test.ESTestCase; + +public class UtilsTests extends ESTestCase { + + public void testDetectLibFolder() { + String location = HdfsPlugin.class.getProtectionDomain().getCodeSource().getLocation().toString(); + assertEquals(location, Utils.detectLibFolder()); + } +} diff --git a/plugins/repository-hdfs/src/test/resources/additional-cfg.xml b/plugins/repository-hdfs/src/test/resources/additional-cfg.xml new file mode 100644 index 00000000000..b1b6611e924 --- /dev/null +++ b/plugins/repository-hdfs/src/test/resources/additional-cfg.xml @@ -0,0 +1,12 @@ + + + + + foo + foo + + + paradise + lost + + diff --git a/plugins/repository-hdfs/src/test/resources/conf-2.xml b/plugins/repository-hdfs/src/test/resources/conf-2.xml new file mode 100644 index 00000000000..b1b6611e924 --- /dev/null +++ b/plugins/repository-hdfs/src/test/resources/conf-2.xml @@ -0,0 +1,12 @@ + + + + + foo + foo + + + paradise + lost + + diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/10_basic.yaml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/10_basic.yaml new file mode 100644 index 00000000000..b7bc644a832 --- /dev/null +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/10_basic.yaml @@ -0,0 +1,16 @@ +# Integration tests for HDFS Repository plugin +# +# Check plugin is installed +# +"HDFS Repository loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.plugins.0.name: repository-hdfs } + - match: { nodes.$master.plugins.0.jvm: true } diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository.disabled b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository.disabled new file mode 100644 index 00000000000..f1f5f7a65e0 --- /dev/null +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository.disabled @@ -0,0 +1,25 @@ +# Integration tests for HDFS Repository plugin +# +# Check plugin is installed +# +"HDFS Repository Config": + - do: + snapshot.create_repository: + repository: test_repo_hdfs_1 + verify: false + body: + type: hdfs + settings: + # local HDFS implementation + conf.fs.es-hdfs.impl: "org.elasticsearch.repositories.hdfs.TestingFs" + uri: "es-hdfs://./build/" + path: "build/data/repo-hdfs" + + # Get repositry + - do: + snapshot.get_repository: + repository: test_repo_hdfs_1 + + - is_true: test_repo_hdfs_1 + - is_true: test_repo_hdfs_1.settings.uri + - match: {test_repo_hdfs_1.settings.path : "build/data/repo-hdfs"} diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle new file mode 100644 index 00000000000..32ad37530c2 --- /dev/null +++ b/plugins/repository-s3/build.gradle @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The S3 repository plugin adds S3 repositories.' + classname 'org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin' +} + +versions << [ + 'aws': '1.10.33' +] + +dependencies { + compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}" + compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}" + compile "com.amazonaws:aws-java-sdk-core:${versions.aws}" + compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "com.fasterxml.jackson.core:jackson-databind:2.5.3" + compile "com.fasterxml.jackson.core:jackson-annotations:2.5.0" +} + +dependencyLicenses { + mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk' + mapping from: /jackson-.*/, to: 'jackson' +} + +compileJava.options.compilerArgs << '-Xlint:-deprecation,-rawtypes' + +test { + // this is needed for insecure plugins, remove if possible! + systemProperty 'tests.artifact', project.name +} diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.19.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.19.jar.sha1 deleted file mode 100644 index 66e418e6fb2..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b53f650323b7242dcced25b679f3e9aa4b494da5 diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 new file mode 100644 index 00000000000..332a8f01035 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 @@ -0,0 +1 @@ +fabedbbe2b834b1add150b6a38395c5ef7380168 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.19.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.19.jar.sha1 deleted file mode 100644 index 1328451c1c0..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c8764f3e61a3c420db429870ec22b31fe755d81d diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 new file mode 100644 index 00000000000..0d7ab9f8381 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 @@ -0,0 +1 @@ +35881245894ecc4d893c074eacdf2e6b56820fda \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.19.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.19.jar.sha1 deleted file mode 100644 index 9932c4676c4..00000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a23dc60d56d54126250c23cab1d01328b1e83678 diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 new file mode 100644 index 00000000000..3328f01c658 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 @@ -0,0 +1 @@ +5665cf77102a932a16e99ebf41d197e03ddbf25c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 new file mode 100644 index 00000000000..3fe8682a1b0 --- /dev/null +++ b/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 @@ -0,0 +1 @@ +4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.6.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.6.jar.sha1 deleted file mode 100644 index bf78aff7364..00000000000 --- a/plugins/repository-s3/licenses/commons-codec-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7f0fc8f61ecadeb3695f0b9464755eee44374d4 diff --git a/plugins/repository-s3/pom.xml b/plugins/repository-s3/pom.xml deleted file mode 100644 index cd68af71da0..00000000000 --- a/plugins/repository-s3/pom.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - repository-s3 - Plugin: Repository: S3 - The S3 repository plugin adds S3 repositories. - - - org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin - 1 - repository_s3 - false - -Xlint:-rawtypes,-deprecation - - - - - - com.amazonaws - aws-java-sdk-s3 - ${amazonaws.version} - - - - - org.apache.httpcomponents - httpclient - compile - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index e5db2ed7357..711b8db9374 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -26,6 +26,56 @@ import org.elasticsearch.common.component.LifecycleComponent; * */ public interface AwsS3Service extends LifecycleComponent { + + final class CLOUD_AWS { + public static final String KEY = "cloud.aws.access_key"; + public static final String SECRET = "cloud.aws.secret_key"; + public static final String PROTOCOL = "cloud.aws.protocol"; + public static final String PROXY_HOST = "cloud.aws.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.proxy.password"; + public static final String SIGNER = "cloud.aws.signer"; + public static final String REGION = "cloud.aws.region"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.proxy_port"; + } + + final class CLOUD_S3 { + public static final String KEY = "cloud.aws.s3.access_key"; + public static final String SECRET = "cloud.aws.s3.secret_key"; + public static final String PROTOCOL = "cloud.aws.s3.protocol"; + public static final String PROXY_HOST = "cloud.aws.s3.proxy.host"; + public static final String PROXY_PORT = "cloud.aws.s3.proxy.port"; + public static final String PROXY_USERNAME = "cloud.aws.s3.proxy.username"; + public static final String PROXY_PASSWORD = "cloud.aws.s3.proxy.password"; + public static final String SIGNER = "cloud.aws.s3.signer"; + public static final String ENDPOINT = "cloud.aws.s3.endpoint"; + @Deprecated + public static final String DEPRECATED_PROXY_HOST = "cloud.aws.s3.proxy_host"; + @Deprecated + public static final String DEPRECATED_PROXY_PORT = "cloud.aws.s3.proxy_port"; + } + + final class REPOSITORY_S3 { + public static final String BUCKET = "repositories.s3.bucket"; + public static final String ENDPOINT = "repositories.s3.endpoint"; + public static final String PROTOCOL = "repositories.s3.protocol"; + public static final String REGION = "repositories.s3.region"; + public static final String SERVER_SIDE_ENCRYPTION = "repositories.s3.server_side_encryption"; + public static final String BUFFER_SIZE = "repositories.s3.buffer_size"; + public static final String MAX_RETRIES = "repositories.s3.max_retries"; + public static final String CHUNK_SIZE = "repositories.s3.chunk_size"; + public static final String COMPRESS = "repositories.s3.compress"; + public static final String STORAGE_CLASS = "repositories.s3.storage_class"; + public static final String CANNED_ACL = "repositories.s3.canned_acl"; + public static final String BASE_PATH = "repositories.s3.base_path"; + } + + + AmazonS3 client(); AmazonS3 client(String endpoint, String protocol, String region, String account, String key); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 81b7e315b69..7d0b72cd63c 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -50,8 +50,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent properties = new HashMap<>(); - - @Before - public void saveProperties() { - for (String p : AWS_INVARIANT_PROPERTIES) { - properties.put(p, System.getProperty(p)); - } - } - - @After - public void restoreProperties() { - for (String p : AWS_INVARIANT_PROPERTIES) { - if (properties.get(p) != null) { - System.setProperty(p, properties.get(p)); - } else { - System.clearProperty(p); - } - } - } - @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder settings = Settings.builder() diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java index 846892b8704..0c9e7535db0 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java @@ -628,4 +628,9 @@ public class AmazonS3Wrapper implements AmazonS3 { public BucketReplicationConfiguration getBucketReplicationConfiguration(GetBucketReplicationConfigurationRequest getBucketReplicationConfigurationRequest) throws AmazonServiceException, AmazonClientException { return delegate.getBucketReplicationConfiguration(getBucketReplicationConfigurationRequest); } + + @Override + public HeadBucketResult headBucket(HeadBucketRequest headBucketRequest) throws AmazonClientException, AmazonServiceException { + return delegate.headBucket(headBucketRequest); + } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreTests.java new file mode 100644 index 00000000000..31e7b2cfb30 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreTests.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws.blobstore; + +import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.StorageClass; +import org.elasticsearch.common.blobstore.BlobStoreException; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class S3BlobStoreTests extends ESTestCase { + public void testInitCannedACL() throws IOException { + String[] aclList = new String[]{ + "private", "public-read", "public-read-write", "authenticated-read", + "log-delivery-write", "bucket-owner-read", "bucket-owner-full-control"}; + + //empty acl + assertThat(S3BlobStore.initCannedACL(null), equalTo(CannedAccessControlList.Private)); + assertThat(S3BlobStore.initCannedACL(""), equalTo(CannedAccessControlList.Private)); + + // it should init cannedACL correctly + for (String aclString : aclList) { + CannedAccessControlList acl = S3BlobStore.initCannedACL(aclString); + assertThat(acl.toString(), equalTo(aclString)); + } + + // it should accept all aws cannedACLs + for (CannedAccessControlList awsList : CannedAccessControlList.values()) { + CannedAccessControlList acl = S3BlobStore.initCannedACL(awsList.toString()); + assertThat(acl, equalTo(awsList)); + } + } + + public void testInvalidCannedACL() throws IOException { + try { + S3BlobStore.initCannedACL("test_invalid"); + fail("CannedACL should fail"); + } catch (BlobStoreException ex) { + assertThat(ex.getMessage(), equalTo("cannedACL is not valid: [test_invalid]")); + } + } + + public void testInitStorageClass() throws IOException { + // it should default to `standard` + assertThat(S3BlobStore.initStorageClass(null), equalTo(StorageClass.Standard)); + assertThat(S3BlobStore.initStorageClass(""), equalTo(StorageClass.Standard)); + + // it should accept [standard, standard_ia, reduced_redundancy] + assertThat(S3BlobStore.initStorageClass("standard"), equalTo(StorageClass.Standard)); + assertThat(S3BlobStore.initStorageClass("standard_ia"), equalTo(StorageClass.StandardInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("reduced_redundancy"), equalTo(StorageClass.ReducedRedundancy)); + } + + public void testCaseInsensitiveStorageClass() throws IOException { + assertThat(S3BlobStore.initStorageClass("sTandaRd"), equalTo(StorageClass.Standard)); + assertThat(S3BlobStore.initStorageClass("sTandaRd_Ia"), equalTo(StorageClass.StandardInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("reduCED_redundancy"), equalTo(StorageClass.ReducedRedundancy)); + } + + public void testInvalidStorageClass() throws IOException { + try { + S3BlobStore.initStorageClass("whatever"); + } catch(BlobStoreException ex) { + assertThat(ex.getMessage(), equalTo("`whatever` is not a valid S3 Storage Class.")); + } + } + + public void testRejectGlacierStorageClass() throws IOException { + try { + S3BlobStore.initStorageClass("glacier"); + } catch(BlobStoreException ex) { + assertThat(ex.getMessage(), equalTo("Glacier storage class is not supported")); + } + } +} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java index f40065c4e2b..f023b64211f 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.cloud.aws.blobstore; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -28,15 +27,14 @@ import java.util.Arrays; import static org.elasticsearch.common.io.Streams.copy; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; /** * Unit test for {@link S3OutputStream}. */ public class S3OutputStreamTests extends ESTestCase { - private static final int BUFFER_SIZE = S3BlobStore.MIN_BUFFER_SIZE.bytesAsInt(); - @Test public void testWriteLessDataThanBufferSize() throws IOException { MockDefaultS3OutputStream out = newS3OutputStream(BUFFER_SIZE); byte[] content = randomUnicodeOfLengthBetween(1, 512).getBytes("UTF-8"); @@ -54,7 +52,6 @@ public class S3OutputStreamTests extends ESTestCase { } - @Test public void testWriteSameDataThanBufferSize() throws IOException { int size = randomIntBetween(BUFFER_SIZE, 2 * BUFFER_SIZE); MockDefaultS3OutputStream out = newS3OutputStream(size); @@ -77,7 +74,6 @@ public class S3OutputStreamTests extends ESTestCase { } - @Test public void testWriteExactlyNTimesMoreDataThanBufferSize() throws IOException { int n = randomIntBetween(2, 3); int length = n * BUFFER_SIZE; @@ -102,7 +98,6 @@ public class S3OutputStreamTests extends ESTestCase { assertTrue(out.isMultipart()); } - @Test public void testWriteRandomNumberOfBytes() throws IOException { Integer randomBufferSize = randomIntBetween(BUFFER_SIZE, 2 * BUFFER_SIZE); MockDefaultS3OutputStream out = newS3OutputStream(randomBufferSize); @@ -129,11 +124,14 @@ public class S3OutputStreamTests extends ESTestCase { } } - @Test(expected = IllegalArgumentException.class) public void testWrongBufferSize() throws IOException { Integer randomBufferSize = randomIntBetween(1, 4 * 1024 * 1024); - MockDefaultS3OutputStream out = newS3OutputStream(randomBufferSize); - fail("Buffer size can't be smaller than 5mb"); + try { + newS3OutputStream(randomBufferSize); + fail("Buffer size can't be smaller than 5mb"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Buffer size can't be smaller than 5mb")); + } } private MockDefaultS3OutputStream newS3OutputStream(int bufferSizeInBytes) { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index c47202dc467..9ffa1286bc6 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -43,18 +43,18 @@ import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.store.MockFSDirectoryService; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; /** */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase { - @Override public Settings indexSettings() { // During restore we frequently restore index to exactly the same state it was before, that might cause the same @@ -83,7 +83,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase cleanRepositoryFiles(basePath); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") public void testSimpleWorkflow() { Client client = client(); Settings.Builder settings = Settings.settingsBuilder() @@ -110,9 +110,9 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); @@ -132,9 +132,9 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); @@ -144,9 +144,9 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); // Test restore after index deletion logger.info("--> delete indices"); @@ -155,13 +155,13 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") public void testEncryption() { Client client = client(); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); @@ -183,9 +183,9 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); @@ -221,9 +221,9 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); @@ -233,9 +233,9 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); - assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L)); // Test restore after index deletion logger.info("--> delete indices"); @@ -244,7 +244,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); @@ -254,20 +254,21 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase * This test verifies that the test configuration is set up in a manner that * does not make the test {@link #testRepositoryWithCustomCredentials()} pointless. */ - @Test(expected = RepositoryVerificationException.class) - public void assertRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredentials() { + public void testRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredentials() { Client client = client(); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.settingsBuilder() - .put("base_path", basePath) - .put("bucket", bucketSettings.get("bucket")) - ).get(); - fail("repository verification should have raise an exception!"); + try { + client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(Settings.settingsBuilder() + .put("base_path", basePath) + .put("bucket", bucketSettings.get("bucket")) + ).get(); + fail("repository verification should have raise an exception!"); + } catch (RepositoryVerificationException e) { + } } - @Test public void testRepositoryWithCustomCredentials() { Client client = client(); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); @@ -285,7 +286,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase assertRepositoryIsOperational(client, "test-repo"); } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") public void testRepositoryWithCustomEndpointProtocol() { Client client = client(); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.external-bucket."); @@ -306,23 +307,24 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase * This test verifies that the test configuration is set up in a manner that * does not make the test {@link #testRepositoryInRemoteRegion()} pointless. */ - @Test(expected = RepositoryVerificationException.class) - public void assertRepositoryInRemoteRegionIsRemote() { + public void testRepositoryInRemoteRegionIsRemote() { Client client = client(); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket."); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.settingsBuilder() - .put("base_path", basePath) - .put("bucket", bucketSettings.get("bucket")) - // Below setting intentionally omitted to assert bucket is not available in default region. - // .put("region", privateBucketSettings.get("region")) - ).get(); - - fail("repository verification should have raise an exception!"); + try { + client.admin().cluster().preparePutRepository("test-repo") + .setType("s3").setSettings(Settings.settingsBuilder() + .put("base_path", basePath) + .put("bucket", bucketSettings.get("bucket")) + // Below setting intentionally omitted to assert bucket is not available in default region. + // .put("region", privateBucketSettings.get("region")) + ).get(); + fail("repository verification should have raise an exception!"); + } catch (RepositoryVerificationException e) { + } } - @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") public void testRepositoryInRemoteRegion() { Client client = client(); Settings settings = internalCluster().getInstance(Settings.class); @@ -342,8 +344,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase /** * Test case for issue #86: https://github.com/elasticsearch/elasticsearch-cloud-aws/issues/86 */ - @Test - public void testNonExistingRepo_86() { + public void testNonExistingRepo86() { Client client = client(); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") @@ -364,8 +365,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase /** * For issue #86: https://github.com/elasticsearch/elasticsearch-cloud-aws/issues/86 */ - @Test - public void testGetDeleteNonExistingSnapshot_86() { + public void testGetDeleteNonExistingSnapshot86() { ClusterAdminClient client = client().admin().cluster(); logger.info("--> creating s3 repository without any path"); PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo") @@ -398,7 +398,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repository, "test-snap").setWaitForCompletion(true).setIndices("test-idx-*").get(); @@ -412,7 +412,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); } refresh(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1").get(); @@ -422,7 +422,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); + assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L)); } diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml index 69b50b66530..34384653a74 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml @@ -11,6 +11,8 @@ bucket: "my_bucket_name" access_key: "AKVAIQBF2RECL7FJWGJQ" secret_key: "vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br" + canned_acl: "public-read" + storage_class: "standard" # Get repositry - do: @@ -21,3 +23,4 @@ - is_true: test_repo_s3_1.settings.bucket - is_false: test_repo_s3_1.settings.access_key - is_false: test_repo_s3_1.settings.secret_key + - match: {test_repo_s3_1.settings.canned_acl : "public-read"} diff --git a/plugins/site-example/build.gradle b/plugins/site-example/build.gradle new file mode 100644 index 00000000000..d2228129025 --- /dev/null +++ b/plugins/site-example/build.gradle @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Demonstrates how to serve resources via elasticsearch.' + jvm false + site true +} + +// no unit tests +test.enabled = false diff --git a/plugins/site-example/licenses/no_deps.txt b/plugins/site-example/licenses/no_deps.txt deleted file mode 100644 index 8cce254d037..00000000000 --- a/plugins/site-example/licenses/no_deps.txt +++ /dev/null @@ -1 +0,0 @@ -This plugin has no third party dependencies diff --git a/plugins/site-example/pom.xml b/plugins/site-example/pom.xml deleted file mode 100644 index 38b1b0d4ca5..00000000000 --- a/plugins/site-example/pom.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - site-example - Plugin: Example site - Demonstrates how to serve resources via elasticsearch. - - - ${project.basedir}/src/main/assemblies/plugin-assembly.xml - true - NA - false - - example - false - true - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - org.apache.maven.plugins - maven-jar-plugin - - - default-jar - none - - - - - - - diff --git a/plugins/site-example/src/main/assemblies/plugin-assembly.xml b/plugins/site-example/src/main/assemblies/plugin-assembly.xml deleted file mode 100644 index 48a0286bf43..00000000000 --- a/plugins/site-example/src/main/assemblies/plugin-assembly.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - plugin - - zip - - false - - - - ${project.basedir}/src/site - - - - - - - ${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties - - true - - - diff --git a/plugins/store-smb/build.gradle b/plugins/store-smb/build.gradle new file mode 100644 index 00000000000..f2238e0a49c --- /dev/null +++ b/plugins/store-smb/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Store SMB plugin adds support for SMB stores.' + classname 'org.elasticsearch.plugin.store.smb.SMBStorePlugin' +} + diff --git a/plugins/store-smb/licenses/no_deps.txt b/plugins/store-smb/licenses/no_deps.txt deleted file mode 100644 index 8cce254d037..00000000000 --- a/plugins/store-smb/licenses/no_deps.txt +++ /dev/null @@ -1 +0,0 @@ -This plugin has no third party dependencies diff --git a/plugins/store-smb/pom.xml b/plugins/store-smb/pom.xml deleted file mode 100644 index 9ce7f4aa33d..00000000000 --- a/plugins/store-smb/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - 4.0.0 - - - org.elasticsearch.plugin - plugins - 3.0.0-SNAPSHOT - - - store-smb - Plugin: Store: SMB - The Store SMB plugin adds support for SMB stores. - - - org.elasticsearch.plugin.store.smb.SMBStorePlugin - 1 - store_smb - false - - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryService.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryService.java index 47efbd03531..744be5e49de 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryService.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryService.java @@ -24,8 +24,7 @@ import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.SmbDirectoryWrapper; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.index.store.IndexStore; @@ -35,8 +34,7 @@ import java.nio.file.Path; public class SmbMmapFsDirectoryService extends FsDirectoryService { - @Inject - public SmbMmapFsDirectoryService(@IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath path) { + public SmbMmapFsDirectoryService(IndexSettings indexSettings, IndexStore indexStore, ShardPath path) { super(indexSettings, indexStore, path); } diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java index 1d1592dcf32..a3686b9b114 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java @@ -19,26 +19,20 @@ package org.elasticsearch.index.store.smbmmapfs; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.indices.store.IndicesStore; +import org.elasticsearch.index.store.IndexStoreConfig; public class SmbMmapFsIndexStore extends IndexStore { - @Inject - public SmbMmapFsIndexStore(Index index, @IndexSettings Settings indexSettings, - IndexSettingsService indexSettingsService, IndicesStore indicesStore) { - super(index, indexSettings, indexSettingsService, indicesStore); + public SmbMmapFsIndexStore(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig) { + super(indexSettings, indexStoreConfig); } @Override public DirectoryService newDirectoryService(ShardPath path) { - return new SmbMmapFsDirectoryService(indexSettings(), this, path); + return new SmbMmapFsDirectoryService(indexSettings, this, path); } } diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryService.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryService.java index 7866090ac55..dc43c627bfd 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryService.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsDirectoryService.java @@ -23,9 +23,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.store.SmbDirectoryWrapper; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.index.store.IndexStore; @@ -35,8 +33,7 @@ import java.nio.file.Path; public class SmbSimpleFsDirectoryService extends FsDirectoryService { - @Inject - public SmbSimpleFsDirectoryService(@IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath path) { + public SmbSimpleFsDirectoryService(IndexSettings indexSettings, IndexStore indexStore, ShardPath path) { super(indexSettings, indexStore, path); } diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java index 67d396a80a5..309ef75e930 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java @@ -19,31 +19,21 @@ package org.elasticsearch.index.store.smbsimplefs; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.indices.store.IndicesStore; +import org.elasticsearch.index.store.IndexStoreConfig; public class SmbSimpleFsIndexStore extends IndexStore { - @Inject - public SmbSimpleFsIndexStore(Index index, @IndexSettings Settings indexSettings, - IndexSettingsService indexSettingsService, IndicesStore indicesStore) { - super(index, indexSettings, indexSettingsService, indicesStore); - } - - public Class shardDirectory() { - return SmbSimpleFsDirectoryService.class; + public SmbSimpleFsIndexStore(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig) { + super(indexSettings, indexStoreConfig); } @Override public DirectoryService newDirectoryService(ShardPath path) { - return new SmbSimpleFsDirectoryService(indexSettings(), this, path); + return new SmbSimpleFsDirectoryService(indexSettings, this, path); } } diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java index 5a5ace336c4..e93bc939e65 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java @@ -19,7 +19,7 @@ package org.elasticsearch.plugin.store.smb; -import org.elasticsearch.index.store.IndexStoreModule; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.store.smbmmapfs.SmbMmapFsIndexStore; import org.elasticsearch.index.store.smbsimplefs.SmbSimpleFsIndexStore; import org.elasticsearch.plugins.Plugin; @@ -36,8 +36,9 @@ public class SMBStorePlugin extends Plugin { return "SMB Store Plugin"; } - public void onModule(IndexStoreModule storeModule) { - storeModule.addIndexStore("smb_mmap_fs", SmbMmapFsIndexStore.class); - storeModule.addIndexStore("smb_simple_fs", SmbSimpleFsIndexStore.class); + @Override + public void onIndexModule(IndexModule indexModule) { + indexModule.addIndexStore("smb_mmap_fs", SmbMmapFsIndexStore::new); + indexModule.addIndexStore("smb_simple_fs", SmbSimpleFsIndexStore::new); } } diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java index 770e819bd20..9e29d6f091a 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java @@ -23,20 +23,17 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.plugin.store.smb.SMBStorePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Test; import java.util.Collection; import static org.hamcrest.Matchers.is; abstract public class AbstractAzureFsTestCase extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(SMBStorePlugin.class); } - @Test public void testAzureFs() { // Create an index and index some documents createIndex("test"); diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 86771979829..00000000000 --- a/pom.xml +++ /dev/null @@ -1,1493 +0,0 @@ - - - 4.0.0 - - org.elasticsearch - parent - 3.0.0-SNAPSHOT - pom - Elasticsearch: Parent POM - Parent POM - 2009 - - - org.sonatype.oss - oss-parent - 7 - - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - scm:git:git@github.com:elastic/elasticsearch.git - scm:git:git@github.com:elastic/elasticsearch.git - http://github.com/elastic/elasticsearch - - - - UTF-8 - - ${project.version} - ${java.home}${file.separator}bin${file.separator}java - 1.8 - 1.8 - - -Xlint:-path - -Xdoclint:-missing - - - 5.4.0 - 1702855 - 5.4.0-snapshot-${lucene.snapshot.revision} - 2.1.17 - 2.6.2 - 1.6.2 - 1.2.17 - 0.7.5.201505241946 - s3://download.elasticsearch.org/elasticsearch/staging/ - - - ${project.build.directory}/dev-tools - ${elasticsearch.tools.directory}/license-check/elasticsearch_license_header.txt - ${elasticsearch.tools.directory}/license-check/license_header_definition.xml - ${elasticsearch.tools.directory}/ant/integration-tests.xml - ${elasticsearch.integ.antfile.default} - - - ${project.basedir}/licenses - ${basedir}/target/releases/${project.build.finalName}.zip - - - auto - true - onerror - - ${project.basedir}/backwards - random - random - ERROR - 512m - ${project.build.directory}/heapdump/ - 5 - .local-${elasticsearch.version}-execution-hints.log - .local-${elasticsearch.version}-integ-execution-hints.log - false - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - fail - ${skipTests} - ${skipTests} - ${project.build.directory}/integ tests - ${project.build.directory}/integ deps - ${integ.scratch}/temp - 9400 - 9500 - 9600 - 9700 - \bno(n|)commit\b - - - - - elasticsearch-releases - http://maven.elasticsearch.org/releases - - true - - - false - - - - oss-snapshots - Sonatype OSS Snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - - Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/${lucene.snapshot.revision} - - - - - - - org.elasticsearch - dev-tools - ${elasticsearch.version} - - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - ${testframework.version} - - - - org.hamcrest - hamcrest-all - 1.3 - - - - junit - junit - 4.11 - - - - com.google.jimfs - jimfs - 1.0 - - - - org.apache.lucene - lucene-test-framework - ${lucene.maven.version} - - - - com.carrotsearch.randomizedtesting - junit4-ant - - - - - - org.elasticsearch - elasticsearch - ${elasticsearch.version} - - - - org.elasticsearch - elasticsearch - ${elasticsearch.version} - test-jar - - - - org.elasticsearch - elasticsearch - ${elasticsearch.version} - zip - - - - org.apache.httpcomponents - httpclient - 4.3.6 - - - - org.apache.lucene - lucene-core - ${lucene.maven.version} - - - org.apache.lucene - lucene-backward-codecs - ${lucene.maven.version} - - - org.apache.lucene - lucene-analyzers-common - ${lucene.maven.version} - - - org.apache.lucene - lucene-queries - ${lucene.maven.version} - - - jakarta-regexp - jakarta-regexp - - - - - org.apache.lucene - lucene-memory - ${lucene.maven.version} - - - org.apache.lucene - lucene-highlighter - ${lucene.maven.version} - - - org.apache.lucene - lucene-queryparser - ${lucene.maven.version} - - - jakarta-regexp - jakarta-regexp - - - - - org.apache.lucene - lucene-suggest - ${lucene.maven.version} - - - org.apache.lucene - lucene-join - ${lucene.maven.version} - - - - org.apache.lucene - lucene-spatial - ${lucene.maven.version} - - - com.spatial4j - spatial4j - 0.4.1 - - - com.vividsolutions - jts - 1.13 - - - xerces - xercesImpl - - - - - - com.github.spullara.mustache.java - compiler - 0.9.1 - - - - - org.apache.lucene - lucene-analyzers-phonetic - ${lucene.maven.version} - - - org.apache.lucene - lucene-analyzers-kuromoji - ${lucene.maven.version} - - - org.apache.lucene - lucene-analyzers-stempel - ${lucene.maven.version} - - - org.apache.lucene - lucene-analyzers-icu - ${lucene.maven.version} - - - org.apache.lucene - lucene-analyzers-smartcn - ${lucene.maven.version} - - - - com.carrotsearch - hppc - 0.7.1 - - - - joda-time - joda-time - - - 2.8.2 - - - org.joda - joda-convert - 1.2 - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - ${jackson.version} - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - ${jackson.version} - - - - io.netty - netty - 3.10.5.Final - - - - com.ning - compress-lzf - 1.0.2 - - - - com.tdunning - t-digest - 3.0 - - - - org.hdrhistogram - HdrHistogram - 2.1.6 - - - - commons-cli - commons-cli - 1.3.1 - - - - log4j - log4j - ${log4j.version} - - - - log4j - apache-log4j-extras - ${log4j.version} - - - - org.slf4j - slf4j-api - ${slf4j.version} - - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - - - - net.java.dev.jna - jna - 4.1.0 - true - - - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 1.4.1 - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - - - org.codehaus.mojo - buildnumber-maven-plugin - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - - - - org.apache.maven.plugins - maven-failsafe-plugin - - - org.apache.maven.plugins - maven-source-plugin - - - org.apache.maven.plugins - maven-remote-resources-plugin - - - de.thetaphi - forbiddenapis - - - org.apache.maven.plugins - maven-antrun-plugin - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.6 - - - org.apache.maven.plugins - maven-clean-plugin - 2.6.1 - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - org.apache.maven.plugins - maven-enforcer-plugin - - - enforce-versions - - enforce - - - - - [${maven.compiler.source},) - - - - - - enforce-maven-version - - enforce - - - - - [3.1.0,) - - - - - - print-versions - validate - - display-info - - - - enforce-java-home-is-set - - enforce - - - - - JAVA_HOME - "JAVA_HOME must be set and point to the jdk to run the tests" - - - true - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.3 - - true - 512m - - false - true - - -XDignore.symbol.file - -Xlint:all - ${xlint.options} - -Xdoclint:all/private - ${doclint.options} - ${javac.werror} - - - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - ${testframework.version} - - - ${jvm.executable} - ${tests.jvm.argline} - 10 - warn - true - ${tests.jvms} - - - - - - - - -Xmx${tests.heap.size} - -Xms${tests.heap.size} - -XX:MaxDirectMemorySize=512m - -Des.logger.prefix= - -XX:+HeapDumpOnOutOfMemoryError - -XX:HeapDumpPath=${tests.heapdump.path} - - ${tests.shuffle} - ${tests.verbose} - ${tests.seed} - - true - - - ./temp - - ${tests.bwc} - ${tests.bwc.path} - ${tests.bwc.version} - ${tests.jvm.argline} - ${tests.appendseed} - ${tests.cluster} - ${tests.iters} - ${project.groupId}:${project.artifactId} - - ${project.artifactId} - ${tests.maxfailures} - ${tests.failfast} - ${tests.class} - ${tests.method} - ${tests.nightly} - ${tests.verbose} - ${tests.badapples} - ${tests.weekly} - ${tests.failfast} - ${tests.awaitsfix} - ${tests.timeoutSuite} - ${tests.showSuccess} - ${tests.thirdparty} - ${tests.config} - ${tests.coverage} - ${project.build.directory} - ${tests.client.ratio} - ${tests.enable_mock_modules} - ${tests.assertion.disabled} - ${tests.rest} - ${tests.rest.suite} - ${tests.rest.blacklist} - ${tests.rest.spec} - ${tests.network} - ${tests.heap.size} - ${tests.filter} - ${elasticsearch.version} - ${tests.locale} - ${tests.rest.load_packaged} - ${tests.timezone} - ${env.ES_TEST_LOCAL} - ${es.node.mode} - ${es.logger.level} - ${tests.security.manager} - ${tests.compatibility} - true - - true - - - - - - - - - - - - - - - - - - - - tests - test - - junit4 - - - ${tests.ifNoTests} - ${skip.unit.tests} - - - - - - - - - - - **/*Tests.class - - - **/*$*.class - - - - - integ-tests - integration-test - - junit4 - - - warn - false - ${skip.integ.tests} - - - - - - - - - - - **/*IT.class - - - - true - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.18.1 - - true - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.18.1 - - ${skip.integ.tests} - true - - - - verify - - verify - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - 1.5 - - - org.elasticsearch:dev-tools:${elasticsearch.version} - org.elasticsearch:rest-api-spec:${elasticsearch.version} - - ${elasticsearch.tools.directory} - - false - false - - - - - process - - - - - - de.thetaphi - forbiddenapis - 2.0 - - - - check-forbidden-apis - - ${maven.compiler.target} - - true - - false - - - jdk-unsafe - jdk-deprecated - jdk-system-out - - - ${elasticsearch.tools.directory}/forbidden/core-signatures.txt - ${elasticsearch.tools.directory}/forbidden/all-signatures.txt - ${elasticsearch.tools.directory}/forbidden/third-party-signatures.txt - - ${forbidden.signatures} - **.SuppressForbidden - - compile - - check - - - - check-forbidden-test-apis - - ${maven.compiler.target} - - true - - false - - - jdk-unsafe - jdk-deprecated - - - ${elasticsearch.tools.directory}/forbidden/test-signatures.txt - ${elasticsearch.tools.directory}/forbidden/all-signatures.txt - - ${forbidden.test.signatures} - **.SuppressForbidden - - test-compile - - testCheck - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.4.1 - - - org.apache.maven.plugins - maven-resources-plugin - 2.7 - - - - ico - - - - - org.apache.maven.plugins - maven-site-plugin - 3.4 - - ${elasticsearch.tools.directory}/site - - - - org.codehaus.mojo - buildnumber-maven-plugin - 1.3 - - - validate - - create - - - - - false - false - - - - - - org.vafer - jdeb - 1.4 - true - - - - org.codehaus.mojo - rpm-maven-plugin - 2.1.3 - true - - - org.apache.maven.plugins - maven-jar-plugin - 2.5 - - - - true - - - ${scmBranch} - ${buildNumber} - ${timestamp} - ${elasticsearch.version} - ${lucene.version} - ${maven.compiler.source} - ${maven.compiler.target} - - - - - - - org.codehaus.mojo - license-maven-plugin - 1.8 - - - - com.mycila - license-maven-plugin - 2.5 - -
    ${elasticsearch.license.header}
    - - ${elasticsearch.license.headerDefinition} - - - src/**/*.java - -     - - - check-license-headers - compile - - check - - - -     - - - org.apache.maven.plugins - maven-eclipse-plugin - 2.10 - - eclipse-build - true - - - .settings/org.eclipse.core.resources.prefs - - =UTF-8 -]]> - - - - .settings/org.eclipse.jdt.core.prefs - - - - - - .settings/org.eclipse.jdt.ui.prefs - - -]]> - - - - - - - - - org.apache.maven.plugins - maven-jxr-plugin - 2.5 - - - org.apache.maven.plugins - maven-pmd-plugin - 3.5 - - - ${elasticsearch.tools.directory}/pmd/custom.xml - - ${maven.compiler.target} - - - - org.codehaus.mojo - findbugs-maven-plugin - 3.0.0 - - true - target/site - true - 2048 - 1800000 - org.elasticsearch.- - - - - org.apache.maven.plugins - maven-project-info-reports-plugin - 2.7 - - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - default-prepare-agent - - prepare-agent - - - - default-report - prepare-package - - report - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.8 - - - - set-werror - validate - - run - - - - - - - - - true - - - - check-invalid-patterns - validate - - run - - - - - - - - - - - - - - - - - - - - - - - - The following files contain tabs or - nocommits:${line.separator}${validate.patternsFound} - - - - - - generate-test-resources - create-heapdump-directory - - - - - - - - run - - - - tests-top-hints - test - - ${skip.unit.tests} - - - - - - - - - run - - - - integ-tests-top-hints - integration-test - - ${skip.integ.tests} - - - - - - - - - run - - - - check-license - verify - - run - - - ${skip.integ.tests} - - Running license check - - - - - - - - - - - - - - - com.carrotsearch.randomizedtesting - junit4-ant - ${testframework.version} - - - - - - org.apache.maven.plugins - maven-invoker-plugin - 2.0.0 - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - -Xdoclint:none - - - - - org.apache.maven.plugins - maven-verifier-plugin - 1.1 - - - org.apache.maven.plugins - maven-checkstyle-plugin - 2.15 - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - true - - -     -     -     - - - default - - true - - - - - com.mycila - license-maven-plugin - - - - - - release - - true - \bno(n|)(release|commit)\b - org.apache.lucene.util.LuceneTestCase$AwaitsFix @ Please fix all bugs before release or mark them as ignored - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - ${gpg.keyname} - ${gpg.passphrase} - ${gpg.keyring} - - - - - - - - - - license - - - license.generation - true - - - - - - - pom - - false - - ${basedir}/src/test/java - - - - true - true - true - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - create-heapdump-directory - none - - - tests-top-hints - none - - - - - - - - - dev - - - - de.thetaphi - forbiddenapis - - - check-forbidden-apis - none - - - check-forbidden-test-apis - none - - - - - com.mycila - license-maven-plugin - - - check-license-headers - none - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - check-invalid-patterns - none - - - - - - - - - coverage - - - tests.coverage - true - - - - - true - - - - - org.jacoco - jacoco-maven-plugin - - - - - - - static - - - tests.static - true - - - - false - - - - - org.codehaus.mojo - findbugs-maven-plugin - - - - - - - org.apache.maven.plugins - maven-jxr-plugin - - - org.apache.maven.plugins - maven-pmd-plugin - - - org.codehaus.mojo - findbugs-maven-plugin - - - org.apache.maven.plugins - maven-project-info-reports-plugin - 2.7 - - - - index - - - - - - - - - - dev-tools - rest-api-spec - core - distribution - plugins - qa - -     diff --git a/core/src/main/java/org/elasticsearch/index/fieldvisitor/AllFieldsVisitor.java b/qa/evil-tests/build.gradle similarity index 67% rename from core/src/main/java/org/elasticsearch/index/fieldvisitor/AllFieldsVisitor.java rename to qa/evil-tests/build.gradle index beb7de2c756..96aa6fb635d 100644 --- a/core/src/main/java/org/elasticsearch/index/fieldvisitor/AllFieldsVisitor.java +++ b/qa/evil-tests/build.gradle @@ -16,22 +16,21 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.fieldvisitor; -import org.apache.lucene.index.FieldInfo; - -import java.io.IOException; - -/** +/* + * Evil tests that need to do unrealistic things: test OS security + * integration, change default filesystem impl, mess with arbitrary + * threads, etc. */ -public class AllFieldsVisitor extends FieldsVisitor { - public AllFieldsVisitor() { - super(true); - } +apply plugin: 'elasticsearch.standalone-test' - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - return Status.YES; - } +dependencies { + testCompile 'com.google.jimfs:jimfs:1.0' +} + +// TODO: give each evil test its own fresh JVM for more isolation. + +test { + systemProperty 'tests.security.manager', 'false' } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java similarity index 94% rename from core/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java index 8b6803420a6..9faa604a18e 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.Build; import org.elasticsearch.Version; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.cli.CliTool.ExitStatus; import org.elasticsearch.common.cli.CliToolTestCase; import org.elasticsearch.common.collect.Tuple; @@ -36,6 +37,7 @@ import java.util.Locale; import static org.elasticsearch.common.cli.CliTool.ExitStatus.*; import static org.hamcrest.Matchers.*; +@SuppressForbidden(reason = "modifies system properties intentionally") public class BootstrapCliParserTests extends CliToolTestCase { private CaptureOutputTerminal terminal = new CaptureOutputTerminal(); @@ -54,8 +56,8 @@ public class BootstrapCliParserTests extends CliToolTestCase { assertStatus(status, OK_AND_EXIT); assertThatTerminalOutput(containsString(Version.CURRENT.toString())); - assertThatTerminalOutput(containsString(Build.CURRENT.hashShort())); - assertThatTerminalOutput(containsString(Build.CURRENT.timestamp())); + assertThatTerminalOutput(containsString(Build.CURRENT.shortHash())); + assertThatTerminalOutput(containsString(Build.CURRENT.date())); assertThatTerminalOutput(containsString(JvmInfo.jvmInfo().version())); } @@ -65,8 +67,8 @@ public class BootstrapCliParserTests extends CliToolTestCase { assertStatus(status, OK_AND_EXIT); assertThatTerminalOutput(containsString(Version.CURRENT.toString())); - assertThatTerminalOutput(containsString(Build.CURRENT.hashShort())); - assertThatTerminalOutput(containsString(Build.CURRENT.timestamp())); + assertThatTerminalOutput(containsString(Build.CURRENT.shortHash())); + assertThatTerminalOutput(containsString(Build.CURRENT.date())); assertThatTerminalOutput(containsString(JvmInfo.jvmInfo().version())); CaptureOutputTerminal terminal = new CaptureOutputTerminal(); @@ -75,8 +77,8 @@ public class BootstrapCliParserTests extends CliToolTestCase { assertStatus(status, OK_AND_EXIT); assertThatTerminalOutput(containsString(Version.CURRENT.toString())); - assertThatTerminalOutput(containsString(Build.CURRENT.hashShort())); - assertThatTerminalOutput(containsString(Build.CURRENT.timestamp())); + assertThatTerminalOutput(containsString(Build.CURRENT.shortHash())); + assertThatTerminalOutput(containsString(Build.CURRENT.date())); assertThatTerminalOutput(containsString(JvmInfo.jvmInfo().version())); } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java new file mode 100644 index 00000000000..220d093301c --- /dev/null +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.test.ESTestCase; + +import java.io.FilePermission; +import java.security.AllPermission; +import java.security.CodeSource; +import java.security.Permission; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.security.ProtectionDomain; +import java.security.cert.Certificate; +import java.util.Collections; + +/** + * Unit tests for ESPolicy: these cannot run with security manager, + * we don't allow messing with the policy + */ +public class ESPolicyUnitTests extends ESTestCase { + /** + * Test policy with null codesource. + *

    + * This can happen when restricting privileges with doPrivileged, + * even though ProtectionDomain's ctor javadocs might make you think + * that the policy won't be consulted. + */ + public void testNullCodeSource() throws Exception { + assumeTrue("test cannot run with security manager", System.getSecurityManager() == null); + // create a policy with AllPermission + Permission all = new AllPermission(); + PermissionCollection allCollection = all.newPermissionCollection(); + allCollection.add(all); + ESPolicy policy = new ESPolicy(allCollection, Collections.emptyMap(), true); + // restrict ourselves to NoPermission + PermissionCollection noPermissions = new Permissions(); + assertFalse(policy.implies(new ProtectionDomain(null, noPermissions), new FilePermission("foo", "read"))); + } + + /** + * test with null location + *

    + * its unclear when/if this happens, see https://bugs.openjdk.java.net/browse/JDK-8129972 + */ + public void testNullLocation() throws Exception { + assumeTrue("test cannot run with security manager", System.getSecurityManager() == null); + PermissionCollection noPermissions = new Permissions(); + ESPolicy policy = new ESPolicy(noPermissions, Collections.emptyMap(), true); + assertFalse(policy.implies(new ProtectionDomain(new CodeSource(null, (Certificate[])null), noPermissions), new FilePermission("foo", "read"))); + } +} diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJarHellTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJarHellTests.java new file mode 100644 index 00000000000..25bf3ac48f9 --- /dev/null +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJarHellTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** Tests for Jarhell that change very important system properties... very evil! */ +@SuppressForbidden(reason = "modifies system properties intentionally") +public class EvilJarHellTests extends ESTestCase { + + URL makeJar(Path dir, String name, Manifest manifest, String... files) throws IOException { + Path jarpath = dir.resolve(name); + ZipOutputStream out; + if (manifest == null) { + out = new JarOutputStream(Files.newOutputStream(jarpath, StandardOpenOption.CREATE)); + } else { + out = new JarOutputStream(Files.newOutputStream(jarpath, StandardOpenOption.CREATE), manifest); + } + for (String file : files) { + out.putNextEntry(new ZipEntry(file)); + } + out.close(); + return jarpath.toUri().toURL(); + } + + public void testBootclasspathLeniency() throws Exception { + Path dir = createTempDir(); + String previousJavaHome = System.getProperty("java.home"); + System.setProperty("java.home", dir.toString()); + URL[] jars = {makeJar(dir, "foo.jar", null, "DuplicateClass.class"), makeJar(dir, "bar.jar", null, "DuplicateClass.class")}; + try { + JarHell.checkJarHell(jars); + } finally { + System.setProperty("java.home", previousJavaHome); + } + } + + public void testRequiredJDKVersionIsOK() throws Exception { + Path dir = createTempDir(); + String previousJavaVersion = System.getProperty("java.specification.version"); + System.setProperty("java.specification.version", "1.7"); + + Manifest manifest = new Manifest(); + Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0"); + attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7"); + URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")}; + try { + JarHell.checkJarHell(jars); + } finally { + System.setProperty("java.specification.version", previousJavaVersion); + } + } + + public void testBadJDKVersionProperty() throws Exception { + Path dir = createTempDir(); + String previousJavaVersion = System.getProperty("java.specification.version"); + System.setProperty("java.specification.version", "bogus"); + + Manifest manifest = new Manifest(); + Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0"); + attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7"); + URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")}; + try { + JarHell.checkJarHell(jars); + } finally { + System.setProperty("java.specification.version", previousJavaVersion); + } + } +} diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java new file mode 100644 index 00000000000..695d2a42321 --- /dev/null +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java @@ -0,0 +1,232 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.FilePermission; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.util.Set; + +@SuppressForbidden(reason = "modifies system properties and attempts to create symbolic links intentionally") +public class EvilSecurityTests extends ESTestCase { + + /** test generated permissions */ + public void testGeneratedPermissions() throws Exception { + Path path = createTempDir(); + // make a fake ES home and ensure we only grant permissions to that. + Path esHome = path.resolve("esHome"); + Settings.Builder settingsBuilder = Settings.builder(); + settingsBuilder.put("path.home", esHome.toString()); + Settings settings = settingsBuilder.build(); + + Path fakeTmpDir = createTempDir(); + String realTmpDir = System.getProperty("java.io.tmpdir"); + Permissions permissions; + try { + System.setProperty("java.io.tmpdir", fakeTmpDir.toString()); + Environment environment = new Environment(settings); + permissions = Security.createPermissions(environment); + } finally { + System.setProperty("java.io.tmpdir", realTmpDir); + } + + // the fake es home + assertNoPermissions(esHome, permissions); + // its parent + assertNoPermissions(esHome.getParent(), permissions); + // some other sibling + assertNoPermissions(esHome.getParent().resolve("other"), permissions); + // double check we overwrote java.io.tmpdir correctly for the test + assertNoPermissions(PathUtils.get(realTmpDir), permissions); + } + + /** test generated permissions for all configured paths */ + public void testEnvironmentPaths() throws Exception { + Path path = createTempDir(); + // make a fake ES home and ensure we only grant permissions to that. + Path esHome = path.resolve("esHome"); + + Settings.Builder settingsBuilder = Settings.builder(); + settingsBuilder.put("path.home", esHome.resolve("home").toString()); + settingsBuilder.put("path.conf", esHome.resolve("conf").toString()); + settingsBuilder.put("path.scripts", esHome.resolve("scripts").toString()); + settingsBuilder.put("path.plugins", esHome.resolve("plugins").toString()); + settingsBuilder.putArray("path.data", esHome.resolve("data1").toString(), esHome.resolve("data2").toString()); + settingsBuilder.put("path.shared_data", esHome.resolve("custom").toString()); + settingsBuilder.put("path.logs", esHome.resolve("logs").toString()); + settingsBuilder.put("pidfile", esHome.resolve("test.pid").toString()); + Settings settings = settingsBuilder.build(); + + Path fakeTmpDir = createTempDir(); + String realTmpDir = System.getProperty("java.io.tmpdir"); + Permissions permissions; + Environment environment; + try { + System.setProperty("java.io.tmpdir", fakeTmpDir.toString()); + environment = new Environment(settings); + permissions = Security.createPermissions(environment); + } finally { + System.setProperty("java.io.tmpdir", realTmpDir); + } + + // the fake es home + assertNoPermissions(esHome, permissions); + // its parent + assertNoPermissions(esHome.getParent(), permissions); + // some other sibling + assertNoPermissions(esHome.getParent().resolve("other"), permissions); + // double check we overwrote java.io.tmpdir correctly for the test + assertNoPermissions(PathUtils.get(realTmpDir), permissions); + + // check that all directories got permissions: + + // bin file: ro + assertExactPermissions(new FilePermission(environment.binFile().toString(), "read,readlink"), permissions); + // lib file: ro + assertExactPermissions(new FilePermission(environment.libFile().toString(), "read,readlink"), permissions); + // modules file: ro + assertExactPermissions(new FilePermission(environment.modulesFile().toString(), "read,readlink"), permissions); + // config file: ro + assertExactPermissions(new FilePermission(environment.configFile().toString(), "read,readlink"), permissions); + // scripts file: ro + assertExactPermissions(new FilePermission(environment.scriptsFile().toString(), "read,readlink"), permissions); + // plugins: ro + assertExactPermissions(new FilePermission(environment.pluginsFile().toString(), "read,readlink"), permissions); + + // data paths: r/w + for (Path dataPath : environment.dataFiles()) { + assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions); + } + for (Path dataPath : environment.dataWithClusterFiles()) { + assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions); + } + assertExactPermissions(new FilePermission(environment.sharedDataFile().toString(), "read,readlink,write,delete"), permissions); + // logs: r/w + assertExactPermissions(new FilePermission(environment.logsFile().toString(), "read,readlink,write,delete"), permissions); + // temp dir: r/w + assertExactPermissions(new FilePermission(fakeTmpDir.toString(), "read,readlink,write,delete"), permissions); + // PID file: delete only (for the shutdown hook) + assertExactPermissions(new FilePermission(environment.pidFile().toString(), "delete"), permissions); + } + + public void testEnsureSymlink() throws IOException { + Path p = createTempDir(); + + Path exists = p.resolve("exists"); + Files.createDirectory(exists); + + // symlink + Path linkExists = p.resolve("linkExists"); + try { + Files.createSymbolicLink(linkExists, exists); + } catch (UnsupportedOperationException | IOException e) { + assumeNoException("test requires filesystem that supports symbolic links", e); + } catch (SecurityException e) { + assumeNoException("test cannot create symbolic links with security manager enabled", e); + } + Security.ensureDirectoryExists(linkExists); + Files.createTempFile(linkExists, null, null); + } + + public void testEnsureBrokenSymlink() throws IOException { + Path p = createTempDir(); + + // broken symlink + Path brokenLink = p.resolve("brokenLink"); + try { + Files.createSymbolicLink(brokenLink, p.resolve("nonexistent")); + } catch (UnsupportedOperationException | IOException e) { + assumeNoException("test requires filesystem that supports symbolic links", e); + } catch (SecurityException e) { + assumeNoException("test cannot create symbolic links with security manager enabled", e); + } + try { + Security.ensureDirectoryExists(brokenLink); + fail("didn't get expected exception"); + } catch (IOException expected) {} + } + + /** When a configured dir is a symlink, test that permissions work on link target */ + public void testSymlinkPermissions() throws IOException { + // see https://github.com/elastic/elasticsearch/issues/12170 + assumeFalse("windows does not automatically grant permission to the target of symlinks", Constants.WINDOWS); + Path dir = createTempDir(); + + Path target = dir.resolve("target"); + Files.createDirectory(target); + + // symlink + Path link = dir.resolve("link"); + try { + Files.createSymbolicLink(link, target); + } catch (UnsupportedOperationException | IOException e) { + assumeNoException("test requires filesystem that supports symbolic links", e); + } catch (SecurityException e) { + assumeNoException("test cannot create symbolic links with security manager enabled", e); + } + Permissions permissions = new Permissions(); + Security.addPath(permissions, "testing", link, "read"); + assertExactPermissions(new FilePermission(link.toString(), "read"), permissions); + assertExactPermissions(new FilePermission(link.resolve("foo").toString(), "read"), permissions); + assertExactPermissions(new FilePermission(target.toString(), "read"), permissions); + assertExactPermissions(new FilePermission(target.resolve("foo").toString(), "read"), permissions); + } + + /** + * checks exact file permissions, meaning those and only those for that path. + */ + static void assertExactPermissions(FilePermission expected, PermissionCollection actual) { + String target = expected.getName(); // see javadocs + Set permissionSet = asSet(expected.getActions().split(",")); + boolean read = permissionSet.remove("read"); + boolean readlink = permissionSet.remove("readlink"); + boolean write = permissionSet.remove("write"); + boolean delete = permissionSet.remove("delete"); + boolean execute = permissionSet.remove("execute"); + assertTrue("unrecognized permission: " + permissionSet, permissionSet.isEmpty()); + assertEquals(read, actual.implies(new FilePermission(target, "read"))); + assertEquals(readlink, actual.implies(new FilePermission(target, "readlink"))); + assertEquals(write, actual.implies(new FilePermission(target, "write"))); + assertEquals(delete, actual.implies(new FilePermission(target, "delete"))); + assertEquals(execute, actual.implies(new FilePermission(target, "execute"))); + } + + /** + * checks that this path has no permissions + */ + static void assertNoPermissions(Path path, PermissionCollection actual) { + String target = path.toString(); + assertFalse(actual.implies(new FilePermission(target, "read"))); + assertFalse(actual.implies(new FilePermission(target, "readlink"))); + assertFalse(actual.implies(new FilePermission(target, "write"))); + assertFalse(actual.implies(new FilePermission(target, "delete"))); + assertFalse(actual.implies(new FilePermission(target, "execute"))); + } +} diff --git a/core/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java similarity index 77% rename from core/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java index 3fbaf5a60e1..a319aaabb70 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java @@ -19,22 +19,36 @@ package org.elasticsearch.bootstrap; +import org.apache.lucene.util.Constants; import org.elasticsearch.test.ESTestCase; /** Simple tests seccomp filter is working. */ public class SeccompTests extends ESTestCase { + /** command to try to run in tests */ + static final String EXECUTABLE = Constants.WINDOWS ? "calc" : "ls"; + @Override public void setUp() throws Exception { super.setUp(); assumeTrue("requires seccomp filter installation", Natives.isSeccompInstalled()); // otherwise security manager will block the execution, no fun assumeTrue("cannot test with security manager enabled", System.getSecurityManager() == null); + // otherwise, since we don't have TSYNC support, rules are not applied to the test thread + // (randomizedrunner class initialization happens in its own thread, after the test thread is created) + // instead we just forcefully run it for the test thread here. + if (!JNANatives.LOCAL_SECCOMP_ALL) { + try { + Seccomp.init(createTempDir()); + } catch (Throwable e) { + throw new RuntimeException("unable to forcefully apply seccomp to test thread", e); + } + } } public void testNoExecution() throws Exception { try { - Runtime.getRuntime().exec("ls"); + Runtime.getRuntime().exec(EXECUTABLE); fail("should not have been able to execute!"); } catch (Exception expected) { // we can't guarantee how its converted, currently its an IOException, like this: @@ -60,7 +74,7 @@ public class SeccompTests extends ESTestCase { @Override public void run() { try { - Runtime.getRuntime().exec("ls"); + Runtime.getRuntime().exec(EXECUTABLE); fail("should not have been able to execute!"); } catch (Exception expected) { // ok diff --git a/core/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java similarity index 98% rename from core/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java index d1480047ad9..95d0789fbf8 100644 --- a/core/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java @@ -19,16 +19,16 @@ package org.elasticsearch.common.cli; -import java.nio.charset.StandardCharsets; import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; + import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; -import org.junit.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; @@ -59,79 +59,66 @@ public class CheckFileCommandTests extends ESTestCase { CHANGE, KEEP, DISABLED } - @Test public void testThatCommandLogsErrorMessageOnFail() throws Exception { executeCommand(jimFsConfiguration, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.CHANGE)); assertThat(captureOutputTerminal.getTerminalOutput(), hasItem(containsString("Please ensure that the user account running Elasticsearch has read access to this file"))); } - @Test public void testThatCommandLogsNothingWhenPermissionRemains() throws Exception { executeCommand(jimFsConfiguration, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.KEEP)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsNothingWhenDisabled() throws Exception { executeCommand(jimFsConfiguration, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsNothingIfFilesystemDoesNotSupportPermissions() throws Exception { executeCommand(jimFsConfigurationWithoutPermissions, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsOwnerChange() throws Exception { executeCommand(jimFsConfiguration, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.CHANGE)); assertThat(captureOutputTerminal.getTerminalOutput(), hasItem(allOf(containsString("Owner of file ["), containsString("] used to be ["), containsString("], but now is [")))); } - @Test public void testThatCommandLogsNothingIfOwnerRemainsSame() throws Exception { executeCommand(jimFsConfiguration, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.KEEP)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsNothingIfOwnerIsDisabled() throws Exception { executeCommand(jimFsConfiguration, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsNothingIfFileSystemDoesNotSupportOwners() throws Exception { executeCommand(jimFsConfigurationWithoutPermissions, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsIfGroupChanges() throws Exception { executeCommand(jimFsConfiguration, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.CHANGE)); assertThat(captureOutputTerminal.getTerminalOutput(), hasItem(allOf(containsString("Group of file ["), containsString("] used to be ["), containsString("], but now is [")))); } - @Test public void testThatCommandLogsNothingIfGroupRemainsSame() throws Exception { executeCommand(jimFsConfiguration, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.KEEP)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsNothingIfGroupIsDisabled() throws Exception { executeCommand(jimFsConfiguration, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandLogsNothingIfFileSystemDoesNotSupportGroups() throws Exception { executeCommand(jimFsConfigurationWithoutPermissions, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED)); assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandDoesNotLogAnythingOnFileCreation() throws Exception { Configuration configuration = randomBoolean() ? jimFsConfiguration : jimFsConfigurationWithoutPermissions; @@ -147,7 +134,6 @@ public class CheckFileCommandTests extends ESTestCase { assertThat(captureOutputTerminal.getTerminalOutput(), hasSize(0)); } - @Test public void testThatCommandWorksIfFileIsDeletedByCommand() throws Exception { Configuration configuration = randomBoolean() ? jimFsConfiguration : jimFsConfigurationWithoutPermissions; diff --git a/core/src/test/java/org/elasticsearch/common/cli/CliToolTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CliToolTests.java similarity index 97% rename from core/src/test/java/org/elasticsearch/common/cli/CliToolTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CliToolTests.java index f275d1db5c3..2613ab9951a 100644 --- a/core/src/test/java/org/elasticsearch/common/cli/CliToolTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CliToolTests.java @@ -22,10 +22,10 @@ package org.elasticsearch.common.cli; import org.apache.commons.cli.CommandLine; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.internal.InternalSettingsPreparer; -import org.junit.Test; import java.io.IOException; import java.util.HashMap; @@ -46,9 +46,8 @@ import static org.hamcrest.Matchers.is; /** * */ +@SuppressForbidden(reason = "modifies system properties intentionally") public class CliToolTests extends CliToolTestCase { - - @Test public void testOK() throws Exception { Terminal terminal = new MockTerminal(); final AtomicReference executed = new AtomicReference<>(false); @@ -65,7 +64,6 @@ public class CliToolTests extends CliToolTestCase { assertCommandHasBeenExecuted(executed); } - @Test public void testUsageError() throws Exception { Terminal terminal = new MockTerminal(); final AtomicReference executed = new AtomicReference<>(false); @@ -82,7 +80,6 @@ public class CliToolTests extends CliToolTestCase { assertCommandHasBeenExecuted(executed); } - @Test public void testIOError() throws Exception { Terminal terminal = new MockTerminal(); final AtomicReference executed = new AtomicReference<>(false); @@ -99,7 +96,6 @@ public class CliToolTests extends CliToolTestCase { assertCommandHasBeenExecuted(executed); } - @Test public void testCodeError() throws Exception { Terminal terminal = new MockTerminal(); final AtomicReference executed = new AtomicReference<>(false); @@ -116,7 +112,6 @@ public class CliToolTests extends CliToolTestCase { assertCommandHasBeenExecuted(executed); } - @Test public void testMultiCommand() { Terminal terminal = new MockTerminal(); int count = randomIntBetween(2, 7); @@ -144,8 +139,7 @@ public class CliToolTests extends CliToolTestCase { } } - @Test - public void testMultiCommand_UnknownCommand() { + public void testMultiCommandUnknownCommand() { Terminal terminal = new MockTerminal(); int count = randomIntBetween(2, 7); final AtomicReference[] executed = new AtomicReference[count]; @@ -171,8 +165,7 @@ public class CliToolTests extends CliToolTestCase { } } - @Test - public void testSingleCommand_ToolHelp() throws Exception { + public void testSingleCommandToolHelp() throws Exception { CaptureOutputTerminal terminal = new CaptureOutputTerminal(); final AtomicReference executed = new AtomicReference<>(false); final NamedCommand cmd = new NamedCommand("cmd1", terminal) { @@ -189,8 +182,7 @@ public class CliToolTests extends CliToolTestCase { assertThat(terminal.getTerminalOutput(), hasItem(containsString("cmd1 help"))); } - @Test - public void testMultiCommand_ToolHelp() { + public void testMultiCommandToolHelp() { CaptureOutputTerminal terminal = new CaptureOutputTerminal(); NamedCommand[] cmds = new NamedCommand[2]; cmds[0] = new NamedCommand("cmd0", terminal) { @@ -212,8 +204,7 @@ public class CliToolTests extends CliToolTestCase { assertThat(terminal.getTerminalOutput(), hasItem(containsString("tool help"))); } - @Test - public void testMultiCommand_CmdHelp() { + public void testMultiCommandCmdHelp() { CaptureOutputTerminal terminal = new CaptureOutputTerminal(); NamedCommand[] cmds = new NamedCommand[2]; cmds[0] = new NamedCommand("cmd0", terminal) { @@ -235,7 +226,6 @@ public class CliToolTests extends CliToolTestCase { assertThat(terminal.getTerminalOutput(), hasItem(containsString("cmd1 help"))); } - @Test public void testThatThrowExceptionCanBeLogged() throws Exception { CaptureOutputTerminal terminal = new CaptureOutputTerminal(); NamedCommand cmd = new NamedCommand("cmd", terminal) { @@ -263,7 +253,6 @@ public class CliToolTests extends CliToolTestCase { } } - @Test public void testMultipleLaunch() throws Exception { Terminal terminal = new MockTerminal(); final AtomicReference executed = new AtomicReference<>(false); @@ -280,7 +269,6 @@ public class CliToolTests extends CliToolTestCase { tool.parse("cmd", Strings.splitStringByCommaToArray("--help")); } - @Test public void testPromptForSetting() throws Exception { final AtomicInteger counter = new AtomicInteger(); final AtomicReference promptedSecretValue = new AtomicReference<>(null); @@ -323,7 +311,6 @@ public class CliToolTests extends CliToolTestCase { assertThat(promptedTextValue.get(), is("replaced")); } - @Test public void testStopAtNonOptionParsing() throws Exception { final CliToolConfig.Cmd lenientCommand = cmd("lenient", CliTool.Command.Exit.class).stopAtNonOption(true).build(); final CliToolConfig.Cmd strictCommand = cmd("strict", CliTool.Command.Exit.class).stopAtNonOption(false).build(); diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/node/internal/EvilInternalSettingsPreparerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/node/internal/EvilInternalSettingsPreparerTests.java new file mode 100644 index 00000000000..52486ba7d62 --- /dev/null +++ b/qa/evil-tests/src/test/java/org/elasticsearch/node/internal/EvilInternalSettingsPreparerTests.java @@ -0,0 +1,145 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.node.internal; + +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.Matchers.*; + +@SuppressForbidden(reason = "modifies system properties intentionally") +public class EvilInternalSettingsPreparerTests extends ESTestCase { + + Map savedProperties = new HashMap<>(); + Settings baseEnvSettings; + + @Before + public void saveSettingsSystemProperties() { + // clear out any properties the settings preparer may look for + savedProperties.clear(); + for (Object propObj : System.getProperties().keySet()) { + String property = (String)propObj; + // NOTE: these prefixes are prefixes of the defaults, so both are handled here + for (String prefix : InternalSettingsPreparer.PROPERTY_PREFIXES) { + if (property.startsWith(prefix)) { + savedProperties.put(property, System.getProperty(property)); + } + } + } + String name = System.getProperty("name"); + if (name != null) { + savedProperties.put("name", name); + } + for (String property : savedProperties.keySet()) { + System.clearProperty(property); + } + } + + @After + public void restoreSettingsSystemProperties() { + for (Map.Entry property : savedProperties.entrySet()) { + System.setProperty(property.getKey(), property.getValue()); + } + } + + @Before + public void createBaseEnvSettings() { + baseEnvSettings = settingsBuilder() + .put("path.home", createTempDir()) + .build(); + } + + @After + public void clearBaseEnvSettings() { + baseEnvSettings = null; + } + + public void testIgnoreSystemProperties() { + try { + System.setProperty("es.node.zone", "foo"); + Settings settings = settingsBuilder() + .put("node.zone", "bar") + .put(baseEnvSettings) + .build(); + Environment env = InternalSettingsPreparer.prepareEnvironment(settings, null); + // Should use setting from the system property + assertThat(env.settings().get("node.zone"), equalTo("foo")); + + settings = settingsBuilder() + .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) + .put("node.zone", "bar") + .put(baseEnvSettings) + .build(); + env = InternalSettingsPreparer.prepareEnvironment(settings, null); + // Should use setting from the system property + assertThat(env.settings().get("node.zone"), equalTo("bar")); + } finally { + System.clearProperty("es.node.zone"); + } + } + + public void testNameSettingsPreference() { + try { + System.setProperty("name", "sys-prop-name"); + // Test system property overrides node.name + Settings settings = settingsBuilder() + .put("node.name", "node-name") + .put(baseEnvSettings) + .build(); + Environment env = InternalSettingsPreparer.prepareEnvironment(settings, null); + assertThat(env.settings().get("name"), equalTo("sys-prop-name")); + + // test name in settings overrides sys prop and node.name + settings = settingsBuilder() + .put("name", "name-in-settings") + .put("node.name", "node-name") + .put(baseEnvSettings) + .build(); + env = InternalSettingsPreparer.prepareEnvironment(settings, null); + assertThat(env.settings().get("name"), equalTo("name-in-settings")); + + // test only node.name in settings + System.clearProperty("name"); + settings = settingsBuilder() + .put("node.name", "node-name") + .put(baseEnvSettings) + .build(); + env = InternalSettingsPreparer.prepareEnvironment(settings, null); + assertThat(env.settings().get("name"), equalTo("node-name")); + + // test no name at all results in name being set + env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null); + assertThat(env.settings().get("name"), not("name-in-settings")); + assertThat(env.settings().get("name"), not("sys-prop-name")); + assertThat(env.settings().get("name"), not("node-name")); + assertThat(env.settings().get("name"), notNullValue()); + } finally { + System.clearProperty("name"); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerPermissionTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerPermissionTests.java similarity index 99% rename from core/src/test/java/org/elasticsearch/plugins/PluginManagerPermissionTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerPermissionTests.java index edf893e1a47..d5a07606e65 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerPermissionTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerPermissionTests.java @@ -43,7 +43,6 @@ import java.util.zip.ZipOutputStream; import static java.nio.file.attribute.PosixFilePermission.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.plugins.PluginInfoTests.writeProperties; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.*; @@ -319,7 +318,7 @@ public class PluginManagerPermissionTests extends ESTestCase { private URL createPlugin(boolean withBinDir, boolean withConfigDir) throws IOException { final Path structure = createTempDir().resolve("fake-plugin"); - writeProperties(structure, "description", "fake desc", + PluginTestUtil.writeProperties(structure, "description", "fake desc", "version", "1.0", "elasticsearch.version", Version.CURRENT.toString(), "jvm", "true", diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java similarity index 92% rename from core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java rename to qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java index e98794cfc99..b2b2c0cff5c 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java @@ -22,6 +22,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.common.Base64; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.cli.CliTool; import org.elasticsearch.common.cli.CliTool.ExitStatus; import org.elasticsearch.common.cli.CliToolTestCase.CaptureOutputTerminal; @@ -57,6 +58,7 @@ import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; @@ -64,14 +66,14 @@ import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import static org.elasticsearch.common.cli.CliTool.ExitStatus.USAGE; import static org.elasticsearch.common.cli.CliToolTestCase.args; -import static org.elasticsearch.common.io.FileSystemUtilsTests.assertFileContent; +import static org.elasticsearch.common.io.FileTestUtils.assertFileContent; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.plugins.PluginInfoTests.writeProperties; import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.*; @@ -81,7 +83,8 @@ import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; @LuceneTestCase.SuppressFileSystems("*") // TODO: clean up this test to allow extra files // TODO: jimfs is really broken here (throws wrong exception from detection method). // if its in your classpath, then do not use plugins!!!!!! -public class PluginManagerIT extends ESIntegTestCase { +@SuppressForbidden(reason = "modifies system properties intentionally") +public class PluginManagerTests extends ESIntegTestCase { private Environment environment; private CaptureOutputTerminal terminal = new CaptureOutputTerminal(); @@ -127,7 +130,7 @@ public class PluginManagerIT extends ESIntegTestCase { /** creates a plugin .zip and returns the url for testing */ private String createPlugin(final Path structure, String... properties) throws IOException { - writeProperties(structure, properties); + PluginTestUtil.writeProperties(structure, properties); Path zip = createTempDir().resolve(structure.getFileName() + ".zip"); try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) { Files.walkFileTree(structure, new SimpleFileVisitor() { @@ -149,7 +152,7 @@ public class PluginManagerIT extends ESIntegTestCase { /** creates a plugin .zip and bad checksum file and returns the url for testing */ private String createPluginWithBadChecksum(final Path structure, String... properties) throws IOException { - writeProperties(structure, properties); + PluginTestUtil.writeProperties(structure, properties); Path zip = createTempDir().resolve(structure.getFileName() + ".zip"); try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) { Files.walkFileTree(structure, new SimpleFileVisitor() { @@ -407,6 +410,45 @@ public class PluginManagerIT extends ESIntegTestCase { assertThatPluginIsListed(pluginName); } + /** + * @deprecated support for this is not going to stick around, seriously. + */ + @Deprecated + public void testAlreadyInstalledNotIsolated() throws Exception { + String pluginName = "fake-plugin"; + Path pluginDir = createTempDir().resolve(pluginName); + Files.createDirectories(pluginDir); + // create a jar file in the plugin + Path pluginJar = pluginDir.resolve("fake-plugin.jar"); + try (ZipOutputStream out = new JarOutputStream(Files.newOutputStream(pluginJar, StandardOpenOption.CREATE))) { + out.putNextEntry(new ZipEntry("foo.class")); + out.closeEntry(); + } + String pluginUrl = createPlugin(pluginDir, + "description", "fake desc", + "name", pluginName, + "version", "1.0", + "elasticsearch.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "isolated", "false", + "jvm", "true", + "classname", "FakePlugin"); + + // install + ExitStatus status = new PluginManagerCliParser(terminal).execute(args("install " + pluginUrl)); + assertEquals("unexpected exit status: output: " + terminal.getTerminalOutput(), ExitStatus.OK, status); + + // install again + status = new PluginManagerCliParser(terminal).execute(args("install " + pluginUrl)); + List output = terminal.getTerminalOutput(); + assertEquals("unexpected exit status: output: " + output, ExitStatus.IO_ERROR, status); + boolean foundExpectedMessage = false; + for (String line : output) { + foundExpectedMessage |= line.contains("already exists"); + } + assertTrue(foundExpectedMessage); + } + public void testInstallSitePluginVerbose() throws IOException { String pluginName = "fake-plugin"; Path pluginDir = createTempDir().resolve(pluginName); @@ -588,10 +630,10 @@ public class PluginManagerIT extends ESIntegTestCase { PluginManager.checkForOfficialPlugins("analysis-smartcn"); PluginManager.checkForOfficialPlugins("analysis-stempel"); PluginManager.checkForOfficialPlugins("delete-by-query"); - PluginManager.checkForOfficialPlugins("lang-expression"); - PluginManager.checkForOfficialPlugins("lang-groovy"); PluginManager.checkForOfficialPlugins("lang-javascript"); + PluginManager.checkForOfficialPlugins("lang-plan-a"); PluginManager.checkForOfficialPlugins("lang-python"); + PluginManager.checkForOfficialPlugins("mapper-attachments"); PluginManager.checkForOfficialPlugins("mapper-murmur3"); PluginManager.checkForOfficialPlugins("mapper-size"); PluginManager.checkForOfficialPlugins("discovery-multicast"); @@ -620,9 +662,14 @@ public class PluginManagerIT extends ESIntegTestCase { SSLSocketFactory defaultSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory(); ServerBootstrap serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory()); - SelfSignedCertificate ssc = new SelfSignedCertificate("localhost"); + SelfSignedCertificate ssc = null; try { + try { + ssc = new SelfSignedCertificate("localhost"); + } catch (Exception e) { + assumeNoException("self signing shenanigans not supported by this JDK", e); + } // Create a trust manager that does not validate certificate chains: SSLContext sc = SSLContext.getInstance("SSL"); @@ -659,7 +706,9 @@ public class PluginManagerIT extends ESIntegTestCase { } finally { HttpsURLConnection.setDefaultSSLSocketFactory(defaultSocketFactory); serverBootstrap.releaseExternalResources(); - ssc.delete(); + if (ssc != null) { + ssc.delete(); + } } } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java similarity index 93% rename from core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java index 81c834ab9ae..266b44ebfbd 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java @@ -21,12 +21,12 @@ package org.elasticsearch.plugins; import org.elasticsearch.Build; import org.elasticsearch.Version; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.http.client.HttpDownloadHelper; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; import org.junit.After; -import org.junit.Test; import java.io.IOException; import java.net.URL; @@ -43,14 +43,13 @@ import static org.hamcrest.Matchers.is; /** * */ +@SuppressForbidden(reason = "modifies system properties intentionally") public class PluginManagerUnitTests extends ESTestCase { - @After public void cleanSystemProperty() { System.clearProperty(PluginManager.PROPERTY_SUPPORT_STAGING_URLS); } - @Test public void testThatConfigDirectoryCanBeOutsideOfElasticsearchHomeDirectory() throws IOException { String pluginName = randomAsciiOfLength(10); Path homeFolder = createTempDir(); @@ -68,7 +67,6 @@ public class PluginManagerUnitTests extends ESTestCase { assertEquals(configDirPath, expectedDirPath); } - @Test public void testSimplifiedNaming() throws IOException { String pluginName = randomAsciiOfLength(10); PluginManager.PluginHandle handle = PluginManager.PluginHandle.parse(pluginName); @@ -82,7 +80,7 @@ public class PluginManagerUnitTests extends ESTestCase { if (supportStagingUrls) { String expectedStagingURL = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip", - Version.CURRENT.number(), Build.CURRENT.hashShort(), pluginName, Version.CURRENT.number(), pluginName, Version.CURRENT.number()); + Version.CURRENT.number(), Build.CURRENT.shortHash(), pluginName, Version.CURRENT.number(), pluginName, Version.CURRENT.number()); assertThat(iterator.next().toExternalForm(), is(expectedStagingURL)); } @@ -93,7 +91,6 @@ public class PluginManagerUnitTests extends ESTestCase { assertThat(iterator.hasNext(), is(false)); } - @Test public void testOfficialPluginName() throws IOException { String randomPluginName = randomFrom(new ArrayList<>(PluginManager.OFFICIAL_PLUGINS)); PluginManager.PluginHandle handle = PluginManager.PluginHandle.parse(randomPluginName); @@ -108,7 +105,7 @@ public class PluginManagerUnitTests extends ESTestCase { if (supportStagingUrls) { String expectedStagingUrl = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%s-%s/org/elasticsearch/plugin/%s/%s/%s-%s.zip", - Version.CURRENT.number(), Build.CURRENT.hashShort(), randomPluginName, Version.CURRENT.number(), randomPluginName, Version.CURRENT.number()); + Version.CURRENT.number(), Build.CURRENT.shortHash(), randomPluginName, Version.CURRENT.number(), randomPluginName, Version.CURRENT.number()); assertThat(iterator.next().toExternalForm(), is(expectedStagingUrl)); } @@ -119,7 +116,6 @@ public class PluginManagerUnitTests extends ESTestCase { assertThat(iterator.hasNext(), is(false)); } - @Test public void testGithubPluginName() throws IOException { String user = randomAsciiOfLength(6); String pluginName = randomAsciiOfLength(10); @@ -129,7 +125,6 @@ public class PluginManagerUnitTests extends ESTestCase { assertThat(handle.urls().get(0).toExternalForm(), is(new URL("https", "github.com", "/" + user + "/" + pluginName + "/" + "archive/master.zip").toExternalForm())); } - @Test public void testDownloadHelperChecksums() throws Exception { // Sanity check to make sure the checksum functions never change how they checksum things assertEquals("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java similarity index 100% rename from core/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java similarity index 80% rename from core/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java rename to qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java index 10b45017b26..1ad972e10ef 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java @@ -22,16 +22,16 @@ package org.elasticsearch.tribe; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.DiscoveryService; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalTestCluster; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Test; import java.nio.file.Path; @@ -43,6 +43,7 @@ import static org.hamcrest.CoreMatchers.equalTo; * all the time, while we need to make the tribe node accept them in this case, so that we can verify that they are not read again as part * of the tribe client nodes initialization. Note that the started nodes will obey to the 'node.mode' settings as the internal cluster does. */ +@SuppressForbidden(reason = "modifies system properties intentionally") public class TribeUnitTests extends ESTestCase { private static Node tribe1; @@ -53,13 +54,24 @@ public class TribeUnitTests extends ESTestCase { @BeforeClass public static void createTribes() { Settings baseSettings = Settings.builder() - .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) .put("http.enabled", false) .put("node.mode", NODE_MODE) .put("path.home", createTempDir()).build(); - tribe1 = NodeBuilder.nodeBuilder().settings(Settings.builder().put(baseSettings).put("cluster.name", "tribe1").put("node.name", "tribe1_node")).node(); - tribe2 = NodeBuilder.nodeBuilder().settings(Settings.builder().put(baseSettings).put("cluster.name", "tribe2").put("node.name", "tribe2_node")).node(); + tribe1 = new TribeClientNode( + Settings.builder() + .put(baseSettings) + .put("cluster.name", "tribe1") + .put("name", "tribe1_node") + .put(DiscoveryService.SETTING_DISCOVERY_SEED, random().nextLong()) + .build()).start(); + tribe2 = new TribeClientNode( + Settings.builder() + .put(baseSettings) + .put("cluster.name", "tribe2") + .put("name", "tribe2_node") + .put(DiscoveryService.SETTING_DISCOVERY_SEED, random().nextLong()) + .build()).start(); } @AfterClass @@ -70,11 +82,12 @@ public class TribeUnitTests extends ESTestCase { tribe2 = null; } - @Test public void testThatTribeClientsIgnoreGlobalSysProps() throws Exception { System.setProperty("es.cluster.name", "tribe_node_cluster"); System.setProperty("es.tribe.t1.cluster.name", "tribe1"); System.setProperty("es.tribe.t2.cluster.name", "tribe2"); + System.setProperty("es.tribe.t1.discovery.id.seed", Long.toString(random().nextLong())); + System.setProperty("es.tribe.t2.discovery.id.seed", Long.toString(random().nextLong())); try { assertTribeNodeSuccesfullyCreated(Settings.EMPTY); @@ -82,10 +95,11 @@ public class TribeUnitTests extends ESTestCase { System.clearProperty("es.cluster.name"); System.clearProperty("es.tribe.t1.cluster.name"); System.clearProperty("es.tribe.t2.cluster.name"); + System.clearProperty("es.tribe.t1.discovery.id.seed"); + System.clearProperty("es.tribe.t2.discovery.id.seed"); } } - @Test public void testThatTribeClientsIgnoreGlobalConfig() throws Exception { Path pathConf = getDataPath("elasticsearch.yml").getParent(); Settings settings = Settings.builder().put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true).put("path.conf", pathConf).build(); @@ -99,7 +113,7 @@ public class TribeUnitTests extends ESTestCase { .put("tribe.t1.node.mode", NODE_MODE).put("tribe.t2.node.mode", NODE_MODE) .put("path.home", createTempDir()).put(extraSettings).build(); - try (Node node = NodeBuilder.nodeBuilder().settings(settings).node()) { + try (Node node = new Node(settings).start()) { try (Client client = node.client()) { assertBusy(new Runnable() { @Override diff --git a/core/src/test/resources/org/elasticsearch/common/cli/tool-cmd1.help b/qa/evil-tests/src/test/resources/org/elasticsearch/common/cli/tool-cmd1.help similarity index 100% rename from core/src/test/resources/org/elasticsearch/common/cli/tool-cmd1.help rename to qa/evil-tests/src/test/resources/org/elasticsearch/common/cli/tool-cmd1.help diff --git a/core/src/test/resources/org/elasticsearch/common/cli/tool.help b/qa/evil-tests/src/test/resources/org/elasticsearch/common/cli/tool.help similarity index 100% rename from core/src/test/resources/org/elasticsearch/common/cli/tool.help rename to qa/evil-tests/src/test/resources/org/elasticsearch/common/cli/tool.help diff --git a/core/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy similarity index 100% rename from core/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy rename to qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy diff --git a/core/src/test/resources/org/elasticsearch/plugins/security/simple-plugin-security.policy b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/simple-plugin-security.policy similarity index 100% rename from core/src/test/resources/org/elasticsearch/plugins/security/simple-plugin-security.policy rename to qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/simple-plugin-security.policy diff --git a/core/src/test/resources/org/elasticsearch/plugins/security/unresolved-plugin-security.policy b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/unresolved-plugin-security.policy similarity index 100% rename from core/src/test/resources/org/elasticsearch/plugins/security/unresolved-plugin-security.policy rename to qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/unresolved-plugin-security.policy diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml b/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml new file mode 100644 index 00000000000..ad1b9be8c89 --- /dev/null +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml @@ -0,0 +1,5 @@ +cluster.name: tribe_node_cluster +tribe.t1.cluster.name: tribe1 +tribe.t2.cluster.name: tribe2 +tribe.t1.discovery.id.seed: 1 +tribe.t2.discovery.id.seed: 2 diff --git a/qa/pom.xml b/qa/pom.xml deleted file mode 100644 index 7cc28fe622e..00000000000 --- a/qa/pom.xml +++ /dev/null @@ -1,131 +0,0 @@ - - - - 4.0.0 - - org.elasticsearch.qa - elasticsearch-qa - 3.0.0-SNAPSHOT - pom - QA: Parent POM - 2015 - - - org.elasticsearch - parent - 3.0.0-SNAPSHOT - - - - - - - - - org.hamcrest - hamcrest-all - test - - - org.apache.lucene - lucene-test-framework - test - - - - - - - - - src/test/resources - true - - rest-api-spec/** - - - - - ${elasticsearch.tools.directory}/rest-api-spec - rest-api-spec - - - api/info.json - api/cluster.health.json - api/cluster.state.json - - api/index.json - api/get.json - api/update.json - api/search.json - api/indices.analyze.json - api/indices.create.json - api/indices.refresh.json - api/nodes.info.json - api/count.json - - - - - ${elasticsearch.tools.directory}/shared-test-resources - false - - - - - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - - - integ-tests - - - 1 - - - 127.0.0.1:${integ.transport.port} - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - check-license - none - - - - - - - - - smoke-test-plugins - smoke-test-multinode - smoke-test-client - - - - - vagrant - - - tests.vagrant - - - - vagrant - - - - diff --git a/qa/smoke-test-client/build.gradle b/qa/smoke-test-client/build.gradle new file mode 100644 index 00000000000..0475243467a --- /dev/null +++ b/qa/smoke-test-client/build.gradle @@ -0,0 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.rest-test' + +// TODO: this test works, but it isn't really a rest test...should we have another plugin for "non rest test that just needs N clusters?" diff --git a/qa/smoke-test-client/pom.xml b/qa/smoke-test-client/pom.xml deleted file mode 100644 index 82495c1c023..00000000000 --- a/qa/smoke-test-client/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - elasticsearch-qa - org.elasticsearch.qa - 3.0.0-SNAPSHOT - - 4.0.0 - - - - smoke-test-client - QA: Smoke Test Client - Test the Java Client against a running cluster - - - true - - - - - org.elasticsearch - elasticsearch - test - - - log4j - log4j - test - - - - - - - src/test/resources - - - - ${elasticsearch.tools.directory}/shared-test-resources - false - - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - - - integ-setup-dependencies - pre-integration-test - - copy - - - ${skip.integ.tests} - true - ${integ.deps}/plugins - - - - - org.elasticsearch.distribution.zip - elasticsearch - ${elasticsearch.version} - zip - true - ${integ.deps} - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - integ-setup - pre-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - integ-teardown - post-integration-test - - run - - - ${skip.integ.tests} - - - - - - - - - - - diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java index 6e912cfab22..95df2d04458 100644 --- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java +++ b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java @@ -20,6 +20,7 @@ package org.elasticsearch.smoketest; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SuppressForbidden; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; @@ -34,7 +35,10 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import java.io.IOException; import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URL; import java.net.UnknownHostException; import java.nio.file.Path; import java.util.Locale; @@ -103,20 +107,14 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase { return client; } - private static Client startClient() throws UnknownHostException { + private static Client startClient() throws IOException { String[] stringAddresses = clusterAddresses.split(","); TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; int i = 0; for (String stringAddress : stringAddresses) { - String[] split = stringAddress.split(":"); - if (split.length < 2) { - throw new IllegalArgumentException("address [" + clusterAddresses + "] not valid"); - } - try { - transportAddresses[i++] = new InetSocketTransportAddress(InetAddress.getByName(split[0]), Integer.valueOf(split[1])); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("port is not valid, expected number but was [" + split[1] + "]"); - } + URL url = new URL("http://" + stringAddress); + InetAddress inetAddress = InetAddress.getByName(url.getHost()); + transportAddresses[i++] = new InetSocketTransportAddress(new InetSocketAddress(inetAddress, url.getPort())); } return startClient(createTempDir(), transportAddresses); } @@ -125,7 +123,7 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase { if (client == null) { try { client = startClient(); - } catch (UnknownHostException e) { + } catch (IOException e) { logger.error("can not start the client", e); } assertThat(client, notNullValue()); diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java index 4c324b0f1a9..4c5b38ca334 100644 --- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java +++ b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java @@ -22,17 +22,14 @@ package org.elasticsearch.smoketest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; -import org.junit.Test; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.greaterThan; public class SmokeTestClientIT extends ESSmokeClientTestCase { - /** * Check that we are connected to a cluster named "elasticsearch". */ - @Test public void testSimpleClient() { Client client = getClient(); @@ -47,7 +44,6 @@ public class SmokeTestClientIT extends ESSmokeClientTestCase { /** * Create an index and index some docs */ - @Test public void testPutDocument() { Client client = getClient(); diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle new file mode 100644 index 00000000000..a30f1d31f73 --- /dev/null +++ b/qa/smoke-test-multinode/build.gradle @@ -0,0 +1,8 @@ + +apply plugin: 'elasticsearch.rest-test' + +integTest { + cluster { + numNodes = 2 + } +} diff --git a/qa/smoke-test-multinode/integration-tests.xml b/qa/smoke-test-multinode/integration-tests.xml deleted file mode 100644 index 1b0425dcec9..00000000000 --- a/qa/smoke-test-multinode/integration-tests.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - Failed to start second node with message: ${failure.message} - - - - - - - - - - - - - - - - diff --git a/qa/smoke-test-multinode/pom.xml b/qa/smoke-test-multinode/pom.xml deleted file mode 100644 index 75bd37f6041..00000000000 --- a/qa/smoke-test-multinode/pom.xml +++ /dev/null @@ -1,284 +0,0 @@ - - - - 4.0.0 - - - org.elasticsearch.qa - elasticsearch-qa - 3.0.0-SNAPSHOT - - - - - smoke-test-multinode - QA: Smoke Test Multi-Node IT - Tests that multi node IT tests work - - - true - ${project.basedir}/integration-tests.xml - smoke_test_multinode - false - - - - org.elasticsearch - elasticsearch - test-jar - test - - - - - org.elasticsearch - elasticsearch - provided - - - org.apache.lucene - lucene-core - provided - - - org.apache.lucene - lucene-backward-codecs - provided - - - org.apache.lucene - lucene-analyzers-common - provided - - - org.apache.lucene - lucene-queries - provided - - - org.apache.lucene - lucene-memory - provided - - - org.apache.lucene - lucene-highlighter - provided - - - org.apache.lucene - lucene-queryparser - provided - - - org.apache.lucene - lucene-suggest - provided - - - org.apache.lucene - lucene-join - provided - - - org.apache.lucene - lucene-spatial - provided - - - com.spatial4j - spatial4j - provided - - - com.vividsolutions - jts - provided - - - com.github.spullara.mustache.java - compiler - provided - - - com.carrotsearch - hppc - provided - - - joda-time - joda-time - provided - - - org.joda - joda-convert - provided - - - com.fasterxml.jackson.core - jackson-core - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - provided - - - io.netty - netty - provided - - - com.ning - compress-lzf - provided - - - com.tdunning - t-digest - provided - - - commons-cli - commons-cli - provided - - - log4j - log4j - provided - - - log4j - apache-log4j-extras - provided - - - org.slf4j - slf4j-api - provided - - - net.java.dev.jna - jna - provided - - - - - - org.apache.httpcomponents - httpclient - test - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - integ-setup-dependencies - pre-integration-test - - copy - - - ${skip.integ.tests} - true - ${integ.deps}/plugins - - - - - org.elasticsearch.distribution.zip - elasticsearch - ${elasticsearch.version} - zip - true - ${integ.deps} - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - integ-setup - pre-integration-test - - run - - - - - - - - - ${skip.integ.tests} - - - - - integ-teardown - post-integration-test - - run - - - - - - ${skip.integ.tests} - - - - - - ant-contrib - ant-contrib - 1.0b3 - - - ant - ant - - - - - org.apache.ant - ant-nodeps - 1.8.1 - - - - - - - diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle new file mode 100644 index 00000000000..70611aed371 --- /dev/null +++ b/qa/smoke-test-plugins/build.gradle @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.MavenFilteringHack + +apply plugin: 'elasticsearch.rest-test' + +ext.pluginsCount = 0 +project.rootProject.subprojects.findAll { it.path.startsWith(':projects:') }.each { subproj -> + integTest { + cluster { + // need to get a non-decorated project object, so must re-lookup the project by path + plugin subproj.name, project(subproj.path) + } + } + pluginCount += 1 +} + +ext.expansions = [ + 'expected.plugins.count': pluginsCount +] + +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + diff --git a/qa/smoke-test-plugins/integration-tests.xml b/qa/smoke-test-plugins/integration-tests.xml deleted file mode 100644 index 4f7c6194e55..00000000000 --- a/qa/smoke-test-plugins/integration-tests.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/qa/smoke-test-plugins/pom.xml b/qa/smoke-test-plugins/pom.xml deleted file mode 100644 index 3e5563139ed..00000000000 --- a/qa/smoke-test-plugins/pom.xml +++ /dev/null @@ -1,478 +0,0 @@ - - - - 4.0.0 - - - org.elasticsearch.qa - elasticsearch-qa - 3.0.0-SNAPSHOT - - - - - smoke-test-plugins - QA: Smoke Test Plugins - Loads up all of our plugins - - - true - ${project.basedir}/integration-tests.xml - smoke_test_plugins - false - - - - org.elasticsearch - elasticsearch - test-jar - test - - - - - org.elasticsearch - elasticsearch - provided - - - org.apache.lucene - lucene-core - provided - - - org.apache.lucene - lucene-backward-codecs - provided - - - org.apache.lucene - lucene-analyzers-common - provided - - - org.apache.lucene - lucene-queries - provided - - - org.apache.lucene - lucene-memory - provided - - - org.apache.lucene - lucene-highlighter - provided - - - org.apache.lucene - lucene-queryparser - provided - - - org.apache.lucene - lucene-suggest - provided - - - org.apache.lucene - lucene-join - provided - - - org.apache.lucene - lucene-spatial - provided - - - com.spatial4j - spatial4j - provided - - - com.vividsolutions - jts - provided - - - com.github.spullara.mustache.java - compiler - provided - - - com.carrotsearch - hppc - provided - - - joda-time - joda-time - provided - - - org.joda - joda-convert - provided - - - com.fasterxml.jackson.core - jackson-core - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - provided - - - io.netty - netty - provided - - - com.ning - compress-lzf - provided - - - com.tdunning - t-digest - provided - - - commons-cli - commons-cli - provided - - - log4j - log4j - provided - - - log4j - apache-log4j-extras - provided - - - org.slf4j - slf4j-api - provided - - - net.java.dev.jna - jna - provided - - - - - - org.apache.httpcomponents - httpclient - test - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - integ-setup-dependencies - pre-integration-test - - copy - - - ${skip.integ.tests} - true - ${integ.deps}/plugins - - - - - org.elasticsearch.distribution.zip - elasticsearch - ${elasticsearch.version} - zip - true - ${integ.deps} - - - - - org.elasticsearch.plugin - analysis-icu - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - analysis-kuromoji - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - analysis-phonetic - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - analysis-smartcn - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - analysis-stempel - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - discovery-gce - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - delete-by-query - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - discovery-azure - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - discovery-ec2 - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - discovery-multicast - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - lang-expression - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - lang-groovy - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - lang-javascript - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - lang-python - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - mapper-murmur3 - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - mapper-size - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - repository-azure - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - repository-s3 - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - store-smb - ${elasticsearch.version} - zip - true - - - - - org.elasticsearch.plugin - jvm-example - ${elasticsearch.version} - zip - true - - - - org.elasticsearch.plugin - site-example - ${elasticsearch.version} - zip - true - - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - count-expected-plugins - validate - - run - - - - - - - - Found ${expected.plugin.count} plugins in ${plugins.dir} - - true - - - - - integ-setup - pre-integration-test - - run - - - - - - - - - - ${skip.integ.tests} - - - - - integ-teardown - post-integration-test - - run - - - - - - ${skip.integ.tests} - - - - - - ant-contrib - ant-contrib - 1.0b3 - - - ant - ant - - - - - org.apache.ant - ant-nodeps - 1.8.1 - - - - - - - diff --git a/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml index dbb09225fce..6a92845a062 100644 --- a/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml +++ b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml @@ -10,4 +10,4 @@ - do: nodes.info: {} - - length: { nodes.$master.plugins: ${expected.plugin.count} } + - length: { nodes.$master.plugins: ${expected.plugins.count} } diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle new file mode 100644 index 00000000000..9d3d424f62e --- /dev/null +++ b/qa/vagrant/build.gradle @@ -0,0 +1,191 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.vagrant.VagrantCommandTask +import org.elasticsearch.gradle.vagrant.BatsOverVagrantTask +import org.elasticsearch.gradle.FileContentsTask +import org.gradle.api.InvalidUserDataException + +String testScripts = '*.bats' +String testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts" +String smokeTestCommand = 'echo I work' +List representativeBoxes = ['ubuntu-1404', 'centos-7'] +List boxes = representativeBoxes + ['ubuntu-1204', 'ubuntu-1504', + 'debian-8', 'centos-6', 'oel-7', 'fedora-22', 'opensuse-13', 'sles-12'] + +/* The version of elasticsearch that we upgrade *from* as part of testing + * upgrades. */ +String upgradeFromVersion = '2.0.0' + +configurations { + test +} + +repositories { + mavenCentral() +} + +dependencies { + test project(path: ':distribution:tar', configuration: 'archives') + test project(path: ':distribution:rpm', configuration: 'archives') + test project(path: ':distribution:deb', configuration: 'archives') + + // Collect all the plugins + for (Project subproj : project.rootProject.subprojects) { + if (subproj.path.startsWith(':plugins:')) { + test project("${subproj.path}") + } + } + + // The version of elasticsearch that we upgrade *from* + test "org.elasticsearch.distribution.deb:elasticsearch:$upgradeFromVersion@deb" + test "org.elasticsearch.distribution.rpm:elasticsearch:$upgradeFromVersion@rpm" +} + +task checkPackages { + group 'Verification' + description 'Check the packages against a representative sample of the ' + + 'linux distributions we have in our Vagrantfile' +} + +task checkPackagesAllDistros { + group 'Verification' + description 'Check the packages against all the linux distributions we ' + + 'have in our Vagrantfile' +} + +task clean(type: Delete) { + group 'Build' + delete buildDir +} + +File testRoot = new File("$buildDir/testroot") +task createTestRoot { + outputs.dir testRoot + doLast { + testRoot.mkdirs() + } +} + +task createVersionFile(type: FileContentsTask) { + dependsOn createTestRoot + file "${testRoot}/version" + contents = version +} + +task createUpgradeFromFile(type: FileContentsTask) { + dependsOn createTestRoot + file "${testRoot}/upgrade_from_version" + contents = upgradeFromVersion +} + +task prepareTestRoot(type: Copy) { + description 'Dump bats test dependencies into the $TESTROOT' + into testRoot + from configurations.test + + dependsOn createVersionFile, createUpgradeFromFile +} + +task checkVagrantVersion(type: Exec) { + group 'Package Verification' + description 'Check that the version of vagrant is ok' + commandLine 'vagrant', '--version' + standardOutput = new ByteArrayOutputStream() + doLast { + String version = standardOutput.toString().trim() + if ((version ==~ /Vagrant 1\.[789]\..+/) == false) { + throw new InvalidUserDataException( + "Illegal version of vagrant [${version}]. Need [Vagrant 1.7+]") + } + } +} + +task vagrantSmokeTest { + group 'Vagrant' + description 'Smoke test some representative distros from the Vagrantfile' +} + +task vagrantSmokeTestAllDistros { + group 'Vagrant' + description 'Smoke test all distros from the Vagrantfile' +} + +// Each box gets it own set of tasks +boxes.each { box -> + String boxTask = taskifyBoxName box + task "vagrantUp${boxTask}"(type: VagrantCommandTask) { + group 'Vagrant' + description "Startup a vagrant VM running ${box}" + boxName box + /* Its important that we try to reprovision the box even if it already + exists. That way updates to the vagrant configuration take automatically. + That isn't to say that the updates will always be compatible. Its ok to + just destroy the boxes if they get busted but that is a manual step + because its slow-ish. */ + /* We lock the provider to virtualbox because the Vagrantfile specifies + lots of boxes that only work properly in virtualbox. Virtualbox is + vagrant's default but its possible to change that default and folks do. + But the boxes that we use are unlikely to work properly with other + virtualization providers. Thus the lock. */ + commandLine 'up', box, '--provision', '--provider', 'virtualbox' + /* It'd be possible to check if the box is already up here and output + SKIPPED but that would require running vagrant status which is slow! */ + dependsOn checkVagrantVersion + } + task "vagrantHalt${boxTask}"(type: VagrantCommandTask) { + group 'Vagrant' + description "Shutdown the vagrant VM running $box" + boxName box + commandLine 'halt', box + } + + task "smokeTest${boxTask}"(type: Exec) { + group 'Vagrant' + description "Smoke test the ${box} VM" + dependsOn "vagrantUp${boxTask}" + finalizedBy "vagrantHalt${boxTask}" + commandLine 'vagrant', 'ssh', box, '--command', + "set -o pipefail && ${smokeTestCommand} | sed -ue \'s/^/ ${box}: /'" + vagrantSmokeTestAllDistros.dependsOn name + if (representativeBoxes.contains(box)) { + vagrantSmokeTest.dependsOn name + } + } + + task "check${boxTask}"(type: BatsOverVagrantTask) { + group 'Package Verification' + description "Run packaging tests against ${box}" + dependsOn "vagrantUp${boxTask}" + finalizedBy "vagrantHalt${boxTask}" + boxName box + command testCommand + dependsOn prepareTestRoot + checkPackagesAllDistros.dependsOn name + if (representativeBoxes.contains(box)) { + checkPackages.dependsOn name + } + } +} + + +// Twists the box name into a sensible task name +String taskifyBoxName(box) { + box.capitalize().replace('-', '') +} diff --git a/qa/vagrant/pom.xml b/qa/vagrant/pom.xml deleted file mode 100644 index e303da5b6a2..00000000000 --- a/qa/vagrant/pom.xml +++ /dev/null @@ -1,413 +0,0 @@ - - - 4.0.0 - - org.elasticsearch.qa - elasticsearch-qa - 3.0.0-SNAPSHOT - - - qa-vagrant - QA: Elasticsearch Vagrant Tests - Tests the Elasticsearch distribution artifacts on virtual - machines using vagrant and bats. - pom - - - - *.bats - sudo bats $BATS/${testScripts} - - precise, trusty, vivid, jessie - centos-6, centos-7, oel-7, fedora-22, opensuse-13, sles-12 - - trusty - centos-7 - - - - - /usr/bin/rpmbuild - - - 2.0.0-beta1 - - - - - - - maven-clean-plugin - - - clean-testroot - pre-integration-test - - clean - - - true - - - ${project.build.directory}/testroot - - - - - - - - - maven-dependency-plugin - - - copy-common-to-testroot - pre-integration-test - - copy - - - ${project.build.directory}/testroot - - - org.elasticsearch.distribution.zip - elasticsearch - ${elasticsearch.version} - zip - - - org.elasticsearch.distribution.tar - elasticsearch - ${elasticsearch.version} - tar.gz - - - org.elasticsearch.distribution.deb - elasticsearch - ${elasticsearch.version} - deb - - - - org.elasticsearch.distribution.deb - elasticsearch - ${upgrade.from.version} - deb - - - org.elasticsearch.plugin - jvm-example - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - analysis-icu - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - analysis-kuromoji - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - analysis-phonetic - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - analysis-smartcn - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - analysis-stempel - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - discovery-gce - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - delete-by-query - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - discovery-azure - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - discovery-ec2 - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - discovery-multicast - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - lang-expression - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - lang-groovy - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - lang-javascript - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - lang-python - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - mapper-murmur3 - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - mapper-size - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - repository-azure - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - repository-s3 - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - store-smb - ${elasticsearch.version} - zip - - - org.elasticsearch.plugin - site-example - ${elasticsearch.version} - zip - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - ant-contrib - ant-contrib - 1.0b3 - - - ant - ant - - - - - - - check-vagrant-version - validate - - run - - - - - - - - - - setup-version-files - pre-integration-test - - run - - - - - - - - - - test-vms - integration-test - - run - - - - - - - - - - - - - - - - - rpm - - - ${packaging.rpm.rpmbuild} - - - - - - maven-dependency-plugin - - - copy-rpm-to-testroot - pre-integration-test - - copy - - - ${project.build.directory}/testroot - - - org.elasticsearch.distribution.rpm - elasticsearch - ${elasticsearch.version} - rpm - - - - org.elasticsearch.distribution.rpm - elasticsearch - ${upgrade.from.version} - rpm - - - - - - - - - - ok - - - - - rpm-via-homebrew - - - /usr/local/bin/rpmbuild - - - - - - maven-dependency-plugin - - - copy-rpm-to-testroot - pre-integration-test - - copy - - - ${project.build.directory}/testroot - - - org.elasticsearch.distribution.rpm - elasticsearch - ${elasticsearch.version} - rpm - - - - org.elasticsearch.distribution.rpm - elasticsearch - ${upgrade.from.version} - rpm - - - - - - - - - - ok - - - - - - - smoke-vms - - echo skipping tests - - - - diff --git a/qa/vagrant/src/dev/ant/vagrant-integration-tests.xml b/qa/vagrant/src/dev/ant/vagrant-integration-tests.xml deleted file mode 100644 index 67b208803b9..00000000000 --- a/qa/vagrant/src/dev/ant/vagrant-integration-tests.xml +++ /dev/null @@ -1,112 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats index 383375f9531..f6a9b22ec3b 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats @@ -92,7 +92,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=elasticsearch install_and_check_plugin lang groovy start_elasticsearch_service run_elasticsearch_tests stop_elasticsearch_service diff --git a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats index 553f867f60d..048b208105e 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats @@ -86,7 +86,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy start_elasticsearch_service run_elasticsearch_tests } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats index 7f447e51cb3..d27622ffdda 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats @@ -81,7 +81,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy start_elasticsearch_service run_elasticsearch_tests } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats index 6558a3831b3..da7b6a180f1 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats @@ -68,7 +68,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy systemctl start elasticsearch.service wait_for_elasticsearch_status assert_file_exist "/var/run/elasticsearch/elasticsearch.pid" diff --git a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats b/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats index 1c5cce59174..fad764eb711 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats @@ -70,7 +70,6 @@ setup() { # Install scripts used to test script filters and search templates before # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy install_elasticsearch_test_scripts service elasticsearch start wait_for_elasticsearch_status diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash index 666e4881eb5..11961e06921 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash @@ -417,9 +417,7 @@ run_elasticsearch_tests() { }' | grep \"count\"\ :\ 1 curl -s -XGET 'http://localhost:9200/library/book/_search/template?pretty' -d '{ - "template": { - "file": "is_guide" - } + "file": "is_guide" }' | grep \"total\"\ :\ 1 curl -s -XDELETE 'http://localhost:9200/_all' diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash index bd9e28e3f09..54978b39605 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash @@ -231,6 +231,10 @@ fi install_and_check_plugin lang groovy } +@test "[$GROUP] install lang-plan-a plugin" { + install_and_check_plugin lang plan-a +} + @test "[$GROUP] install javascript plugin" { install_and_check_plugin lang javascript rhino-*.jar } @@ -239,6 +243,10 @@ fi install_and_check_plugin lang python jython-standalone-*.jar } +@test "[$GROUP] install mapper-attachments plugin" { + install_and_check_plugin mapper attachments +} + @test "[$GROUP] install murmur3 mapper plugin" { install_and_check_plugin mapper murmur3 } @@ -335,6 +343,10 @@ fi remove_plugin lang-groovy } +@test "[$GROUP] remove lang-plan-a plugin" { + remove_plugin lang-plan-a +} + @test "[$GROUP] remove javascript plugin" { remove_plugin lang-javascript } @@ -343,6 +355,10 @@ fi remove_plugin lang-python } +@test "[$GROUP] remove mapper-attachments plugin" { + remove_plugin mapper-attachments +} + @test "[$GROUP] remove murmur3 mapper plugin" { remove_plugin mapper-murmur3 } @@ -423,4 +439,3 @@ fi fi remove_jvm_example } - diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash index 8052fc1340c..7200f28b139 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash @@ -149,7 +149,7 @@ install_and_check_plugin() { # $2 description of the source of the plugin list compare_plugins_list() { cat $1 | sort > /tmp/plugins - ls /elasticsearch/plugins/*/pom.xml | cut -d '/' -f 4 | + ls /elasticsearch/plugins/*/build.gradle | cut -d '/' -f 4 | sort > /tmp/expected echo "Checking plugins from $2 (<) against expected plugins (>):" diff /tmp/expected /tmp/plugins diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle new file mode 100644 index 00000000000..bbfeb03c223 --- /dev/null +++ b/rest-api-spec/build.gradle @@ -0,0 +1 @@ +apply plugin: 'java' diff --git a/rest-api-spec/pom.xml b/rest-api-spec/pom.xml deleted file mode 100644 index e21e55fe89a..00000000000 --- a/rest-api-spec/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - 4.0.0 - org.elasticsearch - rest-api-spec - 3.0.0-SNAPSHOT - Rest API Specification - REST API Specification and tests for use with the Elasticsearch REST Test framework - - org.sonatype.oss - oss-parent - 7 - - - - UTF-8 - s3://download.elasticsearch.org/elasticsearch/staging/ - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - 1.5 - - - - bundle - - - - - - **/* - - - - - - - - release - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - ${gpg.keyname} - ${gpg.passphrase} - ${gpg.keyring} - - - - - - - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json new file mode 100644 index 00000000000..2dcbac5cac9 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json @@ -0,0 +1,38 @@ +{ + "cat.repositories": { + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-repositories.html", + "methods": ["GET"], + "url": { + "path": "/_cat/repositories", + "paths": ["/_cat/repositories"], + "parts": { + }, + "params": { + "local": { + "type" : "boolean", + "description" : "Return local information, do not retrieve the state from master node", + "default": false + }, + "master_timeout": { + "type" : "time", + "description" : "Explicit operation timeout for connection to master node" + }, + "h": { + "type": "list", + "description" : "Comma-separated list of column names to display" + }, + "help": { + "type": "boolean", + "description": "Return help information", + "default": false + }, + "v": { + "type": "boolean", + "description": "Verbose mode. Display column headers", + "default": false + } + } + }, + "body": null + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json new file mode 100644 index 00000000000..4fd5520ecb1 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json @@ -0,0 +1,42 @@ +{ + "cat.snapshots": { + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-snapshots.html", + "methods": ["GET"], + "url": { + "path": "/_cat/snapshots/{repository}", + "paths": ["/_cat/snapshots/{repository}"], + "parts": { + "repository": { + "type" : "list", + "description": "Name of repository from which to fetch the snapshot information" + } + }, + "params": { + "ignore_unavailable": { + "type": "boolean", + "description": "Set to true to ignore unavailable snapshots", + "default": false + }, + "master_timeout": { + "type" : "time", + "description" : "Explicit operation timeout for connection to master node" + }, + "h": { + "type": "list", + "description" : "Comma-separated list of column names to display" + }, + "help": { + "type": "boolean", + "description": "Return help information", + "default": false + }, + "v": { + "type": "boolean", + "description": "Verbose mode. Display column headers", + "default": false + } + } + }, + "body": null + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json index 00b0ec13a5c..9fe9bfe3cad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json @@ -44,6 +44,14 @@ "type" : "string", "description" : "The name of the tokenizer to use for the analysis" }, + "detail": { + "type" : "boolean", + "description" : "With `true`, outputs more advanced details. (default: false)" + }, + "attributes": { + "type" : "list", + "description" : "A comma-separated list of token attributes to output, this parameter works only with `detail=true`" + }, "format": { "type": "enum", "options" : ["detailed","text"], diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.optimize.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json similarity index 91% rename from rest-api-spec/src/main/resources/rest-api-spec/api/indices.optimize.json rename to rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json index c5e63bfe60d..c4170c1962a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.optimize.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json @@ -1,10 +1,10 @@ { - "indices.optimize": { - "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-optimize.html", - "methods": ["POST", "GET"], + "indices.forcemerge": { + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-forcemerge.html", + "methods": ["POST"], "url": { - "path": "/_optimize", - "paths": ["/_optimize", "/{index}/_optimize"], + "path": "/_forcemerge", + "paths": ["/_forcemerge", "/{index}/_forcemerge"], "parts": { "index": { "type" : "list", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json index 123ce1373bb..cc51bdab2c3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json @@ -33,6 +33,11 @@ }, "operation_threading": { "description" : "TODO: ?" + }, + "verbose": { + "type": "boolean", + "description": "Includes detailed memory usage by Lucene.", + "default": false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json index 7ec665ca060..7099f3e2fd2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json @@ -53,8 +53,8 @@ "type" : "list", "description" : "A comma-separated list of document types for the `indexing` index metric" } - }, - "body": null - } + } + }, + "body": null } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json index 874294102c7..fb9ef094f0b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json @@ -15,7 +15,7 @@ "parts": { "metric" : { "type" : "list", - "options" : ["_all", "breaker", "fs", "http", "indices", "jvm", "os", "process", "thread_pool", "transport"], + "options" : ["_all", "breaker", "fs", "http", "indices", "jvm", "os", "process", "thread_pool", "transport", "discovery"], "description" : "Limit the information returned to the specified metrics" }, "index_metric" : { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_exists.json deleted file mode 100644 index a8970467d1c..00000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_exists.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "search_exists": { - "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-exists.html", - "methods": ["POST", "GET"], - "url": { - "path": "/_search/exists", - "paths": ["/_search/exists", "/{index}/_search/exists", "/{index}/{type}/_search/exists"], - "parts": { - "index": { - "type" : "list", - "description" : "A comma-separated list of indices to restrict the results" - }, - "type": { - "type" : "list", - "description" : "A comma-separated list of types to restrict the results" - } - }, - "params": { - "ignore_unavailable": { - "type" : "boolean", - "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)" - }, - "allow_no_indices": { - "type" : "boolean", - "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" - }, - "expand_wildcards": { - "type" : "enum", - "options" : ["open","closed","none","all"], - "default" : "open", - "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." - }, - "min_score": { - "type" : "number", - "description" : "Include only documents with a specific `_score` value in the result" - }, - "preference": { - "type" : "string", - "description" : "Specify the node or shard the operation should be performed on (default: random)" - }, - "routing": { - "type" : "string", - "description" : "Specific routing value" - }, - "q": { - "type" : "string", - "description" : "Query in the Lucene query string syntax" - }, - "analyzer": { - "type" : "string", - "description" : "The analyzer to use for the query string" - }, - "analyze_wildcard": { - "type" : "boolean", - "description" : "Specify whether wildcard and prefix queries should be analyzed (default: false)" - }, - "default_operator": { - "type" : "enum", - "options" : ["AND","OR"], - "default" : "OR", - "description" : "The default operator for query string query (AND or OR)" - }, - "df": { - "type" : "string", - "description" : "The field to use as default where no field prefix is given in the query string" - }, - "lenient": { - "type" : "boolean", - "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" - }, - "lowercase_expanded_terms": { - "type" : "boolean", - "description" : "Specify whether query terms should be lowercased" - } - } - }, - "body": { - "description" : "A query to restrict the results specified with the Query DSL (optional)" - } - } -} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml index 640d77e0183..3ee33b0a67b 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml @@ -148,7 +148,7 @@ index \s+ filter \s+ routing.index \s+ - routing.search \s+ + routing.search \n test_1 \s+ test \s+ @@ -185,6 +185,6 @@ - match: $body: | /^ - index \s+ alias \s+ \n - test \s+ test_1 \s+ \n + index \s+ alias \n + test \s+ test_1 \n $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml index be25839485d..3537da73c81 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml @@ -71,7 +71,7 @@ ( \s* #allow leading spaces to account for right-justified text \d+ \s+ - UNASSIGNED \s+ + UNASSIGNED \n )? $/ @@ -134,7 +134,7 @@ ( \s* #allow leading spaces to account for right-justified text \d+ \s+ - UNASSIGNED \s+ + UNASSIGNED \n )? $/ @@ -156,7 +156,7 @@ disk.percent \s+ host \s+ ip \s+ - node \s+ + node \n ( \s* #allow leading spaces to account for right-justified text @@ -199,7 +199,7 @@ $body: | /^ disk.percent \s+ - node \s+ + node \n ( \s+\d* \s+ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yaml index 1a62ab063d9..87ca75a6092 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yaml @@ -19,7 +19,7 @@ - match: $body: | /# epoch timestamp count - ^ \d+ \s \d{2}:\d{2}:\d{2} \s 0 \s+ \n $/ + ^ \d+ \s \d{2}:\d{2}:\d{2} \s 0 \n$/ - do: index: @@ -35,7 +35,7 @@ - match: $body: | /# epoch timestamp count - ^ \d+ \s \d{2}:\d{2}:\d{2} \s 1 \s+ \n $/ + ^ \d+ \s \d{2}:\d{2}:\d{2} \s 1 \n $/ - do: index: @@ -52,7 +52,7 @@ - match: $body: | /# count - ^ 2 \s+ \n $/ + ^ 2 \n $/ - do: @@ -62,7 +62,7 @@ - match: $body: | /# epoch timestamp count - ^ \d+ \s \d{2}:\d{2}:\d{2} \s 1 \s+ \n $/ + ^ \d+ \s \d{2}:\d{2}:\d{2} \s 1 \n $/ - do: cat.count: @@ -71,5 +71,5 @@ - match: $body: | - /^ epoch \s+ timestamp \s+ count \s+ \n - \d+ \s+ \d{2}:\d{2}:\d{2} \s+ \d+ \s+ \n $/ + /^ epoch \s+ timestamp \s+ count \n + \d+ \s+ \d{2}:\d{2}:\d{2} \s+ \d+ \n $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml index bc362fae58c..dfc580da1dd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml @@ -38,8 +38,8 @@ - match: $body: | - /^ total \s \n - (\s*\d+(\.\d+)?[gmk]?b \s \n)+ $/ + /^ total \n + (\s*\d+(\.\d+)?[gmk]?b \n)+ $/ - do: cat.fielddata: @@ -48,8 +48,8 @@ - match: $body: | - /^ total \s+ foo \s+ \n - (\s*\d+(\.\d+)?[gmk]?b \s+ \d+(\.\d+)?[gmk]?b \s \n)+ \s*$/ + /^ total \s+ foo \n + (\s*\d+(\.\d+)?[gmk]?b \s+ \d+(\.\d+)?[gmk]?b \n)+ $/ - do: cat.fielddata: @@ -59,5 +59,5 @@ - match: $body: | - /^ total \s+ foo \s+ \n - (\s*\d+(\.\d+)?[gmk]?b \s+ \d+(\.\d+)?[gmk]?b \s \n)+ \s*$/ + /^ total \s+ foo \n + (\s*\d+(\.\d+)?[gmk]?b \s+ \d+(\.\d+)?[gmk]?b \n)+ $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml index 9bfde46a371..0692df28a08 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml @@ -46,7 +46,7 @@ \d+ \s+ # unassign \d+ \s+ # pending_tasks (-|\d+[.]\d+ms|s) \s+ # max task waiting time - \d+\.\d+% \s+ # active shards percent + \d+\.\d+% # active shards percent \n )+ $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodeattrs/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodeattrs/10_basic.yaml index 5f6e3e9ae09..8d1dee13893 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodeattrs/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodeattrs/10_basic.yaml @@ -27,8 +27,8 @@ $/ - match: # A specific planted attribute is present and looks good $body: | - /# node\s+ host\s+ ip\s+ attr\s+ value\s* \n - ((\S+\s?){1,10})\s+ (\S+)\s+(\d{1,3}\.){3}\d{1,3}\s+testattr\s+ test \s* \n + /# node\s+ host\s+ ip\s+ attr\s+ value\s* \n + (\S+(\s\S+){0,7})\s+ (\S+)\s+ (\d{1,3}\.){3}\d{1,3}\s+testattr\s+ test \s* \n / # Note for future editors: its quite possible to construct a regex with an # intense amount of backtracking if you use something like (\S\s?)+ to match @@ -44,8 +44,8 @@ $/ - match: # A specific planted attribute is present and looks good $body: | - /# node\s+ host\s+ ip\s+ attr\s+ value\s* \n - ((\S+\s?){1,10})\s+ (\S+)\s+(\d{1,3}\.){3}\d{1,3}\s+testattr\s+ test \s* \n + /# node\s+ host\s+ ip\s+ attr\s+ value\s* \n + (\S+(\s\S+){0,7})\s+ (\S+)\s+ (\d{1,3}\.){3}\d{1,3}\s+ testattr\s+ test \s* \n / - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml index 66145f47b0a..2531e6ef025 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml @@ -6,8 +6,8 @@ - match: $body: | - / #host ip heap.percent ram.percent load node.role master name - ^ (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \s+ \n)+ $/ + / #host ip heap.percent ram.percent cpu load node.role master name + ^ (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \n)+ $/ - do: cat.nodes: @@ -15,8 +15,8 @@ - match: $body: | - /^ host \s+ ip \s+ heap\.percent \s+ ram\.percent \s+ load \s+ node\.role \s+ master \s+ name \s+ \n - (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \s+ \n)+ $/ + /^ host \s+ ip \s+ heap\.percent \s+ ram\.percent \s+ cpu \s+ load \s+ node\.role \s+ master \s+ name \n + (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ \d* \s+ (-)?\d*(\.\d+)? \s+ [-dc] \s+ [-*mx] \s+ (\S+\s?)+ \n)+ $/ - do: cat.nodes: @@ -25,8 +25,8 @@ - match: $body: | - /^ heap\.current \s+ heap\.percent \s+ heap\.max \s+ \n - (\s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+ \s+ \d+(\.\d+)?[ptgmk]?b \s+ \n)+ $/ + /^ heap\.current \s+ heap\.percent \s+ heap\.max \n + (\s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+ \s+ \d+(\.\d+)?[ptgmk]?b \n)+ $/ - do: cat.nodes: @@ -35,8 +35,8 @@ - match: $body: | - /^ heap\.current \s+ heap\.percent \s+ heap\.max \s+ \n - (\s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+ \s+ \d+(\.\d+)?[ptgmk]?b \s+ \n)+ $/ + /^ heap\.current \s+ heap\.percent \s+ heap\.max \n + (\s+ \d+(\.\d+)?[ptgmk]?b \s+ \d+ \s+ \d+(\.\d+)?[ptgmk]?b \n)+ $/ - do: cat.nodes: @@ -46,5 +46,5 @@ - match: # Windows reports -1 for the file descriptor counts. $body: | - /^ file_desc\.current \s+ file_desc\.percent \s+ file_desc\.max \s+ \n - (\s+ (-1|\d+) \s+ \d+ \s+ (-1|\d+) \s+ \n)+ $/ + /^ file_desc\.current \s+ file_desc\.percent \s+ file_desc\.max \n + (\s+ (-1|\d+) \s+ \d+ \s+ (-1|\d+) \n)+ $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml index c34437ce064..b081aa4d8cc 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml @@ -42,7 +42,7 @@ \d+ \s+ # total_bytes \d+ \s+ # translog -?\d+\.\d+% \s+ # translog_percent - -?\d+ \s+ # total_translog + -?\d+ # total_translog \n )+ $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml new file mode 100755 index 00000000000..c7eb9c1f930 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml @@ -0,0 +1,45 @@ +--- +"Help": + - do: + cat.repositories: + help: true + + - match: + $body: | + /^ id .+ \n + type .+ \n + $/ +--- +"Test cat repositories output": + + - do: + cat.repositories: {} + + - match: + $body: | + /^$/ + + - do: + snapshot.create_repository: + repository: test_cat_repo_1 + body: + type: fs + settings: + location: "test_cat_repo_1_loc" + + - do: + snapshot.create_repository: + repository: test_cat_repo_2 + body: + type: fs + settings: + location: "test_cat_repo_2_loc" + + - do: + cat.repositories: {} + + - match: + $body: | + /^ test_cat_repo_1\s+ fs\s*\n + test_cat_repo_2\s+ fs\s*\n + $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml index 766d1782ff1..f264928c21b 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml @@ -15,6 +15,7 @@ ip .+ \n id .+ \n node .+ \n + sync_id .+ \n unassigned.reason .+ \n unassigned.at .+ \n unassigned.for .+ \n @@ -84,6 +85,59 @@ $body: | /^$/ + - do: + indices.create: + index: sync_id_test + body: + settings: + number_of_shards: 5 + number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + - do: + indices.flush_synced: + index: sync_id_test + + - is_false: _shards.failed + + - do: + cat.shards: + index: sync_id_test + h: index,state,sync_id + - match: + $body: | + /^(sync_id_test\s+STARTED\s+[A-Za-z0-9_\-]{20}\n){5}$/ + + - do: + indices.delete: + index: sync_id_test + + - do: + indices.create: + index: sync_id_no_flush_test + body: + settings: + number_of_shards: 5 + number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + + - do: + cat.shards: + index: sync_id_no_flush_test + h: index,state,sync_id + - match: + $body: | + /^(sync_id_no_flush_test\s+STARTED\s+\n){5}$/ + + - do: + indices.delete: + index: sync_id_no_flush_test + - do: indices.create: index: index1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yaml new file mode 100755 index 00000000000..c1a1fc97011 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yaml @@ -0,0 +1,80 @@ +--- +"Help": + - do: + cat.snapshots: + repository: test_cat_snapshots_1 + help: true + + - match: + $body: | + /^ id .+ \n + status .+ \n + start_epoch .+ \n + start_time .+ \n + end_epoch .+ \n + end_time .+ \n + duration .+ \n + indices .+ \n + successful_shards .+ \n + failed_shards .+ \n + total_shards .+ \n + reason .+ \n + $/ +--- +"Test cat snapshots output": + + - do: + snapshot.create_repository: + repository: test_cat_snapshots_1 + body: + type: fs + settings: + location: "test_cat_snapshots_1_loc" + + - do: + cat.snapshots: + repository: test_cat_snapshots_1 + + - match: + $body: | + /^$/ + + - do: + indices.create: + index: index1 + body: + settings: + number_of_shards: "1" + number_of_replicas: "0" + - do: + indices.create: + index: index2 + body: + settings: + number_of_shards: "1" + number_of_replicas: "0" + - do: + cluster.health: + wait_for_status: green + + - do: + snapshot.create: + repository: test_cat_snapshots_1 + snapshot: snap1 + wait_for_completion: true + + - do: + snapshot.create: + repository: test_cat_snapshots_1 + snapshot: snap2 + wait_for_completion: true + + - do: + cat.snapshots: + repository: test_cat_snapshots_1 + + - match: + $body: | + /^ snap1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n + snap2\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n + $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml index 0c8ac5b4028..8d59e7c139c 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml @@ -7,7 +7,7 @@ - match: $body: | / #host ip bulk.active bulk.queue bulk.rejected index.active index.queue index.rejected search.active search.queue search.rejected - ^ (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \n)+ $/ + ^ (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \n)+ $/ - do: cat.thread_pool: @@ -15,8 +15,8 @@ - match: $body: | - /^ host \s+ ip \s+ bulk.active \s+ bulk.queue \s+ bulk.rejected \s+ index.active \s+ index.queue \s+ index.rejected \s+ search.active \s+ search.queue \s+ search.rejected \s+ \n - (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \n)+ $/ + /^ host \s+ ip \s+ bulk.active \s+ bulk.queue \s+ bulk.rejected \s+ index.active \s+ index.queue \s+ index.rejected \s+ search.active \s+ search.queue \s+ search.rejected \n + (\S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \n)+ $/ - do: cat.thread_pool: @@ -25,18 +25,18 @@ - match: $body: | / #pid id host ip port - ^ (\d+ \s+ \S{4} \s+ \S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ (\d+|-) \s+ \n)+ $/ + ^ (\d+ \s+ \S{4} \s+ \S+ \s+ (\d{1,3}\.){3}\d{1,3} \s+ (\d+|-) \n)+ $/ - do: cat.thread_pool: - h: id,ba,fa,gea,ga,ia,maa,ma,oa,pa + h: id,ba,fa,gea,ga,ia,maa,ma,fma,pa v: true full_id: true - match: $body: | - /^ id \s+ ba \s+ fa \s+ gea \s+ ga \s+ ia \s+ maa \s+ oa \s+ pa \s+ \n - (\S+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \n)+ $/ + /^ id \s+ ba \s+ fa \s+ gea \s+ ga \s+ ia \s+ maa \s+ fma \s+ pa \n + (\S+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d+ \n)+ $/ - do: cat.thread_pool: @@ -45,8 +45,8 @@ - match: $body: | - /^ id \s+ bulk.type \s+ bulk.active \s+ bulk.size \s+ bulk.queue \s+ bulk.queueSize \s+ bulk.rejected \s+ bulk.largest \s+ bulk.completed \s+ bulk.min \s+ bulk.max \s+ bulk.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ bulk.type \s+ bulk.active \s+ bulk.size \s+ bulk.queue \s+ bulk.queueSize \s+ bulk.rejected \s+ bulk.largest \s+ bulk.completed \s+ bulk.min \s+ bulk.max \s+ bulk.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -55,8 +55,8 @@ - match: $body: | - /^ id \s+ flush.type \s+ flush.active \s+ flush.size \s+ flush.queue \s+ flush.queueSize \s+ flush.rejected \s+ flush.largest \s+ flush.completed \s+ flush.min \s+ flush.max \s+ flush.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ flush.type \s+ flush.active \s+ flush.size \s+ flush.queue \s+ flush.queueSize \s+ flush.rejected \s+ flush.largest \s+ flush.completed \s+ flush.min \s+ flush.max \s+ flush.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -65,8 +65,8 @@ - match: $body: | - /^ id \s+ generic.type \s+ generic.active \s+ generic.size \s+ generic.queue \s+ generic.queueSize \s+ generic.rejected \s+ generic.largest \s+ generic.completed \s+ generic.min \s+ generic.max \s+ generic.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ generic.type \s+ generic.active \s+ generic.size \s+ generic.queue \s+ generic.queueSize \s+ generic.rejected \s+ generic.largest \s+ generic.completed \s+ generic.min \s+ generic.max \s+ generic.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -75,8 +75,8 @@ - match: $body: | - /^ id \s+ get.type \s+ get.active \s+ get.size \s+ get.queue \s+ get.queueSize \s+ get.rejected \s+ get.largest \s+ get.completed \s+ get.min \s+ get.max \s+ get.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ get.type \s+ get.active \s+ get.size \s+ get.queue \s+ get.queueSize \s+ get.rejected \s+ get.largest \s+ get.completed \s+ get.min \s+ get.max \s+ get.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -85,8 +85,8 @@ - match: $body: | - /^ id \s+ index.type \s+ index.active \s+ index.size \s+ index.queue \s+ index.queueSize \s+ index.rejected \s+ index.largest \s+ index.completed \s+ index.min \s+ index.max \s+ index.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ index.type \s+ index.active \s+ index.size \s+ index.queue \s+ index.queueSize \s+ index.rejected \s+ index.largest \s+ index.completed \s+ index.min \s+ index.max \s+ index.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -95,18 +95,18 @@ - match: $body: | - /^ id \s+ management.type \s+ management.active \s+ management.size \s+ management.queue \s+ management.queueSize \s+ management.rejected \s+ management.largest \s+ management.completed \s+ management.min \s+ management.max \s+ management.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ management.type \s+ management.active \s+ management.size \s+ management.queue \s+ management.queueSize \s+ management.rejected \s+ management.largest \s+ management.completed \s+ management.min \s+ management.max \s+ management.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: - h: id,optimize.type,optimize.active,optimize.size,optimize.queue,optimize.queueSize,optimize.rejected,optimize.largest,optimize.completed,optimize.min,optimize.max,optimize.keepAlive + h: id,force_merge.type,force_merge.active,force_merge.size,force_merge.queue,force_merge.queueSize,force_merge.rejected,force_merge.largest,force_merge.completed,force_merge.min,force_merge.max,force_merge.keepAlive v: true - match: $body: | - /^ id \s+ optimize.type \s+ optimize.active \s+ optimize.size \s+ optimize.queue \s+ optimize.queueSize \s+ optimize.rejected \s+ optimize.largest \s+ optimize.completed \s+ optimize.min \s+ optimize.max \s+ optimize.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ force_merge.type \s+ force_merge.active \s+ force_merge.size \s+ force_merge.queue \s+ force_merge.queueSize \s+ force_merge.rejected \s+ force_merge.largest \s+ force_merge.completed \s+ force_merge.min \s+ force_merge.max \s+ force_merge.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -115,8 +115,8 @@ - match: $body: | - /^ id \s+ percolate.type \s+ percolate.active \s+ percolate.size \s+ percolate.queue \s+ percolate.queueSize \s+ percolate.rejected \s+ percolate.largest \s+ percolate.completed \s+ percolate.min \s+ percolate.max \s+ percolate.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ percolate.type \s+ percolate.active \s+ percolate.size \s+ percolate.queue \s+ percolate.queueSize \s+ percolate.rejected \s+ percolate.largest \s+ percolate.completed \s+ percolate.min \s+ percolate.max \s+ percolate.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -125,8 +125,8 @@ - match: $body: | - /^ id \s+ refresh.type \s+ refresh.active \s+ refresh.size \s+ refresh.queue \s+ refresh.queueSize \s+ refresh.rejected \s+ refresh.largest \s+ refresh.completed \s+ refresh.min \s+ refresh.max \s+ refresh.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ refresh.type \s+ refresh.active \s+ refresh.size \s+ refresh.queue \s+ refresh.queueSize \s+ refresh.rejected \s+ refresh.largest \s+ refresh.completed \s+ refresh.min \s+ refresh.max \s+ refresh.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -135,8 +135,8 @@ - match: $body: | - /^ id \s+ search.type \s+ search.active \s+ search.size \s+ search.queue \s+ search.queueSize \s+ search.rejected \s+ search.largest \s+ search.completed \s+ search.min \s+ search.max \s+ search.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ search.type \s+ search.active \s+ search.size \s+ search.queue \s+ search.queueSize \s+ search.rejected \s+ search.largest \s+ search.completed \s+ search.min \s+ search.max \s+ search.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -145,8 +145,8 @@ - match: $body: | - /^ id \s+ snapshot.type \s+ snapshot.active \s+ snapshot.size \s+ snapshot.queue \s+ snapshot.queueSize \s+ snapshot.rejected \s+ snapshot.largest \s+ snapshot.completed \s+ snapshot.min \s+ snapshot.max \s+ snapshot.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ snapshot.type \s+ snapshot.active \s+ snapshot.size \s+ snapshot.queue \s+ snapshot.queueSize \s+ snapshot.rejected \s+ snapshot.largest \s+ snapshot.completed \s+ snapshot.min \s+ snapshot.max \s+ snapshot.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -155,8 +155,8 @@ - match: $body: | - /^ id \s+ suggest.type \s+ suggest.active \s+ suggest.size \s+ suggest.queue \s+ suggest.queueSize \s+ suggest.rejected \s+ suggest.largest \s+ suggest.completed \s+ suggest.min \s+ suggest.max \s+ suggest.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ suggest.type \s+ suggest.active \s+ suggest.size \s+ suggest.queue \s+ suggest.queueSize \s+ suggest.rejected \s+ suggest.largest \s+ suggest.completed \s+ suggest.min \s+ suggest.max \s+ suggest.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - do: cat.thread_pool: @@ -165,5 +165,5 @@ - match: $body: | - /^ id \s+ warmer.type \s+ warmer.active \s+ warmer.size \s+ warmer.queue \s+ warmer.queueSize \s+ warmer.rejected \s+ warmer.largest \s+ warmer.completed \s+ warmer.min \s+ warmer.max \s+ warmer.keepAlive \s+ \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \s+ \n)+ $/ + /^ id \s+ warmer.type \s+ warmer.active \s+ warmer.size \s+ warmer.queue \s+ warmer.queueSize \s+ warmer.rejected \s+ warmer.largest \s+ warmer.completed \s+ warmer.min \s+ warmer.max \s+ warmer.keepAlive \n + (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yaml index 2495f296121..f3eb0a5fae6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yaml @@ -1,5 +1,4 @@ ---- -"count with body": +setup: - do: indices.create: index: test @@ -14,6 +13,8 @@ indices.refresh: index: [test] +--- +"count with body": - do: count: index: test @@ -35,3 +36,13 @@ foo: test - match: {count : 0} + +--- +"count body without query element": + - do: + catch: request + count: + index: test + body: + match: + foo: bar diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml index 4e6845c17e8..1a6c57848e0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yaml @@ -1,5 +1,14 @@ ---- -"Basic explain": +setup: + - do: + indices.create: + index: test_1 + body: + aliases: + alias_1: {} + - do: + cluster.health: + wait_for_status: yellow + - do: index: index: test_1 @@ -10,6 +19,9 @@ - do: indices.refresh: {} +--- +"Basic explain": + - do: explain: index: test_1 @@ -27,26 +39,6 @@ --- "Basic explain with alias": - - do: - indices.create: - index: test_1 - body: - aliases: - alias_1: {} - - - do: - cluster.health: - wait_for_status: yellow - - - do: - index: - index: test_1 - type: test - id: id_1 - body: { foo: bar, title: howdy } - - - do: - indices.refresh: {} - do: explain: @@ -63,3 +55,14 @@ - match: { _type: test } - match: { _id: id_1 } +--- +"Explain body without query element": + - do: + catch: request + explain: + index: test_1 + type: test + id: id_1 + body: + match_all: {} + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml index a7d5f05f781..165ed9f469c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml @@ -22,7 +22,9 @@ - match: { indices._all.fields.number.max_doc: 1 } - match: { indices._all.fields.number.doc_count: 1 } - match: { indices._all.fields.number.min_value: 123 } + - match: { indices._all.fields.number.min_value_as_string: "123" } - match: { indices._all.fields.number.max_value: 123 } + - match: { indices._all.fields.number.max_value_as_string: "123" } --- "Basic field stats with level set to indices": @@ -49,7 +51,9 @@ - match: { indices.test_1.fields.number.max_doc: 1 } - match: { indices.test_1.fields.number.doc_count: 1 } - match: { indices.test_1.fields.number.min_value: 123 } + - match: { indices.test_1.fields.number.min_value_as_string: "123" } - match: { indices.test_1.fields.number.max_value: 123 } + - match: { indices.test_1.fields.number.max_value_as_string: "123" } --- "Field stats with filtering": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml new file mode 100644 index 00000000000..71229686eed --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml @@ -0,0 +1,30 @@ +--- +"REST test with headers": + - skip: + features: headers + + - do: + index: + index: test_1 + type: test + id: 1 + body: { "body": "foo" } + + - do: + headers: + Content-Type: application/yaml + get: + index: test_1 + type: _all + id: 1 + + - match: + $body: | + /^---\n + _index:\s+\"test_1"\n + _type:\s+"test"\n + _id:\s+"1"\n + _version:\s+1\n + found:\s+true\n + _source:\n + \s+body:\s+"foo"\n$/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml index 49420672861..0b1a090303e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml @@ -71,3 +71,31 @@ setup: - length: {tokens: 2 } - match: { tokens.0.token: foo bar } - match: { tokens.1.token: baz } +--- +"Detail response with Analyzer": + - do: + indices.analyze: + body: {"text": "This is troubled", "analyzer": standard, "explain": true} + - length: { detail.analyzer.tokens: 3 } + - match: { detail.analyzer.name: standard } + - match: { detail.analyzer.tokens.0.token: this } + - match: { detail.analyzer.tokens.1.token: is } + - match: { detail.analyzer.tokens.2.token: troubled } +--- +"Detail output spcified attribute": + - do: + indices.analyze: + body: {"text": "This is troubled", "char_filters": ["html_strip"], "filters": ["snowball"], "tokenizer": standard, "explain": true, "attributes": ["keyword"]} + - length: { detail.charfilters: 1 } + - length: { detail.tokenizer.tokens: 3 } + - length: { detail.tokenfilters.0.tokens: 3 } + - match: { detail.tokenizer.name: standard } + - match: { detail.tokenizer.tokens.0.token: This } + - match: { detail.tokenizer.tokens.1.token: is } + - match: { detail.tokenizer.tokens.2.token: troubled } + - match: { detail.tokenfilters.0.name: snowball } + - match: { detail.tokenfilters.0.tokens.0.token: This } + - match: { detail.tokenfilters.0.tokens.1.token: is } + - match: { detail.tokenfilters.0.tokens.2.token: troubl } + - match: { detail.tokenfilters.0.tokens.2.keyword: false } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.optimize/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/10_basic.yaml similarity index 68% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.optimize/10_basic.yaml rename to rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/10_basic.yaml index 1f24871ac08..6f1c6ea9496 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.optimize/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/10_basic.yaml @@ -1,10 +1,10 @@ --- -"Optimize index tests": +"Force merge index tests": - do: indices.create: index: testing - do: - indices.optimize: + indices.forcemerge: index: testing max_num_segments: 1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/10_basic.yaml index 44313aa9bfa..7e4c57429ec 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/10_basic.yaml @@ -35,7 +35,7 @@ setup: index: test_index name: test_warmer - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} - do: indices.delete_warmer: @@ -55,8 +55,8 @@ setup: - do: indices.get_warmer: {} - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} - - match: {test_idx.warmers.test_warmer2.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} + - match: {test_idx.warmers.test_warmer2.source.query.match_all: {boost: 1.0}} --- @@ -67,8 +67,8 @@ setup: index: '*' name: '*' - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} - - match: {test_idx.warmers.test_warmer2.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} + - match: {test_idx.warmers.test_warmer2.source.query.match_all: {boost: 1.0}} --- "Getting warmers for several indices should work using _all": @@ -78,8 +78,8 @@ setup: index: _all name: _all - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} - - match: {test_idx.warmers.test_warmer2.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} + - match: {test_idx.warmers.test_warmer2.source.query.match_all: {boost: 1.0}} --- "Getting all warmers without specifying index should work": @@ -88,8 +88,8 @@ setup: indices.get_warmer: name: _all - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} - - match: {test_idx.warmers.test_warmer2.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} + - match: {test_idx.warmers.test_warmer2.source.query.match_all: {boost: 1.0}} --- "Getting warmers for several indices should work using prefix*": @@ -99,8 +99,8 @@ setup: index: test_i* name: test_w* - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} - - match: {test_idx.warmers.test_warmer2.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} + - match: {test_idx.warmers.test_warmer2.source.query.match_all: {boost: 1.0}} --- "Getting warmers for several indices should work using comma-separated lists": @@ -110,8 +110,8 @@ setup: index: test_index,test_idx name: test_warmer,test_warmer2 - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} - - match: {test_idx.warmers.test_warmer2.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} + - match: {test_idx.warmers.test_warmer2.source.query.match_all: {boost: 1.0}} --- "Getting a non-existent warmer on an existing index should return an empty body": @@ -131,7 +131,7 @@ setup: index: test_index name: test_warmer,non-existent - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} - is_false: test_index.warmers.non-existent --- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/20_aliases.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/20_aliases.yaml index 96d734475ac..b8a2fa6b27e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/20_aliases.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/20_aliases.yaml @@ -26,5 +26,5 @@ indices.get_warmer: index: test_alias - - match: {test_index.warmers.test_warmer.source.query.match_all: {}} + - match: {test_index.warmers.test_warmer.source.query.match_all: {boost: 1.0}} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/all_path_options.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/all_path_options.yaml index b9c64f7d3a0..ffad427101a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/all_path_options.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_warmer/all_path_options.yaml @@ -38,8 +38,8 @@ setup: - do: indices.get_warmer: { index: _all, name: '*' } - - match: {test_index1.warmers.warmer.source.query.match_all: {}} - - match: {test_index2.warmers.warmer.source.query.match_all: {}} + - match: {test_index1.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {test_index2.warmers.warmer.source.query.match_all: {boost: 1.0}} - is_false: foo --- @@ -54,9 +54,9 @@ setup: - do: indices.get_warmer: { index: _all, name: '*' } - - match: {test_index1.warmers.warmer.source.query.match_all: {}} - - match: {test_index2.warmers.warmer.source.query.match_all: {}} - - match: {foo.warmers.warmer.source.query.match_all: {}} + - match: {test_index1.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {test_index2.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {foo.warmers.warmer.source.query.match_all: {boost: 1.0}} --- "put warmer in * index": @@ -70,9 +70,9 @@ setup: - do: indices.get_warmer: { index: _all, name: '*' } - - match: {test_index1.warmers.warmer.source.query.match_all: {}} - - match: {test_index2.warmers.warmer.source.query.match_all: {}} - - match: {foo.warmers.warmer.source.query.match_all: {}} + - match: {test_index1.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {test_index2.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {foo.warmers.warmer.source.query.match_all: {boost: 1.0}} --- "put warmer prefix* index": @@ -86,8 +86,8 @@ setup: - do: indices.get_warmer: { index: _all, name: '*' } - - match: {test_index1.warmers.warmer.source.query.match_all: {}} - - match: {test_index2.warmers.warmer.source.query.match_all: {}} + - match: {test_index1.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {test_index2.warmers.warmer.source.query.match_all: {boost: 1.0}} - is_false: foo --- @@ -102,8 +102,8 @@ setup: - do: indices.get_warmer: { index: _all, name: '*' } - - match: {test_index1.warmers.warmer.source.query.match_all: {}} - - match: {test_index2.warmers.warmer.source.query.match_all: {}} + - match: {test_index1.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {test_index2.warmers.warmer.source.query.match_all: {boost: 1.0}} - is_false: foo --- @@ -117,9 +117,9 @@ setup: - do: indices.get_warmer: { index: _all, name: '*' } - - match: {test_index1.warmers.warmer.source.query.match_all: {}} - - match: {test_index2.warmers.warmer.source.query.match_all: {}} - - match: {foo.warmers.warmer.source.query.match_all: {}} + - match: {test_index1.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {test_index2.warmers.warmer.source.query.match_all: {boost: 1.0}} + - match: {foo.warmers.warmer.source.query.match_all: {boost: 1.0}} --- "put warmer with missing name": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml index e9af0033677..b14e4cb4885 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml @@ -35,6 +35,27 @@ setup: - do: indices.refresh: {} + - do: + suggest: + index: test1 + body: + result: + text: "b" + completion: + field: bar.completion + + - do: + suggest: + index: test1 + body: + result: + text: "b" + completion: + field: baz.completion + + - do: + indices.refresh: {} + - do: search: sort: bar,baz diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/10_basic.yaml index 5b45f740e44..041f6bb3cc0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.update_aliases/10_basic.yaml @@ -32,3 +32,50 @@ name: test_alias - match: {test_index.aliases.test_alias: {'index_routing': 'routing_value', 'search_routing': 'routing_value'}} + +--- +"Basic test for multiple aliases": + + - do: + indices.create: + index: test_index + + - do: + indices.exists_alias: + name: test_alias1 + + - is_false: '' + + - do: + indices.exists_alias: + name: test_alias2 + + - is_false: '' + + - do: + indices.update_aliases: + body: + actions: + - add: + indices: [test_index] + aliases: [test_alias1, test_alias2] + routing: routing_value + + - do: + indices.exists_alias: + name: test_alias1 + + - is_true: '' + + - do: + indices.exists_alias: + name: test_alias2 + + - is_true: '' + + - do: + indices.get_alias: + index: test_index + + - match: {test_index.aliases.test_alias1: {'index_routing': 'routing_value', 'search_routing': 'routing_value'}} + - match: {test_index.aliases.test_alias2: {'index_routing': 'routing_value', 'search_routing': 'routing_value'}} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml index 2a9ed19221f..d8b3bfea385 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/10_basic.yaml @@ -1,5 +1,4 @@ ---- -"Validate query api": +setup: - do: indices.create: index: testing @@ -11,6 +10,8 @@ cluster.health: wait_for_status: yellow +--- +"Validate query api": - do: indices.validate_query: q: query string @@ -24,6 +25,17 @@ invalid_query: {} - is_false: valid + - is_false: error + + - do: + indices.validate_query: + explain: true + body: + query: + invalid_query: {} + + - is_false: valid + - match: {error: 'org.elasticsearch.common.ParsingException: No query registered for [invalid_query]'} - do: indices.validate_query: @@ -34,3 +46,21 @@ - match: {explanations.0.index: 'testing'} - match: {explanations.0.explanation: '*:*'} +--- +"Validate body without query element": + - do: + indices.validate_query: + body: + match_all: {} + + - is_false: valid + - is_false: error + + - do: + indices.validate_query: + explain: true + body: + match_all: {} + + - is_false: valid + - match: {error: 'org.elasticsearch.common.ParsingException: request does not support [match_all]'} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml new file mode 100644 index 00000000000..a0fb566f893 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml @@ -0,0 +1,25 @@ +--- +"Discovery stats": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.stats: + metric: [ discovery ] + + - is_true: cluster_name + - is_true: nodes + - is_true: nodes.$master.discovery + + - do: + nodes.stats: + filter_path: "nodes.*.discovery" + + - is_false: cluster_name + - is_true: nodes + - is_false: nodes.$master.name + - is_false: nodes.$master.jvm + - is_true: nodes.$master.discovery diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml index a78a5a2a28f..e0ac2aea2df 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml @@ -14,12 +14,12 @@ - do: search: # stringified for boolean value - body: "{ _source: true, query: { match_all: {} } }" + body: { _source: true, query: { match_all: {} } } - length: { hits.hits: 1 } - match: { hits.hits.0._source.count: 1 } - - do: { search: { body: "{ _source: false, query: { match_all: {} } }" } } + - do: { search: { body: { _source: false, query: { match_all: {} } } } } - length: { hits.hits: 1 } - is_false: hits.hits.0._source @@ -79,7 +79,6 @@ body: fields: [ include.field2 ] query: { match_all: {} } - - match: { hits.hits.0.fields: { include.field2 : [v2] }} - is_false: hits.hits.0._source - do: @@ -87,7 +86,7 @@ body: fields: [ include.field2, _source ] query: { match_all: {} } - - match: { hits.hits.0.fields: { include.field2 : [v2] }} + - match: { hits.hits.0._source.include.field2: v2 } - is_true: hits.hits.0._source @@ -95,4 +94,3 @@ search: fielddata_fields: [ "count" ] - match: { hits.hits.0.fields.count: [1] } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yaml index 6921a58d886..5cdde2cb696 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yaml @@ -1,5 +1,4 @@ ---- -"Default index": +setup: - do: indices.create: index: test_2 @@ -24,6 +23,9 @@ indices.refresh: index: [test_1, test_2] +--- +"Basic search": + - do: search: index: _all @@ -62,3 +64,14 @@ - match: {hits.hits.0._index: test_2 } - match: {hits.hits.0._type: test } - match: {hits.hits.0._id: "42" } + +--- +"Search body without query element": + + - do: + catch: request + search: + body: + match: + foo: bar + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_exists/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_exists/10_basic.yaml deleted file mode 100644 index 6045d9b55e4..00000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_exists/10_basic.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -"search_exists with body": - - do: - indices.create: - index: test - - do: - index: - index: test - type: test - id: 1 - body: { foo: bar } - - - do: - indices.refresh: - index: [test] - - - do: - search_exists: - index: test - type: test - body: - query: - match: - foo: bar - - - is_true: exists - - - do: - catch: missing - search_exists: - index: test - type: test - body: - query: - match: - foo: test - - - is_false: exists diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_exists/20_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_exists/20_query_string.yaml deleted file mode 100644 index 11535fd6a26..00000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_exists/20_query_string.yaml +++ /dev/null @@ -1,82 +0,0 @@ ---- -"search_exists with query_string parameters": - - do: - indices.create: - index: test - body: - mappings: - test: - _all: - enabled: false - properties: - number: - type: integer - - - do: - index: - index: test - type: test - id: 1 - body: { field: foo bar} - - - do: - indices.refresh: - index: [test] - - - do: - search_exists: - index: test - q: bar - df: field - - - is_true: exists - - - do: - search_exists: - index: test - q: field:foo field:xyz - - - is_true: exists - - - do: - catch: missing - search_exists: - index: test - q: field:foo field:xyz - default_operator: AND - - - is_false: exists - - - do: - search_exists: - index: test - q: field:bars - analyzer: snowball - - - is_true: exists - - - do: - catch: missing - search_exists: - index: test - q: field:BA* - lowercase_expanded_terms: false - - - is_false: exists - - - do: - search_exists: - index: test - q: field:BA* - analyze_wildcard: true - - - is_true: exists - - - do: - catch: missing - search_exists: - index: test - q: number:foo - lenient: true - - - is_false: exists diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml new file mode 100644 index 00000000000..8eb01023e08 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml @@ -0,0 +1,280 @@ +# This test creates one huge mapping in the setup +# Every test should use its own field to make sure it works + +setup: + + - do: + indices.create: + index: test + body: + mappings: + test: + "properties": + "suggest_1": + "type" : "completion" + "suggest_2": + "type" : "completion" + "suggest_3": + "type" : "completion" + "suggest_4": + "type" : "completion" + "suggest_5a": + "type" : "completion" + "suggest_5b": + "type" : "completion" + "suggest_6": + "type" : "completion" + +--- +"Simple suggestion should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_1: "bar" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_1: "baz" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_1 + + - length: { result: 1 } + - length: { result.0.options: 2 } + +--- +"Simple suggestion array should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_2: ["bar", "foo"] + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "f" + completion: + field: suggest_2 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "foo" } + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_2 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "bar" } + +--- +"Suggestion entry should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_3: + input: "bar" + weight: 2 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_3: + input: "baz" + weight: 3 + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_3 + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "baz" } + - match: { result.0.options.1.text: "bar" } + +--- +"Suggestion entry array should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_4: + - input: "bar" + weight: 3 + - input: "fo" + weight: 3 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_4: + - input: "baz" + weight: 2 + - input: "foo" + weight: 1 + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_4 + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "bar" } + - match: { result.0.options.1.text: "baz" } + + - do: + suggest: + body: + result: + text: "f" + completion: + field: suggest_4 + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "fo" } + - match: { result.0.options.1.text: "foo" } + +--- +"Multiple Completion fields should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_5a: "bar" + suggest_5b: "baz" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_5a + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "bar" } + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_5b + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "baz" } + +--- +"Suggestions with payload fields should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_6: + input: "bar" + weight: 2 + title: "title_bar" + count: 4 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_6: + input: "baz" + weight: 3 + title: "title_baz" + count: 3 + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_6 + payload: [ title, count ] + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "baz" } + - match: { result.0.options.0.payload.title: ["title_baz"] } + - match: { result.0.options.0.payload.count: [3] } + - match: { result.0.options.1.text: "bar" } + - match: { result.0.options.1.payload.title: ["title_bar"] } + - match: { result.0.options.1.payload.count: [4] } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_context.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_context.yaml deleted file mode 100644 index cabd8a39552..00000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_context.yaml +++ /dev/null @@ -1,224 +0,0 @@ -# This test creates one huge mapping in the setup -# Every test should use its own field to make sure it works - -setup: - - - do: - indices.create: - index: test - body: - mappings: - test: - "properties": - "suggest_context": - "type" : "completion" - "context": - "color": - "type" : "category" - "suggest_context_default_hardcoded": - "type" : "completion" - "context": - "color": - "type" : "category" - "default" : "red" - "suggest_context_default_path": - "type" : "completion" - "context": - "color": - "type" : "category" - "default" : "red" - "path" : "color" - "suggest_geo": - "type" : "completion" - "context": - "location": - "type" : "geo" - "precision" : "5km" - ---- -"Simple context suggestion should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_context: - input: "Hoodie red" - context: - color: "red" - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_context: - input: "Hoodie blue" - context: - color: "blue" - - - do: - indices.refresh: {} - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context - context: - color: "red" - - - match: {result.0.options.0.text: "Hoodie red" } - ---- -"Hardcoded category value should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_context_default_hardcoded: - input: "Hoodie red" - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_context_default_hardcoded: - input: "Hoodie blue" - context: - color: "blue" - - - do: - indices.refresh: {} - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context_default_hardcoded - context: - color: "red" - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hoodie red" } - - ---- -"Category suggest context default path should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_context_default_path: - input: "Hoodie red" - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_context_default_path: - input: "Hoodie blue" - color: "blue" - - - do: - indices.refresh: {} - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context_default_path - context: - color: "red" - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hoodie red" } - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context_default_path - context: - color: "blue" - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hoodie blue" } - - ---- -"Geo suggest should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_geo: - input: "Hotel Marriot in Amsterdam" - context: - location: - lat : 52.22 - lon : 4.53 - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_geo: - input: "Hotel Marriot in Berlin" - context: - location: - lat : 53.31 - lon : 13.24 - - - do: - indices.refresh: {} - - - do: - indices.get_mapping: {} - - - do: - suggest: - index: test - body: - result: - text: "hote" - completion: - field: suggest_geo - context: - location: - lat : 52.22 - lon : 4.53 - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hotel Marriot in Amsterdam" } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml new file mode 100644 index 00000000000..da7af85cf9f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml @@ -0,0 +1,267 @@ +# This test creates one huge mapping in the setup +# Every test should use its own field to make sure it works + +setup: + + - do: + indices.create: + index: test + body: + mappings: + test: + "properties": + "location": + "type": "geo_point" + "suggest_context": + "type" : "completion" + "contexts": + - "name" : "color" + "type" : "category" + "suggest_context_with_path": + "type" : "completion" + "contexts": + - "name" : "color" + "type" : "category" + "path" : "color" + "suggest_geo": + "type" : "completion" + "contexts": + - "name" : "location" + "type" : "geo" + "precision" : "5km" + "suggest_multi_contexts": + "type" : "completion" + "contexts": + - "name" : "location" + "type" : "geo" + "precision" : "5km" + "path" : "location" + - "name" : "color" + "type" : "category" + "path" : "color" + +--- +"Simple context suggestion should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_context: + input: "foo red" + contexts: + color: "red" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_context: + input: "foo blue" + contexts: + color: "blue" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context + contexts: + color: "red" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "foo red" } + +--- +"Category suggest context from path should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_context_with_path: + input: "Foo red" + contexts: + color: "red" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_context_with_path: "Foo blue" + color: "blue" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context_with_path + contexts: + color: "red" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Foo red" } + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context_with_path + contexts: + color: "blue" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Foo blue" } + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context_with_path + contexts: + color: ["blue", "red"] + + - length: { result: 1 } + - length: { result.0.options: 2 } + +--- +"Geo suggest should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_geo: + input: "Marriot in Amsterdam" + contexts: + location: + lat : 52.22 + lon : 4.53 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_geo: + input: "Marriot in Berlin" + contexts: + location: + lat : 53.31 + lon : 13.24 + + - do: + indices.refresh: {} + + - do: + indices.get_mapping: {} + + - do: + suggest: + index: test + body: + result: + text: "mar" + completion: + field: suggest_geo + contexts: + location: + lat : 52.2263 + lon : 4.543 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Marriot in Amsterdam" } + +--- +"Multi contexts should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_multi_contexts: "Marriot in Amsterdam" + location: + lat : 52.22 + lon : 4.53 + color: "red" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_multi_contexts: "Marriot in Berlin" + location: + lat : 53.31 + lon : 13.24 + color: "blue" + + - do: + indices.refresh: {} + + - do: + indices.get_mapping: {} + + - do: + suggest: + index: test + body: + result: + text: "mar" + completion: + field: suggest_multi_contexts + contexts: + location: + lat : 52.22 + lon : 4.53 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Marriot in Amsterdam" } + + - do: + suggest: + index: test + body: + result: + text: "mar" + completion: + field: suggest_multi_contexts + contexts: + color: "blue" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Marriot in Berlin" } diff --git a/run.bat b/run.bat deleted file mode 100755 index b0cc48a88ab..00000000000 --- a/run.bat +++ /dev/null @@ -1,5 +0,0 @@ -:: -:: build zip package, but ensuring its from the current source -:: turn off tests and other validation to speed it up -:: TODO: can be sped up more, if shading is moved out of core/ -CALL mvn -am -pl dev-tools,distribution/zip package -DskipTests -Drun -Pdev diff --git a/run.sh b/run.sh deleted file mode 100755 index 16871065918..00000000000 --- a/run.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# -# build zip package, but ensuring its from the current source -# turn off tests and other validation to speed it up -# TODO: can be sped up more, if shading is moved out of core/ -mvn -am -pl dev-tools,distribution/zip package -DskipTests -Drun -Pdev diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 00000000000..e9fb0a043aa --- /dev/null +++ b/settings.gradle @@ -0,0 +1,87 @@ +rootProject.name = 'elasticsearch' + +List projects = [ + 'rest-api-spec', + 'core', + 'distribution:integ-test-zip', + 'distribution:zip', + 'distribution:tar', + 'distribution:deb', + 'distribution:rpm', + 'test-framework', + 'modules:lang-expression', + 'modules:lang-groovy', + 'modules:lang-mustache', + 'plugins:analysis-icu', + 'plugins:analysis-kuromoji', + 'plugins:analysis-phonetic', + 'plugins:analysis-smartcn', + 'plugins:analysis-stempel', + 'plugins:delete-by-query', + 'plugins:discovery-azure', + 'plugins:discovery-ec2', + 'plugins:discovery-gce', + 'plugins:discovery-multicast', + 'plugins:lang-javascript', + 'plugins:lang-plan-a', + 'plugins:lang-python', + 'plugins:mapper-attachments', + 'plugins:mapper-murmur3', + 'plugins:mapper-size', + 'plugins:repository-azure', + 'plugins:repository-hdfs', + 'plugins:repository-s3', + 'plugins:jvm-example', + 'plugins:site-example', + 'plugins:store-smb', + 'qa:evil-tests', + 'qa:smoke-test-client', + 'qa:smoke-test-multinode', + 'qa:smoke-test-plugins', + 'qa:vagrant', +] + +boolean isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse') +if (isEclipse) { + // eclipse cannot handle an intermediate dependency between main and test, so we must create separate projects + // for core-src and core-tests + projects << 'core-tests' +} + +include projects.toArray(new String[0]) + +if (isEclipse) { + project(":core").projectDir = new File(rootProject.projectDir, 'core/src/main') + project(":core").buildFileName = 'eclipse-build.gradle' + project(":core-tests").projectDir = new File(rootProject.projectDir, 'core/src/test') + project(":core-tests").buildFileName = 'eclipse-build.gradle' +} + +/** + * Iterates over sub directories, looking for build.gradle, and adds a project if found + * for that dir with the given path prefix. Note that this requires each level + * of the dir hiearchy to have a build.gradle. Otherwise we would have to iterate + * all files/directories in the source tree to find all projects. + */ +void addSubProjects(String path, File dir) { + if (dir.isDirectory() == false) return; + if (dir.name == 'buildSrc') return; + if (new File(dir, 'build.gradle').exists() == false) return; + + String projectName = "${path}:${dir.name}" + include projectName + for (File subdir : dir.listFiles()) { + addSubProjects(projectName, subdir) + } +} + +// look for extra plugins for elasticsearch +File xplugins = new File(rootProject.projectDir.parentFile, 'x-plugins') +if (xplugins.exists()) { + include ':x-plugins' + project(':x-plugins').projectDir = xplugins + for (File extraPluginDir : xplugins.listFiles()) { + addSubProjects(':x-plugins', extraPluginDir) + } +} + diff --git a/test-framework/build.gradle b/test-framework/build.gradle new file mode 100644 index 00000000000..a423f56c922 --- /dev/null +++ b/test-framework/build.gradle @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import org.elasticsearch.gradle.precommit.PrecommitTasks + +apply plugin: 'elasticsearch.build' +apply plugin: 'com.bmuschko.nexus' + +dependencies { + compile "org.elasticsearch:elasticsearch:${version}" + compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + compile "junit:junit:${versions.junit}" + compile 'org.hamcrest:hamcrest-all:1.3' + compile "org.apache.lucene:lucene-test-framework:${versions.lucene}" + compile "org.apache.lucene:lucene-codecs:${versions.lucene}" + compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile "commons-codec:commons-codec:${versions.commonscodec}" +} + +compileJava.options.compilerArgs << '-Xlint:-cast,-deprecation,-fallthrough,-overrides,-rawtypes,-serial,-try,-unchecked' +compileTestJava.options.compilerArgs << '-Xlint:-rawtypes' + +// the main files are actually test files, so use the appopriate forbidden api sigs +forbiddenApisMain { + bundledSignatures = ['jdk-unsafe', 'jdk-deprecated'] + signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'), + PrecommitTasks.getResource('/forbidden/test-signatures.txt')] +} + +// TODO: should we have licenses for our test deps? +dependencyLicenses.enabled = false + diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test-framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java similarity index 55% rename from core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java rename to test-framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java index 3d19c5fb296..68784083797 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +++ b/test-framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java @@ -19,28 +19,39 @@ package org.elasticsearch.bootstrap; +import com.carrotsearch.randomizedtesting.RandomizedRunner; + +import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestSecurityManager; +import org.elasticsearch.SecureSM; import org.elasticsearch.bootstrap.Bootstrap; import org.elasticsearch.bootstrap.ESPolicy; import org.elasticsearch.bootstrap.Security; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.plugins.PluginInfo; +import org.junit.Assert; import java.io.FilePermission; import java.io.InputStream; -import java.net.URI; +import java.net.SocketPermission; import java.net.URL; import java.nio.file.Path; import java.security.Permission; -import java.security.PermissionCollection; import java.security.Permissions; import java.security.Policy; -import java.security.URIParameter; +import java.security.ProtectionDomain; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Properties; +import java.util.Set; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; @@ -73,6 +84,9 @@ public class BootstrapForTesting { // initialize probes Bootstrap.initializeProbes(); + // initialize sysprops + BootstrapInfo.getSystemProperties(); + // check for jar hell try { JarHell.checkJarHell(); @@ -83,16 +97,12 @@ public class BootstrapForTesting { // install security manager if requested if (systemPropertyAsBoolean("tests.security.manager", true)) { try { - Security.setCodebaseProperties(); // initialize paths the same exact way as bootstrap Permissions perms = new Permissions(); - // add permissions to everything in classpath + Security.addClasspathPermissions(perms); + // crazy jython for (URL url : JarHell.parseClassPath()) { Path path = PathUtils.get(url.toURI()); - // resource itself - perms.add(new FilePermission(path.toString(), "read,readlink")); - // classes underneath - perms.add(new FilePermission(path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink")); // crazy jython... String filename = path.getFileName().toString(); @@ -120,32 +130,26 @@ public class BootstrapForTesting { if (System.getProperty("tests.maven") == null) { perms.add(new RuntimePermission("setIO")); } - - final Policy policy; - // if its a plugin with special permissions, we use a wrapper policy impl to try - // to simulate what happens with a real distribution - List pluginPolicies = Collections.list(BootstrapForTesting.class.getClassLoader().getResources(PluginInfo.ES_PLUGIN_POLICY)); - if (!pluginPolicies.isEmpty()) { - Permissions extra = new Permissions(); - for (URL url : pluginPolicies) { - URI uri = url.toURI(); - Policy pluginPolicy = Policy.getInstance("JavaPolicy", new URIParameter(uri)); - PermissionCollection permissions = pluginPolicy.getPermissions(BootstrapForTesting.class.getProtectionDomain()); - // this method is supported with the specific implementation we use, but just check for safety. - if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) { - throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions"); - } - for (Permission permission : Collections.list(permissions.elements())) { - extra.add(permission); - } + + // add bind permissions for testing + // ephemeral ports (note, on java 7 before update 51, this is a different permission) + // this should really be the only one allowed for tests, otherwise they have race conditions + perms.add(new SocketPermission("localhost:0", "listen,resolve")); + // ... but tests are messy. like file permissions, just let them live in a fantasy for now. + // TODO: cut over all tests to bind to ephemeral ports + perms.add(new SocketPermission("localhost:1024-", "listen,resolve")); + + // read test-framework permissions + final Policy testFramework = Security.readPolicy(Bootstrap.class.getResource("test-framework.policy"), JarHell.parseClassPath()); + final Policy esPolicy = new ESPolicy(perms, getPluginPermissions(), true); + Policy.setPolicy(new Policy() { + @Override + public boolean implies(ProtectionDomain domain, Permission permission) { + // implements union + return esPolicy.implies(domain, permission) || testFramework.implies(domain, permission); } - // TODO: try to get rid of this class now that the world is simpler? - policy = new MockPluginPolicy(perms, extra); - } else { - policy = new ESPolicy(perms, Collections.emptyMap()); - } - Policy.setPolicy(policy); - System.setSecurityManager(new TestSecurityManager()); + }); + System.setSecurityManager(new SecureSM(true)); Security.selfTest(); // guarantee plugin classes are initialized first, in case they have one-time hacks. @@ -168,6 +172,71 @@ public class BootstrapForTesting { } } + /** + * we dont know which codesources belong to which plugin, so just remove the permission from key codebases + * like core, test-framework, etc. this way tests fail if accesscontroller blocks are missing. + */ + @SuppressForbidden(reason = "accesses fully qualified URLs to configure security") + static Map getPluginPermissions() throws Exception { + List pluginPolicies = Collections.list(BootstrapForTesting.class.getClassLoader().getResources(PluginInfo.ES_PLUGIN_POLICY)); + if (pluginPolicies.isEmpty()) { + return Collections.emptyMap(); + } + + // compute classpath minus obvious places, all other jars will get the permission. + Set codebases = new HashSet<>(Arrays.asList(parseClassPathWithSymlinks())); + Set excluded = new HashSet<>(Arrays.asList( + // es core + Bootstrap.class.getProtectionDomain().getCodeSource().getLocation(), + // es test framework + BootstrapForTesting.class.getProtectionDomain().getCodeSource().getLocation(), + // lucene test framework + LuceneTestCase.class.getProtectionDomain().getCodeSource().getLocation(), + // randomized runner + RandomizedRunner.class.getProtectionDomain().getCodeSource().getLocation(), + // junit library + Assert.class.getProtectionDomain().getCodeSource().getLocation() + )); + codebases.removeAll(excluded); + + // parse each policy file, with codebase substitution from the classpath + final List policies = new ArrayList<>(); + for (URL policyFile : pluginPolicies) { + policies.add(Security.readPolicy(policyFile, codebases.toArray(new URL[codebases.size()]))); + } + + // consult each policy file for those codebases + Map map = new HashMap<>(); + for (URL url : codebases) { + map.put(url.getFile(), new Policy() { + @Override + public boolean implies(ProtectionDomain domain, Permission permission) { + // implements union + for (Policy p : policies) { + if (p.implies(domain, permission)) { + return true; + } + } + return false; + } + }); + } + return Collections.unmodifiableMap(map); + } + + /** + * return parsed classpath, but with symlinks resolved to destination files for matching + * this is for matching the toRealPath() in the code where we have a proper plugin structure + */ + @SuppressForbidden(reason = "does evil stuff with paths and urls because devs and jenkins do evil stuff with paths and urls") + static URL[] parseClassPathWithSymlinks() throws Exception { + URL raw[] = JarHell.parseClassPath(); + for (int i = 0; i < raw.length; i++) { + raw[i] = PathUtils.get(raw[i].toURI()).toRealPath().toUri().toURL(); + } + return raw; + } + // does nothing, just easy way to make sure the class is loaded. public static void ensureInitialized() {} } diff --git a/core/src/test/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java b/test-framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java similarity index 100% rename from core/src/test/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java rename to test-framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java diff --git a/core/src/test/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test-framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java similarity index 99% rename from core/src/test/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java rename to test-framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index dd1cb0b9eff..6ac2101fe52 100644 --- a/core/src/test/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test-framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -73,7 +73,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService { null, null, null, null, null, fsInfo, null, null, null, - null); + null, null); } @Inject diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test-framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java similarity index 100% rename from core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java rename to test-framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java diff --git a/core/src/test/java/org/elasticsearch/common/cli/CliToolTestCase.java b/test-framework/src/main/java/org/elasticsearch/common/cli/CliToolTestCase.java similarity index 96% rename from core/src/test/java/org/elasticsearch/common/cli/CliToolTestCase.java rename to test-framework/src/main/java/org/elasticsearch/common/cli/CliToolTestCase.java index 38515ff14c9..ab304c28c54 100644 --- a/core/src/test/java/org/elasticsearch/common/cli/CliToolTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/common/cli/CliToolTestCase.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.cli; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.StreamsUtils; import org.junit.After; @@ -40,11 +41,13 @@ import static org.hamcrest.Matchers.hasSize; public abstract class CliToolTestCase extends ESTestCase { @Before + @SuppressForbidden(reason = "sets es.default.path.home during tests") public void setPathHome() { System.setProperty("es.default.path.home", createTempDir().toString()); } @After + @SuppressForbidden(reason = "clears es.default.path.home during tests") public void clearPathHome() { System.clearProperty("es.default.path.home"); } diff --git a/test-framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java b/test-framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java new file mode 100644 index 00000000000..50d677c600c --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.io; + +import org.junit.Assert; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; + +/** test helper methods for working with files */ +public class FileTestUtils { + + /** + * Check that a file contains a given String + * @param dir root dir for file + * @param filename relative path from root dir to file + * @param expected expected content (if null, we don't expect any file) + */ + public static void assertFileContent(Path dir, String filename, String expected) throws IOException { + Assert.assertThat(Files.exists(dir), is(true)); + Path file = dir.resolve(filename); + if (expected == null) { + Assert.assertThat("file [" + file + "] should not exist.", Files.exists(file), is(false)); + } else { + assertFileExists(file); + String fileContent = new String(Files.readAllBytes(file), java.nio.charset.StandardCharsets.UTF_8); + // trim the string content to prevent different handling on windows vs. unix and CR chars... + Assert.assertThat(fileContent.trim(), equalTo(expected.trim())); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/common/io/PathUtilsForTesting.java b/test-framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java similarity index 100% rename from core/src/test/java/org/elasticsearch/common/io/PathUtilsForTesting.java rename to test-framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java diff --git a/core/src/test/java/org/elasticsearch/common/util/MockBigArrays.java b/test-framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java similarity index 100% rename from core/src/test/java/org/elasticsearch/common/util/MockBigArrays.java rename to test-framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java diff --git a/test-framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test-framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java new file mode 100644 index 00000000000..8b529f9fc8f --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; + + +public class MapperTestUtils { + + public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException { + IndicesModule indicesModule = new IndicesModule(); + return newMapperService(tempDir, indexSettings, indicesModule); + } + + public static MapperService newMapperService(Path tempDir, Settings settings, IndicesModule indicesModule) throws IOException { + Settings.Builder settingsBuilder = Settings.builder() + .put("path.home", tempDir) + .put(settings); + if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) { + settingsBuilder.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + } + Settings finalSettings = settingsBuilder.build(); + MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(new Index("test"), finalSettings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(finalSettings)).build(indexSettings); + SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); + return new MapperService(indexSettings, + analysisService, + similarityService, + mapperRegistry); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/MockEngineFactoryPlugin.java b/test-framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java similarity index 62% rename from core/src/test/java/org/elasticsearch/index/MockEngineFactoryPlugin.java rename to test-framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java index 94ddde0e3fb..db63d137f32 100644 --- a/core/src/test/java/org/elasticsearch/index/MockEngineFactoryPlugin.java +++ b/test-framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java @@ -18,12 +18,12 @@ */ package org.elasticsearch.index; +import org.apache.lucene.index.AssertingDirectoryReader; +import org.apache.lucene.index.FilterDirectoryReader; +import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexModule; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.engine.MockEngineFactory; -import org.elasticsearch.test.engine.MockEngineSupportModule; import java.util.Collection; import java.util.Collections; @@ -38,11 +38,27 @@ public class MockEngineFactoryPlugin extends Plugin { public String description() { return "a mock engine factory for testing"; } + + private Class readerWrapper = AssertingDirectoryReader.class; + @Override - public Collection indexModules(Settings indexSettings) { - return Collections.singletonList(new MockEngineSupportModule()); + public void onIndexModule(IndexModule module) { + module.engineFactory.set(new MockEngineFactory(readerWrapper)); } - public void onModule(IndexModule module) { - module.engineFactoryImpl = MockEngineFactory.class; + + @Override + public Collection nodeModules() { + return Collections.singleton(new MockEngineReaderModule()); + } + + public class MockEngineReaderModule extends AbstractModule { + + public void setReaderClass(Class readerWrapper) { + MockEngineFactoryPlugin.this.readerWrapper = readerWrapper; + } + + @Override + protected void configure() { + } } } diff --git a/core/src/test/java/org/elasticsearch/node/MockNode.java b/test-framework/src/main/java/org/elasticsearch/node/MockNode.java similarity index 91% rename from core/src/test/java/org/elasticsearch/node/MockNode.java rename to test-framework/src/main/java/org/elasticsearch/node/MockNode.java index c5592fef48d..57dcc08f4fe 100644 --- a/core/src/test/java/org/elasticsearch/node/MockNode.java +++ b/test-framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -21,6 +21,7 @@ package org.elasticsearch.node; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.plugins.Plugin; import java.util.Collection; @@ -39,7 +40,7 @@ public class MockNode extends Node { private Collection> plugins; public MockNode(Settings settings, Version version, Collection> classpathPlugins) { - super(settings, version, classpathPlugins); + super(InternalSettingsPreparer.prepareEnvironment(settings, null), version, classpathPlugins); this.version = version; this.plugins = classpathPlugins; } diff --git a/core/src/test/java/org/elasticsearch/node/NodeMocksPlugin.java b/test-framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java similarity index 100% rename from core/src/test/java/org/elasticsearch/node/NodeMocksPlugin.java rename to test-framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java b/test-framework/src/main/java/org/elasticsearch/percolator/PercolatorTestUtil.java similarity index 100% rename from core/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java rename to test-framework/src/main/java/org/elasticsearch/percolator/PercolatorTestUtil.java diff --git a/test-framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java b/test-framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java new file mode 100644 index 00000000000..10f4de2482a --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +/** Utility methods for testing plugins */ +public class PluginTestUtil { + + /** convenience method to write a plugin properties file */ + public static void writeProperties(Path pluginDir, String... stringProps) throws IOException { + assert stringProps.length % 2 == 0; + Files.createDirectories(pluginDir); + Path propertiesFile = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES); + Properties properties = new Properties(); + for (int i = 0; i < stringProps.length; i += 2) { + properties.put(stringProps[i], stringProps[i + 1]); + } + try (OutputStream out = Files.newOutputStream(propertiesFile)) { + properties.store(out, ""); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/script/MockScriptEngine.java b/test-framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java similarity index 90% rename from core/src/test/java/org/elasticsearch/script/MockScriptEngine.java rename to test-framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 1cdac14f3ef..bfd40900456 100644 --- a/core/src/test/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test-framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -21,6 +21,8 @@ package org.elasticsearch.script; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.lookup.SearchLookup; @@ -71,12 +73,17 @@ public class MockScriptEngine implements ScriptEngineService { @Override public Object compile(String script) { - return Integer.parseInt(script); + return script; } @Override public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map vars) { - return null; + return new AbstractExecutableScript() { + @Override + public Object run() { + return new BytesArray((String)compiledScript.compiled()); + } + }; } @Override diff --git a/core/src/test/java/org/elasticsearch/search/MockSearchService.java b/test-framework/src/main/java/org/elasticsearch/search/MockSearchService.java similarity index 100% rename from core/src/test/java/org/elasticsearch/search/MockSearchService.java rename to test-framework/src/main/java/org/elasticsearch/search/MockSearchService.java diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java b/test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java similarity index 100% rename from core/src/test/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java rename to test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java b/test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java similarity index 100% rename from core/src/test/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java rename to test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java b/test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java similarity index 100% rename from core/src/test/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java rename to test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java b/test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java similarity index 100% rename from core/src/test/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java rename to test-framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java b/test-framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java similarity index 77% rename from core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java rename to test-framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java index d60a8706109..ece26be8239 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java @@ -31,7 +31,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; */ @ESIntegTestCase.SuiteScopeTestCase public abstract class AbstractNumericTestCase extends ESIntegTestCase { - protected static long minValue, maxValue, minValues, maxValues; @Override @@ -77,30 +76,25 @@ public abstract class AbstractNumericTestCase extends ESIntegTestCase { public abstract void testSingleValuedField() throws Exception; - public abstract void testSingleValuedField_getProperty() throws Exception; + public abstract void testSingleValuedFieldGetProperty() throws Exception; - public abstract void testSingleValuedField_PartiallyUnmapped() throws Exception; + public abstract void testSingleValuedFieldPartiallyUnmapped() throws Exception; - public abstract void testSingleValuedField_WithValueScript() throws Exception; + public abstract void testSingleValuedFieldWithValueScript() throws Exception; - public abstract void testSingleValuedField_WithValueScript_WithParams() throws Exception; + public abstract void testSingleValuedFieldWithValueScriptWithParams() throws Exception; public abstract void testMultiValuedField() throws Exception; - public abstract void testMultiValuedField_WithValueScript() throws Exception; + public abstract void testMultiValuedFieldWithValueScript() throws Exception; - public abstract void testMultiValuedField_WithValueScript_WithParams() throws Exception; + public abstract void testMultiValuedFieldWithValueScriptWithParams() throws Exception; - public abstract void testScript_SingleValued() throws Exception; + public abstract void testScriptSingleValued() throws Exception; - public abstract void testScript_SingleValued_WithParams() throws Exception; + public abstract void testScriptSingleValuedWithParams() throws Exception; - public abstract void testScript_ExplicitSingleValued_WithParams() throws Exception; - - public abstract void testScript_MultiValued() throws Exception; - - public abstract void testScript_ExplicitMultiValued() throws Exception; - - public abstract void testScript_MultiValued_WithParams() throws Exception; + public abstract void testScriptMultiValued() throws Exception; + public abstract void testScriptMultiValuedWithParams() throws Exception; } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/test/BackgroundIndexer.java b/test-framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java similarity index 99% rename from core/src/test/java/org/elasticsearch/test/BackgroundIndexer.java rename to test-framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java index 764c85657d7..933f26e6e81 100644 --- a/core/src/test/java/org/elasticsearch/test/BackgroundIndexer.java +++ b/test-framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java @@ -195,6 +195,7 @@ public class BackgroundIndexer implements AutoCloseable { XContentBuilder builder = XContentFactory.smileBuilder(); builder.startObject().field("test", "value" + id) .field("text", text.toString()) + .field("id", id) .endObject(); return builder; diff --git a/core/src/test/java/org/elasticsearch/test/CompositeTestCluster.java b/test-framework/src/main/java/org/elasticsearch/test/CompositeTestCluster.java similarity index 96% rename from core/src/test/java/org/elasticsearch/test/CompositeTestCluster.java rename to test-framework/src/main/java/org/elasticsearch/test/CompositeTestCluster.java index 8b14ef06b06..caa414e071c 100644 --- a/core/src/test/java/org/elasticsearch/test/CompositeTestCluster.java +++ b/test-framework/src/main/java/org/elasticsearch/test/CompositeTestCluster.java @@ -63,19 +63,14 @@ public class CompositeTestCluster extends TestCluster { } @Override - public synchronized void beforeTest(Random random, double transportClientRatio) throws IOException { + public synchronized void beforeTest(Random random, double transportClientRatio) throws IOException, InterruptedException { super.beforeTest(random, transportClientRatio); cluster.beforeTest(random, transportClientRatio); Settings defaultSettings = cluster.getDefaultSettings(); final Client client = cluster.size() > 0 ? cluster.client() : cluster.clientNodeClient(); for (int i = 0; i < externalNodes.length; i++) { if (!externalNodes[i].running()) { - try { - externalNodes[i] = externalNodes[i].start(client, defaultSettings, NODE_PREFIX + i, cluster.getClusterName(), i); - } catch (InterruptedException e) { - Thread.interrupted(); - return; - } + externalNodes[i] = externalNodes[i].start(client, defaultSettings, NODE_PREFIX + i, cluster.getClusterName(), i); } externalNodes[i].reset(random.nextLong()); } diff --git a/core/src/test/java/org/elasticsearch/test/CorruptionUtils.java b/test-framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/CorruptionUtils.java rename to test-framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java diff --git a/core/src/test/java/org/elasticsearch/test/DummyShardLock.java b/test-framework/src/main/java/org/elasticsearch/test/DummyShardLock.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/DummyShardLock.java rename to test-framework/src/main/java/org/elasticsearch/test/DummyShardLock.java diff --git a/core/src/test/java/org/elasticsearch/test/ESAllocationTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java similarity index 64% rename from core/src/test/java/org/elasticsearch/test/ESAllocationTestCase.java rename to test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java index d12f4d68752..e82823ae997 100644 --- a/core/src/test/java/org/elasticsearch/test/ESAllocationTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java @@ -25,26 +25,30 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.gateway.AsyncShardFetch; +import org.elasticsearch.gateway.GatewayAllocator; +import org.elasticsearch.gateway.ReplicaShardAllocator; +import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.gateway.NoopGatewayAllocator; import java.lang.reflect.Constructor; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; +import java.util.*; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; @@ -55,30 +59,36 @@ import static org.hamcrest.CoreMatchers.is; */ public abstract class ESAllocationTestCase extends ESTestCase { - public static AllocationService createAllocationService() { + public static MockAllocationService createAllocationService() { return createAllocationService(Settings.Builder.EMPTY_SETTINGS); } - public static AllocationService createAllocationService(Settings settings) { + public static MockAllocationService createAllocationService(Settings settings) { return createAllocationService(settings, getRandom()); } - public static AllocationService createAllocationService(Settings settings, Random random) { + public static MockAllocationService createAllocationService(Settings settings, Random random) { return createAllocationService(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), random); } - public static AllocationService createAllocationService(Settings settings, NodeSettingsService nodeSettingsService, Random random) { - return new AllocationService(settings, + public static MockAllocationService createAllocationService(Settings settings, NodeSettingsService nodeSettingsService, Random random) { + return new MockAllocationService(settings, randomAllocationDeciders(settings, nodeSettingsService, random), new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE); } - public static AllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) { - return new AllocationService(settings, + public static MockAllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) { + return new MockAllocationService(settings, randomAllocationDeciders(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), getRandom()), new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), clusterInfoService); } + public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator allocator) { + return new MockAllocationService(settings, + randomAllocationDeciders(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), getRandom()), + new ShardsAllocators(settings, allocator), EmptyClusterInfoService.INSTANCE); + } + public static AllocationDeciders randomAllocationDeciders(Settings settings, NodeSettingsService nodeSettingsService, Random random) { @@ -102,7 +112,7 @@ public abstract class ESAllocationTestCase extends ESTestCase { for (AllocationDecider d : list) { assertThat(defaultAllocationDeciders.contains(d.getClass()), is(true)); } - Collections.shuffle(list, random); + Randomness.shuffle(list); return new AllocationDeciders(settings, list.toArray(new AllocationDecider[0])); } @@ -172,4 +182,60 @@ public abstract class ESAllocationTestCase extends ESTestCase { return decision; } } + + /** A lock {@link AllocationService} allowing tests to override time */ + protected static class MockAllocationService extends AllocationService { + + private Long nanoTimeOverride = null; + + public MockAllocationService(Settings settings, AllocationDeciders allocationDeciders, ShardsAllocators shardsAllocators, ClusterInfoService clusterInfoService) { + super(settings, allocationDeciders, shardsAllocators, clusterInfoService); + } + + public void setNanoTimeOverride(long nanoTime) { + this.nanoTimeOverride = nanoTime; + } + + @Override + protected long currentNanoTime() { + return nanoTimeOverride == null ? super.currentNanoTime() : nanoTimeOverride; + } + } + + /** + * Mocks behavior in ReplicaShardAllocator to remove delayed shards from list of unassigned shards so they don't get reassigned yet. + */ + protected static class DelayedShardsMockGatewayAllocator extends GatewayAllocator { + private final ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator(Settings.EMPTY) { + @Override + protected AsyncShardFetch.FetchResult fetchData(ShardRouting shard, RoutingAllocation allocation) { + return new AsyncShardFetch.FetchResult<>(shard.shardId(), null, Collections.emptySet(), Collections.emptySet()); + } + }; + + + public DelayedShardsMockGatewayAllocator() { + super(Settings.EMPTY, null, null); + } + + @Override + public void applyStartedShards(StartedRerouteAllocation allocation) {} + + @Override + public void applyFailedShards(FailedRerouteAllocation allocation) {} + + @Override + public boolean allocateUnassigned(RoutingAllocation allocation) { + final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator(); + boolean changed = false; + while (unassignedIterator.hasNext()) { + ShardRouting shard = unassignedIterator.next(); + if (shard.primary() || shard.allocatedPostIndexCreate() == false) { + continue; + } + changed |= replicaShardAllocator.ignoreUnassignedIfDelayed(unassignedIterator, shard); + } + return changed; + } + } } diff --git a/core/src/test/java/org/elasticsearch/test/ESBackcompatTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java similarity index 93% rename from core/src/test/java/org/elasticsearch/test/ESBackcompatTestCase.java rename to test-framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java index 47e163a6291..3e5c903a1ba 100644 --- a/core/src/test/java/org/elasticsearch/test/ESBackcompatTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java @@ -131,16 +131,6 @@ public abstract class ESBackcompatTestCase extends ESIntegTestCase { return file; } - @Override - protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) { - if (globalCompatibilityVersion().before(Version.V_1_3_2)) { - // if we test against nodes before 1.3.2 we disable all the compression due to a known bug - // see #7210 - builder.put(RecoverySettings.INDICES_RECOVERY_COMPRESS, false); - } - return builder; - } - /** * Retruns the tests compatibility version. */ @@ -250,13 +240,6 @@ public abstract class ESBackcompatTestCase extends ESIntegTestCase { Settings.Builder builder = Settings.builder().put(requiredSettings()); builder.put(TransportModule.TRANSPORT_TYPE_KEY, "netty"); // run same transport / disco as external builder.put("node.mode", "network"); - - if (compatibilityVersion().before(Version.V_1_3_2)) { - // if we test against nodes before 1.3.2 we disable all the compression due to a known bug - // see #7210 - builder.put(Transport.TransportSettings.TRANSPORT_TCP_COMPRESS, false) - .put(RecoverySettings.INDICES_RECOVERY_COMPRESS, false); - } return builder.build(); } diff --git a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java similarity index 95% rename from core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java rename to test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 0597ad9078a..37a8fd388b8 100644 --- a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -24,9 +24,11 @@ import com.carrotsearch.randomizedtesting.Randomness; import com.carrotsearch.randomizedtesting.annotations.TestGroup; import com.carrotsearch.randomizedtesting.generators.RandomInts; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.http.impl.client.HttpClients; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SuppressForbidden; import org.apache.lucene.util.TestUtil; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; @@ -34,7 +36,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -43,7 +45,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; @@ -96,7 +98,6 @@ import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.codec.CodecService; -import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Loading; @@ -109,7 +110,6 @@ import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.index.translog.TranslogWriter; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.cache.request.IndicesRequestCache; -import org.elasticsearch.indices.flush.IndicesSyncedFlushResult; import org.elasticsearch.indices.flush.SyncedFlushService; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.node.Node; @@ -129,15 +129,35 @@ import org.junit.BeforeClass; import java.io.IOException; import java.io.InputStream; -import java.lang.annotation.*; +import java.lang.annotation.Annotation; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URL; import java.net.UnknownHostException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; @@ -149,8 +169,15 @@ import static org.elasticsearch.common.util.CollectionUtils.eagerPartition; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.XContentTestUtils.convertToMap; import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; +import static org.hamcrest.Matchers.emptyArray; +import static org.hamcrest.Matchers.emptyIterable; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; /** * {@link ESIntegTestCase} is an abstract base class to run integration @@ -168,8 +195,8 @@ import static org.hamcrest.Matchers.*; * should be used, here is an example: * 

      *
- * @ClusterScope(scope=Scope.TEST) public class SomeIT extends ESIntegTestCase {
- * @Test public void testMethod() {}
+ * {@literal @}ClusterScope(scope=Scope.TEST) public class SomeIT extends ESIntegTestCase {
+ * public void testMethod() {}
  * }
  *
    *

    @@ -179,9 +206,9 @@ import static org.hamcrest.Matchers.*; * determined at random and can change across tests. The {@link ClusterScope} allows configuring the initial number of nodes * that are created before the tests start. * 

    - * @ClusterScope(scope=Scope.SUITE, numDataNodes=3)
+ * {@literal @}ClusterScope(scope=Scope.SUITE, numDataNodes=3)
  * public class SomeIT extends ESIntegTestCase {
- * @Test public void testMethod() {}
+ * public void testMethod() {}
  * }
  *
    *

    @@ -301,29 +328,21 @@ public abstract class ESIntegTestCase extends ESTestCase { } protected final void beforeInternal() throws Exception { - assert Thread.getDefaultUncaughtExceptionHandler() instanceof ElasticsearchUncaughtExceptionHandler; - try { - final Scope currentClusterScope = getCurrentClusterScope(); - switch (currentClusterScope) { - case SUITE: - assert SUITE_SEED != null : "Suite seed was not initialized"; - currentCluster = buildAndPutCluster(currentClusterScope, SUITE_SEED); - break; - case TEST: - currentCluster = buildAndPutCluster(currentClusterScope, randomLong()); - break; - default: - fail("Unknown Scope: [" + currentClusterScope + "]"); - } - cluster().beforeTest(getRandom(), getPerTestTransportClientRatio()); - cluster().wipe(excludeTemplates()); - randomIndexTemplate(); - } catch (OutOfMemoryError e) { - if (e.getMessage().contains("unable to create new native thread")) { - ESTestCase.printStackDump(logger); - } - throw e; + final Scope currentClusterScope = getCurrentClusterScope(); + switch (currentClusterScope) { + case SUITE: + assert SUITE_SEED != null : "Suite seed was not initialized"; + currentCluster = buildAndPutCluster(currentClusterScope, SUITE_SEED); + break; + case TEST: + currentCluster = buildAndPutCluster(currentClusterScope, randomLong()); + break; + default: + fail("Unknown Scope: [" + currentClusterScope + "]"); } + cluster().beforeTest(getRandom(), getPerTestTransportClientRatio()); + cluster().wipe(excludeTemplates()); + randomIndexTemplate(); } private void printTestMessage(String message) { @@ -384,7 +403,6 @@ public abstract class ESIntegTestCase extends ESTestCase { .startObject("template-longs") .field("match_mapping_type", "long") .startObject("mapping") - .field("doc_values", randomBoolean()) .startObject("fielddata") .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)) .endObject() @@ -395,7 +413,6 @@ public abstract class ESIntegTestCase extends ESTestCase { .startObject("template-doubles") .field("match_mapping_type", "double") .startObject("mapping") - .field("doc_values", randomBoolean()) .startObject("fielddata") .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)) .endObject() @@ -406,7 +423,6 @@ public abstract class ESIntegTestCase extends ESTestCase { .startObject("template-geo_points") .field("match_mapping_type", "geo_point") .startObject("mapping") - .field("doc_values", randomBoolean()) .startObject("fielddata") .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)) .endObject() @@ -418,7 +434,6 @@ public abstract class ESIntegTestCase extends ESTestCase { .field("match_mapping_type", "boolean") .startObject("mapping") .startObject("fielddata") - .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")) .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)) .endObject() .endObject() @@ -438,7 +453,7 @@ public abstract class ESIntegTestCase extends ESTestCase { .setOrder(0) .setSettings(randomSettingsBuilder); if (mappings != null) { - logger.info("test using _default_ mappings: [{}]", mappings.bytesStream().bytes().toUtf8()); + logger.info("test using _default_ mappings: [{}]", mappings.bytes().toUtf8()); putTemplate.addMapping("_default_", mappings); } assertAcked(putTemplate.execute().actionGet()); @@ -521,7 +536,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } private TestCluster buildWithPrivateContext(final Scope scope, final long seed) throws Exception { - return RandomizedContext.current().runWithPrivateRandomness(new Randomness(seed), new Callable() { + return RandomizedContext.current().runWithPrivateRandomness(new com.carrotsearch.randomizedtesting.Randomness(seed), new Callable() { @Override public TestCluster call() throws Exception { return buildTestCluster(scope, seed); @@ -993,7 +1008,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } if (lastKnownCount.get() >= numDocs) { try { - long count = client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount(); + long count = client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet().getHits().totalHits(); if (count == lastKnownCount.get()) { // no progress - try to refresh for the next time client().admin().indices().prepareRefresh().get(); @@ -1023,7 +1038,7 @@ public abstract class ESIntegTestCase extends ESTestCase { /** * Sets the cluster's minimum master node and make sure the response is acknowledge. - * Note: this doesn't guaranty the new settings is in effect, just that it has been received bu all nodes. + * Note: this doesn't guarantee that the new setting has taken effect, just that it has been received by all nodes. */ public void setMinimumMasterNodes(int n) { assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings( @@ -1251,11 +1266,11 @@ public abstract class ESIntegTestCase extends ESTestCase { } /** - * Waits for all relocations and optimized all indices in the cluster to 1 segment. + * Waits for all relocations and force merge all indices in the cluster to 1 segment. */ - protected OptimizeResponse optimize() { + protected ForceMergeResponse forceMerge() { waitForRelocation(); - OptimizeResponse actionGet = client().admin().indices().prepareOptimize().setMaxNumSegments(1).execute().actionGet(); + ForceMergeResponse actionGet = client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet(); assertNoFailures(actionGet); return actionGet; } @@ -1376,7 +1391,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } } final String[] indices = indicesSet.toArray(new String[indicesSet.size()]); - Collections.shuffle(builders, random); + Collections.shuffle(builders, random()); final CopyOnWriteArrayList> errors = new CopyOnWriteArrayList<>(); List inFlightAsyncOperations = new ArrayList<>(); // If you are indexing just a few documents then frequently do it one at a time. If many then frequently in bulk. @@ -1473,7 +1488,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } /** - * Maybe refresh, optimize, or flush then always make sure there aren't too many in flight async operations. + * Maybe refresh, force merge, or flush then always make sure there aren't too many in flight async operations. */ private void postIndexAsyncActions(String[] indices, List inFlightAsyncOperations, boolean maybeFlush) throws InterruptedException { if (rarely()) { @@ -1489,8 +1504,8 @@ public abstract class ESIntegTestCase extends ESTestCase { new LatchedActionListener<>(newLatch(inFlightAsyncOperations))); } } else if (rarely()) { - client().admin().indices().prepareOptimize(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).setMaxNumSegments(between(1, 10)).setFlush(maybeFlush && randomBoolean()).execute( - new LatchedActionListener<>(newLatch(inFlightAsyncOperations))); + client().admin().indices().prepareForceMerge(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).setMaxNumSegments(between(1, 10)).setFlush(maybeFlush && randomBoolean()).execute( + new LatchedActionListener(newLatch(inFlightAsyncOperations))); } } while (inFlightAsyncOperations.size() > MAX_IN_FLIGHT_ASYNC_INDEXES) { @@ -1715,20 +1730,14 @@ public abstract class ESIntegTestCase extends ESTestCase { return Settings.EMPTY; } - private ExternalTestCluster buildExternalCluster(String clusterAddresses) throws UnknownHostException { + private ExternalTestCluster buildExternalCluster(String clusterAddresses) throws IOException { String[] stringAddresses = clusterAddresses.split(","); TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; int i = 0; for (String stringAddress : stringAddresses) { - String[] split = stringAddress.split(":"); - if (split.length < 2) { - throw new IllegalArgumentException("address [" + clusterAddresses + "] not valid"); - } - try { - transportAddresses[i++] = new InetSocketTransportAddress(InetAddress.getByName(split[0]), Integer.valueOf(split[1])); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("port is not valid, expected number but was [" + split[1] + "]"); - } + URL url = new URL("http://" + stringAddress); + InetAddress inetAddress = InetAddress.getByName(url.getHost()); + transportAddresses[i++] = new InetSocketTransportAddress(new InetSocketAddress(inetAddress, url.getPort())); } return new ExternalTestCluster(createTempDir(), externalClusterClientSettings(), transportClientPlugins(), transportAddresses); } @@ -1839,13 +1848,6 @@ public abstract class ESIntegTestCase extends ESTestCase { return perTestRatio; } - /** - * Returns a random numeric field data format from the choices of "array" or "doc_values". - */ - public static String randomNumericFieldDataFormat() { - return randomFrom(Arrays.asList("array", "doc_values")); - } - /** * Returns a random JODA Time Zone based on Java Time Zones */ @@ -1892,8 +1894,8 @@ public abstract class ESIntegTestCase extends ESTestCase { protected NumShards getNumShards(String index) { MetaData metaData = client().admin().cluster().prepareState().get().getState().metaData(); assertThat(metaData.hasIndex(index), equalTo(true)); - int numShards = Integer.valueOf(metaData.index(index).settings().get(SETTING_NUMBER_OF_SHARDS)); - int numReplicas = Integer.valueOf(metaData.index(index).settings().get(SETTING_NUMBER_OF_REPLICAS)); + int numShards = Integer.valueOf(metaData.index(index).getSettings().get(SETTING_NUMBER_OF_SHARDS)); + int numReplicas = Integer.valueOf(metaData.index(index).getSettings().get(SETTING_NUMBER_OF_REPLICAS)); return new NumShards(numShards, numReplicas); } @@ -2091,6 +2093,7 @@ public abstract class ESIntegTestCase extends ESTestCase { */ @Retention(RetentionPolicy.RUNTIME) @Inherited + @Target(ElementType.TYPE) public @interface SuiteScopeTestCase { } @@ -2099,6 +2102,7 @@ public abstract class ESIntegTestCase extends ESTestCase { */ @Retention(RetentionPolicy.RUNTIME) @Inherited + @Target(ElementType.TYPE) public @interface SuppressLocalMode { } @@ -2107,6 +2111,7 @@ public abstract class ESIntegTestCase extends ESTestCase { */ @Retention(RetentionPolicy.RUNTIME) @Inherited + @Target(ElementType.TYPE) public @interface SuppressNetworkMode { } diff --git a/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java similarity index 96% rename from core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java rename to test-framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index c877da877d4..287bd121c90 100644 --- a/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -19,13 +19,12 @@ package org.elasticsearch.test; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -39,7 +38,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.internal.SearchContext; @@ -49,7 +47,9 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; /** * A test that keep a singleton node started for all tests that can be used to get @@ -120,7 +120,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { } private static Node newNode() { - Node build = NodeBuilder.nodeBuilder().local(true).data(true).settings(Settings.builder() + Node build = new Node(Settings.builder() .put(ClusterName.SETTING, InternalTestCluster.clusterName("single-node-cluster", randomLong())) .put("path.home", createTempDir()) // TODO: use a consistent data path for custom paths @@ -133,8 +133,11 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { .put("script.indexed", "on") .put(EsExecutors.PROCESSORS, 1) // limit the number of threads created .put("http.enabled", false) + .put("node.local", true) + .put("node.data", true) .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) // make sure we get what we set :) - ).build(); + .build() + ); build.start(); assertThat(DiscoveryNode.localNode(build.settings()), is(true)); return build; diff --git a/core/src/test/java/org/elasticsearch/test/ESTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESTestCase.java similarity index 88% rename from core/src/test/java/org/elasticsearch/test/ESTestCase.java rename to test-framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 78d004f43e4..e1443110c0d 100644 --- a/core/src/test/java/org/elasticsearch/test/ESTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -36,11 +36,13 @@ import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.TestRuleMarkFailure; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.bootstrap.BootstrapForTesting; import org.elasticsearch.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; import org.elasticsearch.common.logging.ESLogger; @@ -145,20 +147,6 @@ public abstract class ESTestCase extends LuceneTestCase { PathUtilsForTesting.teardown(); } - // setup a default exception handler which knows when and how to print a stacktrace - private static Thread.UncaughtExceptionHandler defaultHandler; - - @BeforeClass - public static void setDefaultExceptionHandler() throws Exception { - defaultHandler = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler(defaultHandler)); - } - - @AfterClass - public static void restoreDefaultExceptionHandler() throws Exception { - Thread.setDefaultUncaughtExceptionHandler(defaultHandler); - } - // randomize content type for request builders @BeforeClass @@ -176,6 +164,7 @@ public abstract class ESTestCase extends LuceneTestCase { // randomize and override the number of cpus so tests reproduce regardless of real number of cpus @BeforeClass + @SuppressForbidden(reason = "sets the number of cpus during tests") public static void setProcessors() { int numCpu = TestUtil.nextInt(random(), 1, 4); System.setProperty(EsExecutors.DEFAULT_SYSPROP, Integer.toString(numCpu)); @@ -183,6 +172,7 @@ public abstract class ESTestCase extends LuceneTestCase { } @AfterClass + @SuppressForbidden(reason = "clears the number of cpus during tests") public static void restoreProcessors() { System.clearProperty(EsExecutors.DEFAULT_SYSPROP); } @@ -551,60 +541,6 @@ public abstract class ESTestCase extends LuceneTestCase { return builder; } - // ----------------------------------------------------------------- - // Failure utilities - // ----------------------------------------------------------------- - - static final class ElasticsearchUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler { - - private final Thread.UncaughtExceptionHandler parent; - private final ESLogger logger = Loggers.getLogger(getClass()); - - private ElasticsearchUncaughtExceptionHandler(Thread.UncaughtExceptionHandler parent) { - this.parent = parent; - } - - @Override - public void uncaughtException(Thread t, Throwable e) { - if (e instanceof EsRejectedExecutionException) { - if (e.getMessage() != null && ((EsRejectedExecutionException) e).isExecutorShutdown()) { - return; // ignore the EsRejectedExecutionException when a node shuts down - } - } else if (e instanceof OutOfMemoryError) { - if (e.getMessage() != null && e.getMessage().contains("unable to create new native thread")) { - printStackDump(logger); - } - } - parent.uncaughtException(t, e); - } - } - - protected static final void printStackDump(ESLogger logger) { - // print stack traces if we can't create any native thread anymore - Map allStackTraces = Thread.getAllStackTraces(); - logger.error(formatThreadStacks(allStackTraces)); - } - - /** Dump threads and their current stack trace. */ - public static String formatThreadStacks(Map threads) { - StringBuilder message = new StringBuilder(); - int cnt = 1; - final Formatter f = new Formatter(message, Locale.ENGLISH); - for (Map.Entry e : threads.entrySet()) { - if (e.getKey().isAlive()) { - f.format(Locale.ENGLISH, "\n %2d) %s", cnt++, threadName(e.getKey())).flush(); - } - if (e.getValue().length == 0) { - message.append("\n at (empty stack)"); - } else { - for (StackTraceElement ste : e.getValue()) { - message.append("\n at ").append(ste); - } - } - } - return message.toString(); - } - private static String threadName(Thread t) { return "Thread[" + "id=" + t.getId() + @@ -630,7 +566,7 @@ public abstract class ESTestCase extends LuceneTestCase { throw new IllegalArgumentException("Can\'t pick " + size + " random objects from a list of " + values.length + " objects"); } List list = arrayAsArrayList(values); - Collections.shuffle(list); + Collections.shuffle(list, random()); return list.subList(0, size); } @@ -679,9 +615,26 @@ public abstract class ESTestCase extends LuceneTestCase { sb.append("]"); assertThat(count + " files exist that should have been cleaned:\n" + sb.toString(), count, equalTo(0)); } - + /** Returns the suite failure marker: internal use only! */ public static TestRuleMarkFailure getSuiteFailureMarker() { return suiteFailureMarker; } + + /** Compares two stack traces, ignoring module (which is not yet serialized) */ + public static void assertArrayEquals(StackTraceElement expected[], StackTraceElement actual[]) { + assertEquals(expected.length, actual.length); + for (int i = 0; i < expected.length; i++) { + assertEquals(expected[i], actual[i]); + } + } + + /** Compares two stack trace elements, ignoring module (which is not yet serialized) */ + public static void assertEquals(StackTraceElement expected, StackTraceElement actual) { + assertEquals(expected.getClassName(), actual.getClassName()); + assertEquals(expected.getMethodName(), actual.getMethodName()); + assertEquals(expected.getFileName(), actual.getFileName()); + assertEquals(expected.getLineNumber(), actual.getLineNumber()); + assertEquals(expected.isNativeMethod(), actual.isNativeMethod()); + } } diff --git a/core/src/test/java/org/elasticsearch/test/ESTokenStreamTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESTokenStreamTestCase.java similarity index 99% rename from core/src/test/java/org/elasticsearch/test/ESTokenStreamTestCase.java rename to test-framework/src/main/java/org/elasticsearch/test/ESTokenStreamTestCase.java index 29a1a3362d9..ed54ae60fbd 100644 --- a/core/src/test/java/org/elasticsearch/test/ESTokenStreamTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESTokenStreamTestCase.java @@ -54,4 +54,5 @@ public abstract class ESTokenStreamTestCase extends BaseTokenStreamTestCase { public Settings.Builder newAnalysisSettingsBuilder() { return Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); } + } diff --git a/core/src/test/java/org/elasticsearch/test/ExternalNode.java b/test-framework/src/main/java/org/elasticsearch/test/ExternalNode.java similarity index 97% rename from core/src/test/java/org/elasticsearch/test/ExternalNode.java rename to test-framework/src/main/java/org/elasticsearch/test/ExternalNode.java index 3aeb473b965..6ab39a5b139 100644 --- a/core/src/test/java/org/elasticsearch/test/ExternalNode.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ExternalNode.java @@ -206,7 +206,7 @@ final class ExternalNode implements Closeable { this.random.setSeed(seed); } - synchronized void stop() { + synchronized void stop() throws InterruptedException { if (running()) { try { if (this.client != null) { @@ -214,11 +214,7 @@ final class ExternalNode implements Closeable { } } finally { process.destroy(); - try { - process.waitFor(); - } catch (InterruptedException e) { - Thread.interrupted(); - } + process.waitFor(); process = null; nodeInfo = null; @@ -233,7 +229,11 @@ final class ExternalNode implements Closeable { @Override public void close() { - stop(); + try { + stop(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } synchronized String getName() { diff --git a/core/src/test/java/org/elasticsearch/test/ExternalTestCluster.java b/test-framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java similarity index 80% rename from core/src/test/java/org/elasticsearch/test/ExternalTestCluster.java rename to test-framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java index 90ca7818b9f..34b6bfbfb14 100644 --- a/core/src/test/java/org/elasticsearch/test/ExternalTestCluster.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java @@ -81,26 +81,33 @@ public final class ExternalTestCluster extends TestCluster { for (Class pluginClass : pluginClasses) { transportClientBuilder.addPlugin(pluginClass); } - this.client = transportClientBuilder.build().addTransportAddresses(transportAddresses); + TransportClient client = transportClientBuilder.build(); - NodesInfoResponse nodeInfos = this.client.admin().cluster().prepareNodesInfo().clear().setSettings(true).setHttp(true).get(); - httpAddresses = new InetSocketAddress[nodeInfos.getNodes().length]; - this.clusterName = nodeInfos.getClusterName().value(); - int dataNodes = 0; - int masterAndDataNodes = 0; - for (int i = 0; i < nodeInfos.getNodes().length; i++) { - NodeInfo nodeInfo = nodeInfos.getNodes()[i]; - httpAddresses[i] = ((InetSocketTransportAddress) nodeInfo.getHttp().address().publishAddress()).address(); - if (DiscoveryNode.dataNode(nodeInfo.getSettings())) { - dataNodes++; - masterAndDataNodes++; - } else if (DiscoveryNode.masterNode(nodeInfo.getSettings())) { - masterAndDataNodes++; + try { + client.addTransportAddresses(transportAddresses); + NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear().setSettings(true).setHttp(true).get(); + httpAddresses = new InetSocketAddress[nodeInfos.getNodes().length]; + this.clusterName = nodeInfos.getClusterName().value(); + int dataNodes = 0; + int masterAndDataNodes = 0; + for (int i = 0; i < nodeInfos.getNodes().length; i++) { + NodeInfo nodeInfo = nodeInfos.getNodes()[i]; + httpAddresses[i] = ((InetSocketTransportAddress) nodeInfo.getHttp().address().publishAddress()).address(); + if (DiscoveryNode.dataNode(nodeInfo.getSettings())) { + dataNodes++; + masterAndDataNodes++; + } else if (DiscoveryNode.masterNode(nodeInfo.getSettings())) { + masterAndDataNodes++; + } } + this.numDataNodes = dataNodes; + this.numMasterAndDataNodes = masterAndDataNodes; + this.client = client; + logger.info("Setup ExternalTestCluster [{}] made of [{}] nodes", nodeInfos.getClusterName().value(), size()); + } catch (Exception e) { + client.close(); + throw e; } - this.numDataNodes = dataNodes; - this.numMasterAndDataNodes = masterAndDataNodes; - logger.info("Setup ExternalTestCluster [{}] made of [{}] nodes", nodeInfos.getClusterName().value(), size()); } @Override diff --git a/test-framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java b/test-framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java new file mode 100644 index 00000000000..5ce620166c1 --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldFilterLeafReader; +import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.LeafReader; + +import java.io.IOException; +import java.util.Collections; + +public class FieldMaskingReader extends FilterDirectoryReader { + private final String field; + public FieldMaskingReader(String field, DirectoryReader in) throws IOException { + super(in, new FilterDirectoryReader.SubReaderWrapper() { + @Override + public LeafReader wrap(LeafReader reader) { + return new FieldFilterLeafReader(reader, Collections.singleton(field), true); + } + }); + this.field = field; + + } + + @Override + protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { + return new FieldMaskingReader(field, in); + } + + @Override + public Object getCoreCacheKey() { + return in.getCoreCacheKey(); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsModule.java b/test-framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java similarity index 55% rename from core/src/main/java/org/elasticsearch/index/settings/IndexSettingsModule.java rename to test-framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java index d4a0637358f..39e1857f412 100644 --- a/core/src/main/java/org/elasticsearch/index/settings/IndexSettingsModule.java +++ b/test-framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java @@ -16,31 +16,44 @@ * specific language governing permissions and limitations * under the License. */ +package org.elasticsearch.test; -package org.elasticsearch.index.settings; - +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; + +import java.util.Collections; -/** - * - */ public class IndexSettingsModule extends AbstractModule { private final Index index; - private final Settings settings; public IndexSettingsModule(Index index, Settings settings) { - this.index = index; this.settings = settings; + this.index = index; + } @Override protected void configure() { - IndexSettingsService indexSettingsService = new IndexSettingsService(index, settings); - bind(IndexSettingsService.class).toInstance(indexSettingsService); - bind(Settings.class).annotatedWith(IndexSettings.class).toProvider(new IndexSettingsProvider(indexSettingsService)); + bind(IndexSettings.class).toInstance(newIndexSettings(index, settings)); + } + + public static IndexSettings newIndexSettings(String index, Settings settings) { + return newIndexSettings(new Index(index), settings); + } + + public static IndexSettings newIndexSettings(Index index, Settings settings) { + Settings build = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(settings) + .build(); + IndexMetaData metaData = IndexMetaData.builder(index.getName()).settings(build).build(); + return new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList()); } } diff --git a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/test-framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java similarity index 95% rename from core/src/test/java/org/elasticsearch/test/InternalTestCluster.java rename to test-framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 22973d425a3..7ae3226b66a 100644 --- a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test-framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -60,23 +60,22 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.discovery.DiscoveryService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.cache.IndexCacheModule; +import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineClosedException; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.IndexStoreConfig; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; -import org.elasticsearch.indices.cache.request.IndicesRequestCache; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeMocksPlugin; @@ -90,7 +89,6 @@ import org.elasticsearch.test.disruption.ServiceDisruptionScheme; import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.AssertingLocalTransport; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.netty.NettyTransport; @@ -100,20 +98,11 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.*; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; @@ -121,15 +110,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static junit.framework.Assert.fail; -import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY; -import static org.apache.lucene.util.LuceneTestCase.rarely; -import static org.apache.lucene.util.LuceneTestCase.usually; +import static org.apache.lucene.util.LuceneTestCase.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.ESTestCase.assertBusy; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.*; import static org.junit.Assert.assertThat; /** @@ -406,18 +391,6 @@ public final class InternalTestCluster extends TestCluster { if (random.nextBoolean()) { // sometimes set a builder.put(SearchService.DEFAULT_KEEPALIVE_KEY, TimeValue.timeValueSeconds(100 + random.nextInt(5 * 60))); } - if (random.nextBoolean()) { - // change threadpool types to make sure we don't have components that rely on the type of thread pools - for (String name : Arrays.asList(ThreadPool.Names.BULK, ThreadPool.Names.FLUSH, ThreadPool.Names.GET, - ThreadPool.Names.INDEX, ThreadPool.Names.MANAGEMENT, ThreadPool.Names.OPTIMIZE, - ThreadPool.Names.PERCOLATE, ThreadPool.Names.REFRESH, ThreadPool.Names.SEARCH, ThreadPool.Names.SNAPSHOT, - ThreadPool.Names.SUGGEST, ThreadPool.Names.WARMER)) { - if (random.nextBoolean()) { - final String type = RandomPicks.randomFrom(random, Arrays.asList("fixed", "cached", "scaling")); - builder.put(ThreadPool.THREADPOOL_GROUP + name + ".type", type); - } - } - } if (random.nextInt(10) == 0) { // node gets an extra cpu this time @@ -448,22 +421,22 @@ public final class InternalTestCluster extends TestCluster { } if (random.nextBoolean()) { - builder.put(IndexCacheModule.QUERY_CACHE_TYPE, random.nextBoolean() ? IndexCacheModule.INDEX_QUERY_CACHE : IndexCacheModule.NONE_QUERY_CACHE); + builder.put(IndexModule.QUERY_CACHE_TYPE, random.nextBoolean() ? IndexModule.INDEX_QUERY_CACHE : IndexModule.NONE_QUERY_CACHE); } if (random.nextBoolean()) { - builder.put(IndexCacheModule.QUERY_CACHE_EVERYTHING, random.nextBoolean()); + builder.put(IndexModule.QUERY_CACHE_EVERYTHING, random.nextBoolean()); } if (random.nextBoolean()) { if (random.nextInt(10) == 0) { // do something crazy slow here - builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB)); + builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB)); } else { - builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB)); + builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB)); } } if (random.nextBoolean()) { - builder.put(IndicesStore.INDICES_STORE_THROTTLE_TYPE, RandomPicks.randomFrom(random, StoreRateLimiting.Type.values())); + builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_TYPE, RandomPicks.randomFrom(random, StoreRateLimiting.Type.values())); } if (random.nextBoolean()) { @@ -474,14 +447,6 @@ public final class InternalTestCluster extends TestCluster { } } - if (random.nextBoolean()) { - builder.put(RecoverySettings.INDICES_RECOVERY_COMPRESS, random.nextBoolean()); - } - - if (random.nextBoolean()) { - builder.put(IndicesRequestCache.INDICES_CACHE_QUERY_CONCURRENCY_LEVEL, RandomInts.randomIntBetween(random, 1, 32)); - builder.put(IndicesFieldDataCache.FIELDDATA_CACHE_CONCURRENCY_LEVEL, RandomInts.randomIntBetween(random, 1, 32)); - } if (random.nextBoolean()) { builder.put(NettyTransport.PING_SCHEDULE, RandomInts.randomIntBetween(random, 100, 2000) + "ms"); } @@ -627,7 +592,7 @@ public final class InternalTestCluster extends TestCluster { .put("path.home", baseDir) // allow overriding path.home .put(settings) .put("name", name) - .put("discovery.id.seed", seed) + .put(DiscoveryService.SETTING_DISCOVERY_SEED, seed) .build(); MockNode node = new MockNode(finalSettings, version, plugins); return new NodeAndClient(name, node); @@ -874,7 +839,8 @@ public final class InternalTestCluster extends TestCluster { IOUtils.rm(nodeEnv.nodeDataPaths()); } } - Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).build(); + final long newIdSeed = node.settings().getAsLong(DiscoveryService.SETTING_DISCOVERY_SEED, 0l) + 1; // use a new seed to make sure we have new node id + Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(DiscoveryService.SETTING_DISCOVERY_SEED, newIdSeed).build(); Collection> plugins = node.getPlugins(); Version version = node.getVersion(); node = new MockNode(finalSettings, version, plugins); @@ -940,7 +906,7 @@ public final class InternalTestCluster extends TestCluster { } @Override - public synchronized void beforeTest(Random random, double transportClientRatio) throws IOException { + public synchronized void beforeTest(Random random, double transportClientRatio) throws IOException, InterruptedException { super.beforeTest(random, transportClientRatio); reset(true); } @@ -1262,7 +1228,7 @@ public final class InternalTestCluster extends TestCluster { * Restarts a random data node in the cluster */ public void restartRandomDataNode() throws Exception { - restartRandomNode(EMPTY_CALLBACK); + restartRandomDataNode(EMPTY_CALLBACK); } /** @@ -1584,7 +1550,7 @@ public final class InternalTestCluster extends TestCluster { for (int i = 0; i < numNodes; i++) { asyncs.add(startNodeAsync(settings, version)); } - + return () -> { List ids = new ArrayList<>(); for (Async async : asyncs) { @@ -1738,7 +1704,7 @@ public final class InternalTestCluster extends TestCluster { ClusterService clusterService = getInstanceFromNode(ClusterService.class, node); IndexService indexService = indicesService.indexService(index); if (indexService != null) { - assertThat(indexService.settingsService().getSettings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), greaterThan(shard)); + assertThat(indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), greaterThan(shard)); OperationRouting operationRouting = getInstanceFromNode(OperationRouting.class, node); while (true) { String routing = RandomStrings.randomAsciiOfLength(random, 10); @@ -1879,7 +1845,7 @@ public final class InternalTestCluster extends TestCluster { } NodeService nodeService = getInstanceFromNode(NodeService.class, nodeAndClient.node); - NodeStats stats = nodeService.stats(CommonStatsFlags.ALL, false, false, false, false, false, false, false, false, false); + NodeStats stats = nodeService.stats(CommonStatsFlags.ALL, false, false, false, false, false, false, false, false, false, false); assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l)); assertThat("Query cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0l)); assertThat("FixedBitSet cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getSegments().getBitsetMemoryInBytes(), equalTo(0l)); diff --git a/test-framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test-framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java new file mode 100644 index 00000000000..c02c1d8503f --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test; + +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.shard.IndexEventListener; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardState; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collection; +import java.util.Collections; + +/** + * This is a testing plugin that registers a generic {@link org.elasticsearch.test.MockIndexEventListener.TestEventListener} as a node level service as well as a listener + * on every index. Tests can access it like this: + * 

    + *     TestEventListener listener = internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node1);
+ *     listener.setNewDelegate(new IndexEventListener() {
+ *        // do some stuff
+ *     });
+ *
    + * This allows tests to use the listener without registering their own plugins. + */ +public final class MockIndexEventListener { + + public static class TestPlugin extends Plugin { + private final TestEventListener listener = new TestEventListener(); + @Override + public String name() { + return "mock-index-listener"; + } + @Override + public String description() { + return "a mock index listener for testing only"; + } + + @Override + public void onIndexModule(IndexModule module) { + module.addIndexEventListener(listener); + } + + @Override + public Collection nodeModules() { + return Collections.singleton(binder -> binder.bind(TestEventListener.class).toInstance(listener)); + } + } + + public static class TestEventListener implements IndexEventListener { + private volatile IndexEventListener delegate = new IndexEventListener() {}; + + public void setNewDelegate(IndexEventListener listener) { + delegate = listener == null ? new IndexEventListener() {} : listener; + } + + @Override + public void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting oldRouting, ShardRouting newRouting) { + delegate.shardRoutingChanged(indexShard, oldRouting, newRouting); + } + + @Override + public void afterIndexShardCreated(IndexShard indexShard) { + delegate.afterIndexShardCreated(indexShard); + } + + @Override + public void afterIndexShardStarted(IndexShard indexShard) { + delegate.afterIndexShardStarted(indexShard); + } + + @Override + public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { + delegate.beforeIndexShardClosed(shardId, indexShard, indexSettings); + } + + @Override + public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { + delegate.afterIndexShardClosed(shardId, indexShard, indexSettings); + } + + @Override + public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) { + delegate.indexShardStateChanged(indexShard, previousState, currentState, reason); + } + + @Override + public void onShardInactive(IndexShard indexShard) { + delegate.onShardInactive(indexShard); + } + + @Override + public void beforeIndexCreated(Index index, Settings indexSettings) { + delegate.beforeIndexCreated(index, indexSettings); + } + + @Override + public void afterIndexCreated(IndexService indexService) { + delegate.afterIndexCreated(indexService); + } + + @Override + public void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) { + delegate.beforeIndexShardCreated(shardId, indexSettings); + } + + @Override + public void beforeIndexClosed(IndexService indexService) { + delegate.beforeIndexClosed(indexService); + } + + @Override + public void afterIndexClosed(Index index, Settings indexSettings) { + delegate.afterIndexClosed(index, indexSettings); + } + + @Override + public void beforeIndexShardDeleted(ShardId shardId, Settings indexSettings) { + delegate.beforeIndexShardDeleted(shardId, indexSettings); + } + + @Override + public void afterIndexShardDeleted(ShardId shardId, Settings indexSettings) { + delegate.afterIndexShardDeleted(shardId, indexSettings); + } + + @Override + public void afterIndexDeleted(Index index, Settings indexSettings) { + delegate.afterIndexDeleted(index, indexSettings); + } + + @Override + public void beforeIndexDeleted(IndexService indexService) { + delegate.beforeIndexDeleted(indexService); + } + + @Override + public void beforeIndexAddedToCluster(Index index, Settings indexSettings) { + delegate.beforeIndexAddedToCluster(index, indexSettings); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/test/NodeConfigurationSource.java b/test-framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/NodeConfigurationSource.java rename to test-framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java diff --git a/core/src/test/java/org/elasticsearch/test/StreamsUtils.java b/test-framework/src/main/java/org/elasticsearch/test/StreamsUtils.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/StreamsUtils.java rename to test-framework/src/main/java/org/elasticsearch/test/StreamsUtils.java diff --git a/core/src/test/java/org/elasticsearch/test/TestCluster.java b/test-framework/src/main/java/org/elasticsearch/test/TestCluster.java similarity index 95% rename from core/src/test/java/org/elasticsearch/test/TestCluster.java rename to test-framework/src/main/java/org/elasticsearch/test/TestCluster.java index 60fb248420d..858fbab9ab5 100644 --- a/core/src/test/java/org/elasticsearch/test/TestCluster.java +++ b/test-framework/src/main/java/org/elasticsearch/test/TestCluster.java @@ -63,7 +63,7 @@ public abstract class TestCluster implements Iterable, Closeable { /** * This method should be executed before each test to reset the cluster to its initial state. */ - public void beforeTest(Random random, double transportClientRatio) throws IOException { + public void beforeTest(Random random, double transportClientRatio) throws IOException, InterruptedException { assert transportClientRatio >= 0.0 && transportClientRatio <= 1.0; logger.debug("Reset test cluster with transport client ratio: [{}]", transportClientRatio); this.transportClientRatio = transportClientRatio; @@ -153,12 +153,6 @@ public abstract class TestCluster implements Iterable, Closeable { assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(String.class))); } } - } catch (AssertionError ae) { - // Try to see what threads are doing when we hit the "Delete index failed - not acked": - logger.info("dump all threads on AssertionError"); - ESTestCase.printStackDump(logger); - logger.info("done dump all threads on AssertionError"); - throw ae; } } } diff --git a/core/src/test/java/org/elasticsearch/test/TestSearchContext.java b/test-framework/src/main/java/org/elasticsearch/test/TestSearchContext.java similarity index 96% rename from core/src/test/java/org/elasticsearch/test/TestSearchContext.java rename to test-framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 35b6ab2f835..468b1877250 100644 --- a/core/src/test/java/org/elasticsearch/test/TestSearchContext.java +++ b/test-framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -21,7 +21,6 @@ package org.elasticsearch.test; import com.carrotsearch.hppc.ObjectObjectAssociativeContainer; import org.apache.lucene.search.Collector; -import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.util.Counter; @@ -42,7 +41,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; @@ -106,7 +104,7 @@ public class TestSearchContext extends SearchContext { this.bigArrays = bigArrays.withCircuitBreaking(); this.indexService = indexService; this.indexFieldDataService = indexService.fieldData(); - this.fixedBitSetFilterCache = indexService.bitsetFilterCache(); + this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.threadPool = threadPool; this.indexShard = indexService.getShardOrNull(0); this.scriptService = scriptService; @@ -133,7 +131,7 @@ public class TestSearchContext extends SearchContext { } @Override - public Filter searchFilter(String[] types) { + public Query searchFilter(String[] types) { return null; } @@ -312,14 +310,7 @@ public class TestSearchContext extends SearchContext { } @Override - public AnalysisService analysisService() { - return indexService.analysisService(); - } - - @Override - public IndexQueryParserService queryParserService() { - return indexService.queryParserService(); - } + public AnalysisService analysisService() { return indexService.analysisService();} @Override public SimilarityService similarityService() { @@ -413,7 +404,7 @@ public class TestSearchContext extends SearchContext { } @Override - public Filter aliasFilter() { + public Query aliasFilter() { return null; } @@ -659,7 +650,7 @@ public class TestSearchContext extends SearchContext { @Override public Set getHeaders() { - return Collections.EMPTY_SET; + return Collections.emptySet(); } @Override diff --git a/core/src/test/java/org/elasticsearch/test/VersionUtils.java b/test-framework/src/main/java/org/elasticsearch/test/VersionUtils.java similarity index 98% rename from core/src/test/java/org/elasticsearch/test/VersionUtils.java rename to test-framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 30a89e4fb5f..93eef969b43 100644 --- a/core/src/test/java/org/elasticsearch/test/VersionUtils.java +++ b/test-framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -35,7 +35,7 @@ public class VersionUtils { private static final List SORTED_VERSIONS; static { - Field[] declaredFields = Version.class.getDeclaredFields(); + Field[] declaredFields = Version.class.getFields(); Set ids = new HashSet<>(); for (Field field : declaredFields) { final int mod = field.getModifiers(); diff --git a/core/src/test/java/org/elasticsearch/test/XContentTestUtils.java b/test-framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/XContentTestUtils.java rename to test-framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java diff --git a/core/src/test/java/org/elasticsearch/test/client/RandomizingClient.java b/test-framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/client/RandomizingClient.java rename to test-framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java diff --git a/core/src/test/java/org/elasticsearch/test/cluster/NoopClusterService.java b/test-framework/src/main/java/org/elasticsearch/test/cluster/NoopClusterService.java similarity index 95% rename from core/src/test/java/org/elasticsearch/test/cluster/NoopClusterService.java rename to test-framework/src/main/java/org/elasticsearch/test/cluster/NoopClusterService.java index 834e7d540c4..cb3d643f555 100644 --- a/core/src/test/java/org/elasticsearch/test/cluster/NoopClusterService.java +++ b/test-framework/src/main/java/org/elasticsearch/test/cluster/NoopClusterService.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.service.PendingClusterTask; -import org.elasticsearch.common.Priority; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.transport.DummyTransportAddress; @@ -115,12 +114,12 @@ public class NoopClusterService implements ClusterService { } @Override - public void submitStateUpdateTask(String source, Priority priority, ClusterStateUpdateTask updateTask) { + public void submitStateUpdateTask(String source, ClusterStateUpdateTask updateTask) { } @Override - public void submitStateUpdateTask(String source, ClusterStateUpdateTask updateTask) { + public void submitStateUpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor executor, ClusterStateTaskListener listener) { } diff --git a/core/src/test/java/org/elasticsearch/test/cluster/TestClusterService.java b/test-framework/src/main/java/org/elasticsearch/test/cluster/TestClusterService.java similarity index 87% rename from core/src/test/java/org/elasticsearch/test/cluster/TestClusterService.java rename to test-framework/src/main/java/org/elasticsearch/test/cluster/TestClusterService.java index fa62dd4b6ee..5dc8cce99c6 100644 --- a/core/src/test/java/org/elasticsearch/test/cluster/TestClusterService.java +++ b/test-framework/src/main/java/org/elasticsearch/test/cluster/TestClusterService.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.elasticsearch.cluster.service.PendingClusterTask; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Priority; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.logging.ESLogger; @@ -40,10 +39,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.threadpool.ThreadPool; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Queue; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -51,7 +47,7 @@ import java.util.concurrent.ScheduledFuture; public class TestClusterService implements ClusterService { volatile ClusterState state; - private final Collection listeners = new CopyOnWriteArrayList<>(); + private final List listeners = new CopyOnWriteArrayList<>(); private final Queue onGoingTimeouts = ConcurrentCollections.newQueue(); private final ThreadPool threadPool; private final ESLogger logger = Loggers.getLogger(getClass(), Settings.EMPTY); @@ -135,7 +131,7 @@ public class TestClusterService implements ClusterService { @Override public void addFirst(ClusterStateListener listener) { - throw new UnsupportedOperationException(); + listeners.add(0, listener); } @Override @@ -183,31 +179,35 @@ public class TestClusterService implements ClusterService { } @Override - synchronized public void submitStateUpdateTask(String source, Priority priority, ClusterStateUpdateTask updateTask) { - logger.debug("processing [{}]", source); - if (state().nodes().localNodeMaster() == false && updateTask.runOnlyOnMaster()) { - updateTask.onNoLongerMaster(source); - logger.debug("failed [{}], no longer master", source); - return; - } - ClusterState newState; - ClusterState previousClusterState = state; - try { - newState = updateTask.execute(previousClusterState); - } catch (Exception e) { - updateTask.onFailure(source, new ElasticsearchException("failed to process cluster state update task [" + source + "]", e)); - return; - } - setStateAndNotifyListeners(newState); - if (updateTask instanceof ClusterStateUpdateTask) { - ((ClusterStateUpdateTask) updateTask).clusterStateProcessed(source, previousClusterState, newState); - } - logger.debug("finished [{}]", source); + public void submitStateUpdateTask(String source, ClusterStateUpdateTask updateTask) { + submitStateUpdateTask(source, null, updateTask, updateTask, updateTask); } @Override - public void submitStateUpdateTask(String source, ClusterStateUpdateTask updateTask) { - submitStateUpdateTask(source, Priority.NORMAL, updateTask); + synchronized public void submitStateUpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor executor, ClusterStateTaskListener listener) { + logger.debug("processing [{}]", source); + if (state().nodes().localNodeMaster() == false && executor.runOnlyOnMaster()) { + listener.onNoLongerMaster(source); + logger.debug("failed [{}], no longer master", source); + return; + } + ClusterStateTaskExecutor.BatchResult batchResult; + ClusterState previousClusterState = state; + try { + batchResult = executor.execute(previousClusterState, Arrays.asList(task)); + } catch (Exception e) { + batchResult = ClusterStateTaskExecutor.BatchResult.builder().failure(task, e).build(previousClusterState); + } + + batchResult.executionResults.get(task).handle( + () -> {}, + ex -> listener.onFailure(source, new ElasticsearchException("failed to process cluster state update task [" + source + "]", ex)) + ); + + setStateAndNotifyListeners(batchResult.resultingState); + listener.clusterStateProcessed(source, previousClusterState, batchResult.resultingState); + logger.debug("finished [{}]", source); + } @Override diff --git a/core/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/test-framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java rename to test-framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java diff --git a/core/src/test/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java similarity index 98% rename from core/src/test/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java index 8154abfbd33..e318843e84f 100644 --- a/core/src/test/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java +++ b/test-framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java @@ -58,7 +58,7 @@ public class BlockClusterStateProcessing extends SingleNodeDisruption { boolean success = disruptionLatch.compareAndSet(null, new CountDownLatch(1)); assert success : "startDisrupting called without waiting on stopDistrupting to complete"; final CountDownLatch started = new CountDownLatch(1); - clusterService.submitStateUpdateTask("service_disruption_block", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("service_disruption_block", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public boolean runOnlyOnMaster() { diff --git a/core/src/test/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java diff --git a/core/src/test/java/org/elasticsearch/test/disruption/LongGCDisruption.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/disruption/LongGCDisruption.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java diff --git a/core/src/test/java/org/elasticsearch/test/disruption/NetworkDelaysPartition.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkDelaysPartition.java similarity index 89% rename from core/src/test/java/org/elasticsearch/test/disruption/NetworkDelaysPartition.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkDelaysPartition.java index 8439f6e8f76..c422b042721 100644 --- a/core/src/test/java/org/elasticsearch/test/disruption/NetworkDelaysPartition.java +++ b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkDelaysPartition.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.test.disruption; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.transport.MockTransportService; @@ -78,10 +77,9 @@ public class NetworkDelaysPartition extends NetworkPartition { } @Override - void applyDisruption(DiscoveryNode node1, MockTransportService transportService1, - DiscoveryNode node2, MockTransportService transportService2) { - transportService1.addUnresponsiveRule(node1, duration); - transportService1.addUnresponsiveRule(node2, duration); + void applyDisruption(MockTransportService transportService1, MockTransportService transportService2) { + transportService1.addUnresponsiveRule(transportService1, duration); + transportService1.addUnresponsiveRule(transportService2, duration); } @Override diff --git a/core/src/test/java/org/elasticsearch/test/disruption/NetworkDisconnectPartition.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisconnectPartition.java similarity index 82% rename from core/src/test/java/org/elasticsearch/test/disruption/NetworkDisconnectPartition.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisconnectPartition.java index 8653b50f749..ed0aa17cfcf 100644 --- a/core/src/test/java/org/elasticsearch/test/disruption/NetworkDisconnectPartition.java +++ b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisconnectPartition.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.test.disruption; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.transport.MockTransportService; @@ -46,10 +45,9 @@ public class NetworkDisconnectPartition extends NetworkPartition { } @Override - void applyDisruption(DiscoveryNode node1, MockTransportService transportService1, - DiscoveryNode node2, MockTransportService transportService2) { - transportService1.addFailToSendNoConnectRule(node2); - transportService2.addFailToSendNoConnectRule(node1); + void applyDisruption(MockTransportService transportService1, MockTransportService transportService2) { + transportService1.addFailToSendNoConnectRule(transportService2); + transportService2.addFailToSendNoConnectRule(transportService1); } @Override diff --git a/core/src/test/java/org/elasticsearch/test/disruption/NetworkPartition.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkPartition.java similarity index 82% rename from core/src/test/java/org/elasticsearch/test/disruption/NetworkPartition.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkPartition.java index 88bcb9024a1..9a65fc579f0 100644 --- a/core/src/test/java/org/elasticsearch/test/disruption/NetworkPartition.java +++ b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkPartition.java @@ -18,10 +18,8 @@ */ package org.elasticsearch.test.disruption; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.discovery.Discovery; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; @@ -29,7 +27,6 @@ import org.elasticsearch.transport.TransportService; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Random; import java.util.Set; @@ -140,7 +137,6 @@ public abstract class NetworkPartition implements ServiceDisruptionScheme { @Override public synchronized void removeFromNode(String node, InternalTestCluster cluster) { MockTransportService transportService = (MockTransportService) cluster.getInstance(TransportService.class, node); - DiscoveryNode discoveryNode = discoveryNode(node); Set otherSideNodes; if (nodesSideOne.contains(node)) { otherSideNodes = nodesSideTwo; @@ -153,8 +149,7 @@ public abstract class NetworkPartition implements ServiceDisruptionScheme { } for (String node2 : otherSideNodes) { MockTransportService transportService2 = (MockTransportService) cluster.getInstance(TransportService.class, node2); - DiscoveryNode discoveryNode2 = discoveryNode(node2); - removeDisruption(discoveryNode, transportService, discoveryNode2, transportService2); + removeDisruption(transportService, transportService2); } } @@ -165,11 +160,6 @@ public abstract class NetworkPartition implements ServiceDisruptionScheme { protected abstract String getPartitionDescription(); - - protected DiscoveryNode discoveryNode(String node) { - return cluster.getInstance(Discovery.class, node).localNode(); - } - @Override public synchronized void startDisrupting() { if (nodesSideOne.size() == 0 || nodesSideTwo.size() == 0) { @@ -179,11 +169,9 @@ public abstract class NetworkPartition implements ServiceDisruptionScheme { activeDisruption = true; for (String node1 : nodesSideOne) { MockTransportService transportService1 = (MockTransportService) cluster.getInstance(TransportService.class, node1); - DiscoveryNode discoveryNode1 = discoveryNode(node1); for (String node2 : nodesSideTwo) { - DiscoveryNode discoveryNode2 = discoveryNode(node2); MockTransportService transportService2 = (MockTransportService) cluster.getInstance(TransportService.class, node2); - applyDisruption(discoveryNode1, transportService1, discoveryNode2, transportService2); + applyDisruption(transportService1, transportService2); } } } @@ -197,24 +185,20 @@ public abstract class NetworkPartition implements ServiceDisruptionScheme { logger.info("restoring partition between nodes {} & nodes {}", nodesSideOne, nodesSideTwo); for (String node1 : nodesSideOne) { MockTransportService transportService1 = (MockTransportService) cluster.getInstance(TransportService.class, node1); - DiscoveryNode discoveryNode1 = discoveryNode(node1); for (String node2 : nodesSideTwo) { - DiscoveryNode discoveryNode2 = discoveryNode(node2); MockTransportService transportService2 = (MockTransportService) cluster.getInstance(TransportService.class, node2); - removeDisruption(discoveryNode1, transportService1, discoveryNode2, transportService2); + removeDisruption(transportService1, transportService2); } } activeDisruption = false; } - abstract void applyDisruption(DiscoveryNode node1, MockTransportService transportService1, - DiscoveryNode node2, MockTransportService transportService2); + abstract void applyDisruption(MockTransportService transportService1, MockTransportService transportService2); - protected void removeDisruption(DiscoveryNode node1, MockTransportService transportService1, - DiscoveryNode node2, MockTransportService transportService2) { - transportService1.clearRule(node2); - transportService2.clearRule(node1); + protected void removeDisruption(MockTransportService transportService1, MockTransportService transportService2) { + transportService1.clearRule(transportService2); + transportService2.clearRule(transportService1); } } diff --git a/core/src/test/java/org/elasticsearch/test/disruption/NetworkPartitionIT.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkPartitionIT.java similarity index 98% rename from core/src/test/java/org/elasticsearch/test/disruption/NetworkPartitionIT.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkPartitionIT.java index fc2b9469a73..b1ce97374a4 100644 --- a/core/src/test/java/org/elasticsearch/test/disruption/NetworkPartitionIT.java +++ b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkPartitionIT.java @@ -17,27 +17,22 @@ * under the License. */ - package org.elasticsearch.test.disruption; - import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.transport.MockTransportService; -import org.junit.Test; import java.io.IOException; import java.util.Collection; public class NetworkPartitionIT extends ESIntegTestCase { - @Override protected Collection> nodePlugins() { return pluginList(MockTransportService.TestPlugin.class); } - @Test public void testNetworkPartitionWithNodeShutdown() throws IOException { internalCluster().ensureAtLeastNumDataNodes(2); String[] nodeNames = internalCluster().getNodeNames(); diff --git a/core/src/test/java/org/elasticsearch/test/disruption/NetworkUnresponsivePartition.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkUnresponsivePartition.java similarity index 83% rename from core/src/test/java/org/elasticsearch/test/disruption/NetworkUnresponsivePartition.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkUnresponsivePartition.java index 1feb56c46c7..b69b7af3e5e 100644 --- a/core/src/test/java/org/elasticsearch/test/disruption/NetworkUnresponsivePartition.java +++ b/test-framework/src/main/java/org/elasticsearch/test/disruption/NetworkUnresponsivePartition.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.test.disruption; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.transport.MockTransportService; @@ -45,10 +44,9 @@ public class NetworkUnresponsivePartition extends NetworkPartition { } @Override - void applyDisruption(DiscoveryNode node1, MockTransportService transportService1, - DiscoveryNode node2, MockTransportService transportService2) { - transportService1.addUnresponsiveRule(node2); - transportService2.addUnresponsiveRule(node1); + void applyDisruption(MockTransportService transportService1, MockTransportService transportService2) { + transportService1.addUnresponsiveRule(transportService2); + transportService2.addUnresponsiveRule(transportService1); } @Override diff --git a/core/src/test/java/org/elasticsearch/test/disruption/NoOpDisruptionScheme.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/NoOpDisruptionScheme.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/disruption/NoOpDisruptionScheme.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/NoOpDisruptionScheme.java diff --git a/core/src/test/java/org/elasticsearch/test/disruption/ServiceDisruptionScheme.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/ServiceDisruptionScheme.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/disruption/ServiceDisruptionScheme.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/ServiceDisruptionScheme.java diff --git a/core/src/test/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java diff --git a/core/src/test/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java b/test-framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java similarity index 99% rename from core/src/test/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java rename to test-framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java index 3c56f8305c0..b9c663686b1 100644 --- a/core/src/test/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java +++ b/test-framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java @@ -102,7 +102,7 @@ public class SlowClusterStateProcessing extends SingleNodeDisruption { return false; } final AtomicBoolean stopped = new AtomicBoolean(false); - clusterService.submitStateUpdateTask("service_disruption_delay", Priority.IMMEDIATE, new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("service_disruption_delay", new ClusterStateUpdateTask(Priority.IMMEDIATE) { @Override public boolean runOnlyOnMaster() { diff --git a/core/src/test/java/org/elasticsearch/test/engine/AssertingSearcher.java b/test-framework/src/main/java/org/elasticsearch/test/engine/AssertingSearcher.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/engine/AssertingSearcher.java rename to test-framework/src/main/java/org/elasticsearch/test/engine/AssertingSearcher.java diff --git a/core/src/test/java/org/elasticsearch/test/engine/MockEngineFactory.java b/test-framework/src/main/java/org/elasticsearch/test/engine/MockEngineFactory.java similarity index 87% rename from core/src/test/java/org/elasticsearch/test/engine/MockEngineFactory.java rename to test-framework/src/main/java/org/elasticsearch/test/engine/MockEngineFactory.java index 360849542f6..de51670f57d 100644 --- a/core/src/test/java/org/elasticsearch/test/engine/MockEngineFactory.java +++ b/test-framework/src/main/java/org/elasticsearch/test/engine/MockEngineFactory.java @@ -33,16 +33,10 @@ import static java.lang.annotation.ElementType.PARAMETER; import static java.lang.annotation.RetentionPolicy.RUNTIME; public final class MockEngineFactory implements EngineFactory { - @BindingAnnotation - @Target({FIELD, PARAMETER}) - @Retention(RUNTIME) - public @interface MockReaderType { - } - private Class wrapper; + private final Class wrapper; - @Inject - public MockEngineFactory(@MockReaderType Class wrapper) { + public MockEngineFactory(Class wrapper) { this.wrapper = wrapper; } diff --git a/core/src/test/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test-framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java similarity index 94% rename from core/src/test/java/org/elasticsearch/test/engine/MockEngineSupport.java rename to test-framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java index ab570afdd9d..70dfa6847b4 100644 --- a/core/src/test/java/org/elasticsearch/test/engine/MockEngineSupport.java +++ b/test-framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java @@ -21,11 +21,7 @@ package org.elasticsearch.test.engine; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.AssertingIndexSearcher; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.QueryCache; -import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.*; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.logging.ESLogger; @@ -79,18 +75,18 @@ public final class MockEngineSupport { } public MockEngineSupport(EngineConfig config, Class wrapper) { - Settings indexSettings = config.getIndexSettings(); + Settings settings = config.getIndexSettings().getSettings(); shardId = config.getShardId(); filterCache = config.getQueryCache(); filterCachingPolicy = config.getQueryCachingPolicy(); - final long seed = indexSettings.getAsLong(ESIntegTestCase.SETTING_INDEX_SEED, 0l); + final long seed = settings.getAsLong(ESIntegTestCase.SETTING_INDEX_SEED, 0l); Random random = new Random(seed); - final double ratio = indexSettings.getAsDouble(WRAP_READER_RATIO, 0.0d); // DISABLED by default - AssertingDR is crazy slow + final double ratio = settings.getAsDouble(WRAP_READER_RATIO, 0.0d); // DISABLED by default - AssertingDR is crazy slow boolean wrapReader = random.nextDouble() < ratio; if (logger.isTraceEnabled()) { logger.trace("Using [{}] for shard [{}] seed: [{}] wrapReader: [{}]", this.getClass().getName(), shardId, seed, wrapReader); } - mockContext = new MockContext(random, wrapReader, wrapper, indexSettings); + mockContext = new MockContext(random, wrapReader, wrapper, settings); this.searcherCloseable = new SearcherCloseable(); LuceneTestCase.closeAfterSuite(searcherCloseable); // only one suite closeable per Engine } diff --git a/core/src/test/java/org/elasticsearch/test/engine/MockInternalEngine.java b/test-framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java similarity index 94% rename from core/src/test/java/org/elasticsearch/test/engine/MockInternalEngine.java rename to test-framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java index 616d873786e..15bb2918680 100644 --- a/core/src/test/java/org/elasticsearch/test/engine/MockInternalEngine.java +++ b/test-framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java @@ -21,7 +21,6 @@ package org.elasticsearch.test.engine; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SearcherManager; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.engine.InternalEngine; @@ -35,7 +34,7 @@ final class MockInternalEngine extends InternalEngine { MockInternalEngine(EngineConfig config, boolean skipInitialTranslogRecovery, Class wrapper) throws EngineException { super(config, skipInitialTranslogRecovery); - randomizeFlushOnClose = IndexMetaData.isOnSharedFilesystem(config.getIndexSettings()) == false; + randomizeFlushOnClose = config.getIndexSettings().isOnSharedFilesystem() == false; wrapperClass = wrapper; } diff --git a/core/src/test/java/org/elasticsearch/test/engine/MockShadowEngine.java b/test-framework/src/main/java/org/elasticsearch/test/engine/MockShadowEngine.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/engine/MockShadowEngine.java rename to test-framework/src/main/java/org/elasticsearch/test/engine/MockShadowEngine.java diff --git a/core/src/test/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test-framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java rename to test-framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java diff --git a/core/src/test/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java b/test-framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java similarity index 97% rename from core/src/test/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java rename to test-framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java index ac582511032..825b203022d 100644 --- a/core/src/test/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java +++ b/test-framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java @@ -32,7 +32,7 @@ public class NoopGatewayAllocator extends GatewayAllocator { public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator(); - private NoopGatewayAllocator() { + protected NoopGatewayAllocator() { super(Settings.EMPTY, null, null); } diff --git a/core/src/test/java/org/elasticsearch/test/hamcrest/CollectionAssertions.java b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionAssertions.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/hamcrest/CollectionAssertions.java rename to test-framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionAssertions.java diff --git a/core/src/test/java/org/elasticsearch/test/hamcrest/CollectionMatchers.java b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionMatchers.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/hamcrest/CollectionMatchers.java rename to test-framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionMatchers.java diff --git a/core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java similarity index 94% rename from core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java rename to test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index 5772543dc6b..9d8ad7f7dcf 100644 --- a/core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -30,14 +30,12 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.exists.ExistsResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; @@ -53,6 +51,8 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -82,13 +82,7 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.elasticsearch.test.ESTestCase.assertArrayEquals; -import static org.elasticsearch.test.ESTestCase.assertEquals; -import static org.elasticsearch.test.ESTestCase.assertFalse; -import static org.elasticsearch.test.ESTestCase.assertNotNull; -import static org.elasticsearch.test.ESTestCase.assertTrue; -import static org.elasticsearch.test.ESTestCase.fail; -import static org.elasticsearch.test.ESTestCase.random; +import static org.apache.lucene.util.LuceneTestCase.random; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -102,6 +96,12 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @@ -204,17 +204,6 @@ public class ElasticsearchAssertions { return msg; } - /* - * assertions - */ - public static void assertHitCount(SearchResponse searchResponse, long expectedHitCount) { - if (searchResponse.getHits().totalHits() != expectedHitCount) { - fail("Hit count is " + searchResponse.getHits().totalHits() + " but " + expectedHitCount + " was expected. " - + formatShardStatus(searchResponse)); - } - assertVersionSerializable(searchResponse); - } - public static void assertNoSearchHits(SearchResponse searchResponse) { assertEquals(0, searchResponse.getHits().getHits().length); } @@ -254,20 +243,13 @@ public class ElasticsearchAssertions { assertVersionSerializable(searchResponse); } - public static void assertHitCount(CountResponse countResponse, long expectedHitCount) { - if (countResponse.getCount() != expectedHitCount) { - fail("Count is " + countResponse.getCount() + " but " + expectedHitCount + " was expected. " + formatShardStatus(countResponse)); + public static void assertHitCount(SearchResponse countResponse, long expectedHitCount) { + if (countResponse.getHits().totalHits() != expectedHitCount) { + fail("Count is " + countResponse.getHits().totalHits() + " but " + expectedHitCount + " was expected. " + formatShardStatus(countResponse)); } assertVersionSerializable(countResponse); } - public static void assertExists(ExistsResponse existsResponse, boolean expected) { - if (existsResponse.exists() != expected) { - fail("Exist is " + existsResponse.exists() + " but " + expected + " was expected " + formatShardStatus(existsResponse)); - } - assertVersionSerializable(existsResponse); - } - public static void assertMatchCount(PercolateResponse percolateResponse, long expectedHitCount) { if (percolateResponse.getCount() != expectedHitCount) { fail("Count is " + percolateResponse.getCount() + " but " + expectedHitCount + " was expected. " + formatShardStatus(percolateResponse)); @@ -663,6 +645,10 @@ public class ElasticsearchAssertions { } public static void assertVersionSerializable(Version version, Streamable streamable) { + assertVersionSerializable(version, streamable, null); + } + + public static void assertVersionSerializable(Version version, Streamable streamable, NamedWriteableRegistry namedWriteableRegistry) { try { Streamable newInstance = tryCreateNewInstance(streamable); if (newInstance == null) { @@ -674,10 +660,15 @@ public class ElasticsearchAssertions { } BytesReference orig = serialize(version, streamable); StreamInput input = StreamInput.wrap(orig); + if (namedWriteableRegistry != null) { + input = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry); + } input.setVersion(version); newInstance.readFrom(input); - assertThat("Stream should be fully read with version [" + version + "] for streamable [" + streamable + "]", input.available(), equalTo(0)); - assertThat("Serialization failed with version [" + version + "] bytes should be equal for streamable [" + streamable + "]", serialize(version, streamable), equalTo(orig)); + assertThat("Stream should be fully read with version [" + version + "] for streamable [" + streamable + "]", input.available(), + equalTo(0)); + assertThat("Serialization failed with version [" + version + "] bytes should be equal for streamable [" + streamable + "]", + serialize(version, streamable), equalTo(orig)); } catch (Throwable ex) { throw new RuntimeException("failed to check serialization - version [" + version + "] for streamable [" + streamable + "]", ex); } @@ -714,7 +705,7 @@ public class ElasticsearchAssertions { IllegalAccessException, InvocationTargetException { try { Class clazz = streamable.getClass(); - Constructor constructor = clazz.getDeclaredConstructor(); + Constructor constructor = clazz.getConstructor(); assertThat(constructor, Matchers.notNullValue()); Streamable newInstance = constructor.newInstance(); return newInstance; @@ -750,7 +741,7 @@ public class ElasticsearchAssertions { Assert.assertThat(response.getNodesMap().get(nodeId), notNullValue()); - PluginsInfo plugins = response.getNodesMap().get(nodeId).getPlugins(); + PluginsAndModules plugins = response.getNodesMap().get(nodeId).getPlugins(); Assert.assertThat(plugins, notNullValue()); List pluginNames = filterAndMap(plugins, jvmPluginPredicate, nameFunction); @@ -770,7 +761,7 @@ public class ElasticsearchAssertions { boolean anyHaveUrls = plugins - .getInfos() + .getPluginInfos() .stream() .filter(jvmPluginPredicate.and(sitePluginPredicate.negate())) .map(urlFunction) @@ -800,8 +791,8 @@ public class ElasticsearchAssertions { } } - private static List filterAndMap(PluginsInfo pluginsInfo, Predicate predicate, Function function) { - return pluginsInfo.getInfos().stream().filter(predicate).map(function).collect(Collectors.toList()); + private static List filterAndMap(PluginsAndModules pluginsInfo, Predicate predicate, Function function) { + return pluginsInfo.getPluginInfos().stream().filter(predicate).map(function).collect(Collectors.toList()); } private static Predicate jvmPluginPredicate = p -> p.isJvm(); diff --git a/core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java rename to test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java diff --git a/core/src/test/java/org/elasticsearch/test/hamcrest/RegexMatcher.java b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/RegexMatcher.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/hamcrest/RegexMatcher.java rename to test-framework/src/main/java/org/elasticsearch/test/hamcrest/RegexMatcher.java diff --git a/core/src/test/java/org/elasticsearch/test/junit/annotations/Network.java b/test-framework/src/main/java/org/elasticsearch/test/junit/annotations/Network.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/junit/annotations/Network.java rename to test-framework/src/main/java/org/elasticsearch/test/junit/annotations/Network.java diff --git a/core/src/test/java/org/elasticsearch/test/junit/annotations/TestLogging.java b/test-framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/junit/annotations/TestLogging.java rename to test-framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java diff --git a/core/src/test/java/org/elasticsearch/test/junit/listeners/LoggingListener.java b/test-framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/junit/listeners/LoggingListener.java rename to test-framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java diff --git a/core/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test-framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java similarity index 86% rename from core/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java rename to test-framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java index 0a52a376610..47a77dfc9d2 100644 --- a/core/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java +++ b/test-framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java @@ -76,40 +76,25 @@ public class ReproduceInfoPrinter extends RunListener { return; } - final StringBuilder b = new StringBuilder(); - if (inVerifyPhase()) { - b.append("REPRODUCE WITH: mvn verify -Pdev -Dskip.unit.tests" ); - } else { - b.append("REPRODUCE WITH: mvn test -Pdev"); - } - String project = System.getProperty("tests.project"); - if (project != null) { - b.append(" -pl " + project); - } - MavenMessageBuilder mavenMessageBuilder = new MavenMessageBuilder(b); - mavenMessageBuilder.appendAllOpts(failure.getDescription()); + final StringBuilder b = new StringBuilder("REPRODUCE WITH: gradle "); + String task = System.getProperty("tests.task"); + // TODO: enforce (intellij still runs the runner?) or use default "test" but that wont' work for integ + b.append(task); + + GradleMessageBuilder gradleMessageBuilder = new GradleMessageBuilder(b); + gradleMessageBuilder.appendAllOpts(failure.getDescription()); //Rest tests are a special case as they allow for additional parameters if (failure.getDescription().getTestClass().isAnnotationPresent(Rest.class)) { - mavenMessageBuilder.appendRestTestsProperties(); + gradleMessageBuilder.appendRestTestsProperties(); } System.err.println(b.toString()); } - protected TraceFormatting traces() { - TraceFormatting traces = new TraceFormatting(); - try { - traces = RandomizedContext.current().getRunner().getTraceFormatting(); - } catch (IllegalStateException e) { - // Ignore if no context. - } - return traces; - } + protected static class GradleMessageBuilder extends ReproduceErrorMessageBuilder { - protected static class MavenMessageBuilder extends ReproduceErrorMessageBuilder { - - public MavenMessageBuilder(StringBuilder b) { + public GradleMessageBuilder(StringBuilder b) { super(b); } diff --git a/core/src/test/java/org/elasticsearch/test/junit/rule/RepeatOnExceptionRule.java b/test-framework/src/main/java/org/elasticsearch/test/junit/rule/RepeatOnExceptionRule.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/junit/rule/RepeatOnExceptionRule.java rename to test-framework/src/main/java/org/elasticsearch/test/junit/rule/RepeatOnExceptionRule.java diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/BlacklistedPathPatternMatcher.java b/test-framework/src/main/java/org/elasticsearch/test/rest/BlacklistedPathPatternMatcher.java new file mode 100644 index 00000000000..e5bb75955cf --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/BlacklistedPathPatternMatcher.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test.rest; + +import java.util.regex.Pattern; + +/** + * Matches blacklist patterns. + * + * Currently the following syntax is supported: + * + *
      + *
    • Exact matches, as in cat.aliases/10_basic/Empty cluster
      • + *
    • Wildcard matches within the same segment of a path , as in indices.get/10_basic/*allow_no_indices*. This will + * match indices.get/10_basic/allow_no_indices, indices.get/10_basic/allow_no_indices_at_all but not + * indices.get/10_basic/advanced/allow_no_indices (contains an additional segment)
      • + *
    + * + * Each blacklist pattern is a suffix match on the path. Empty patterns are not allowed. + */ +final class BlacklistedPathPatternMatcher { + private final Pattern pattern; + + /** + * Constructs a new BlacklistedPathPatternMatcher instance from the provided suffix pattern. + * + * @param p The suffix pattern. Must be a non-empty string. + */ + BlacklistedPathPatternMatcher(String p) { + // guard against accidentally matching everything as an empty string lead to the pattern ".*" which matches everything + if (p == null || p.trim().isEmpty()) { + throw new IllegalArgumentException("Empty blacklist patterns are not supported"); + } + // very simple transformation from wildcard to a proper regex + String finalPattern = p + .replaceAll("\\*", "[^/]*") // support wildcard matches (within a single path segment) + .replaceAll("\\\\,", ","); // restore previously escaped ',' in paths. + + // suffix match + pattern = Pattern.compile(".*" + finalPattern); + } + + /** + * Checks whether the provided path matches the suffix pattern, i.e. "/foo/bar" will match the pattern "bar". + * + * @param path The path to match. Must not be null. + * @return true iff this path is a suffix match. + */ + public boolean isSuffixMatch(String path) { + return pattern.matcher(path).matches(); + } +} diff --git a/core/src/test/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java similarity index 92% rename from core/src/test/java/org/elasticsearch/test/rest/ESRestTestCase.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 33ddea019ae..266f8e8038c 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -22,13 +22,13 @@ package org.elasticsearch.test.rest; import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.annotations.TestGroup; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase.SuppressFsync; import org.apache.lucene.util.TimeUnits; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.node.Node; @@ -49,7 +49,6 @@ import org.elasticsearch.test.rest.support.FileUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import java.io.IOException; import java.io.InputStream; @@ -65,7 +64,6 @@ import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.PathMatcher; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collections; @@ -123,9 +121,18 @@ public abstract class ESRestTestCase extends ESIntegTestCase { private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test"; private static final String DEFAULT_SPEC_PATH = "/rest-api-spec/api"; - private static final String PATHS_SEPARATOR = ","; + /** + * This separator pattern matches ',' except it is preceded by a '\'. This allows us to support ',' within paths when it is escaped with + * a slash. + * + * For example, the path string "/a/b/c\,d/e/f,/foo/bar,/baz" is separated to "/a/b/c\,d/e/f", "/foo/bar" and "/baz". + * + * For reference, this regular expression feature is known as zero-width negative look-behind. + * + */ + private static final String PATHS_SEPARATOR = "(? blacklistPathMatchers = new ArrayList<>(); private static RestTestExecutionContext restTestExecutionContext; private final RestTestCandidate testCandidate; @@ -133,14 +140,8 @@ public abstract class ESRestTestCase extends ESIntegTestCase { public ESRestTestCase(RestTestCandidate testCandidate) { this.testCandidate = testCandidate; String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); - if (blacklist != null) { - blacklistPathMatchers = new PathMatcher[blacklist.length]; - int i = 0; - for (String glob : blacklist) { - blacklistPathMatchers[i++] = PathUtils.getDefaultFileSystem().getPathMatcher("glob:" + glob); - } - } else { - blacklistPathMatchers = new PathMatcher[0]; + for (String entry : blacklist) { + this.blacklistPathMatchers.add(new BlacklistedPathPatternMatcher(entry)); } } @@ -158,7 +159,7 @@ public abstract class ESRestTestCase extends ESIntegTestCase { .put("node.testattr", "test") .put(super.nodeSettings(nodeOrdinal)).build(); } - + public static Iterable createParameters(int id, int count) throws IOException, RestTestParseException { TestGroup testGroup = Rest.class.getAnnotation(TestGroup.class); String sysProperty = TestGroup.Utilities.getSysProperty(Rest.class); @@ -217,7 +218,7 @@ public abstract class ESRestTestCase extends ESIntegTestCase { return testCandidates; } - + private static boolean mustExecute(String test, int id, int count) { int hash = (int) (Math.abs((long)test.hashCode()) % count); return hash == id; @@ -226,7 +227,7 @@ public abstract class ESRestTestCase extends ESIntegTestCase { private static String[] resolvePathsProperty(String propertyName, String defaultValue) { String property = System.getProperty(propertyName); if (!Strings.hasLength(property)) { - return defaultValue == null ? null : new String[]{defaultValue}; + return defaultValue == null ? Strings.EMPTY_ARRAY : new String[]{defaultValue}; } else { return property.split(PATHS_SEPARATOR); } @@ -239,13 +240,13 @@ public abstract class ESRestTestCase extends ESIntegTestCase { @SuppressForbidden(reason = "proper use of URL, hack around a JDK bug") static FileSystem getFileSystem() throws IOException { // REST suite handling is currently complicated, with lots of filtering and so on - // For now, to work embedded in a jar, return a ZipFileSystem over the jar contents. + // For now, to work embedded in a jar, return a ZipFileSystem over the jar contents. URL codeLocation = FileUtils.class.getProtectionDomain().getCodeSource().getLocation(); boolean loadPackaged = RandomizedTest.systemPropertyAsBoolean(REST_LOAD_PACKAGED_TESTS, true); if (codeLocation.getFile().endsWith(".jar") && loadPackaged) { try { // hack around a bug in the zipfilesystem implementation before java 9, - // its checkWritable was incorrect and it won't work without write permissions. + // its checkWritable was incorrect and it won't work without write permissions. // if we add the permission, it will open jars r/w, which is too scary! so copy to a safe r-w location. Path tmp = Files.createTempFile(null, ".jar"); try (InputStream in = codeLocation.openStream()) { @@ -324,11 +325,9 @@ public abstract class ESRestTestCase extends ESIntegTestCase { @Before public void reset() throws IOException, RestException { //skip test if it matches one of the blacklist globs - for (PathMatcher blacklistedPathMatcher : blacklistPathMatchers) { - //we need to replace a few characters otherwise the test section name can't be parsed as a path on windows - String testSection = testCandidate.getTestSection().getName().replace("*", "").replace("\\", "/").replaceAll("\\s+/", "/").replace(":", "").trim(); - String testPath = testCandidate.getSuitePath() + "/" + testSection; - assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher.matches(PathUtils.get(testPath))); + for (BlacklistedPathPatternMatcher blacklistedPathMatcher : blacklistPathMatchers) { + String testPath = testCandidate.getSuitePath() + "/" + testCandidate.getTestSection().getName(); + assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher.isSuffixMatch(testPath)); } //The client needs non static info to get initialized, therefore it can't be initialized in the before class restTestExecutionContext.initClient(cluster().httpAddresses(), restClientSettings()); @@ -352,7 +351,6 @@ public abstract class ESRestTestCase extends ESIntegTestCase { return messageBuilder.toString(); } - @Test public void test() throws IOException { //let's check that there is something to run, otherwise there might be a problem with the test section if (testCandidate.getTestSection().getExecutableSections().size() == 0) { diff --git a/core/src/test/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test-framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/FakeRestRequest.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java b/test-framework/src/main/java/org/elasticsearch/test/rest/RestTestCandidate.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/RestTestCandidate.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test-framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java similarity index 94% rename from core/src/test/java/org/elasticsearch/test/rest/RestTestExecutionContext.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java index b7dad93d593..4054b8efce1 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/RestTestExecutionContext.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java @@ -62,7 +62,8 @@ public class RestTestExecutionContext implements Closeable { * Saves the obtained response in the execution context. * @throws RestException if the returned status code is non ok */ - public RestResponse callApi(String apiName, Map params, List> bodies) throws IOException, RestException { + public RestResponse callApi(String apiName, Map params, List> bodies, + Map headers) throws IOException, RestException { //makes a copy of the parameters before modifying them for this specific request HashMap requestParams = new HashMap<>(params); for (Map.Entry entry : requestParams.entrySet()) { @@ -74,7 +75,7 @@ public class RestTestExecutionContext implements Closeable { String body = actualBody(bodies); try { - response = callApiInternal(apiName, requestParams, body); + response = callApiInternal(apiName, requestParams, body, headers); //we always stash the last response body stash.stashValue("body", response.getBody()); return response; @@ -104,8 +105,8 @@ public class RestTestExecutionContext implements Closeable { return XContentFactory.jsonBuilder().map(body).string(); } - private RestResponse callApiInternal(String apiName, Map params, String body) throws IOException, RestException { - return restClient.callApi(apiName, params, body); + private RestResponse callApiInternal(String apiName, Map params, String body, Map headers) throws IOException, RestException { + return restClient.callApi(apiName, params, body, headers); } /** diff --git a/core/src/test/java/org/elasticsearch/test/rest/Stash.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Stash.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/Stash.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/Stash.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/RestClient.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java similarity index 96% rename from core/src/test/java/org/elasticsearch/test/rest/client/RestClient.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java index b7173db0838..63a8b397c45 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/client/RestClient.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java @@ -132,7 +132,7 @@ public class RestClient implements Closeable { * @throws RestException if the obtained status code is non ok, unless the specific error code needs to be ignored * according to the ignore parameter received as input (which won't get sent to elasticsearch) */ - public RestResponse callApi(String apiName, Map params, String body) throws IOException, RestException { + public RestResponse callApi(String apiName, Map params, String body, Map headers) throws IOException, RestException { List ignores = new ArrayList<>(); Map requestParams = null; @@ -151,6 +151,9 @@ public class RestClient implements Closeable { } HttpRequestBuilder httpRequestBuilder = callApiBuilder(apiName, requestParams, body); + for (Map.Entry header : headers.entrySet()) { + httpRequestBuilder.addHeader(header.getKey(), header.getValue()); + } logger.debug("calling api [{}]", apiName); HttpResponse httpResponse = httpRequestBuilder.execute(); @@ -230,8 +233,9 @@ public class RestClient implements Closeable { httpRequestBuilder.method(RandomizedTest.randomFrom(supportedMethods)); } - //the http method is randomized (out of the available ones with the chosen api) - return httpRequestBuilder.path(RandomizedTest.randomFrom(restApi.getFinalPaths(pathParts))); + //the rest path to use is randomized out of the matching ones (if more than one) + RestPath restPath = RandomizedTest.randomFrom(restApi.getFinalPaths(pathParts)); + return httpRequestBuilder.pathParts(restPath.getPathParts()); } private RestApi restApi(String apiName) { diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/RestException.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/RestException.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/client/RestException.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/client/RestException.java diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/client/RestPath.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/RestPath.java new file mode 100644 index 00000000000..f6e3ddabd5e --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/client/RestPath.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest.client; + +import java.util.*; + +public class RestPath { + private final List parts; + private final List placeholders; + + public RestPath(List parts) { + List pathParts = new ArrayList<>(parts.size()); + for (String part : parts) { + pathParts.add(new PathPart(part, false)); + } + this.parts = pathParts; + this.placeholders = Collections.emptyList(); + } + + public RestPath(String path) { + String[] pathParts = path.split("/"); + List placeholders = new ArrayList<>(); + List parts = new ArrayList<>(); + for (String pathPart : pathParts) { + if (pathPart.length() > 0) { + if (pathPart.startsWith("{")) { + if (pathPart.indexOf('}') != pathPart.length() - 1) { + throw new IllegalArgumentException("more than one parameter found in the same path part: [" + pathPart + "]"); + } + String placeholder = pathPart.substring(1, pathPart.length() - 1); + parts.add(new PathPart(placeholder, true)); + placeholders.add(placeholder); + } else { + parts.add(new PathPart(pathPart, false)); + } + } + } + this.placeholders = placeholders; + this.parts = parts; + } + + public String[] getPathParts() { + String[] parts = new String[this.parts.size()]; + int i = 0; + for (PathPart part : this.parts) { + parts[i++] = part.pathPart; + } + return parts; + } + + public boolean matches(Set params) { + return placeholders.size() == params.size() && placeholders.containsAll(params); + } + + public RestPath replacePlaceholders(Map params) { + List finalPathParts = new ArrayList<>(parts.size()); + for (PathPart pathPart : parts) { + if (pathPart.isPlaceholder) { + String value = params.get(pathPart.pathPart); + if (value == null) { + throw new IllegalArgumentException("parameter [" + pathPart.pathPart + "] missing"); + } + finalPathParts.add(value); + } else { + finalPathParts.add(pathPart.pathPart); + } + } + return new RestPath(finalPathParts); + } + + private static class PathPart { + private final boolean isPlaceholder; + private final String pathPart; + + private PathPart(String pathPart, boolean isPlaceholder) { + this.isPlaceholder = isPlaceholder; + this.pathPart = pathPart; + } + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/RestResponse.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/client/RestResponse.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java similarity index 79% rename from core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java index 2f42488a151..34665efa0f1 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java @@ -86,14 +86,42 @@ public class HttpRequestBuilder { return this; } + /** + * Sets the path to send the request to. Url encoding needs to be applied by the caller. + * Use {@link #pathParts(String...)} instead if the path needs to be encoded, part by part. + */ public HttpRequestBuilder path(String path) { this.path = path; return this; } + /** + * Sets the path by providing the different parts (without slashes), which will be properly encoded. + */ + public HttpRequestBuilder pathParts(String... path) { + //encode rules for path and query string parameters are different. We use URI to encode the path, and URLEncoder for each query string parameter (see addParam). + //We need to encode each path part separately though, as each one might contain slashes that need to be escaped, which needs to be done manually. + if (path.length == 0) { + this.path = "/"; + return this; + } + StringBuilder finalPath = new StringBuilder(); + for (String pathPart : path) { + try { + finalPath.append('/'); + URI uri = new URI(null, null, null, -1, pathPart, null, null); + //manually escape any slash that each part may contain + finalPath.append(uri.getRawPath().replaceAll("/", "%2F")); + } catch(URISyntaxException e) { + throw new RuntimeException("unable to build uri", e); + } + } + this.path = finalPath.toString(); + return this; + } + public HttpRequestBuilder addParam(String name, String value) { try { - //manually url encode params, since URI does it only partially (e.g. '+' stays as is) this.params.put(name, URLEncoder.encode(value, "utf-8")); return this; } catch (UnsupportedEncodingException e) { @@ -181,19 +209,12 @@ public class HttpRequestBuilder { } private URI buildUri() { - try { - //url encode rules for path and query params are different. We use URI to encode the path, but we manually encode each query param through URLEncoder. - URI uri = new URI(protocol, null, host, port, path, null, null); - //String concatenation FTW. If we use the nicer multi argument URI constructor query parameters will get only partially encoded - //(e.g. '+' will stay as is) hence when trying to properly encode params manually they will end up double encoded (+ becomes %252B instead of %2B). - StringBuilder uriBuilder = new StringBuilder(protocol).append("://").append(host).append(":").append(port).append(uri.getRawPath()); - if (params.size() > 0) { - uriBuilder.append("?").append(params.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"))); - } - return URI.create(uriBuilder.toString()); - } catch(URISyntaxException e) { - throw new IllegalArgumentException("unable to build uri", e); + StringBuilder uriBuilder = new StringBuilder(protocol).append("://").append(host).append(":").append(port).append(path); + if (params.size() > 0) { + uriBuilder.append("?").append(params.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"))); } + //using this constructor no url encoding happens, as we did everything upfront in addParam and pathPart methods + return URI.create(uriBuilder.toString()); } private HttpEntityEnclosingRequestBase addOptionalBody(HttpEntityEnclosingRequestBase requestBase) { diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpResponse.java b/test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/client/http/HttpResponse.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/json/JsonPath.java b/test-framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/json/JsonPath.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/DoSectionParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/DoSectionParser.java similarity index 74% rename from core/src/test/java/org/elasticsearch/test/rest/parser/DoSectionParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/DoSectionParser.java index ec5aef54459..2a20e0f3146 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/parser/DoSectionParser.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/DoSectionParser.java @@ -25,6 +25,8 @@ import org.elasticsearch.test.rest.section.ApiCallSection; import org.elasticsearch.test.rest.section.DoSection; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; /** * Parser for do sections @@ -40,6 +42,8 @@ public class DoSectionParser implements RestTestFragmentParser { XContentParser.Token token; DoSection doSection = new DoSection(); + ApiCallSection apiCallSection = null; + Map headers = new HashMap<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -49,8 +53,17 @@ public class DoSectionParser implements RestTestFragmentParser { doSection.setCatch(parser.text()); } } else if (token == XContentParser.Token.START_OBJECT) { - if (currentFieldName != null) { - ApiCallSection apiCallSection = new ApiCallSection(currentFieldName); + if ("headers".equals(currentFieldName)) { + String headerName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + headerName = parser.currentName(); + } else if (token.isValue()) { + headers.put(headerName, parser.text()); + } + } + } else if (currentFieldName != null) { // must be part of API call then + apiCallSection = new ApiCallSection(currentFieldName); String paramName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -73,17 +86,20 @@ public class DoSectionParser implements RestTestFragmentParser { } } } - doSection.setApiCallSection(apiCallSection); } } } - - parser.nextToken(); - - if (doSection.getApiCallSection() == null) { - throw new RestTestParseException("client call section is mandatory within a do section"); + try { + if (apiCallSection == null) { + throw new RestTestParseException("client call section is mandatory within a do section"); + } + if (headers.isEmpty() == false) { + apiCallSection.addHeaders(headers); + } + doSection.setApiCallSection(apiCallSection); + } finally { + parser.nextToken(); } - return doSection; } } diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/IsFalseParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/IsFalseParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/IsFalseParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/IsFalseParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/IsTrueParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/IsTrueParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/IsTrueParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/IsTrueParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/LengthParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/LengthParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/LengthParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/LengthParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/LessThanParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/LessThanParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/MatchParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/MatchParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/MatchParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/MatchParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/RestTestFragmentParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestFragmentParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/RestTestFragmentParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestFragmentParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/RestTestParseException.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestParseException.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/RestTestParseException.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestParseException.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/SetSectionParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/SetSectionParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/SetSectionParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/SetSectionParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/SetupSectionParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/SetupSectionParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/SetupSectionParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/SetupSectionParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/parser/SkipSectionParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/parser/SkipSectionParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/parser/SkipSectionParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/parser/SkipSectionParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/ApiCallSection.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/ApiCallSection.java similarity index 85% rename from core/src/test/java/org/elasticsearch/test/rest/section/ApiCallSection.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/ApiCallSection.java index da6c0b3be2c..030469148ed 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/section/ApiCallSection.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/section/ApiCallSection.java @@ -33,6 +33,7 @@ public class ApiCallSection { private final String api; private final Map params = new HashMap<>(); + private final Map headers = new HashMap<>(); private final List> bodies = new ArrayList<>(); public ApiCallSection(String api) { @@ -56,6 +57,18 @@ public class ApiCallSection { this.params.put(key, value); } + public void addHeaders(Map otherHeaders) { + this.headers.putAll(otherHeaders); + } + + public void addHeader(String key, String value) { + this.headers.put(key, value); + } + + public Map getHeaders() { + return unmodifiableMap(headers); + } + public List> getBodies() { return Collections.unmodifiableList(bodies); } diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/Assertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/Assertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/DoSection.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java similarity index 96% rename from core/src/test/java/org/elasticsearch/test/rest/section/DoSection.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java index 9a1bf1c9267..38504c4af5f 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/section/DoSection.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java @@ -45,6 +45,9 @@ import static org.junit.Assert.fail; * * - do: * catch: missing + * headers: + * Authorization: Basic user:pass + * Content-Type: application/json * update: * index: test_1 * type: test @@ -86,7 +89,8 @@ public class DoSection implements ExecutableSection { } try { - RestResponse restResponse = executionContext.callApi(apiCallSection.getApi(), apiCallSection.getParams(), apiCallSection.getBodies()); + RestResponse restResponse = executionContext.callApi(apiCallSection.getApi(), apiCallSection.getParams(), + apiCallSection.getBodies(), apiCallSection.getHeaders()); if (Strings.hasLength(catchParam)) { String catchStatusCode; if (catches.containsKey(catchParam)) { diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/ExecutableSection.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/ExecutableSection.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/ExecutableSection.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/ExecutableSection.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/IsFalseAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/IsFalseAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/IsFalseAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/IsFalseAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/IsTrueAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/IsTrueAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/IsTrueAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/IsTrueAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/LengthAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/LengthAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/LessThanAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/LessThanAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/MatchAssertion.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/MatchAssertion.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/RestTestSuite.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/RestTestSuite.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/RestTestSuite.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/RestTestSuite.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/SetSection.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/SetSection.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/SetSection.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/SetSection.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/SetupSection.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/SetupSection.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/SetupSection.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/SetupSection.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/SkipSection.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/SkipSection.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/SkipSection.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/SkipSection.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/TestSection.java b/test-framework/src/main/java/org/elasticsearch/test/rest/section/TestSection.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/section/TestSection.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/section/TestSection.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/spec/RestApi.java b/test-framework/src/main/java/org/elasticsearch/test/rest/spec/RestApi.java similarity index 69% rename from core/src/test/java/org/elasticsearch/test/rest/spec/RestApi.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/spec/RestApi.java index bebdbfc645e..60c39b64411 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/spec/RestApi.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/spec/RestApi.java @@ -20,14 +20,12 @@ package org.elasticsearch.test.rest.spec; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.test.rest.client.RestPath; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Represents an elasticsearch REST endpoint (api) @@ -41,7 +39,7 @@ public class RestApi { private List params = new ArrayList<>(); private BODY body = BODY.NOT_SUPPORTED; - public static enum BODY { + public enum BODY { NOT_SUPPORTED, OPTIONAL, REQUIRED } @@ -131,28 +129,18 @@ public class RestApi { * Finds the best matching rest path given the current parameters and replaces * placeholders with their corresponding values received as arguments */ - public String[] getFinalPaths(Map pathParams) { - + public RestPath[] getFinalPaths(Map pathParams) { List matchingRestPaths = findMatchingRestPaths(pathParams.keySet()); if (matchingRestPaths == null || matchingRestPaths.isEmpty()) { throw new IllegalArgumentException("unable to find matching rest path for api [" + name + "] and path params " + pathParams); } - String[] paths = new String[matchingRestPaths.size()]; + RestPath[] restPaths = new RestPath[matchingRestPaths.size()]; for (int i = 0; i < matchingRestPaths.size(); i++) { RestPath restPath = matchingRestPaths.get(i); - String path = restPath.path; - for (Map.Entry paramEntry : restPath.parts.entrySet()) { - // replace path placeholders with actual values - String value = pathParams.get(paramEntry.getValue()); - if (value == null) { - throw new IllegalArgumentException("parameter [" + paramEntry.getValue() + "] missing"); - } - path = path.replace(paramEntry.getKey(), value); - } - paths[i] = path; + restPaths[i] = restPath.replacePlaceholders(pathParams); } - return paths; + return restPaths; } /** @@ -165,15 +153,11 @@ public class RestApi { List matchingRestPaths = new ArrayList<>(); RestPath[] restPaths = buildRestPaths(); - for (RestPath restPath : restPaths) { - if (restPath.parts.size() == restParams.size()) { - if (restPath.parts.values().containsAll(restParams)) { - matchingRestPaths.add(restPath); - } + if (restPath.matches(restParams)) { + matchingRestPaths.add(restPath); } } - return matchingRestPaths; } @@ -184,33 +168,4 @@ public class RestApi { } return restPaths; } - - private static class RestPath { - private static final Pattern PLACEHOLDERS_PATTERN = Pattern.compile("(\\{(.*?)})"); - - final String path; - //contains param to replace (e.g. {index}) and param key to use for lookup in the current values map (e.g. index) - final Map parts; - - RestPath(String path) { - this.path = path; - this.parts = extractParts(path); - } - - private static Map extractParts(String input) { - Map parts = new HashMap<>(); - Matcher matcher = PLACEHOLDERS_PATTERN.matcher(input); - while (matcher.find()) { - //key is e.g. {index} - String key = input.substring(matcher.start(), matcher.end()); - if (matcher.groupCount() != 2) { - throw new IllegalArgumentException("no lookup key found for param [" + key + "]"); - } - //to be replaced with current value found with key e.g. index - String value = matcher.group(2); - parts.put(key, value); - } - return parts; - } - } } diff --git a/core/src/test/java/org/elasticsearch/test/rest/spec/RestApiParser.java b/test-framework/src/main/java/org/elasticsearch/test/rest/spec/RestApiParser.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/spec/RestApiParser.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/spec/RestApiParser.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/spec/RestSpec.java b/test-framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/spec/RestSpec.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/support/Features.java b/test-framework/src/main/java/org/elasticsearch/test/rest/support/Features.java similarity index 97% rename from core/src/test/java/org/elasticsearch/test/rest/support/Features.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/support/Features.java index 018d2413737..0f51f72e8e5 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/support/Features.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/support/Features.java @@ -34,7 +34,7 @@ import java.util.List; */ public final class Features { - private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting"); + private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers"); private Features() { diff --git a/core/src/test/java/org/elasticsearch/test/rest/support/FileUtils.java b/test-framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java similarity index 97% rename from core/src/test/java/org/elasticsearch/test/rest/support/FileUtils.java rename to test-framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java index e73906efff7..69acae55fdc 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/support/FileUtils.java +++ b/test-framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java @@ -94,7 +94,7 @@ public final class FileUtils { String newPath = optionalPathPrefix + "/" + path; file = findFile(fileSystem, newPath, optionalFileSuffix); if (!lenientExists(file)) { - throw new NoSuchFileException(path); + throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: " + optionalFileSuffix); } } return file; diff --git a/core/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test-framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java similarity index 76% rename from core/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java rename to test-framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java index 43049672602..27a2e6fb22e 100644 --- a/core/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java +++ b/test-framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java @@ -27,20 +27,18 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.*; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestRuleMarkFailure; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.*; import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.index.store.IndexStoreModule; import org.elasticsearch.index.store.Store; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESTestCase; @@ -54,33 +52,27 @@ import java.util.*; public class MockFSDirectoryService extends FsDirectoryService { - public static final String CHECK_INDEX_ON_CLOSE = "index.store.mock.check_index_on_close"; public static final String RANDOM_IO_EXCEPTION_RATE_ON_OPEN = "index.store.mock.random.io_exception_rate_on_open"; public static final String RANDOM_PREVENT_DOUBLE_WRITE = "index.store.mock.random.prevent_double_write"; public static final String RANDOM_NO_DELETE_OPEN_FILE = "index.store.mock.random.no_delete_open_file"; public static final String CRASH_INDEX = "index.store.mock.random.crash_index"; - private static final EnumSet validCheckIndexStates = EnumSet.of( - IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY - ); - private final FsDirectoryService delegateService; - private final boolean checkIndexOnClose; private final Random random; private final double randomIOExceptionRate; private final double randomIOExceptionRateOnOpen; private final MockDirectoryWrapper.Throttling throttle; - private final Settings indexSettings; private final boolean preventDoubleWrite; private final boolean noDeleteOpenFile; private final boolean crashIndex; @Inject - public MockFSDirectoryService(@IndexSettings Settings indexSettings, IndexStore indexStore, final IndicesService service, final ShardPath path) { - super(indexSettings, indexStore, path); + public MockFSDirectoryService(IndexSettings idxSettings, IndexStore indexStore, final ShardPath path) { + super(idxSettings, indexStore, path); + Settings indexSettings = idxSettings.getSettings(); final long seed = indexSettings.getAsLong(ESIntegTestCase.SETTING_INDEX_SEED, 0l); this.random = new Random(seed); - checkIndexOnClose = indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, true); + randomIOExceptionRate = indexSettings.getAsDouble(RANDOM_IO_EXCEPTION_RATE, 0.0d); randomIOExceptionRateOnOpen = indexSettings.getAsDouble(RANDOM_IO_EXCEPTION_RATE_ON_OPEN, 0.0d); preventDoubleWrite = indexSettings.getAsBoolean(RANDOM_PREVENT_DOUBLE_WRITE, true); // true is default in MDW @@ -93,35 +85,7 @@ public class MockFSDirectoryService extends FsDirectoryService { logger.debug("Using MockDirWrapper with seed [{}] throttle: [{}] crashIndex: [{}]", SeedUtils.formatSeed(seed), throttle, crashIndex); } - this.indexSettings = indexSettings; delegateService = randomDirectorService(indexStore, path); - if (checkIndexOnClose) { - final IndicesLifecycle.Listener listener = new IndicesLifecycle.Listener() { - - boolean canRun = false; - - @Override - public void beforeIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - if (indexShard != null && shardId.equals(sid)) { - if (validCheckIndexStates.contains(indexShard.state()) && IndexMetaData.isOnSharedFilesystem(indexSettings) == false) { - canRun = true; - } - } - } - - @Override - public void afterIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard, - @IndexSettings Settings indexSettings) { - if (shardId.equals(sid) && indexShard != null && canRun) { - assert indexShard.state() == IndexShardState.CLOSED : "Current state must be closed"; - checkIndex(indexShard.store(), sid); - } - service.indicesLifecycle().removeListener(this); - } - }; - service.indicesLifecycle().addListener(listener); - } } @@ -135,7 +99,7 @@ public class MockFSDirectoryService extends FsDirectoryService { throw new UnsupportedOperationException(); } - public void checkIndex(Store store, ShardId shardId) { + public static void checkIndex(ESLogger logger, Store store, ShardId shardId) { if (store.tryIncRef()) { logger.info("start check index"); try { @@ -207,10 +171,10 @@ public class MockFSDirectoryService extends FsDirectoryService { } private FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) { - Settings.Builder builder = Settings.settingsBuilder(); - builder.put(indexSettings); - builder.put(IndexStoreModule.STORE_TYPE, RandomPicks.randomFrom(random, IndexStoreModule.Type.values()).getSettingsKey()); - return new FsDirectoryService(builder.build(), indexStore, path); + final IndexSettings indexSettings = indexStore.getIndexSettings(); + final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexModule.STORE_TYPE, RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey())).build(); + final IndexSettings newIndexSettings = new IndexSettings(build, indexSettings.getNodeSettings(), Collections.emptyList()); + return new FsDirectoryService(newIndexSettings, indexStore, path); } public static final class ElasticsearchMockDirectoryWrapper extends MockDirectoryWrapper { diff --git a/test-framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java b/test-framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java new file mode 100644 index 00000000000..86cf0ddb563 --- /dev/null +++ b/test-framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.store; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.shard.*; +import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.IndexStore; +import org.elasticsearch.index.store.IndexStoreConfig; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.IdentityHashMap; +import java.util.Map; + +public class MockFSIndexStore extends IndexStore { + + public static final String CHECK_INDEX_ON_CLOSE = "index.store.mock.check_index_on_close"; + + public static class TestPlugin extends Plugin { + @Override + public String name() { + return "mock-index-store"; + } + @Override + public String description() { + return "a mock index store for testing"; + } + @Override + public Settings additionalSettings() { + return Settings.builder().put(IndexModule.STORE_TYPE, "mock").build(); + } + + @Override + public void onIndexModule(IndexModule indexModule) { + Settings indexSettings = indexModule.getSettings(); + if ("mock".equals(indexSettings.get(IndexModule.STORE_TYPE))) { + if (indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, true)) { + indexModule.addIndexEventListener(new Listener()); + } + indexModule.addIndexStore("mock", MockFSIndexStore::new); + } + } + } + + MockFSIndexStore(IndexSettings indexSettings, + IndexStoreConfig config) { + super(indexSettings, config); + } + + public DirectoryService newDirectoryService(ShardPath path) { + return new MockFSDirectoryService(indexSettings, this, path); + } + + private static final EnumSet validCheckIndexStates = EnumSet.of( + IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY + ); + private static final class Listener implements IndexEventListener { + + private final Map shardSet = Collections.synchronizedMap(new IdentityHashMap<>()); + @Override + public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { + if (indexShard != null) { + Boolean remove = shardSet.remove(indexShard); + if (remove == Boolean.TRUE) { + ESLogger logger = Loggers.getLogger(getClass(), indexShard.indexSettings().getSettings(), indexShard.shardId()); + MockFSDirectoryService.checkIndex(logger, indexShard.store(), indexShard.shardId()); + } + } + } + + @Override + public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) { + if (currentState == IndexShardState.CLOSED && validCheckIndexStates.contains(previousState) && indexShard.indexSettings().isOnSharedFilesystem() == false) { + shardSet.put(indexShard, Boolean.TRUE); + } + + } + } + +} diff --git a/core/src/test/java/org/elasticsearch/test/transport/AssertingLocalTransport.java b/test-framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java similarity index 97% rename from core/src/test/java/org/elasticsearch/test/transport/AssertingLocalTransport.java rename to test-framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java index c253a752d6b..64cc401cb5f 100644 --- a/core/src/test/java/org/elasticsearch/test/transport/AssertingLocalTransport.java +++ b/test-framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java @@ -77,13 +77,15 @@ public class AssertingLocalTransport extends LocalTransport { @Override protected void handleParsedResponse(final TransportResponse response, final TransportResponseHandler handler) { - ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersionBetween(random, minVersion, maxVersion), response); + ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersionBetween(random, minVersion, maxVersion), response, + namedWriteableRegistry); super.handleParsedResponse(response, handler); } @Override public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { - ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersionBetween(random, minVersion, maxVersion), request); + ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersionBetween(random, minVersion, maxVersion), request, + namedWriteableRegistry); super.sendRequest(node, requestId, action, request, options); } } diff --git a/core/src/test/java/org/elasticsearch/test/transport/CapturingTransport.java b/test-framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java similarity index 99% rename from core/src/test/java/org/elasticsearch/test/transport/CapturingTransport.java rename to test-framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index 476b89aa1a9..2363d98a113 100644 --- a/core/src/test/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test-framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -178,6 +178,6 @@ public class CapturingTransport implements Transport { @Override public List getLocalAddresses() { - return Collections.EMPTY_LIST; + return Collections.emptyList(); } } diff --git a/core/src/test/java/org/elasticsearch/test/transport/MockTransportService.java b/test-framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java similarity index 77% rename from core/src/test/java/org/elasticsearch/test/transport/MockTransportService.java rename to test-framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index f5da216da5d..e1efd6c3745 100644 --- a/core/src/test/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test-framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -55,6 +55,14 @@ import java.util.concurrent.CopyOnWriteArrayList; /** * A mock transport service that allows to simulate different network topology failures. + * Internally it maps TransportAddress objects to rules that inject failures. + * Adding rules for a node is done by adding rules for all bound addresses of a node + * (and the publish address, if different). + * Matching requests to rules is based on the transport address associated with the + * discovery node of the request, namely by DiscoveryNode.getAddress(). + * This address is usually the publish address of the node but can also be a different one + * (for example, @see org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing, which constructs + * fake DiscoveryNode instances where the publish address is one of the bound addresses). */ public class MockTransportService extends TransportService { @@ -82,7 +90,14 @@ public class MockTransportService extends TransportService { public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool) { super(settings, new LookupTestTransport(transport), threadPool); this.original = transport; + } + public static TransportAddress[] extractTransportAddresses(TransportService transportService) { + HashSet transportAddresses = new HashSet<>(); + BoundTransportAddress boundTransportAddress = transportService.boundAddress(); + transportAddresses.addAll(Arrays.asList(boundTransportAddress.boundAddresses())); + transportAddresses.add(boundTransportAddress.publishAddress()); + return transportAddresses.toArray(new TransportAddress[transportAddresses.size()]); } /** @@ -93,10 +108,19 @@ public class MockTransportService extends TransportService { } /** - * Clears the rule associated with the provided node. + * Clears the rule associated with the provided transport service. */ - public void clearRule(DiscoveryNode node) { - transport().transports.remove(node.getAddress()); + public void clearRule(TransportService transportService) { + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + clearRule(transportAddress); + } + } + + /** + * Clears the rule associated with the provided transport address. + */ + public void clearRule(TransportAddress transportAddress) { + transport().transports.remove(transportAddress); } /** @@ -110,8 +134,18 @@ public class MockTransportService extends TransportService { * Adds a rule that will cause every send request to fail, and each new connect since the rule * is added to fail as well. */ - public void addFailToSendNoConnectRule(DiscoveryNode node) { - addDelegate(node, new DelegateTransport(original) { + public void addFailToSendNoConnectRule(TransportService transportService) { + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + addFailToSendNoConnectRule(transportAddress); + } + } + + /** + * Adds a rule that will cause every send request to fail, and each new connect since the rule + * is added to fail as well. + */ + public void addFailToSendNoConnectRule(TransportAddress transportAddress) { + addDelegate(transportAddress, new DelegateTransport(original) { @Override public void connectToNode(DiscoveryNode node) throws ConnectTransportException { throw new ConnectTransportException(node, "DISCONNECT: simulated"); @@ -132,16 +166,32 @@ public class MockTransportService extends TransportService { /** * Adds a rule that will cause matching operations to throw ConnectTransportExceptions */ - public void addFailToSendNoConnectRule(DiscoveryNode node, final String... blockedActions) { - addFailToSendNoConnectRule(node, new HashSet<>(Arrays.asList(blockedActions))); + public void addFailToSendNoConnectRule(TransportService transportService, final String... blockedActions) { + addFailToSendNoConnectRule(transportService, new HashSet<>(Arrays.asList(blockedActions))); } /** * Adds a rule that will cause matching operations to throw ConnectTransportExceptions */ - public void addFailToSendNoConnectRule(DiscoveryNode node, final Set blockedActions) { + public void addFailToSendNoConnectRule(TransportAddress transportAddress, final String... blockedActions) { + addFailToSendNoConnectRule(transportAddress, new HashSet<>(Arrays.asList(blockedActions))); + } - addDelegate(node, new DelegateTransport(original) { + /** + * Adds a rule that will cause matching operations to throw ConnectTransportExceptions + */ + public void addFailToSendNoConnectRule(TransportService transportService, final Set blockedActions) { + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + addFailToSendNoConnectRule(transportAddress, blockedActions); + } + } + + /** + * Adds a rule that will cause matching operations to throw ConnectTransportExceptions + */ + public void addFailToSendNoConnectRule(TransportAddress transportAddress, final Set blockedActions) { + + addDelegate(transportAddress, new DelegateTransport(original) { @Override public void connectToNode(DiscoveryNode node) throws ConnectTransportException { original.connectToNode(node); @@ -167,8 +217,18 @@ public class MockTransportService extends TransportService { * Adds a rule that will cause ignores each send request, simulating an unresponsive node * and failing to connect once the rule was added. */ - public void addUnresponsiveRule(DiscoveryNode node) { - addDelegate(node, new DelegateTransport(original) { + public void addUnresponsiveRule(TransportService transportService) { + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + addUnresponsiveRule(transportAddress); + } + } + + /** + * Adds a rule that will cause ignores each send request, simulating an unresponsive node + * and failing to connect once the rule was added. + */ + public void addUnresponsiveRule(TransportAddress transportAddress) { + addDelegate(transportAddress, new DelegateTransport(original) { @Override public void connectToNode(DiscoveryNode node) throws ConnectTransportException { throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); @@ -192,10 +252,22 @@ public class MockTransportService extends TransportService { * * @param duration the amount of time to delay sending and connecting. */ - public void addUnresponsiveRule(DiscoveryNode node, final TimeValue duration) { + public void addUnresponsiveRule(TransportService transportService, final TimeValue duration) { + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + addUnresponsiveRule(transportAddress, duration); + } + } + + /** + * Adds a rule that will cause ignores each send request, simulating an unresponsive node + * and failing to connect once the rule was added. + * + * @param duration the amount of time to delay sending and connecting. + */ + public void addUnresponsiveRule(TransportAddress transportAddress, final TimeValue duration) { final long startTime = System.currentTimeMillis(); - addDelegate(node, new DelegateTransport(original) { + addDelegate(transportAddress, new DelegateTransport(original) { TimeValue getDelay() { return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime)); @@ -280,12 +352,25 @@ public class MockTransportService extends TransportService { } /** - * Adds a new delegate transport that is used for communication with the given node. + * Adds a new delegate transport that is used for communication with the given transport service. * - * @return true iff no other delegate was registered for this node before, otherwise false + * @return true iff no other delegate was registered for any of the addresses bound by transport service, otherwise false */ - public boolean addDelegate(DiscoveryNode node, DelegateTransport transport) { - return transport().transports.put(node.getAddress(), transport) == null; + public boolean addDelegate(TransportService transportService, DelegateTransport transport) { + boolean noRegistered = true; + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + noRegistered &= addDelegate(transportAddress, transport); + } + return noRegistered; + } + + /** + * Adds a new delegate transport that is used for communication with the given transport address. + * + * @return true iff no other delegate was registered for this address before, otherwise false + */ + public boolean addDelegate(TransportAddress transportAddress, DelegateTransport transport) { + return transport().transports.put(transportAddress, transport) == null; } private LookupTestTransport transport() { diff --git a/dev-tools/src/main/resources/shared-test-resources/log4j.properties b/test-framework/src/main/resources/log4j.properties similarity index 100% rename from dev-tools/src/main/resources/shared-test-resources/log4j.properties rename to test-framework/src/main/resources/log4j.properties diff --git a/test-framework/src/test/java/org/elasticsearch/test/rest/BlacklistedPathPatternMatcherTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/BlacklistedPathPatternMatcherTests.java new file mode 100644 index 00000000000..9414a2219cf --- /dev/null +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/BlacklistedPathPatternMatcherTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test.rest; + + +import org.elasticsearch.test.ESTestCase; + +public class BlacklistedPathPatternMatcherTests extends ESTestCase { + + public void testMatchesExact() { + // suffix match + assertMatch("cat.aliases/10_basic/Empty cluster", "/some/suite_path/cat.aliases/10_basic/Empty cluster"); + // exact match + assertMatch("cat.aliases/10_basic/Empty cluster", "cat.aliases/10_basic/Empty cluster"); + // additional text at the end should not match + assertNoMatch("cat.aliases/10_basic/Empty cluster", "cat.aliases/10_basic/Empty clusters in here"); + } + + public void testMatchesSimpleWildcardPatterns() { + assertMatch("termvector/20_issue7121/*", "/suite/termvector/20_issue7121/test_first"); + assertMatch("termvector/20_issue7121/*", "/suite/termvector/20_issue7121/"); + // do not cross segment boundaries + assertNoMatch("termvector/20_issue7121/*", "/suite/termvector/20_issue7121/test/first"); + } + + public void testMatchesMultiWildcardPatterns() { + assertMatch("indices.get/10_basic/*allow_no_indices*", "/suite/indices.get/10_basic/we_allow_no_indices"); + assertMatch("indices.get/10_basic/*allow_no_indices*", "/suite/indices.get/10_basic/we_allow_no_indices_at_all"); + assertNoMatch("indices.get/10_basic/*allow_no_indices*", "/suite/indices.get/10_basic/we_allow_no_indices_at_all/here"); + assertMatch("indices.get/*/*allow_no_indices*", "/suite/indices.get/10_basic/we_allow_no_indices_at_all"); + assertMatch("indices.get/*/*allow_no_indices*", "/suite/indices.get/20_basic/we_allow_no_indices_at_all"); + assertMatch("*/*/*allow_no_indices*", "/suite/path/to/test/indices.get/20_basic/we_allow_no_indices_at_all"); + } + + public void testMatchesPatternsWithEscapedCommas() { + assertMatch("indices.get/10_basic\\,20_advanced/foo", "/suite/indices.get/10_basic,20_advanced/foo"); + } + + public void testMatchesMixedPatterns() { + assertMatch("indices.get/*/10_basic\\,20_advanced/*foo*", "/suite/indices.get/all/10_basic,20_advanced/foo"); + assertMatch("indices.get/*/10_basic\\,20_advanced/*foo*", "/suite/indices.get/all/10_basic,20_advanced/my_foo"); + assertMatch("indices.get/*/10_basic\\,20_advanced/*foo*", "/suite/indices.get/all/10_basic,20_advanced/foo_bar"); + } + + + + private void assertMatch(String pattern, String path) { + BlacklistedPathPatternMatcher matcher = new BlacklistedPathPatternMatcher(pattern); + assertTrue("Pattern [" + pattern + "] should have matched path [" + path + "]", matcher.isSuffixMatch(path)); + } + + private void assertNoMatch(String pattern, String path) { + BlacklistedPathPatternMatcher matcher = new BlacklistedPathPatternMatcher(pattern); + assertFalse("Pattern [" + pattern + "] should not have matched path [" + path + "]", matcher.isSuffixMatch(path)); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/AbstractParserTestCase.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/AbstractParserTestCase.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/test/AbstractParserTestCase.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/AbstractParserTestCase.java diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/AssertionParsersTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/AssertionParsersTests.java similarity index 88% rename from core/src/test/java/org/elasticsearch/test/rest/test/AssertionParsersTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/AssertionParsersTests.java index b316ae0face..68b84b99639 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/AssertionParsersTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/AssertionParsersTests.java @@ -19,18 +19,28 @@ package org.elasticsearch.test.rest.test; import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.rest.parser.*; -import org.elasticsearch.test.rest.section.*; -import org.junit.Test; +import org.elasticsearch.test.rest.parser.GreaterThanParser; +import org.elasticsearch.test.rest.parser.IsFalseParser; +import org.elasticsearch.test.rest.parser.IsTrueParser; +import org.elasticsearch.test.rest.parser.LengthParser; +import org.elasticsearch.test.rest.parser.LessThanParser; +import org.elasticsearch.test.rest.parser.MatchParser; +import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext; +import org.elasticsearch.test.rest.section.GreaterThanAssertion; +import org.elasticsearch.test.rest.section.IsFalseAssertion; +import org.elasticsearch.test.rest.section.IsTrueAssertion; +import org.elasticsearch.test.rest.section.LengthAssertion; +import org.elasticsearch.test.rest.section.LessThanAssertion; +import org.elasticsearch.test.rest.section.MatchAssertion; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; public class AssertionParsersTests extends AbstractParserTestCase { - - @Test public void testParseIsTrue() throws Exception { parser = YamlXContent.yamlXContent.createParser( "get.fields._timestamp" @@ -43,7 +53,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat(trueAssertion.getField(), equalTo("get.fields._timestamp")); } - @Test public void testParseIsFalse() throws Exception { parser = YamlXContent.yamlXContent.createParser( "docs.1._source" @@ -56,7 +65,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat(falseAssertion.getField(), equalTo("docs.1._source")); } - @Test public void testParseGreaterThan() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ field: 3}" @@ -70,7 +78,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(3)); } - @Test public void testParseLessThan() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ field: 3}" @@ -84,7 +91,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(3)); } - @Test public void testParseLength() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ _id: 22}" @@ -98,8 +104,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(22)); } - @Test - @SuppressWarnings("unchecked") public void testParseMatchSimpleIntegerValue() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ field: 10 }" @@ -114,8 +118,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat((Integer) matchAssertion.getExpectedValue(), equalTo(10)); } - @Test - @SuppressWarnings("unchecked") public void testParseMatchSimpleStringValue() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ foo: bar }" @@ -130,8 +132,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat(matchAssertion.getExpectedValue().toString(), equalTo("bar")); } - @Test - @SuppressWarnings("unchecked") public void testParseMatchArray() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{'matches': ['test_percolator_1', 'test_percolator_2']}" @@ -149,7 +149,6 @@ public class AssertionParsersTests extends AbstractParserTestCase { assertThat(strings.get(1).toString(), equalTo("test_percolator_2")); } - @Test @SuppressWarnings("unchecked") public void testParseMatchSourceValues() throws Exception { parser = YamlXContent.yamlXContent.createParser( diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/DoSectionParserTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/DoSectionParserTests.java similarity index 91% rename from core/src/test/java/org/elasticsearch/test/rest/test/DoSectionParserTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/DoSectionParserTests.java index c3aba8114a9..3c65fda94ca 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/DoSectionParserTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/DoSectionParserTests.java @@ -27,18 +27,16 @@ import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext; import org.elasticsearch.test.rest.section.ApiCallSection; import org.elasticsearch.test.rest.section.DoSection; import org.hamcrest.MatcherAssert; -import org.junit.Test; import java.io.IOException; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class DoSectionParserTests extends AbstractParserTestCase { - - @Test public void testParseDoSectionNoBody() throws Exception { parser = YamlXContent.yamlXContent.createParser( "get:\n" + @@ -60,7 +58,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertThat(apiCallSection.hasBody(), equalTo(false)); } - @Test public void testParseDoSectionNoParamsNoBody() throws Exception { parser = YamlXContent.yamlXContent.createParser( "cluster.node_info: {}" @@ -76,7 +73,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertThat(apiCallSection.hasBody(), equalTo(false)); } - @Test public void testParseDoSectionWithJsonBody() throws Exception { String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }"; parser = YamlXContent.yamlXContent.createParser( @@ -102,7 +98,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertJsonEquals(apiCallSection.getBodies().get(0), body); } - @Test public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception { String bodies[] = new String[]{ "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n", @@ -132,7 +127,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertThat(apiCallSection.getBodies().size(), equalTo(4)); } - @Test public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception { String[] bodies = new String[] { "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }", @@ -162,7 +156,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { } } - @Test public void testParseDoSectionWithYamlBody() throws Exception { parser = YamlXContent.yamlXContent.createParser( "search:\n" + @@ -184,7 +177,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertJsonEquals(apiCallSection.getBodies().get(0), body); } - @Test public void testParseDoSectionWithYamlMultipleBodies() throws Exception { parser = YamlXContent.yamlXContent.createParser( "bulk:\n" + @@ -225,7 +217,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { } } - @Test public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception { parser = YamlXContent.yamlXContent.createParser( "bulk:\n" + @@ -259,7 +250,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { } } - @Test public void testParseDoSectionWithYamlBodyMultiGet() throws Exception { parser = YamlXContent.yamlXContent.createParser( "mget:\n" + @@ -285,7 +275,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertJsonEquals(apiCallSection.getBodies().get(0), body); } - @Test public void testParseDoSectionWithBodyStringified() throws Exception { parser = YamlXContent.yamlXContent.createParser( "index:\n" + @@ -311,7 +300,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertJsonEquals(apiCallSection.getBodies().get(0), "{ _source: true, query: { match_all: {} } }"); } - @Test public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception { parser = YamlXContent.yamlXContent.createParser( "index:\n" + @@ -335,7 +323,6 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertJsonEquals(apiCallSection.getBodies().get(1), body); } - @Test public void testParseDoSectionWithCatch() throws Exception { parser = YamlXContent.yamlXContent.createParser( "catch: missing\n" + @@ -354,17 +341,43 @@ public class DoSectionParserTests extends AbstractParserTestCase { assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); } - @Test (expected = RestTestParseException.class) + public void testParseDoSectionWithHeaders() throws Exception { + parser = YamlXContent.yamlXContent.createParser( + "headers:\n" + + " Authorization: \"thing one\"\n" + + " Content-Type: \"application/json\"\n" + + "indices.get_warmer:\n" + + " index: test_index\n" + + " name: test_warmer" + ); + + DoSectionParser doSectionParser = new DoSectionParser(); + DoSection doSection = doSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + assertThat(doSection.getApiCallSection().getHeaders(), notNullValue()); + assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one")); + assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json")); + } + public void testParseDoSectionWithoutClientCallSection() throws Exception { parser = YamlXContent.yamlXContent.createParser( "catch: missing\n" ); DoSectionParser doSectionParser = new DoSectionParser(); - doSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + try { + doSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + fail("Expected RestTestParseException"); + } catch (RestTestParseException e) { + assertThat(e.getMessage(), is("client call section is mandatory within a do section")); + } } - @Test public void testParseDoSectionMultivaluedField() throws Exception { parser = YamlXContent.yamlXContent.createParser( "indices.get_field_mapping:\n" + diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java similarity index 76% rename from core/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java index 2aad874774f..128cb862e57 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.test.rest.test; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.support.FileUtils; -import org.junit.Test; import java.nio.file.Files; import java.nio.file.Path; @@ -32,48 +31,45 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.greaterThan; public class FileUtilsTests extends ESTestCase { - - @Test public void testLoadSingleYamlSuite() throws Exception { - Map> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "/rest-api-spec/test/get/10_basic"); - assertSingleFile(yamlSuites, "get", "10_basic.yaml"); + Map> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "/rest-api-spec/test/suite1/10_basic"); + assertSingleFile(yamlSuites, "suite1", "10_basic.yaml"); //the path prefix is optional - yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "get/10_basic.yaml"); - assertSingleFile(yamlSuites, "get", "10_basic.yaml"); + yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1/10_basic.yaml"); + assertSingleFile(yamlSuites, "suite1", "10_basic.yaml"); //extension .yaml is optional - yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "get/10_basic"); - assertSingleFile(yamlSuites, "get", "10_basic.yaml"); + yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1/10_basic"); + assertSingleFile(yamlSuites, "suite1", "10_basic.yaml"); } - @Test public void testLoadMultipleYamlSuites() throws Exception { //single directory - Map> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "get"); + Map> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1"); assertThat(yamlSuites, notNullValue()); assertThat(yamlSuites.size(), equalTo(1)); - assertThat(yamlSuites.containsKey("get"), equalTo(true)); - assertThat(yamlSuites.get("get").size(), greaterThan(1)); + assertThat(yamlSuites.containsKey("suite1"), equalTo(true)); + assertThat(yamlSuites.get("suite1").size(), greaterThan(1)); //multiple directories - yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "get", "index"); + yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1", "suite2"); assertThat(yamlSuites, notNullValue()); assertThat(yamlSuites.size(), equalTo(2)); - assertThat(yamlSuites.containsKey("get"), equalTo(true)); - assertThat(yamlSuites.get("get").size(), greaterThan(1)); - assertThat(yamlSuites.containsKey("index"), equalTo(true)); - assertThat(yamlSuites.get("index").size(), greaterThan(1)); + assertThat(yamlSuites.containsKey("suite1"), equalTo(true)); + assertEquals(2, yamlSuites.get("suite1").size()); + assertThat(yamlSuites.containsKey("suite2"), equalTo(true)); + assertEquals(2, yamlSuites.get("suite2").size()); //multiple paths, which can be both directories or yaml test suites (with optional file extension) - yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "indices.optimize/10_basic", "index"); + yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite2/10_basic", "suite1"); assertThat(yamlSuites, notNullValue()); assertThat(yamlSuites.size(), equalTo(2)); - assertThat(yamlSuites.containsKey("indices.optimize"), equalTo(true)); - assertThat(yamlSuites.get("indices.optimize").size(), equalTo(1)); - assertSingleFile(yamlSuites.get("indices.optimize"), "indices.optimize", "10_basic.yaml"); - assertThat(yamlSuites.containsKey("index"), equalTo(true)); - assertThat(yamlSuites.get("index").size(), greaterThan(1)); + assertThat(yamlSuites.containsKey("suite2"), equalTo(true)); + assertThat(yamlSuites.get("suite2").size(), equalTo(1)); + assertSingleFile(yamlSuites.get("suite2"), "suite2", "10_basic.yaml"); + assertThat(yamlSuites.containsKey("suite1"), equalTo(true)); + assertThat(yamlSuites.get("suite1").size(), greaterThan(1)); //files can be loaded from classpath and from file system too Path dir = createTempDir(); diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java similarity index 96% rename from core/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java index dfc12253ed1..fefcd57af79 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java @@ -21,17 +21,19 @@ package org.elasticsearch.test.rest.test; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.Stash; import org.elasticsearch.test.rest.json.JsonPath; -import org.junit.Test; import java.util.List; import java.util.Map; import java.util.Set; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class JsonPathTests extends ESTestCase { - - @Test public void testEvaluateObjectPathEscape() throws Exception { String json = "{ \"field1\": { \"field2.field3\" : \"value2\" } }"; JsonPath jsonPath = new JsonPath(json); @@ -40,7 +42,6 @@ public class JsonPathTests extends ESTestCase { assertThat((String)object, equalTo("value2")); } - @Test public void testEvaluateObjectPathWithDoubleDot() throws Exception { String json = "{ \"field1\": { \"field2\" : \"value2\" } }"; JsonPath jsonPath = new JsonPath(json); @@ -49,7 +50,6 @@ public class JsonPathTests extends ESTestCase { assertThat((String)object, equalTo("value2")); } - @Test public void testEvaluateObjectPathEndsWithDot() throws Exception { String json = "{ \"field1\": { \"field2\" : \"value2\" } }"; JsonPath jsonPath = new JsonPath(json); @@ -58,7 +58,6 @@ public class JsonPathTests extends ESTestCase { assertThat((String)object, equalTo("value2")); } - @Test public void testEvaluateString() throws Exception { String json = "{ \"field1\": { \"field2\" : \"value2\" } }"; JsonPath jsonPath = new JsonPath(json); @@ -67,7 +66,6 @@ public class JsonPathTests extends ESTestCase { assertThat((String)object, equalTo("value2")); } - @Test public void testEvaluateInteger() throws Exception { String json = "{ \"field1\": { \"field2\" : 333 } }"; JsonPath jsonPath = new JsonPath(json); @@ -76,7 +74,6 @@ public class JsonPathTests extends ESTestCase { assertThat((Integer)object, equalTo(333)); } - @Test public void testEvaluateDouble() throws Exception { String json = "{ \"field1\": { \"field2\" : 3.55 } }"; JsonPath jsonPath = new JsonPath(json); @@ -85,7 +82,6 @@ public class JsonPathTests extends ESTestCase { assertThat((Double)object, equalTo(3.55)); } - @Test public void testEvaluateArray() throws Exception { String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }"; JsonPath jsonPath = new JsonPath(json); @@ -99,7 +95,6 @@ public class JsonPathTests extends ESTestCase { assertThat((String)list.get(1), equalTo("value2")); } - @Test public void testEvaluateArrayElement() throws Exception { String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }"; JsonPath jsonPath = new JsonPath(json); @@ -108,7 +103,6 @@ public class JsonPathTests extends ESTestCase { assertThat((String)object, equalTo("value2")); } - @Test public void testEvaluateArrayElementObject() throws Exception { String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }"; JsonPath jsonPath = new JsonPath(json); @@ -117,7 +111,6 @@ public class JsonPathTests extends ESTestCase { assertThat((String)object, equalTo("value2")); } - @Test public void testEvaluateArrayElementObjectWrongPath() throws Exception { String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }"; JsonPath jsonPath = new JsonPath(json); @@ -125,7 +118,6 @@ public class JsonPathTests extends ESTestCase { assertThat(object, nullValue()); } - @Test @SuppressWarnings("unchecked") public void testEvaluateObjectKeys() throws Exception { String json = "{ \"metadata\": { \"templates\" : {\"template_1\": { \"field\" : \"value\"}, \"template_2\": { \"field\" : \"value\"} } } }"; @@ -138,7 +130,6 @@ public class JsonPathTests extends ESTestCase { assertThat(strings, contains("template_1", "template_2")); } - @Test @SuppressWarnings("unchecked") public void testEvaluateEmptyPath() throws Exception { String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }"; @@ -149,7 +140,6 @@ public class JsonPathTests extends ESTestCase { assertThat(((Map)object).containsKey("field1"), equalTo(true)); } - @Test public void testEvaluateStashInPropertyName() throws Exception { String json = "{ \"field1\": { \"elements\" : {\"element1\": \"value1\"}}}"; JsonPath jsonPath = new JsonPath(json); diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java similarity index 93% rename from core/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java index c25ebd42ffd..e2f321c81c5 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.spec.RestApiParser; -import org.junit.Test; import java.io.IOException; @@ -32,14 +31,11 @@ import static org.hamcrest.Matchers.containsString; * */ public class RestApiParserFailingTests extends ESTestCase { - - @Test - public void brokenSpecShouldThrowUsefulExceptionWhenParsingFailsOnParams() throws Exception { + public void testBrokenSpecShouldThrowUsefulExceptionWhenParsingFailsOnParams() throws Exception { parseAndExpectFailure(BROKEN_SPEC_PARAMS, "Expected params field in rest api definition to contain an object"); } - @Test - public void brokenSpecShouldThrowUsefulExceptionWhenParsingFailsOnParts() throws Exception { + public void testBrokenSpecShouldThrowUsefulExceptionWhenParsingFailsOnParts() throws Exception { parseAndExpectFailure(BROKEN_SPEC_PARTS, "Expected parts field in rest api definition to contain an object"); } @@ -51,7 +47,6 @@ public class RestApiParserFailingTests extends ESTestCase { } catch (IOException e) { assertThat(e.getMessage(), containsString(expectedErrorMessage)); } - } // see params section is broken, an inside param is missing diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java similarity index 99% rename from core/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java index 7f4cdf10abc..262b155c668 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java @@ -21,15 +21,12 @@ package org.elasticsearch.test.rest.test; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.spec.RestApi; import org.elasticsearch.test.rest.spec.RestApiParser; -import org.junit.Test; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; public class RestApiParserTests extends AbstractParserTestCase { - - @Test public void testParseRestSpecIndexApi() throws Exception { parser = JsonXContent.jsonXContent.createParser(REST_SPEC_INDEX_API); RestApi restApi = new RestApiParser().parse(parser); @@ -52,7 +49,6 @@ public class RestApiParserTests extends AbstractParserTestCase { assertThat(restApi.isBodyRequired(), equalTo(true)); } - @Test public void testParseRestSpecGetTemplateApi() throws Exception { parser = JsonXContent.jsonXContent.createParser(REST_SPEC_GET_TEMPLATE_API); RestApi restApi = new RestApiParser().parse(parser); @@ -70,7 +66,6 @@ public class RestApiParserTests extends AbstractParserTestCase { assertThat(restApi.isBodyRequired(), equalTo(false)); } - @Test public void testParseRestSpecCountApi() throws Exception { parser = JsonXContent.jsonXContent.createParser(REST_SPEC_COUNT_API); RestApi restApi = new RestApiParser().parse(parser); diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java similarity index 97% rename from core/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java index 5a31bfb0ba8..e15b62147cf 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java @@ -30,14 +30,16 @@ import org.elasticsearch.test.rest.section.IsTrueAssertion; import org.elasticsearch.test.rest.section.MatchAssertion; import org.elasticsearch.test.rest.section.RestTestSuite; import org.junit.After; -import org.junit.Test; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class RestTestParserTests extends ESTestCase { - private XContentParser parser; @Override @@ -52,7 +54,6 @@ public class RestTestParserTests extends ESTestCase { parser.close(); } - @Test public void testParseTestSetupAndSections() throws Exception { parser = YamlXContent.yamlXContent.createParser( "setup:\n" + @@ -139,7 +140,6 @@ public class RestTestParserTests extends ESTestCase { assertThat(matchAssertion.getExpectedValue().toString(), equalTo("whitespace")); } - @Test public void testParseTestSingleTestSection() throws Exception { parser = YamlXContent.yamlXContent.createParser( "---\n" + @@ -238,7 +238,6 @@ public class RestTestParserTests extends ESTestCase { assertThat(((Map) matchAssertion.getExpectedValue()).get("foo").toString(), equalTo("bar")); } - @Test public void testParseTestMultipleTestSections() throws Exception { parser = YamlXContent.yamlXContent.createParser( "---\n" + @@ -330,7 +329,6 @@ public class RestTestParserTests extends ESTestCase { assertThat(doSection.getApiCallSection().hasBody(), equalTo(true)); } - @Test(expected = RestTestParseException.class) public void testParseTestDuplicateTestSections() throws Exception { parser = YamlXContent.yamlXContent.createParser( "---\n" + @@ -361,6 +359,11 @@ public class RestTestParserTests extends ESTestCase { ); RestTestSuiteParser testParser = new RestTestSuiteParser(); - testParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + try { + testParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + fail("Expected RestTestParseException"); + } catch (RestTestParseException e) { + assertThat(e.getMessage(), containsString("duplicate test section")); + } } } diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/SetSectionParserTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/SetSectionParserTests.java similarity index 89% rename from core/src/test/java/org/elasticsearch/test/rest/test/SetSectionParserTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/SetSectionParserTests.java index ba28a486632..c2b66375664 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/SetSectionParserTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/SetSectionParserTests.java @@ -23,14 +23,12 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext; import org.elasticsearch.test.rest.parser.SetSectionParser; import org.elasticsearch.test.rest.section.SetSection; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; public class SetSectionParserTests extends AbstractParserTestCase { - - @Test public void testParseSetSectionSingleValue() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ _id: id }" @@ -46,7 +44,6 @@ public class SetSectionParserTests extends AbstractParserTestCase { assertThat(setSection.getStash().get("_id"), equalTo("id")); } - @Test public void testParseSetSectionMultipleValues() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ _id: id, _type: type, _index: index }" @@ -64,14 +61,17 @@ public class SetSectionParserTests extends AbstractParserTestCase { assertThat(setSection.getStash().get("_index"), equalTo("index")); } - @Test(expected = RestTestParseException.class) public void testParseSetSectionNoValues() throws Exception { parser = YamlXContent.yamlXContent.createParser( "{ }" ); SetSectionParser setSectionParser = new SetSectionParser(); - - setSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + try { + setSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + fail("Expected RestTestParseException"); + } catch (RestTestParseException e) { + assertThat(e.getMessage(), is("set section must set at least a value")); + } } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/SetupSectionParserTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/SetupSectionParserTests.java similarity index 99% rename from core/src/test/java/org/elasticsearch/test/rest/test/SetupSectionParserTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/SetupSectionParserTests.java index beb7449d830..9dd388056d5 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/SetupSectionParserTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/SetupSectionParserTests.java @@ -23,16 +23,12 @@ import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext; import org.elasticsearch.test.rest.parser.SetupSectionParser; import org.elasticsearch.test.rest.section.SetupSection; -import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; public class SetupSectionParserTests extends AbstractParserTestCase { - - @Test public void testParseSetupSection() throws Exception { - parser = YamlXContent.yamlXContent.createParser( " - do:\n" + " index1:\n" + @@ -58,9 +54,7 @@ public class SetupSectionParserTests extends AbstractParserTestCase { assertThat(setupSection.getDoSections().get(1).getApiCallSection().getApi(), equalTo("index2")); } - @Test public void testParseSetupAndSkipSectionNoSkip() throws Exception { - parser = YamlXContent.yamlXContent.createParser( " - skip:\n" + " version: \"0.90.0 - 0.90.7\"\n" + diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/SkipSectionParserTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/SkipSectionParserTests.java similarity index 82% rename from core/src/test/java/org/elasticsearch/test/rest/test/SkipSectionParserTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/SkipSectionParserTests.java index a697427f853..5864e78134d 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/SkipSectionParserTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/SkipSectionParserTests.java @@ -25,13 +25,13 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext; import org.elasticsearch.test.rest.parser.SkipSectionParser; import org.elasticsearch.test.rest.section.SkipSection; -import org.junit.Test; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class SkipSectionParserTests extends AbstractParserTestCase { - - @Test public void testParseSkipSectionVersionNoFeature() throws Exception { parser = YamlXContent.yamlXContent.createParser( "version: \" - 0.90.2\"\n" + @@ -66,7 +66,6 @@ public class SkipSectionParserTests extends AbstractParserTestCase { assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param")); } - @Test public void testParseSkipSectionFeatureNoVersion() throws Exception { parser = YamlXContent.yamlXContent.createParser( "features: regex" @@ -83,7 +82,6 @@ public class SkipSectionParserTests extends AbstractParserTestCase { assertThat(skipSection.getReason(), nullValue()); } - @Test public void testParseSkipSectionFeaturesNoVersion() throws Exception { parser = YamlXContent.yamlXContent.createParser( "features: [regex1,regex2,regex3]" @@ -102,7 +100,6 @@ public class SkipSectionParserTests extends AbstractParserTestCase { assertThat(skipSection.getReason(), nullValue()); } - @Test(expected = RestTestParseException.class) public void testParseSkipSectionBothFeatureAndVersion() throws Exception { parser = YamlXContent.yamlXContent.createParser( "version: \" - 0.90.2\"\n" + @@ -112,26 +109,39 @@ public class SkipSectionParserTests extends AbstractParserTestCase { SkipSectionParser skipSectionParser = new SkipSectionParser(); - skipSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + try { + skipSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + fail("Expected RestTestParseException"); + } catch (RestTestParseException e) { + assertThat(e.getMessage(), is("version or features are mutually exclusive")); + } } - @Test(expected = RestTestParseException.class) public void testParseSkipSectionNoReason() throws Exception { parser = YamlXContent.yamlXContent.createParser( "version: \" - 0.90.2\"\n" ); SkipSectionParser skipSectionParser = new SkipSectionParser(); - skipSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + try { + skipSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + fail("Expected RestTestParseException"); + } catch (RestTestParseException e) { + assertThat(e.getMessage(), is("reason is mandatory within skip version section")); + } } - @Test(expected = RestTestParseException.class) public void testParseSkipSectionNoVersionNorFeature() throws Exception { parser = YamlXContent.yamlXContent.createParser( "reason: Delete ignores the parent param\n" ); SkipSectionParser skipSectionParser = new SkipSectionParser(); - skipSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + try { + skipSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser)); + fail("Expected RestTestParseException"); + } catch (RestTestParseException e) { + assertThat(e.getMessage(), is("version or features is mandatory within skip section")); + } } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/test/rest/test/TestSectionParserTests.java b/test-framework/src/test/java/org/elasticsearch/test/rest/test/TestSectionParserTests.java similarity index 94% rename from core/src/test/java/org/elasticsearch/test/rest/test/TestSectionParserTests.java rename to test-framework/src/test/java/org/elasticsearch/test/rest/test/TestSectionParserTests.java index ac6a3c129fa..c157610b645 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/test/TestSectionParserTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/rest/test/TestSectionParserTests.java @@ -22,16 +22,25 @@ import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.test.rest.parser.RestTestSectionParser; import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext; -import org.elasticsearch.test.rest.section.*; -import org.junit.Test; +import org.elasticsearch.test.rest.section.DoSection; +import org.elasticsearch.test.rest.section.GreaterThanAssertion; +import org.elasticsearch.test.rest.section.IsFalseAssertion; +import org.elasticsearch.test.rest.section.IsTrueAssertion; +import org.elasticsearch.test.rest.section.LengthAssertion; +import org.elasticsearch.test.rest.section.LessThanAssertion; +import org.elasticsearch.test.rest.section.MatchAssertion; +import org.elasticsearch.test.rest.section.SetSection; +import org.elasticsearch.test.rest.section.SkipSection; +import org.elasticsearch.test.rest.section.TestSection; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class TestSectionParserTests extends AbstractParserTestCase { - - @Test public void testParseTestSectionWithDoSection() throws Exception { parser = YamlXContent.yamlXContent.createParser( "\"First test section\": \n" + @@ -57,7 +66,6 @@ public class TestSectionParserTests extends AbstractParserTestCase { assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); } - @Test public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception { String yaml = "\"First test section\": \n" + @@ -94,7 +102,6 @@ public class TestSectionParserTests extends AbstractParserTestCase { assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id")); } - @Test public void testParseTestSectionWithMultipleDoSections() throws Exception { parser = YamlXContent.yamlXContent.createParser( "\"Basic\":\n" + @@ -133,7 +140,6 @@ public class TestSectionParserTests extends AbstractParserTestCase { assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); } - @Test public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception { parser = YamlXContent.yamlXContent.createParser( "\"Basic\":\n" + @@ -228,9 +234,7 @@ public class TestSectionParserTests extends AbstractParserTestCase { assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10)); } - @Test public void testSmallSection() throws Exception { - parser = YamlXContent.yamlXContent.createParser( "\"node_info test\":\n" + " - do:\n" + diff --git a/core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test-framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java similarity index 99% rename from core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java rename to test-framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 4d167d761e9..1514e254a7f 100644 --- a/core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -94,7 +94,7 @@ public class InternalTestClusterTests extends ESTestCase { } } - public void testBeforeTest() throws IOException { + public void testBeforeTest() throws Exception { long clusterSeed = randomLong(); int minNumDataNodes = randomIntBetween(0, 3); int maxNumDataNodes = randomIntBetween(minNumDataNodes, 4); diff --git a/core/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java b/test-framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java similarity index 95% rename from core/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java rename to test-framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java index e4c5f2edaf3..bb0722365cf 100644 --- a/core/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java +++ b/test-framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.junit.listeners.LoggingListener; -import org.junit.Test; import org.junit.runner.Description; import org.junit.runner.Result; @@ -34,10 +33,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; public class LoggingListenerTests extends ESTestCase { - - @Test public void testCustomLevelPerMethod() throws Exception { - LoggingListener loggingListener = new LoggingListener(); Description suiteDescription = Description.createSuiteDescription(TestClass.class); @@ -51,7 +47,7 @@ public class LoggingListenerTests extends ESTestCase { assertThat(xyzLogger.getLevel(), nullValue()); assertThat(abcLogger.getLevel(), nullValue()); - Method method = TestClass.class.getDeclaredMethod("annotatedTestMethod"); + Method method = TestClass.class.getMethod("annotatedTestMethod"); TestLogging annotation = method.getAnnotation(TestLogging.class); Description testDescription = Description.createTestDescription(LoggingListenerTests.class, "annotatedTestMethod", annotation); loggingListener.testStarted(testDescription); @@ -67,9 +63,7 @@ public class LoggingListenerTests extends ESTestCase { assertThat(abcLogger.getLevel(), nullValue()); } - @Test public void testCustomLevelPerClass() throws Exception { - LoggingListener loggingListener = new LoggingListener(); Description suiteDescription = Description.createSuiteDescription(AnnotatedTestClass.class); @@ -97,9 +91,7 @@ public class LoggingListenerTests extends ESTestCase { assertThat(xyzLogger.getLevel(), nullValue()); } - @Test public void testCustomLevelPerClassAndPerMethod() throws Exception { - LoggingListener loggingListener = new LoggingListener(); Description suiteDescription = Description.createSuiteDescription(AnnotatedTestClass.class); @@ -113,7 +105,7 @@ public class LoggingListenerTests extends ESTestCase { assertThat(abcLogger.getLevel(), equalTo("ERROR")); assertThat(xyzLogger.getLevel(), nullValue()); - Method method = TestClass.class.getDeclaredMethod("annotatedTestMethod"); + Method method = TestClass.class.getMethod("annotatedTestMethod"); TestLogging annotation = method.getAnnotation(TestLogging.class); Description testDescription = Description.createTestDescription(LoggingListenerTests.class, "annotatedTestMethod", annotation); loggingListener.testStarted(testDescription); @@ -124,7 +116,7 @@ public class LoggingListenerTests extends ESTestCase { assertThat(abcLogger.getLevel(), equalTo("ERROR")); assertThat(xyzLogger.getLevel(), nullValue()); - Method method2 = TestClass.class.getDeclaredMethod("annotatedTestMethod2"); + Method method2 = TestClass.class.getMethod("annotatedTestMethod2"); TestLogging annotation2 = method2.getAnnotation(TestLogging.class); Description testDescription2 = Description.createTestDescription(LoggingListenerTests.class, "annotatedTestMethod2", annotation2); loggingListener.testStarted(testDescription2); diff --git a/core/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java b/test-framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java rename to test-framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java diff --git a/core/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java b/test-framework/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java similarity index 98% rename from core/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java rename to test-framework/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java index 5233bae7c7f..8c3c18454cf 100644 --- a/core/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java +++ b/test-framework/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java @@ -20,7 +20,6 @@ package org.elasticsearch.test.test; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.TestCluster; -import org.junit.Test; import java.io.IOException; @@ -36,7 +35,6 @@ public class TestScopeClusterIT extends ESIntegTestCase { private static long[] SEQUENCE = new long[100]; private static Long CLUSTER_SEED = null; - @Test public void testReproducible() throws IOException { if (ITER++ == 0) { CLUSTER_SEED = cluster().seed(); diff --git a/core/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java b/test-framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java rename to test-framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java diff --git a/test-framework/src/test/resources/rest-api-spec/test/suite1/10_basic.yaml b/test-framework/src/test/resources/rest-api-spec/test/suite1/10_basic.yaml new file mode 100644 index 00000000000..0689f714d64 --- /dev/null +++ b/test-framework/src/test/resources/rest-api-spec/test/suite1/10_basic.yaml @@ -0,0 +1,31 @@ +--- +"Basic": + + - do: + index: + index: test_1 + type: test + id: 中文 + body: { "foo": "Hello: 中文" } + + - do: + get: + index: test_1 + type: test + id: 中文 + + - match: { _index: test_1 } + - match: { _type: test } + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } + + - do: + get: + index: test_1 + type: _all + id: 中文 + + - match: { _index: test_1 } + - match: { _type: test } + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } diff --git a/test-framework/src/test/resources/rest-api-spec/test/suite1/20_another_test.yaml b/test-framework/src/test/resources/rest-api-spec/test/suite1/20_another_test.yaml new file mode 100644 index 00000000000..5e08112253e --- /dev/null +++ b/test-framework/src/test/resources/rest-api-spec/test/suite1/20_another_test.yaml @@ -0,0 +1,21 @@ +--- +"Default values": + + - do: + index: + index: test_1 + type: test + id: 1 + body: { "foo": "bar" } + + - do: + get: + index: test_1 + type: _all + id: 1 + + - match: { _index: test_1 } + - match: { _type: test } + - match: { _id: '1' } + - match: { _source: { foo: "bar" } } + diff --git a/test-framework/src/test/resources/rest-api-spec/test/suite2/10_basic.yaml b/test-framework/src/test/resources/rest-api-spec/test/suite2/10_basic.yaml new file mode 100644 index 00000000000..745e1117402 --- /dev/null +++ b/test-framework/src/test/resources/rest-api-spec/test/suite2/10_basic.yaml @@ -0,0 +1,26 @@ +--- +"Index with ID": + + - do: + index: + index: test-weird-index-中文 + type: weird.type + id: 1 + body: { foo: bar } + + - match: { _index: test-weird-index-中文 } + - match: { _type: weird.type } + - match: { _id: "1"} + - match: { _version: 1} + + - do: + get: + index: test-weird-index-中文 + type: weird.type + id: 1 + + - match: { _index: test-weird-index-中文 } + - match: { _type: weird.type } + - match: { _id: "1"} + - match: { _version: 1} + - match: { _source: { foo: bar }} diff --git a/test-framework/src/test/resources/rest-api-spec/test/suite2/15_test2.yaml b/test-framework/src/test/resources/rest-api-spec/test/suite2/15_test2.yaml new file mode 100644 index 00000000000..3fff0512b96 --- /dev/null +++ b/test-framework/src/test/resources/rest-api-spec/test/suite2/15_test2.yaml @@ -0,0 +1,26 @@ +--- +"Index without ID": + + - do: + index: + index: test_1 + type: test + body: { foo: bar } + + - is_true: _id + - match: { _index: test_1 } + - match: { _type: test } + - match: { _version: 1 } + - set: { _id: id } + + - do: + get: + index: test_1 + type: test + id: '$id' + + - match: { _index: test_1 } + - match: { _type: test } + - match: { _id: $id } + - match: { _version: 1 } + - match: { _source: { foo: bar }}